script/dom/html/embedded_content/media_elements/
media_frame_renderer.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::sync::{Arc, Mutex, RwLock, Weak};
6
7use euclid::default::Size2D;
8use layout_api::MediaFrame;
9use media::{GLPlayerMsg, GLPlayerMsgForward, WindowGLContext};
10use paint_api::{CrossProcessPaintApi, ImageUpdate, SerializableImageData};
11use pixels::RasterImage;
12use servo_base::generic_channel::{GenericCallback, GenericSharedMemory};
13use servo_base::id::WebViewId;
14use servo_media::player::video::{VideoFrame, VideoFrameRenderer};
15use webrender_api::{
16    ExternalImageData, ExternalImageId, ExternalImageType, ImageBufferKind, ImageDescriptor,
17    ImageDescriptorFlags, ImageFormat, ImageKey,
18};
19
20use crate::tasks::task_source::SendableTaskSource;
21
22#[derive(MallocSizeOf, PartialEq)]
23enum FrameStatus {
24    Locked,
25    Unlocked,
26}
27
28#[derive(MallocSizeOf)]
29pub(crate) struct FrameHolder(FrameStatus, VideoFrame);
30
31impl FrameHolder {
32    fn new(frame: VideoFrame) -> FrameHolder {
33        FrameHolder(FrameStatus::Unlocked, frame)
34    }
35
36    fn lock(&mut self) {
37        if self.0 == FrameStatus::Unlocked {
38            self.0 = FrameStatus::Locked;
39        };
40    }
41
42    fn unlock(&mut self) {
43        if self.0 == FrameStatus::Locked {
44            self.0 = FrameStatus::Unlocked;
45        };
46    }
47
48    fn set(&mut self, new_frame: VideoFrame) {
49        if self.0 == FrameStatus::Unlocked {
50            self.1 = new_frame
51        };
52    }
53
54    fn get(&self) -> (u32, Size2D<i32>, usize) {
55        if self.0 == FrameStatus::Locked {
56            (
57                self.1.get_texture_id(),
58                Size2D::new(self.1.get_width(), self.1.get_height()),
59                0,
60            )
61        } else {
62            unreachable!();
63        }
64    }
65
66    pub(crate) fn get_frame(&self) -> VideoFrame {
67        self.1.clone()
68    }
69}
70
71#[derive(MallocSizeOf)]
72pub(crate) struct MediaFrameRenderer {
73    webview_id: WebViewId,
74    player_id: Option<usize>,
75    #[conditional_malloc_size_of]
76    glplayer_id: Arc<RwLock<Option<u64>>>,
77    paint_api: CrossProcessPaintApi,
78    player_context: WindowGLContext,
79    current_frame: Option<MediaFrame>,
80    old_frame: Option<ImageKey>,
81    very_old_frame: Option<ImageKey>,
82    current_frame_holder: Option<FrameHolder>,
83    /// <https://html.spec.whatwg.org/multipage/#poster-frame>
84    poster_frame: Option<MediaFrame>,
85}
86
87impl MediaFrameRenderer {
88    pub(crate) fn new(
89        webview_id: WebViewId,
90        paint_api: CrossProcessPaintApi,
91        player_context: WindowGLContext,
92    ) -> Self {
93        Self {
94            webview_id,
95            player_id: None,
96            glplayer_id: Arc::new(RwLock::new(None)),
97            paint_api,
98            player_context,
99            current_frame: None,
100            old_frame: None,
101            very_old_frame: None,
102            current_frame_holder: None,
103            poster_frame: None,
104        }
105    }
106
107    pub(crate) fn current_frame(&self) -> Option<MediaFrame> {
108        self.current_frame
109    }
110
111    pub(crate) fn poster_frame(&self) -> Option<MediaFrame> {
112        self.poster_frame
113    }
114
115    pub(crate) fn current_frame_holder(&self) -> Option<&FrameHolder> {
116        self.current_frame_holder.as_ref()
117    }
118
119    pub(crate) fn setup(
120        &mut self,
121        player_id: usize,
122        task_source: SendableTaskSource,
123        weak_video_renderer: Weak<Mutex<MediaFrameRenderer>>,
124    ) {
125        self.player_id = Some(player_id);
126        let glplayer_id = self.glplayer_id.clone();
127        let callback = GenericCallback::new(move |message| {
128            let message = message.unwrap();
129            let weak_video_renderer = weak_video_renderer.clone();
130
131            let glplayer_id = glplayer_id.clone();
132            task_source.queue(task!(handle_glplayer_message: move || {
133                trace!("GLPlayer message {:?}", message);
134
135                let Some(video_renderer) = weak_video_renderer.upgrade() else {
136                    return;
137                };
138
139                match message {
140                    GLPlayerMsgForward::Lock(sender) => {
141                        if let Some(holder) = video_renderer
142                            .lock()
143                            .unwrap()
144                            .current_frame_holder
145                            .as_mut() {
146                                holder.lock();
147                                sender.send(holder.get()).unwrap();
148                            };
149                    },
150                    GLPlayerMsgForward::Unlock() => {
151                        if let Some(holder) = video_renderer
152                            .lock()
153                            .unwrap()
154                            .current_frame_holder
155                            .as_mut() { holder.unlock() }
156                    },
157                    GLPlayerMsgForward::PlayerId(id) => {
158                        let mut glplayer_id = glplayer_id.write().unwrap();
159                        if let Some(already_set_id) = *glplayer_id {
160                            error!("Player id already set to {already_set_id} will be replaced with {id}");
161                        }
162                        *glplayer_id = Some(id);
163                    },
164                }
165            }));
166        })
167        .expect("Could not create callback");
168
169        if let Some(glplayer_thread_sender) = &self.player_context.glplayer_thread_sender {
170            glplayer_thread_sender
171                .send(GLPlayerMsg::RegisterPlayer(callback))
172                .unwrap();
173        }
174    }
175
176    pub(crate) fn reset(&mut self) {
177        self.player_id = None;
178
179        if let Some(glplayer_id) = self.glplayer_id.write().unwrap().take() {
180            self.player_context
181                .send(GLPlayerMsg::UnregisterPlayer(glplayer_id));
182        }
183
184        self.current_frame_holder = None;
185
186        let mut updates = smallvec::smallvec![];
187
188        if let Some(current_frame) = self.current_frame.take() {
189            updates.push(ImageUpdate::DeleteImage(current_frame.image_key));
190        }
191
192        if let Some(old_image_key) = self.old_frame.take() {
193            updates.push(ImageUpdate::DeleteImage(old_image_key));
194        }
195
196        if let Some(very_old_image_key) = self.very_old_frame.take() {
197            updates.push(ImageUpdate::DeleteImage(very_old_image_key));
198        }
199
200        if !updates.is_empty() {
201            self.paint_api
202                .update_images(self.webview_id.into(), updates);
203        }
204    }
205
206    pub(crate) fn set_poster_frame(&mut self, image: Option<Arc<RasterImage>>) {
207        self.poster_frame = image.and_then(|image| {
208            image.id.map(|image_key| MediaFrame {
209                image_key,
210                width: image.metadata.width as i32,
211                height: image.metadata.height as i32,
212            })
213        });
214    }
215}
216
217impl Drop for MediaFrameRenderer {
218    fn drop(&mut self) {
219        self.reset();
220    }
221}
222
223impl VideoFrameRenderer for MediaFrameRenderer {
224    fn render(&mut self, frame: VideoFrame) {
225        if self.player_id.is_none() ||
226            (frame.is_gl_texture() && self.glplayer_id.read().unwrap().is_none())
227        {
228            return;
229        }
230
231        let mut updates = smallvec::smallvec![];
232
233        if let Some(old_image_key) =
234            std::mem::replace(&mut self.very_old_frame, self.old_frame.take())
235        {
236            updates.push(ImageUpdate::DeleteImage(old_image_key));
237        }
238
239        let descriptor = ImageDescriptor::new(
240            frame.get_width(),
241            frame.get_height(),
242            ImageFormat::BGRA8,
243            ImageDescriptorFlags::empty(),
244        );
245
246        match &mut self.current_frame {
247            Some(current_frame)
248                if current_frame.width == frame.get_width() &&
249                    current_frame.height == frame.get_height() =>
250            {
251                if !frame.is_gl_texture() {
252                    updates.push(ImageUpdate::UpdateImage(
253                        current_frame.image_key,
254                        descriptor,
255                        SerializableImageData::Raw(GenericSharedMemory::from_arc_vec(
256                            frame.get_data(),
257                        )),
258                        None,
259                    ));
260                }
261
262                self.current_frame_holder
263                    .get_or_insert_with(|| FrameHolder::new(frame.clone()))
264                    .set(frame);
265
266                if let Some(old_image_key) = self.old_frame.take() {
267                    updates.push(ImageUpdate::DeleteImage(old_image_key));
268                }
269            },
270            Some(current_frame) => {
271                self.old_frame = Some(current_frame.image_key);
272
273                let Some(new_image_key) =
274                    self.paint_api.generate_image_key_blocking(self.webview_id)
275                else {
276                    return;
277                };
278
279                /* update current_frame */
280                current_frame.image_key = new_image_key;
281                current_frame.width = frame.get_width();
282                current_frame.height = frame.get_height();
283
284                // FIXME: This code is duplicated below this branch
285                let image_data = self
286                    .glplayer_id
287                    .read()
288                    .unwrap()
289                    .filter(|_| frame.is_gl_texture())
290                    .map(|glplayer_id| {
291                        let texture_target = if frame.is_external_oes() {
292                            ImageBufferKind::TextureExternal
293                        } else {
294                            ImageBufferKind::Texture2D
295                        };
296
297                        SerializableImageData::External(ExternalImageData {
298                            id: ExternalImageId(glplayer_id),
299                            channel_index: 0,
300                            image_type: ExternalImageType::TextureHandle(texture_target),
301                            normalized_uvs: false,
302                        })
303                    })
304                    .unwrap_or_else(|| {
305                        SerializableImageData::Raw(GenericSharedMemory::from_arc_vec(
306                            frame.get_data(),
307                        ))
308                    });
309
310                self.current_frame_holder
311                    .get_or_insert_with(|| FrameHolder::new(frame.clone()))
312                    .set(frame);
313
314                updates.push(ImageUpdate::AddImage(
315                    new_image_key,
316                    descriptor,
317                    image_data,
318                    false,
319                ));
320            },
321            None => {
322                let Some(image_key) = self.paint_api.generate_image_key_blocking(self.webview_id)
323                else {
324                    return;
325                };
326
327                self.current_frame = Some(MediaFrame {
328                    image_key,
329                    width: frame.get_width(),
330                    height: frame.get_height(),
331                });
332
333                let image_data = self
334                    .glplayer_id
335                    .read()
336                    .unwrap()
337                    .filter(|_| frame.is_gl_texture())
338                    .map(|glplayer_id| {
339                        let texture_target = if frame.is_external_oes() {
340                            ImageBufferKind::TextureExternal
341                        } else {
342                            ImageBufferKind::Texture2D
343                        };
344
345                        SerializableImageData::External(ExternalImageData {
346                            id: ExternalImageId(glplayer_id),
347                            channel_index: 0,
348                            image_type: ExternalImageType::TextureHandle(texture_target),
349                            normalized_uvs: false,
350                        })
351                    })
352                    .unwrap_or_else(|| {
353                        SerializableImageData::Raw(GenericSharedMemory::from_arc_vec(
354                            frame.get_data(),
355                        ))
356                    });
357
358                self.current_frame_holder = Some(FrameHolder::new(frame));
359
360                updates.push(ImageUpdate::AddImage(
361                    image_key, descriptor, image_data, false,
362                ));
363            },
364        }
365        self.paint_api
366            .update_images(self.webview_id.into(), updates);
367    }
368}
script/dom/html/embedded_content/media_elements/media_frame_renderer.rs

script/dom/html/embedded_content/media_elements/
media_frame_renderer.rs
