servo_media_gstreamer_render_unix/
lib.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5//! `RenderUnix` is a `Render` implementation for Unix-based
6//! platforms. It implements an OpenGL mechanism shared by Linux and
7//! many of the BSD flavors.
8//!
9//! Internally it uses GStreamer's *glsinkbin* element as *videosink*
10//! wrapping the *appsink* from the Player. And the shared frames are
11//! mapped as texture IDs.
12
13use std::sync::{Arc, Mutex};
14
15use gstreamer_gl::prelude::*;
16use servo_media_gstreamer_render::Render;
17use servo_media_player::PlayerError;
18use servo_media_player::context::{GlApi, GlContext, NativeDisplay, PlayerGLContext};
19use servo_media_player::video::{Buffer, VideoFrame, VideoFrameData};
20
21struct GStreamerBuffer {
22    is_external_oes: bool,
23    frame: gstreamer_gl::GLVideoFrame<gstreamer_gl::gl_video_frame::Readable>,
24}
25
26impl Buffer for GStreamerBuffer {
27    fn to_vec(&self) -> Option<VideoFrameData> {
28        // packed formats are guaranteed to be in a single plane
29        if self.frame.format() == gstreamer_video::VideoFormat::Rgba {
30            let tex_id = self.frame.texture_id(0).ok()?;
31            Some(if self.is_external_oes {
32                VideoFrameData::OESTexture(tex_id)
33            } else {
34                VideoFrameData::Texture(tex_id)
35            })
36        } else {
37            None
38        }
39    }
40}
41
42pub struct RenderUnix {
43    display: gstreamer_gl::GLDisplay,
44    app_context: gstreamer_gl::GLContext,
45    gst_context: Arc<Mutex<Option<gstreamer_gl::GLContext>>>,
46    gl_upload: Arc<Mutex<Option<gstreamer::Element>>>,
47}
48
49impl RenderUnix {
50    /// Tries to create a new intance of the `RenderUnix`
51    ///
52    /// # Arguments
53    ///
54    /// * `context` - is the PlayerContext trait object from application.
55    pub fn new(app_gl_context: Box<dyn PlayerGLContext>) -> Option<RenderUnix> {
56        // Check that we actually have the elements that we
57        // need to make this work.
58        gstreamer::ElementFactory::find("glsinkbin")?;
59
60        let display_native = app_gl_context.get_native_display();
61        let gl_context = app_gl_context.get_gl_context();
62        let gl_api = match app_gl_context.get_gl_api() {
63            GlApi::OpenGL => gstreamer_gl::GLAPI::OPENGL,
64            GlApi::OpenGL3 => gstreamer_gl::GLAPI::OPENGL3,
65            GlApi::Gles1 => gstreamer_gl::GLAPI::GLES1,
66            GlApi::Gles2 => gstreamer_gl::GLAPI::GLES2,
67            GlApi::None => return None,
68        };
69
70        let (wrapped_context, display) = match gl_context {
71            GlContext::Egl(context) => {
72                let display = match display_native {
73                    #[cfg(feature = "gl-egl")]
74                    NativeDisplay::Egl(display_native) => {
75                        unsafe { gstreamer_gl_egl::GLDisplayEGL::with_egl_display(display_native) }
76                            .map(|display| display.upcast())
77                            .ok()
78                    },
79                    #[cfg(feature = "gl-wayland")]
80                    NativeDisplay::Wayland(display_native) => unsafe {
81                        gstreamer_gl_wayland::GLDisplayWayland::with_display(display_native)
82                    }
83                    .map(|display| display.upcast())
84                    .ok(),
85                    _ => None,
86                };
87
88                RenderUnix::create_wrapped_context(
89                    display,
90                    context,
91                    gstreamer_gl::GLPlatform::EGL,
92                    gl_api,
93                )
94            },
95            GlContext::Glx(context) => {
96                let display = match display_native {
97                    #[cfg(feature = "gl-x11")]
98                    NativeDisplay::X11(display_native) => {
99                        unsafe { gstreamer_gl_x11::GLDisplayX11::with_display(display_native) }
100                            .map(|display| display.upcast())
101                            .ok()
102                    },
103                    _ => None,
104                };
105
106                RenderUnix::create_wrapped_context(
107                    display,
108                    context,
109                    gstreamer_gl::GLPlatform::GLX,
110                    gl_api,
111                )
112            },
113            GlContext::Unknown => (None, None),
114        };
115
116        match wrapped_context {
117            Some(app_context) => {
118                let cat = gstreamer::DebugCategory::get("servoplayer").unwrap();
119                let _: Result<(), ()> = app_context
120                    .activate(true)
121                    .and_then(|_| {
122                        app_context.fill_info().or_else(|err| {
123                            gstreamer::warning!(
124                                cat,
125                                "Couldn't fill the wrapped app GL context: {}",
126                                err.to_string()
127                            );
128                            Ok(())
129                        })
130                    })
131                    .or_else(|_| {
132                        gstreamer::warning!(cat, "Couldn't activate the wrapped app GL context");
133                        Ok(())
134                    });
135                Some(RenderUnix {
136                    display: display.unwrap(),
137                    app_context,
138                    gst_context: Arc::new(Mutex::new(None)),
139                    gl_upload: Arc::new(Mutex::new(None)),
140                })
141            },
142            _ => None,
143        }
144    }
145
146    fn create_wrapped_context(
147        display: Option<gstreamer_gl::GLDisplay>,
148        handle: usize,
149        platform: gstreamer_gl::GLPlatform,
150        api: gstreamer_gl::GLAPI,
151    ) -> (
152        Option<gstreamer_gl::GLContext>,
153        Option<gstreamer_gl::GLDisplay>,
154    ) {
155        match display {
156            Some(display) => {
157                let wrapped_context = unsafe {
158                    gstreamer_gl::GLContext::new_wrapped(&display, handle, platform, api)
159                };
160                (wrapped_context, Some(display))
161            },
162            _ => (None, None),
163        }
164    }
165}
166
167impl Render for RenderUnix {
168    fn is_gl(&self) -> bool {
169        true
170    }
171
172    fn build_frame(&self, sample: gstreamer::Sample) -> Option<VideoFrame> {
173        if self.gst_context.lock().unwrap().is_none() && self.gl_upload.lock().unwrap().is_some() {
174            *self.gst_context.lock().unwrap() = self
175                .gl_upload
176                .lock()
177                .unwrap()
178                .as_ref()
179                .map(|glupload| glupload.property::<gstreamer_gl::GLContext>("context"));
180        }
181
182        let buffer = sample.buffer_owned()?;
183        let caps = sample.caps()?;
184
185        let is_external_oes = caps
186            .structure(0)
187            .and_then(|s| {
188                s.get::<&str>("texture-target").ok().and_then(|target| {
189                    if target == "external-oes" {
190                        Some(s)
191                    } else {
192                        None
193                    }
194                })
195            })
196            .is_some();
197
198        let info = gstreamer_video::VideoInfo::from_caps(caps).ok()?;
199        let frame = gstreamer_gl::GLVideoFrame::from_buffer_readable(buffer, &info).ok()?;
200        VideoFrame::new(
201            info.width() as i32,
202            info.height() as i32,
203            Arc::new(GStreamerBuffer {
204                is_external_oes,
205                frame,
206            }),
207        )
208    }
209
210    fn build_video_sink(
211        &self,
212        appsink: &gstreamer::Element,
213        pipeline: &gstreamer::Element,
214    ) -> Result<(), PlayerError> {
215        if self.gl_upload.lock().unwrap().is_some() {
216            return Err(PlayerError::Backend(
217                "render unix already setup the video sink".to_owned(),
218            ));
219        }
220
221        let vsinkbin = gstreamer::ElementFactory::make("glsinkbin")
222            .name("servo-media-vsink")
223            .build()
224            .map_err(|error| {
225                PlayerError::Backend(format!("glupload creation failed: {error:?}"))
226            })?;
227
228        let caps = gstreamer::Caps::builder("video/x-raw")
229            .features([gstreamer_gl::CAPS_FEATURE_MEMORY_GL_MEMORY])
230            .field("format", gstreamer_video::VideoFormat::Rgba.to_str())
231            .field(
232                "texture-target",
233                gstreamer::List::new(["2D", "external-oes"]),
234            )
235            .build();
236        appsink.set_property("caps", caps);
237
238        vsinkbin.set_property("sink", appsink);
239
240        pipeline.set_property("video-sink", &vsinkbin);
241
242        let bus = pipeline.bus().expect("pipeline with no bus");
243        let display_ = self.display.clone();
244        let context_ = self.app_context.clone();
245        bus.set_sync_handler(move |_, msg| {
246            if let gstreamer::MessageView::NeedContext(ctxt) = msg.view() &&
247                let Some(el) = msg
248                    .src()
249                    .map(|s| s.clone().downcast::<gstreamer::Element>().unwrap())
250            {
251                let context_type = ctxt.context_type();
252                if context_type == *gstreamer_gl::GL_DISPLAY_CONTEXT_TYPE {
253                    let ctxt = gstreamer::Context::new(context_type, true);
254                    ctxt.set_gl_display(&display_);
255                    el.set_context(&ctxt);
256                } else if context_type == "gst.gl.app_context" {
257                    let mut ctxt = gstreamer::Context::new(context_type, true);
258                    {
259                        let s = ctxt.get_mut().unwrap().structure_mut();
260                        s.set_value("context", context_.to_send_value());
261                    }
262                    el.set_context(&ctxt);
263                }
264            }
265
266            gstreamer::BusSyncReply::Pass
267        });
268
269        let mut iter = vsinkbin
270            .dynamic_cast::<gstreamer::Bin>()
271            .unwrap()
272            .iterate_elements();
273        *self.gl_upload.lock().unwrap() = loop {
274            match iter.next() {
275                Ok(Some(element)) => {
276                    if "glupload" == element.factory().unwrap().name() {
277                        break Some(element);
278                    }
279                },
280                Err(gstreamer::IteratorError::Resync) => iter.resync(),
281                _ => break None,
282            }
283        };
284
285        Ok(())
286    }
287}
servo_media_gstreamer_render_unix/lib.rs

servo_media_gstreamer_render_unix/
lib.rs
