servo_media_gstreamer_render_unix/
lib.rs

1//! `RenderUnix` is a `Render` implementation for Unix-based
2//! platforms. It implements an OpenGL mechanism shared by Linux and
3//! many of the BSD flavors.
4//!
5//! Internally it uses GStreamer's *glsinkbin* element as *videosink*
6//! wrapping the *appsink* from the Player. And the shared frames are
7//! mapped as texture IDs.
8
9use gst::prelude::*;
10use gst_gl::prelude::*;
11use sm_gst_render;
12use sm_gst_render::Render;
13use sm_player;
14use sm_player::context::{GlApi, GlContext, NativeDisplay, PlayerGLContext};
15use sm_player::video::{Buffer, VideoFrame, VideoFrameData};
16use sm_player::PlayerError;
17use std::sync::{Arc, Mutex};
18
19struct GStreamerBuffer {
20    is_external_oes: bool,
21    frame: gst_gl::GLVideoFrame<gst_gl::gl_video_frame::Readable>,
22}
23
24impl Buffer for GStreamerBuffer {
25    fn to_vec(&self) -> Result<VideoFrameData, ()> {
26        // packed formats are guaranteed to be in a single plane
27        if self.frame.format() == gst_video::VideoFormat::Rgba {
28            let tex_id = self.frame.texture_id(0).map_err(|_| ())?;
29            Ok(if self.is_external_oes {
30                VideoFrameData::OESTexture(tex_id)
31            } else {
32                VideoFrameData::Texture(tex_id)
33            })
34        } else {
35            Err(())
36        }
37    }
38}
39
40pub struct RenderUnix {
41    display: gst_gl::GLDisplay,
42    app_context: gst_gl::GLContext,
43    gst_context: Arc<Mutex<Option<gst_gl::GLContext>>>,
44    gl_upload: Arc<Mutex<Option<gst::Element>>>,
45}
46
47impl RenderUnix {
48    /// Tries to create a new intance of the `RenderUnix`
49    ///
50    /// # Arguments
51    ///
52    /// * `context` - is the PlayerContext trait object from
53    /// application.
54    pub fn new(app_gl_context: Box<dyn PlayerGLContext>) -> Option<RenderUnix> {
55        // Check that we actually have the elements that we
56        // need to make this work.
57        if gst::ElementFactory::find("glsinkbin").is_none() {
58            return None;
59        }
60
61        let display_native = app_gl_context.get_native_display();
62        let gl_context = app_gl_context.get_gl_context();
63        let gl_api = match app_gl_context.get_gl_api() {
64            GlApi::OpenGL => gst_gl::GLAPI::OPENGL,
65            GlApi::OpenGL3 => gst_gl::GLAPI::OPENGL3,
66            GlApi::Gles1 => gst_gl::GLAPI::GLES1,
67            GlApi::Gles2 => gst_gl::GLAPI::GLES2,
68            GlApi::None => return None,
69        };
70
71        let (wrapped_context, display) = match gl_context {
72            GlContext::Egl(context) => {
73                let display = match display_native {
74                    #[cfg(feature = "gl-egl")]
75                    NativeDisplay::Egl(display_native) => {
76                        unsafe { gstreamer_gl_egl::GLDisplayEGL::with_egl_display(display_native) }
77                            .map(|display| display.upcast())
78                            .ok()
79                    }
80                    #[cfg(feature = "gl-wayland")]
81                    NativeDisplay::Wayland(display_native) => unsafe {
82                        gstreamer_gl_wayland::GLDisplayWayland::with_display(display_native)
83                    }
84                    .map(|display| display.upcast())
85                    .ok(),
86                    _ => None,
87                };
88
89                RenderUnix::create_wrapped_context(
90                    display,
91                    context,
92                    gst_gl::GLPlatform::EGL,
93                    gl_api,
94                )
95            }
96            GlContext::Glx(context) => {
97                let display = match display_native {
98                    #[cfg(feature = "gl-x11")]
99                    NativeDisplay::X11(display_native) => {
100                        unsafe { gstreamer_gl_x11::GLDisplayX11::with_display(display_native) }
101                            .map(|display| display.upcast())
102                            .ok()
103                    }
104                    _ => None,
105                };
106
107                RenderUnix::create_wrapped_context(
108                    display,
109                    context,
110                    gst_gl::GLPlatform::GLX,
111                    gl_api,
112                )
113            }
114            GlContext::Unknown => (None, None),
115        };
116
117        if let Some(app_context) = wrapped_context {
118            let cat = gst::DebugCategory::get("servoplayer").unwrap();
119            let _: Result<(), ()> = app_context
120                .activate(true)
121                .and_then(|_| {
122                    app_context.fill_info().or_else(|err| {
123                        gst::warning!(
124                            cat,
125                            "Couldn't fill the wrapped app GL context: {}",
126                            err.to_string()
127                        );
128                        Ok(())
129                    })
130                })
131                .or_else(|_| {
132                    gst::warning!(cat, "Couldn't activate the wrapped app GL context");
133                    Ok(())
134                });
135            Some(RenderUnix {
136                display: display.unwrap(),
137                app_context,
138                gst_context: Arc::new(Mutex::new(None)),
139                gl_upload: Arc::new(Mutex::new(None)),
140            })
141        } else {
142            None
143        }
144    }
145
146    fn create_wrapped_context(
147        display: Option<gst_gl::GLDisplay>,
148        handle: usize,
149        platform: gst_gl::GLPlatform,
150        api: gst_gl::GLAPI,
151    ) -> (Option<gst_gl::GLContext>, Option<gst_gl::GLDisplay>) {
152        if let Some(display) = display {
153            let wrapped_context =
154                unsafe { gst_gl::GLContext::new_wrapped(&display, handle, platform, api) };
155            (wrapped_context, Some(display))
156        } else {
157            (None, None)
158        }
159    }
160}
161
162impl Render for RenderUnix {
163    fn is_gl(&self) -> bool {
164        true
165    }
166
167    fn build_frame(&self, sample: gst::Sample) -> Result<VideoFrame, ()> {
168        if self.gst_context.lock().unwrap().is_none() && self.gl_upload.lock().unwrap().is_some() {
169            *self.gst_context.lock().unwrap() =
170                if let Some(glupload) = self.gl_upload.lock().unwrap().as_ref() {
171                    Some(glupload.property::<gst_gl::GLContext>("context"))
172                } else {
173                    None
174                };
175        }
176
177        let buffer = sample.buffer_owned().ok_or(())?;
178        let caps = sample.caps().ok_or(())?;
179
180        let is_external_oes = caps
181            .structure(0)
182            .and_then(|s| {
183                s.get::<&str>("texture-target").ok().and_then(|target| {
184                    if target == "external-oes" {
185                        Some(s)
186                    } else {
187                        None
188                    }
189                })
190            })
191            .is_some();
192
193        let info = gst_video::VideoInfo::from_caps(caps).map_err(|_| ())?;
194        let frame = gst_gl::GLVideoFrame::from_buffer_readable(buffer, &info).map_err(|_| ())?;
195
196        VideoFrame::new(
197            info.width() as i32,
198            info.height() as i32,
199            Arc::new(GStreamerBuffer {
200                is_external_oes,
201                frame,
202            }),
203        )
204    }
205
206    fn build_video_sink(
207        &self,
208        appsink: &gst::Element,
209        pipeline: &gst::Element,
210    ) -> Result<(), PlayerError> {
211        if self.gl_upload.lock().unwrap().is_some() {
212            return Err(PlayerError::Backend(
213                "render unix already setup the video sink".to_owned(),
214            ));
215        }
216
217        let vsinkbin = gst::ElementFactory::make("glsinkbin")
218            .name("servo-media-vsink")
219            .build()
220            .map_err(|error| {
221                PlayerError::Backend(format!("glupload creation failed: {error:?}"))
222            })?;
223
224        let caps = gst::Caps::builder("video/x-raw")
225            .features([gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY])
226            .field("format", gst_video::VideoFormat::Rgba.to_str())
227            .field("texture-target", gst::List::new(["2D", "external-oes"]))
228            .build();
229        appsink.set_property("caps", caps);
230
231        vsinkbin.set_property("sink", appsink);
232
233        pipeline.set_property("video-sink", &vsinkbin);
234
235        let bus = pipeline.bus().expect("pipeline with no bus");
236        let display_ = self.display.clone();
237        let context_ = self.app_context.clone();
238        bus.set_sync_handler(move |_, msg| {
239            match msg.view() {
240                gst::MessageView::NeedContext(ctxt) => {
241                    if let Some(el) = msg
242                        .src()
243                        .map(|s| s.clone().downcast::<gst::Element>().unwrap())
244                    {
245                        let context_type = ctxt.context_type();
246                        if context_type == *gst_gl::GL_DISPLAY_CONTEXT_TYPE {
247                            let ctxt = gst::Context::new(context_type, true);
248                            ctxt.set_gl_display(&display_);
249                            el.set_context(&ctxt);
250                        } else if context_type == "gst.gl.app_context" {
251                            let mut ctxt = gst::Context::new(context_type, true);
252                            {
253                                let s = ctxt.get_mut().unwrap().structure_mut();
254                                s.set_value("context", context_.to_send_value());
255                            }
256                            el.set_context(&ctxt);
257                        }
258                    }
259                }
260                _ => (),
261            }
262
263            gst::BusSyncReply::Pass
264        });
265
266        let mut iter = vsinkbin
267            .dynamic_cast::<gst::Bin>()
268            .unwrap()
269            .iterate_elements();
270        *self.gl_upload.lock().unwrap() = loop {
271            match iter.next() {
272                Ok(Some(element)) => {
273                    if "glupload" == element.factory().unwrap().name() {
274                        break Some(element);
275                    }
276                }
277                Err(gst::IteratorError::Resync) => iter.resync(),
278                _ => break None,
279            }
280        };
281
282        Ok(())
283    }
284}
servo_media_gstreamer_render_unix/lib.rs

servo_media_gstreamer_render_unix/
lib.rs
