pixels/
snapshot.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::ops::{Bound, Deref, DerefMut, Range, RangeBounds};
6use std::sync::Arc;
7
8use euclid::default::{Rect, Size2D};
9use image::codecs::jpeg::JpegEncoder;
10use image::codecs::png::PngEncoder;
11use image::codecs::webp::WebPEncoder;
12use image::{ExtendedColorType, GenericImageView, ImageEncoder, ImageError, Rgb};
13use ipc_channel::ipc::IpcSharedMemory;
14use malloc_size_of_derive::MallocSizeOf;
15use serde::{Deserialize, Serialize};
16
17use crate::{EncodedImageType, Multiply, rgba8_get_rect, transform_inplace};
18
19#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, MallocSizeOf, PartialEq, Serialize)]
20pub enum SnapshotPixelFormat {
21    #[default]
22    RGBA,
23    BGRA,
24}
25
26#[derive(Clone, Copy, Debug, Deserialize, Eq, MallocSizeOf, PartialEq, Serialize)]
27pub enum Alpha {
28    Premultiplied,
29    NotPremultiplied,
30    /// This is used for opaque textures for which the presence of alpha in the
31    /// output data format does not matter.
32    DontCare,
33}
34
35impl Alpha {
36    pub const fn from_premultiplied(is_premultiplied: bool) -> Self {
37        if is_premultiplied {
38            Self::Premultiplied
39        } else {
40            Self::NotPremultiplied
41        }
42    }
43
44    pub const fn needs_alpha_multiplication(&self) -> bool {
45        match self {
46            Alpha::Premultiplied => false,
47            Alpha::NotPremultiplied => true,
48            Alpha::DontCare => false,
49        }
50    }
51}
52
53#[derive(Clone, Copy, Debug, Deserialize, Eq, MallocSizeOf, PartialEq, Serialize)]
54pub enum SnapshotAlphaMode {
55    /// Internal data is opaque (alpha is cleared to 1)
56    Opaque,
57    /// Internal data should be threated as opaque (does not mean it actually is)
58    AsOpaque { premultiplied: bool },
59    /// Data is not opaque
60    Transparent { premultiplied: bool },
61}
62
63impl Default for SnapshotAlphaMode {
64    fn default() -> Self {
65        Self::Transparent {
66            premultiplied: true,
67        }
68    }
69}
70
71impl SnapshotAlphaMode {
72    pub const fn alpha(&self) -> Alpha {
73        match self {
74            SnapshotAlphaMode::Opaque => Alpha::DontCare,
75            SnapshotAlphaMode::AsOpaque { premultiplied } => {
76                Alpha::from_premultiplied(*premultiplied)
77            },
78            SnapshotAlphaMode::Transparent { premultiplied } => {
79                Alpha::from_premultiplied(*premultiplied)
80            },
81        }
82    }
83}
84
85/// The data in a [`Snapshot`]. If created via shared memory, this will be
86/// the `SharedMemory` variant, but otherwise it is the `Owned` variant.
87/// any attempt to mutate the [`Snapshot`] will convert it to the `Owned`
88/// variant.
89#[derive(Clone, Debug, MallocSizeOf)]
90pub enum SnapshotData {
91    SharedMemory(
92        #[conditional_malloc_size_of] Arc<IpcSharedMemory>,
93        Range<usize>,
94    ),
95    Owned(Vec<u8>),
96}
97
98impl SnapshotData {
99    fn to_vec(&self) -> Vec<u8> {
100        match &self {
101            SnapshotData::SharedMemory(data, byte_range) => Vec::from(&data[byte_range.clone()]),
102            SnapshotData::Owned(data) => data.clone(),
103        }
104    }
105}
106
107impl DerefMut for SnapshotData {
108    fn deref_mut(&mut self) -> &mut Self::Target {
109        match self {
110            SnapshotData::SharedMemory(..) => {
111                *self = SnapshotData::Owned(self.to_vec());
112                &mut *self
113            },
114            SnapshotData::Owned(items) => items,
115        }
116    }
117}
118
119impl Deref for SnapshotData {
120    type Target = [u8];
121
122    fn deref(&self) -> &Self::Target {
123        match &self {
124            SnapshotData::SharedMemory(data, byte_range) => &data[byte_range.clone()],
125            SnapshotData::Owned(items) => items,
126        }
127    }
128}
129
130/// Represents image bitmap with metadata, usually as snapshot of canvas
131///
132/// This allows us to hold off conversions (BGRA <-> RGBA, (un)premultiply)
133/// to when/if they are actually needed (WebGL/WebGPU can load both BGRA and RGBA).
134///
135/// Inspired by snapshot for concept in WebGPU spec:
136/// <https://gpuweb.github.io/gpuweb/#abstract-opdef-get-a-copy-of-the-image-contents-of-a-context>
137#[derive(Clone, Debug, MallocSizeOf)]
138pub struct Snapshot {
139    size: Size2D<u32>,
140    /// internal data (can be any format it will be converted on use if needed)
141    data: SnapshotData,
142    /// RGBA/BGRA (reflect internal data)
143    format: SnapshotPixelFormat,
144    /// How to treat alpha channel
145    alpha_mode: SnapshotAlphaMode,
146}
147
148impl Snapshot {
149    pub const fn size(&self) -> Size2D<u32> {
150        self.size
151    }
152
153    pub const fn format(&self) -> SnapshotPixelFormat {
154        self.format
155    }
156
157    pub const fn alpha_mode(&self) -> SnapshotAlphaMode {
158        self.alpha_mode
159    }
160
161    pub fn empty() -> Self {
162        Self {
163            size: Size2D::zero(),
164            data: SnapshotData::Owned(vec![]),
165            format: SnapshotPixelFormat::RGBA,
166            alpha_mode: SnapshotAlphaMode::Transparent {
167                premultiplied: true,
168            },
169        }
170    }
171
172    /// Returns snapshot with provided size that is black transparent alpha
173    pub fn cleared(size: Size2D<u32>) -> Self {
174        Self {
175            size,
176            data: SnapshotData::Owned(vec![0; size.area() as usize * 4]),
177            format: SnapshotPixelFormat::RGBA,
178            alpha_mode: SnapshotAlphaMode::Transparent {
179                premultiplied: true,
180            },
181        }
182    }
183
184    pub fn from_vec(
185        size: Size2D<u32>,
186        format: SnapshotPixelFormat,
187        alpha_mode: SnapshotAlphaMode,
188        data: Vec<u8>,
189    ) -> Self {
190        Self {
191            size,
192            data: SnapshotData::Owned(data),
193            format,
194            alpha_mode,
195        }
196    }
197
198    pub fn from_shared_memory(
199        size: Size2D<u32>,
200        format: SnapshotPixelFormat,
201        alpha_mode: SnapshotAlphaMode,
202        data: Arc<IpcSharedMemory>,
203        byte_range_bounds: impl RangeBounds<usize>,
204    ) -> Self {
205        let range_start = match byte_range_bounds.start_bound() {
206            Bound::Included(bound) => *bound,
207            Bound::Excluded(bound) => *bound + 1,
208            Bound::Unbounded => 0,
209        };
210        let range_end = match byte_range_bounds.end_bound() {
211            Bound::Included(bound) => *bound + 1,
212            Bound::Excluded(bound) => *bound,
213            Bound::Unbounded => data.len(),
214        };
215        Self {
216            size,
217            data: SnapshotData::SharedMemory(data, range_start..range_end),
218            format,
219            alpha_mode,
220        }
221    }
222
223    pub fn get_rect(&self, rect: Rect<u32>) -> Self {
224        let data = rgba8_get_rect(self.as_raw_bytes(), self.size(), rect).to_vec();
225        Self::from_vec(rect.size, self.format, self.alpha_mode, data)
226    }
227
228    /// Convert inner data of snapshot to target format and alpha mode.
229    /// If data is already in target format and alpha mode no work will be done.
230    pub fn transform(
231        &mut self,
232        target_alpha_mode: SnapshotAlphaMode,
233        target_format: SnapshotPixelFormat,
234    ) {
235        if self.alpha_mode == target_alpha_mode && target_format == self.format {
236            return;
237        }
238
239        let swap_rb = target_format != self.format;
240        let multiply = match (self.alpha_mode, target_alpha_mode) {
241            (SnapshotAlphaMode::Opaque, _) => Multiply::None,
242            (alpha_mode, SnapshotAlphaMode::Opaque)
243                if alpha_mode.alpha() == Alpha::Premultiplied =>
244            {
245                Multiply::UnMultiply
246            },
247            (_, SnapshotAlphaMode::Opaque) => Multiply::None,
248            (
249                SnapshotAlphaMode::Transparent { premultiplied } |
250                SnapshotAlphaMode::AsOpaque { premultiplied },
251                SnapshotAlphaMode::Transparent {
252                    premultiplied: target_premultiplied,
253                } |
254                SnapshotAlphaMode::AsOpaque {
255                    premultiplied: target_premultiplied,
256                },
257            ) => {
258                if premultiplied == target_premultiplied {
259                    Multiply::None
260                } else if target_premultiplied {
261                    Multiply::PreMultiply
262                } else {
263                    Multiply::UnMultiply
264                }
265            },
266        };
267
268        let clear_alpha = !matches!(self.alpha_mode, SnapshotAlphaMode::Opaque) &&
269            matches!(target_alpha_mode, SnapshotAlphaMode::Opaque);
270
271        if matches!(multiply, Multiply::None) && !swap_rb && !clear_alpha {
272            return;
273        }
274
275        transform_inplace(self.data.deref_mut(), multiply, swap_rb, clear_alpha);
276        self.alpha_mode = target_alpha_mode;
277        self.format = target_format;
278    }
279
280    pub fn as_raw_bytes(&self) -> &[u8] {
281        &self.data
282    }
283
284    pub fn as_raw_bytes_mut(&mut self) -> &mut [u8] {
285        &mut self.data
286    }
287
288    pub fn to_shared(&self) -> SharedSnapshot {
289        let (data, byte_range) = match &self.data {
290            SnapshotData::SharedMemory(data, byte_range) => (data.clone(), byte_range.clone()),
291            SnapshotData::Owned(data) => {
292                (Arc::new(IpcSharedMemory::from_bytes(data)), 0..data.len())
293            },
294        };
295        SharedSnapshot {
296            size: self.size,
297            data,
298            byte_range,
299            format: self.format,
300            alpha_mode: self.alpha_mode,
301        }
302    }
303
304    pub fn encode_for_mime_type<W: std::io::Write>(
305        &mut self,
306        image_type: &EncodedImageType,
307        quality: Option<f64>,
308        encoder: &mut W,
309    ) -> Result<(), ImageError> {
310        let width = self.size.width;
311        let height = self.size.height;
312        let alpha_mode = match image_type {
313            EncodedImageType::Jpeg => SnapshotAlphaMode::AsOpaque {
314                premultiplied: true,
315            },
316            _ => SnapshotAlphaMode::Transparent {
317                premultiplied: false,
318            },
319        };
320
321        self.transform(alpha_mode, SnapshotPixelFormat::RGBA);
322        let data = &self.data;
323
324        match image_type {
325            EncodedImageType::Png => {
326                // FIXME(nox): https://github.com/image-rs/image-png/issues/86
327                // FIXME(nox): https://github.com/image-rs/image-png/issues/87
328                PngEncoder::new(encoder).write_image(data, width, height, ExtendedColorType::Rgba8)
329            },
330            EncodedImageType::Jpeg => {
331                let mut jpeg_encoder = if let Some(quality) = quality {
332                    // The specification allows quality to be in [0.0..1.0] but the JPEG encoder
333                    // expects it to be in [1..100]
334                    if (0.0..=1.0).contains(&quality) {
335                        JpegEncoder::new_with_quality(
336                            encoder,
337                            (quality * 100.0).round().clamp(1.0, 100.0) as u8,
338                        )
339                    } else {
340                        JpegEncoder::new(encoder)
341                    }
342                } else {
343                    JpegEncoder::new(encoder)
344                };
345
346                // JPEG doesn't support transparency, so simply calling jpeg_encoder.write_image fails here.
347                // Instead we have to create a struct to translate from rgba to rgb.
348                struct RgbaDataForJpegEncoder<'a> {
349                    width: u32,
350                    height: u32,
351                    data: &'a [u8],
352                }
353
354                impl<'a> GenericImageView for RgbaDataForJpegEncoder<'a> {
355                    type Pixel = Rgb<u8>;
356
357                    fn dimensions(&self) -> (u32, u32) {
358                        (self.width, self.height)
359                    }
360
361                    fn get_pixel(&self, x: u32, y: u32) -> Self::Pixel {
362                        let offset = (self.width * y + x) as usize * 4;
363                        Rgb([
364                            self.data[offset],
365                            self.data[offset + 1],
366                            self.data[offset + 2],
367                        ])
368                    }
369                }
370
371                let image = RgbaDataForJpegEncoder {
372                    width,
373                    height,
374                    data,
375                };
376
377                jpeg_encoder.encode_image(&image)
378            },
379            EncodedImageType::Webp => {
380                // No quality support because of https://github.com/image-rs/image/issues/1984
381                WebPEncoder::new_lossless(encoder).write_image(
382                    data,
383                    width,
384                    height,
385                    ExtendedColorType::Rgba8,
386                )
387            },
388        }
389    }
390}
391
392impl From<Snapshot> for Vec<u8> {
393    fn from(value: Snapshot) -> Self {
394        match value.data {
395            SnapshotData::SharedMemory(..) => Vec::from(value.as_raw_bytes()),
396            SnapshotData::Owned(data) => data,
397        }
398    }
399}
400
401/// A version of [`Snapshot`] that can be sent across IPC channels.
402#[derive(Clone, Debug, Deserialize, MallocSizeOf, Serialize)]
403pub struct SharedSnapshot {
404    /// The physical size of this [`SharedSnapshot`].
405    size: Size2D<u32>,
406    /// The shared data of this [`SharedSnapshot`].
407    #[conditional_malloc_size_of]
408    data: Arc<IpcSharedMemory>,
409    /// The byte range of the data within the shared memory segment. This is used to
410    /// send individual image frames of animated images.
411    byte_range: Range<usize>,
412    /// The [`SnapshotPixelFormat`] of this [`SharedSnapshot`]
413    format: SnapshotPixelFormat,
414    /// The [`SnapshotAlphaMode`] of this [`SharedSnapshot`].
415    alpha_mode: SnapshotAlphaMode,
416}
417
418impl SharedSnapshot {
419    pub fn to_owned(&self) -> Snapshot {
420        Snapshot {
421            size: self.size,
422            data: SnapshotData::SharedMemory(self.data.clone(), self.byte_range.clone()),
423            format: self.format,
424            alpha_mode: self.alpha_mode,
425        }
426    }
427
428    /// Returns snapshot with provided size that is black transparent alpha
429    pub fn cleared(size: Size2D<u32>) -> Self {
430        let length_in_bytes = size.area() as usize * 4;
431        Self {
432            size,
433            data: Arc::new(IpcSharedMemory::from_byte(0, length_in_bytes)),
434            byte_range: 0..length_in_bytes,
435            format: SnapshotPixelFormat::RGBA,
436            alpha_mode: SnapshotAlphaMode::Transparent {
437                premultiplied: true,
438            },
439        }
440    }
441
442    pub const fn size(&self) -> Size2D<u32> {
443        self.size
444    }
445
446    pub const fn format(&self) -> SnapshotPixelFormat {
447        self.format
448    }
449
450    pub const fn alpha_mode(&self) -> SnapshotAlphaMode {
451        self.alpha_mode
452    }
453
454    pub fn data(&self) -> &[u8] {
455        &(&self.data)[self.byte_range.clone()]
456    }
457
458    pub fn shared_memory(&self) -> IpcSharedMemory {
459        (*self.data).clone()
460    }
461}
pixels/snapshot.rs

pixels/
snapshot.rs
