pixels/
snapshot.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
4
5use std::ops::{Bound, Deref, DerefMut, Range, RangeBounds};
6use std::sync::Arc;
7
8use euclid::default::{Rect, Size2D};
9use image::codecs::jpeg::JpegEncoder;
10use image::codecs::png::PngEncoder;
11use image::codecs::webp::WebPEncoder;
12use image::{ExtendedColorType, GenericImageView, ImageEncoder, ImageError, Rgb};
13use ipc_channel::ipc::IpcSharedMemory;
14use malloc_size_of_derive::MallocSizeOf;
15use serde::{Deserialize, Serialize};
16
17use crate::{EncodedImageType, Multiply, rgba8_get_rect, transform_inplace};
18
19#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, MallocSizeOf, PartialEq, Serialize)]
20pub enum SnapshotPixelFormat {
21    #[default]
22    RGBA,
23    BGRA,
24}
25
26#[derive(Clone, Copy, Debug, Deserialize, Eq, MallocSizeOf, PartialEq, Serialize)]
27pub enum Alpha {
28    Premultiplied,
29    NotPremultiplied,
30    /// This is used for opaque textures for which the presence of alpha in the
31    /// output data format does not matter.
32    DontCare,
33}
34
35impl Alpha {
36    pub const fn from_premultiplied(is_premultiplied: bool) -> Self {
37        if is_premultiplied {
38            Self::Premultiplied
39        } else {
40            Self::NotPremultiplied
41        }
42    }
43
44    pub const fn needs_alpha_multiplication(&self) -> bool {
45        match self {
46            Alpha::Premultiplied => false,
47            Alpha::NotPremultiplied => true,
48            Alpha::DontCare => false,
49        }
50    }
51}
52
53#[derive(Clone, Copy, Debug, Deserialize, Eq, MallocSizeOf, PartialEq, Serialize)]
54pub enum SnapshotAlphaMode {
55    /// Internal data is opaque (alpha is cleared to 1)
56    Opaque,
57    /// Internal data should be treated as opaque (does not mean it actually is)
58    AsOpaque { premultiplied: bool },
59    /// Data is not opaque
60    Transparent { premultiplied: bool },
61}
62
63impl Default for SnapshotAlphaMode {
64    fn default() -> Self {
65        Self::Transparent {
66            premultiplied: true,
67        }
68    }
69}
70
71impl SnapshotAlphaMode {
72    pub const fn alpha(&self) -> Alpha {
73        match self {
74            SnapshotAlphaMode::Opaque => Alpha::DontCare,
75            SnapshotAlphaMode::AsOpaque { premultiplied } => {
76                Alpha::from_premultiplied(*premultiplied)
77            },
78            SnapshotAlphaMode::Transparent { premultiplied } => {
79                Alpha::from_premultiplied(*premultiplied)
80            },
81        }
82    }
83}
84
85/// The data in a [`Snapshot`]. If created via shared memory, this will be
86/// the `SharedMemory` variant, but otherwise it is the `Owned` variant.
87/// any attempt to mutate the [`Snapshot`] will convert it to the `Owned`
88/// variant.
89#[derive(Clone, Debug, MallocSizeOf)]
90pub enum SnapshotData {
91    SharedMemory(
92        #[conditional_malloc_size_of] Arc<IpcSharedMemory>,
93        Range<usize>,
94    ),
95    SharedVec(#[conditional_malloc_size_of] Arc<Vec<u8>>, Range<usize>),
96    Owned(Vec<u8>),
97}
98
99impl SnapshotData {
100    fn to_vec(&self) -> Vec<u8> {
101        match &self {
102            SnapshotData::SharedMemory(data, byte_range) => Vec::from(&data[byte_range.clone()]),
103            SnapshotData::SharedVec(data, byte_range) => Vec::from(&data[byte_range.clone()]),
104            SnapshotData::Owned(data) => data.clone(),
105        }
106    }
107}
108
109impl DerefMut for SnapshotData {
110    fn deref_mut(&mut self) -> &mut Self::Target {
111        match self {
112            SnapshotData::SharedMemory(..) | SnapshotData::SharedVec(..) => {
113                *self = SnapshotData::Owned(self.to_vec());
114                &mut *self
115            },
116            SnapshotData::Owned(items) => items,
117        }
118    }
119}
120
121impl Deref for SnapshotData {
122    type Target = [u8];
123
124    fn deref(&self) -> &Self::Target {
125        match &self {
126            SnapshotData::SharedMemory(data, byte_range) => &data[byte_range.clone()],
127            SnapshotData::SharedVec(data, byte_range) => &data[byte_range.clone()],
128            SnapshotData::Owned(items) => items,
129        }
130    }
131}
132
133/// Represents image bitmap with metadata, usually as snapshot of canvas
134///
135/// This allows us to hold off conversions (BGRA <-> RGBA, (un)premultiply)
136/// to when/if they are actually needed (WebGL/WebGPU can load both BGRA and RGBA).
137///
138/// Inspired by snapshot for concept in WebGPU spec:
139/// <https://gpuweb.github.io/gpuweb/#abstract-opdef-get-a-copy-of-the-image-contents-of-a-context>
140#[derive(Clone, Debug, MallocSizeOf)]
141pub struct Snapshot {
142    size: Size2D<u32>,
143    /// internal data (can be any format it will be converted on use if needed)
144    data: SnapshotData,
145    /// RGBA/BGRA (reflect internal data)
146    format: SnapshotPixelFormat,
147    /// How to treat alpha channel
148    alpha_mode: SnapshotAlphaMode,
149}
150
151impl Snapshot {
152    pub const fn size(&self) -> Size2D<u32> {
153        self.size
154    }
155
156    pub const fn format(&self) -> SnapshotPixelFormat {
157        self.format
158    }
159
160    pub const fn alpha_mode(&self) -> SnapshotAlphaMode {
161        self.alpha_mode
162    }
163
164    pub fn empty() -> Self {
165        Self {
166            size: Size2D::zero(),
167            data: SnapshotData::Owned(vec![]),
168            format: SnapshotPixelFormat::RGBA,
169            alpha_mode: SnapshotAlphaMode::Transparent {
170                premultiplied: true,
171            },
172        }
173    }
174
175    /// Returns snapshot with provided size that is black transparent alpha
176    pub fn cleared(size: Size2D<u32>) -> Self {
177        Self {
178            size,
179            data: SnapshotData::Owned(vec![0; size.area() as usize * 4]),
180            format: SnapshotPixelFormat::RGBA,
181            alpha_mode: SnapshotAlphaMode::Transparent {
182                premultiplied: true,
183            },
184        }
185    }
186
187    pub fn from_vec(
188        size: Size2D<u32>,
189        format: SnapshotPixelFormat,
190        alpha_mode: SnapshotAlphaMode,
191        data: Vec<u8>,
192    ) -> Self {
193        Self {
194            size,
195            data: SnapshotData::Owned(data),
196            format,
197            alpha_mode,
198        }
199    }
200
201    pub fn from_arc_vec(
202        size: Size2D<u32>,
203        format: SnapshotPixelFormat,
204        alpha_mode: SnapshotAlphaMode,
205        data: Arc<Vec<u8>>,
206        byte_range_bounds: impl RangeBounds<usize>,
207    ) -> Self {
208        let range_start = match byte_range_bounds.start_bound() {
209            Bound::Included(bound) => *bound,
210            Bound::Excluded(bound) => *bound + 1,
211            Bound::Unbounded => 0,
212        };
213        let range_end = match byte_range_bounds.end_bound() {
214            Bound::Included(bound) => *bound + 1,
215            Bound::Excluded(bound) => *bound,
216            Bound::Unbounded => data.len(),
217        };
218        Self {
219            size,
220            data: SnapshotData::SharedVec(data, range_start..range_end),
221            format,
222            alpha_mode,
223        }
224    }
225
226    pub fn get_rect(&self, rect: Rect<u32>) -> Self {
227        let data = rgba8_get_rect(self.as_raw_bytes(), self.size(), rect).to_vec();
228        Self::from_vec(rect.size, self.format, self.alpha_mode, data)
229    }
230
231    /// Convert inner data of snapshot to target format and alpha mode.
232    /// If data is already in target format and alpha mode no work will be done.
233    pub fn transform(
234        &mut self,
235        target_alpha_mode: SnapshotAlphaMode,
236        target_format: SnapshotPixelFormat,
237    ) {
238        if self.alpha_mode == target_alpha_mode && target_format == self.format {
239            return;
240        }
241
242        let swap_rb = target_format != self.format;
243        let multiply = match (self.alpha_mode, target_alpha_mode) {
244            (SnapshotAlphaMode::Opaque, _) => Multiply::None,
245            (alpha_mode, SnapshotAlphaMode::Opaque)
246                if alpha_mode.alpha() == Alpha::Premultiplied =>
247            {
248                Multiply::UnMultiply
249            },
250            (_, SnapshotAlphaMode::Opaque) => Multiply::None,
251            (
252                SnapshotAlphaMode::Transparent { premultiplied } |
253                SnapshotAlphaMode::AsOpaque { premultiplied },
254                SnapshotAlphaMode::Transparent {
255                    premultiplied: target_premultiplied,
256                } |
257                SnapshotAlphaMode::AsOpaque {
258                    premultiplied: target_premultiplied,
259                },
260            ) => {
261                if premultiplied == target_premultiplied {
262                    Multiply::None
263                } else if target_premultiplied {
264                    Multiply::PreMultiply
265                } else {
266                    Multiply::UnMultiply
267                }
268            },
269        };
270
271        let clear_alpha = !matches!(self.alpha_mode, SnapshotAlphaMode::Opaque) &&
272            matches!(target_alpha_mode, SnapshotAlphaMode::Opaque);
273
274        if matches!(multiply, Multiply::None) && !swap_rb && !clear_alpha {
275            return;
276        }
277
278        transform_inplace(self.data.deref_mut(), multiply, swap_rb, clear_alpha);
279        self.alpha_mode = target_alpha_mode;
280        self.format = target_format;
281    }
282
283    pub fn as_raw_bytes(&self) -> &[u8] {
284        &self.data
285    }
286
287    pub fn as_raw_bytes_mut(&mut self) -> &mut [u8] {
288        &mut self.data
289    }
290
291    pub fn to_shared(&self) -> SharedSnapshot {
292        let (data, byte_range) = match &self.data {
293            SnapshotData::SharedMemory(data, byte_range) => (data.clone(), byte_range.clone()),
294            SnapshotData::SharedVec(data, byte_range) => (
295                Arc::new(IpcSharedMemory::from_bytes(data)),
296                byte_range.clone(),
297            ),
298            SnapshotData::Owned(data) => {
299                (Arc::new(IpcSharedMemory::from_bytes(data)), 0..data.len())
300            },
301        };
302        SharedSnapshot {
303            size: self.size,
304            data,
305            byte_range,
306            format: self.format,
307            alpha_mode: self.alpha_mode,
308        }
309    }
310
311    pub fn encode_for_mime_type<W: std::io::Write>(
312        &mut self,
313        image_type: &EncodedImageType,
314        quality: Option<f64>,
315        encoder: &mut W,
316    ) -> Result<(), ImageError> {
317        let width = self.size.width;
318        let height = self.size.height;
319        let alpha_mode = match image_type {
320            EncodedImageType::Jpeg => SnapshotAlphaMode::AsOpaque {
321                premultiplied: true,
322            },
323            _ => SnapshotAlphaMode::Transparent {
324                premultiplied: false,
325            },
326        };
327
328        self.transform(alpha_mode, SnapshotPixelFormat::RGBA);
329        let data = &self.data;
330
331        match image_type {
332            EncodedImageType::Png => {
333                // FIXME(nox): https://github.com/image-rs/image-png/issues/86
334                // FIXME(nox): https://github.com/image-rs/image-png/issues/87
335                PngEncoder::new(encoder).write_image(data, width, height, ExtendedColorType::Rgba8)
336            },
337            EncodedImageType::Jpeg => {
338                let mut jpeg_encoder = if let Some(quality) = quality {
339                    // The specification allows quality to be in [0.0..1.0] but the JPEG encoder
340                    // expects it to be in [1..100]
341                    if (0.0..=1.0).contains(&quality) {
342                        JpegEncoder::new_with_quality(
343                            encoder,
344                            (quality * 100.0).round().clamp(1.0, 100.0) as u8,
345                        )
346                    } else {
347                        JpegEncoder::new(encoder)
348                    }
349                } else {
350                    JpegEncoder::new(encoder)
351                };
352
353                // JPEG doesn't support transparency, so simply calling jpeg_encoder.write_image fails here.
354                // Instead we have to create a struct to translate from rgba to rgb.
355                struct RgbaDataForJpegEncoder<'a> {
356                    width: u32,
357                    height: u32,
358                    data: &'a [u8],
359                }
360
361                impl<'a> GenericImageView for RgbaDataForJpegEncoder<'a> {
362                    type Pixel = Rgb<u8>;
363
364                    fn dimensions(&self) -> (u32, u32) {
365                        (self.width, self.height)
366                    }
367
368                    fn get_pixel(&self, x: u32, y: u32) -> Self::Pixel {
369                        let offset = (self.width * y + x) as usize * 4;
370                        Rgb([
371                            self.data[offset],
372                            self.data[offset + 1],
373                            self.data[offset + 2],
374                        ])
375                    }
376                }
377
378                let image = RgbaDataForJpegEncoder {
379                    width,
380                    height,
381                    data,
382                };
383
384                jpeg_encoder.encode_image(&image)
385            },
386            EncodedImageType::Webp => {
387                // No quality support because of https://github.com/image-rs/image/issues/1984
388                WebPEncoder::new_lossless(encoder).write_image(
389                    data,
390                    width,
391                    height,
392                    ExtendedColorType::Rgba8,
393                )
394            },
395        }
396    }
397}
398
399impl From<Snapshot> for Vec<u8> {
400    fn from(value: Snapshot) -> Self {
401        match value.data {
402            SnapshotData::SharedMemory(..) => Vec::from(value.as_raw_bytes()),
403            SnapshotData::SharedVec(..) => Vec::from(value.as_raw_bytes()),
404            SnapshotData::Owned(data) => data,
405        }
406    }
407}
408
409/// A version of [`Snapshot`] that can be sent across IPC channels.
410#[derive(Clone, Debug, Deserialize, MallocSizeOf, Serialize)]
411pub struct SharedSnapshot {
412    /// The physical size of this [`SharedSnapshot`].
413    size: Size2D<u32>,
414    /// The shared data of this [`SharedSnapshot`].
415    #[conditional_malloc_size_of]
416    data: Arc<IpcSharedMemory>,
417    /// The byte range of the data within the shared memory segment. This is used to
418    /// send individual image frames of animated images.
419    byte_range: Range<usize>,
420    /// The [`SnapshotPixelFormat`] of this [`SharedSnapshot`]
421    format: SnapshotPixelFormat,
422    /// The [`SnapshotAlphaMode`] of this [`SharedSnapshot`].
423    alpha_mode: SnapshotAlphaMode,
424}
425
426impl SharedSnapshot {
427    pub fn to_owned(&self) -> Snapshot {
428        Snapshot {
429            size: self.size,
430            data: SnapshotData::SharedMemory(self.data.clone(), self.byte_range.clone()),
431            format: self.format,
432            alpha_mode: self.alpha_mode,
433        }
434    }
435
436    /// Returns snapshot with provided size that is black transparent alpha
437    pub fn cleared(size: Size2D<u32>) -> Self {
438        let length_in_bytes = size.area() as usize * 4;
439        Self {
440            size,
441            data: Arc::new(IpcSharedMemory::from_byte(0, length_in_bytes)),
442            byte_range: 0..length_in_bytes,
443            format: SnapshotPixelFormat::RGBA,
444            alpha_mode: SnapshotAlphaMode::Transparent {
445                premultiplied: true,
446            },
447        }
448    }
449
450    pub const fn size(&self) -> Size2D<u32> {
451        self.size
452    }
453
454    pub const fn format(&self) -> SnapshotPixelFormat {
455        self.format
456    }
457
458    pub const fn alpha_mode(&self) -> SnapshotAlphaMode {
459        self.alpha_mode
460    }
461
462    pub fn data(&self) -> &[u8] {
463        &(&self.data)[self.byte_range.clone()]
464    }
465
466    pub fn shared_memory(&self) -> IpcSharedMemory {
467        (*self.data).clone()
468    }
469}
pixels/snapshot.rs

pixels/
snapshot.rs
