moxcms/conversions/avx/
rgb_xyz_q2_13_opt.rs

1/*
2 * // Copyright (c) Radzivon Bartoshyk 3/2025. All rights reserved.
3 * //
4 * // Redistribution and use in source and binary forms, with or without modification,
5 * // are permitted provided that the following conditions are met:
6 * //
7 * // 1.  Redistributions of source code must retain the above copyright notice, this
8 * // list of conditions and the following disclaimer.
9 * //
10 * // 2.  Redistributions in binary form must reproduce the above copyright notice,
11 * // this list of conditions and the following disclaimer in the documentation
12 * // and/or other materials provided with the distribution.
13 * //
14 * // 3.  Neither the name of the copyright holder nor the names of its
15 * // contributors may be used to endorse or promote products derived from
16 * // this software without specific prior written permission.
17 * //
18 * // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 * // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22 * // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24 * // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25 * // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29#![cfg(feature = "avx_shaper_fixed_point_paths")]
30use crate::conversions::avx::AvxAlignedU16;
31use crate::conversions::rgbxyz_fixed::TransformMatrixShaperFpOptVec;
32use crate::transform::PointeeSizeExpressible;
33use crate::{CmsError, Layout, TransformExecutor};
34use num_traits::AsPrimitive;
35use std::arch::x86_64::*;
36
37#[inline(always)]
38pub(crate) unsafe fn _xmm_broadcast_epi32(f: &i32) -> __m128i {
39    let float_ref: &f32 = unsafe { &*(f as *const i32 as *const f32) };
40    unsafe { _mm_castps_si128(_mm_broadcast_ss(float_ref)) }
41}
42
43pub(crate) struct TransformShaperRgbQ2_13OptAvx<
44    T: Copy,
45    const SRC_LAYOUT: u8,
46    const DST_LAYOUT: u8,
47    const PRECISION: i32,
48> {
49    pub(crate) profile: TransformMatrixShaperFpOptVec<i32, i16, T>,
50    pub(crate) bit_depth: usize,
51    pub(crate) gamma_lut: usize,
52}
53
54impl<
55    T: Copy + PointeeSizeExpressible + 'static,
56    const SRC_LAYOUT: u8,
57    const DST_LAYOUT: u8,
58    const PRECISION: i32,
59> TransformShaperRgbQ2_13OptAvx<T, SRC_LAYOUT, DST_LAYOUT, PRECISION>
60where
61    u32: AsPrimitive<T>,
62{
63    #[target_feature(enable = "avx2")]
64    unsafe fn transform_avx2(&self, src: &[T], dst: &mut [T]) -> Result<(), CmsError> {
65        let src_cn = Layout::from(SRC_LAYOUT);
66        let dst_cn = Layout::from(DST_LAYOUT);
67        let src_channels = src_cn.channels();
68        let dst_channels = dst_cn.channels();
69
70        let mut temporary0 = AvxAlignedU16([0; 16]);
71
72        if src.len() / src_channels != dst.len() / dst_channels {
73            return Err(CmsError::LaneSizeMismatch);
74        }
75        if src.len() % src_channels != 0 {
76            return Err(CmsError::LaneMultipleOfChannels);
77        }
78        if dst.len() % dst_channels != 0 {
79            return Err(CmsError::LaneMultipleOfChannels);
80        }
81
82        let t = self.profile.adaptation_matrix.transpose();
83
84        let max_colors = ((1 << self.bit_depth) - 1).as_();
85
86        // safety precondition for linearization table
87        if T::FINITE {
88            let cap = (1 << self.bit_depth) - 1;
89            assert!(self.profile.linear.len() >= cap);
90        } else {
91            assert!(self.profile.linear.len() >= T::NOT_FINITE_LINEAR_TABLE_SIZE);
92        }
93
94        let lut_lin = &self.profile.linear;
95
96        unsafe {
97            let m0 = _mm256_setr_epi16(
98                t.v[0][0], t.v[1][0], t.v[0][1], t.v[1][1], t.v[0][2], t.v[1][2], 0, 0, t.v[0][0],
99                t.v[1][0], t.v[0][1], t.v[1][1], t.v[0][2], t.v[1][2], 0, 0,
100            );
101            let m2 = _mm256_setr_epi16(
102                t.v[2][0], 1, t.v[2][1], 1, t.v[2][2], 1, 0, 0, t.v[2][0], 1, t.v[2][1], 1,
103                t.v[2][2], 1, 0, 0,
104            );
105
106            let rnd_val = ((1i32 << (PRECISION - 1)) as i16).to_ne_bytes();
107            let rnd = _mm256_set1_epi32(i32::from_ne_bytes([0, 0, rnd_val[0], rnd_val[1]]));
108
109            let zeros = _mm256_setzero_si256();
110
111            let v_max_value = _mm256_set1_epi32(self.gamma_lut as i32 - 1);
112
113            let (mut r0, mut g0, mut b0, mut a0);
114            let (mut r1, mut g1, mut b1, mut a1);
115
116            let mut src_iter = src.chunks_exact(src_channels * 2);
117
118            if let Some(src0) = src_iter.next() {
119                r0 = _xmm_broadcast_epi32(lut_lin.get_unchecked(src0[src_cn.r_i()]._as_usize()));
120                g0 = _xmm_broadcast_epi32(lut_lin.get_unchecked(src0[src_cn.g_i()]._as_usize()));
121                b0 = _xmm_broadcast_epi32(lut_lin.get_unchecked(src0[src_cn.b_i()]._as_usize()));
122
123                r1 = _xmm_broadcast_epi32(
124                    lut_lin.get_unchecked(src0[src_cn.r_i() + src_channels]._as_usize()),
125                );
126                g1 = _xmm_broadcast_epi32(
127                    lut_lin.get_unchecked(src0[src_cn.g_i() + src_channels]._as_usize()),
128                );
129                b1 = _xmm_broadcast_epi32(
130                    lut_lin.get_unchecked(src0[src_cn.b_i() + src_channels]._as_usize()),
131                );
132
133                a0 = if src_channels == 4 {
134                    src0[src_cn.a_i()]
135                } else {
136                    max_colors
137                };
138                a1 = if src_channels == 4 {
139                    src0[src_cn.a_i() + src_channels]
140                } else {
141                    max_colors
142                };
143            } else {
144                r0 = _mm_setzero_si128();
145                g0 = _mm_setzero_si128();
146                b0 = _mm_setzero_si128();
147                a0 = max_colors;
148                r1 = _mm_setzero_si128();
149                g1 = _mm_setzero_si128();
150                b1 = _mm_setzero_si128();
151                a1 = max_colors;
152            }
153
154            for (src, dst) in src_iter.zip(dst.chunks_exact_mut(dst_channels * 2)) {
155                let zr0 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(r0), r1);
156                let mut zg0 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(g0), g1);
157                let zb0 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(b0), b1);
158                zg0 = _mm256_slli_epi32::<16>(zg0);
159
160                let zrg0 = _mm256_or_si256(zr0, zg0);
161                let zbz0 = _mm256_or_si256(zb0, rnd);
162
163                let va0 = _mm256_madd_epi16(zrg0, m0);
164                let va1 = _mm256_madd_epi16(zbz0, m2);
165
166                let mut v0 = _mm256_add_epi32(va0, va1);
167
168                v0 = _mm256_srai_epi32::<PRECISION>(v0);
169                v0 = _mm256_max_epi32(v0, zeros);
170                v0 = _mm256_min_epi32(v0, v_max_value);
171
172                _mm256_store_si256(temporary0.0.as_mut_ptr() as *mut _, v0);
173
174                r0 = _xmm_broadcast_epi32(lut_lin.get_unchecked(src[src_cn.r_i()]._as_usize()));
175                g0 = _xmm_broadcast_epi32(lut_lin.get_unchecked(src[src_cn.g_i()]._as_usize()));
176                b0 = _xmm_broadcast_epi32(lut_lin.get_unchecked(src[src_cn.b_i()]._as_usize()));
177
178                r1 = _xmm_broadcast_epi32(
179                    lut_lin.get_unchecked(src[src_cn.r_i() + src_channels]._as_usize()),
180                );
181                g1 = _xmm_broadcast_epi32(
182                    lut_lin.get_unchecked(src[src_cn.g_i() + src_channels]._as_usize()),
183                );
184                b1 = _xmm_broadcast_epi32(
185                    lut_lin.get_unchecked(src[src_cn.b_i() + src_channels]._as_usize()),
186                );
187
188                dst[dst_cn.r_i()] = self.profile.gamma[temporary0.0[0] as usize];
189                dst[dst_cn.g_i()] = self.profile.gamma[temporary0.0[2] as usize];
190                dst[dst_cn.b_i()] = self.profile.gamma[temporary0.0[4] as usize];
191                if dst_channels == 4 {
192                    dst[dst_cn.a_i()] = a0;
193                }
194
195                dst[dst_cn.r_i() + dst_channels] = self.profile.gamma[temporary0.0[8] as usize];
196                dst[dst_cn.g_i() + dst_channels] = self.profile.gamma[temporary0.0[10] as usize];
197                dst[dst_cn.b_i() + dst_channels] = self.profile.gamma[temporary0.0[12] as usize];
198                if dst_channels == 4 {
199                    dst[dst_cn.a_i() + dst_channels] = a1;
200                }
201
202                a0 = if src_channels == 4 {
203                    src[src_cn.a_i()]
204                } else {
205                    max_colors
206                };
207                a1 = if src_channels == 4 {
208                    src[src_cn.a_i() + src_channels]
209                } else {
210                    max_colors
211                };
212            }
213
214            if let Some(dst) = dst.chunks_exact_mut(dst_channels * 2).last() {
215                let zr0 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(r0), r1);
216                let mut zg0 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(g0), g1);
217                let zb0 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(b0), b1);
218                zg0 = _mm256_slli_epi32::<16>(zg0);
219
220                let zrg0 = _mm256_or_si256(zr0, zg0);
221                let zbz0 = _mm256_or_si256(zb0, rnd);
222
223                let va0 = _mm256_madd_epi16(zrg0, m0);
224                let va1 = _mm256_madd_epi16(zbz0, m2);
225
226                let mut v0 = _mm256_add_epi32(va0, va1);
227
228                v0 = _mm256_srai_epi32::<PRECISION>(v0);
229                v0 = _mm256_max_epi32(v0, zeros);
230                v0 = _mm256_min_epi32(v0, v_max_value);
231
232                _mm256_store_si256(temporary0.0.as_mut_ptr() as *mut _, v0);
233
234                dst[dst_cn.r_i()] = self.profile.gamma[temporary0.0[0] as usize];
235                dst[dst_cn.g_i()] = self.profile.gamma[temporary0.0[2] as usize];
236                dst[dst_cn.b_i()] = self.profile.gamma[temporary0.0[4] as usize];
237                if dst_channels == 4 {
238                    dst[dst_cn.a_i()] = a0;
239                }
240
241                dst[dst_cn.r_i() + dst_channels] = self.profile.gamma[temporary0.0[8] as usize];
242                dst[dst_cn.g_i() + dst_channels] = self.profile.gamma[temporary0.0[10] as usize];
243                dst[dst_cn.b_i() + dst_channels] = self.profile.gamma[temporary0.0[12] as usize];
244                if dst_channels == 4 {
245                    dst[dst_cn.a_i() + dst_channels] = a1;
246                }
247            }
248
249            let src = src.chunks_exact(src_channels * 2).remainder();
250            let dst = dst.chunks_exact_mut(dst_channels * 2).into_remainder();
251
252            for (src, dst) in src
253                .chunks_exact(src_channels)
254                .zip(dst.chunks_exact_mut(dst_channels))
255            {
256                let r = _xmm_broadcast_epi32(lut_lin.get_unchecked(src[src_cn.r_i()]._as_usize()));
257                let mut g =
258                    _xmm_broadcast_epi32(lut_lin.get_unchecked(src[src_cn.g_i()]._as_usize()));
259                let b = _xmm_broadcast_epi32(lut_lin.get_unchecked(src[src_cn.b_i()]._as_usize()));
260
261                g = _mm_slli_epi32::<16>(g);
262
263                let a = if src_channels == 4 {
264                    src[src_cn.a_i()]
265                } else {
266                    max_colors
267                };
268
269                let zrg0 = _mm_or_si128(r, g);
270                let zbz0 = _mm_or_si128(b, _mm256_castsi256_si128(rnd));
271
272                let v0 = _mm_madd_epi16(zrg0, _mm256_castsi256_si128(m0));
273                let v1 = _mm_madd_epi16(zbz0, _mm256_castsi256_si128(m2));
274
275                let mut v = _mm_add_epi32(v0, v1);
276
277                v = _mm_srai_epi32::<PRECISION>(v);
278                v = _mm_max_epi32(v, _mm_setzero_si128());
279                v = _mm_min_epi32(v, _mm256_castsi256_si128(v_max_value));
280
281                _mm_store_si128(temporary0.0.as_mut_ptr() as *mut _, v);
282
283                dst[dst_cn.r_i()] = self.profile.gamma[temporary0.0[0] as usize];
284                dst[dst_cn.g_i()] = self.profile.gamma[temporary0.0[2] as usize];
285                dst[dst_cn.b_i()] = self.profile.gamma[temporary0.0[4] as usize];
286                if dst_channels == 4 {
287                    dst[dst_cn.a_i()] = a;
288                }
289            }
290        }
291
292        Ok(())
293    }
294
295    #[cfg(feature = "in_place")]
296    #[target_feature(enable = "avx2")]
297    unsafe fn transform_in_place_avx2(&self, in_out: &mut [T]) -> Result<(), CmsError> {
298        let src_cn = Layout::from(SRC_LAYOUT);
299        assert_eq!(
300            SRC_LAYOUT, DST_LAYOUT,
301            "This is in-place transform, layout must not diverge"
302        );
303        let src_channels = src_cn.channels();
304
305        let mut temporary0 = AvxAlignedU16([0; 16]);
306
307        if in_out.len() % src_channels != 0 {
308            return Err(CmsError::LaneMultipleOfChannels);
309        }
310
311        let t = self.profile.adaptation_matrix.transpose();
312
313        let max_colors = ((1 << self.bit_depth) - 1).as_();
314
315        // safety precondition for linearization table
316        if T::FINITE {
317            let cap = (1 << self.bit_depth) - 1;
318            assert!(self.profile.linear.len() >= cap);
319        } else {
320            assert!(self.profile.linear.len() >= T::NOT_FINITE_LINEAR_TABLE_SIZE);
321        }
322
323        let lut_lin = &self.profile.linear;
324
325        unsafe {
326            let m0 = _mm256_setr_epi16(
327                t.v[0][0], t.v[1][0], t.v[0][1], t.v[1][1], t.v[0][2], t.v[1][2], 0, 0, t.v[0][0],
328                t.v[1][0], t.v[0][1], t.v[1][1], t.v[0][2], t.v[1][2], 0, 0,
329            );
330            let m2 = _mm256_setr_epi16(
331                t.v[2][0], 1, t.v[2][1], 1, t.v[2][2], 1, 0, 0, t.v[2][0], 1, t.v[2][1], 1,
332                t.v[2][2], 1, 0, 0,
333            );
334
335            let rnd_val = ((1i32 << (PRECISION - 1)) as i16).to_ne_bytes();
336            let rnd = _mm256_set1_epi32(i32::from_ne_bytes([0, 0, rnd_val[0], rnd_val[1]]));
337
338            let zeros = _mm256_setzero_si256();
339
340            let v_max_value = _mm256_set1_epi32(self.gamma_lut as i32 - 1);
341
342            let (mut r0, mut g0, mut b0, mut a0);
343            let (mut r1, mut g1, mut b1, mut a1);
344
345            for dst in in_out.chunks_exact_mut(src_channels * 2) {
346                r0 = _xmm_broadcast_epi32(lut_lin.get_unchecked(dst[src_cn.r_i()]._as_usize()));
347                g0 = _xmm_broadcast_epi32(lut_lin.get_unchecked(dst[src_cn.g_i()]._as_usize()));
348                b0 = _xmm_broadcast_epi32(lut_lin.get_unchecked(dst[src_cn.b_i()]._as_usize()));
349
350                r1 = _xmm_broadcast_epi32(
351                    lut_lin.get_unchecked(dst[src_cn.r_i() + src_channels]._as_usize()),
352                );
353                g1 = _xmm_broadcast_epi32(
354                    lut_lin.get_unchecked(dst[src_cn.g_i() + src_channels]._as_usize()),
355                );
356                b1 = _xmm_broadcast_epi32(
357                    lut_lin.get_unchecked(dst[src_cn.b_i() + src_channels]._as_usize()),
358                );
359
360                a0 = if src_channels == 4 {
361                    dst[src_cn.a_i()]
362                } else {
363                    max_colors
364                };
365                a1 = if src_channels == 4 {
366                    dst[src_cn.a_i() + src_channels]
367                } else {
368                    max_colors
369                };
370
371                let zr0 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(r0), r1);
372                let mut zg0 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(g0), g1);
373                let zb0 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(b0), b1);
374                zg0 = _mm256_slli_epi32::<16>(zg0);
375
376                let zrg0 = _mm256_or_si256(zr0, zg0);
377                let zbz0 = _mm256_or_si256(zb0, rnd);
378
379                let va0 = _mm256_madd_epi16(zrg0, m0);
380                let va1 = _mm256_madd_epi16(zbz0, m2);
381
382                let mut v0 = _mm256_add_epi32(va0, va1);
383
384                v0 = _mm256_srai_epi32::<PRECISION>(v0);
385                v0 = _mm256_max_epi32(v0, zeros);
386                v0 = _mm256_min_epi32(v0, v_max_value);
387
388                _mm256_store_si256(temporary0.0.as_mut_ptr() as *mut _, v0);
389
390                dst[src_cn.r_i()] = self.profile.gamma[temporary0.0[0] as usize];
391                dst[src_cn.g_i()] = self.profile.gamma[temporary0.0[2] as usize];
392                dst[src_cn.b_i()] = self.profile.gamma[temporary0.0[4] as usize];
393                if src_channels == 4 {
394                    dst[src_cn.a_i()] = a0;
395                }
396
397                dst[src_cn.r_i() + src_channels] = self.profile.gamma[temporary0.0[8] as usize];
398                dst[src_cn.g_i() + src_channels] = self.profile.gamma[temporary0.0[10] as usize];
399                dst[src_cn.b_i() + src_channels] = self.profile.gamma[temporary0.0[12] as usize];
400                if src_channels == 4 {
401                    dst[src_cn.a_i() + src_channels] = a1;
402                }
403            }
404
405            let dst = in_out.chunks_exact_mut(src_channels * 2).into_remainder();
406
407            for dst in dst.chunks_exact_mut(src_channels) {
408                let r = _xmm_broadcast_epi32(lut_lin.get_unchecked(dst[src_cn.r_i()]._as_usize()));
409                let mut g =
410                    _xmm_broadcast_epi32(lut_lin.get_unchecked(dst[src_cn.g_i()]._as_usize()));
411                let b = _xmm_broadcast_epi32(lut_lin.get_unchecked(dst[src_cn.b_i()]._as_usize()));
412
413                g = _mm_slli_epi32::<16>(g);
414
415                let a = if src_channels == 4 {
416                    dst[src_cn.a_i()]
417                } else {
418                    max_colors
419                };
420
421                let zrg0 = _mm_or_si128(r, g);
422                let zbz0 = _mm_or_si128(b, _mm256_castsi256_si128(rnd));
423
424                let v0 = _mm_madd_epi16(zrg0, _mm256_castsi256_si128(m0));
425                let v1 = _mm_madd_epi16(zbz0, _mm256_castsi256_si128(m2));
426
427                let mut v = _mm_add_epi32(v0, v1);
428
429                v = _mm_srai_epi32::<PRECISION>(v);
430                v = _mm_max_epi32(v, _mm_setzero_si128());
431                v = _mm_min_epi32(v, _mm256_castsi256_si128(v_max_value));
432
433                _mm_store_si128(temporary0.0.as_mut_ptr() as *mut _, v);
434
435                dst[src_cn.r_i()] = self.profile.gamma[temporary0.0[0] as usize];
436                dst[src_cn.g_i()] = self.profile.gamma[temporary0.0[2] as usize];
437                dst[src_cn.b_i()] = self.profile.gamma[temporary0.0[4] as usize];
438                if src_channels == 4 {
439                    dst[src_cn.a_i()] = a;
440                }
441            }
442        }
443
444        Ok(())
445    }
446}
447
448impl<
449    T: Copy + PointeeSizeExpressible + 'static + Default,
450    const SRC_LAYOUT: u8,
451    const DST_LAYOUT: u8,
452    const PRECISION: i32,
453> TransformExecutor<T> for TransformShaperRgbQ2_13OptAvx<T, SRC_LAYOUT, DST_LAYOUT, PRECISION>
454where
455    u32: AsPrimitive<T>,
456{
457    fn transform(&self, src: &[T], dst: &mut [T]) -> Result<(), CmsError> {
458        unsafe { self.transform_avx2(src, dst) }
459    }
460}
461
462#[cfg(feature = "in_place")]
463use crate::InPlaceTransformExecutor;
464
465#[cfg(feature = "in_place")]
466impl<
467    T: Copy + PointeeSizeExpressible + 'static + Default,
468    const SRC_LAYOUT: u8,
469    const DST_LAYOUT: u8,
470    const PRECISION: i32,
471> InPlaceTransformExecutor<T>
472    for TransformShaperRgbQ2_13OptAvx<T, SRC_LAYOUT, DST_LAYOUT, PRECISION>
473where
474    u32: AsPrimitive<T>,
475{
476    fn transform(&self, in_out: &mut [T]) -> Result<(), CmsError> {
477        unsafe { self.transform_in_place_avx2(in_out) }
478    }
479}
moxcms/conversions/avx/rgb_xyz_q2_13_opt.rs

moxcms/conversions/avx/
rgb_xyz_q2_13_opt.rs
