moxcms/conversions/avx/
rgb_xyz.rs

1/*
2 * // Copyright (c) Radzivon Bartoshyk 3/2025. All rights reserved.
3 * //
4 * // Redistribution and use in source and binary forms, with or without modification,
5 * // are permitted provided that the following conditions are met:
6 * //
7 * // 1.  Redistributions of source code must retain the above copyright notice, this
8 * // list of conditions and the following disclaimer.
9 * //
10 * // 2.  Redistributions in binary form must reproduce the above copyright notice,
11 * // this list of conditions and the following disclaimer in the documentation
12 * // and/or other materials provided with the distribution.
13 * //
14 * // 3.  Neither the name of the copyright holder nor the names of its
15 * // contributors may be used to endorse or promote products derived from
16 * // this software without specific prior written permission.
17 * //
18 * // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 * // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22 * // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24 * // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25 * // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29#![cfg(feature = "avx_shaper_paths")]
30use crate::conversions::TransformMatrixShaper;
31use crate::conversions::avx::AvxAlignedU16;
32use crate::transform::PointeeSizeExpressible;
33use crate::{CmsError, Layout, TransformExecutor};
34use num_traits::AsPrimitive;
35use std::arch::x86_64::*;
36
37pub(crate) struct TransformShaperRgbAvx<
38    T: Clone + Copy + 'static + PointeeSizeExpressible + Default,
39    const SRC_LAYOUT: u8,
40    const DST_LAYOUT: u8,
41    // deleting linear cap is in effective here
42    const LINEAR_CAP: usize,
43> {
44    pub(crate) profile: TransformMatrixShaper<T, LINEAR_CAP>,
45    pub(crate) bit_depth: usize,
46    pub(crate) gamma_lut: usize,
47}
48
49impl<
50    T: Clone + Copy + 'static + PointeeSizeExpressible + Default,
51    const SRC_LAYOUT: u8,
52    const DST_LAYOUT: u8,
53    const LINEAR_CAP: usize,
54> TransformShaperRgbAvx<T, SRC_LAYOUT, DST_LAYOUT, LINEAR_CAP>
55where
56    u32: AsPrimitive<T>,
57{
58    #[target_feature(enable = "avx2", enable = "fma")]
59    unsafe fn transform_impl<const FMA: bool>(
60        &self,
61        src: &[T],
62        dst: &mut [T],
63    ) -> Result<(), CmsError> {
64        let src_cn = Layout::from(SRC_LAYOUT);
65        let dst_cn = Layout::from(DST_LAYOUT);
66        let src_channels = src_cn.channels();
67        let dst_channels = dst_cn.channels();
68
69        let mut temporary0 = AvxAlignedU16([0; 16]);
70
71        if src.len() / src_channels != dst.len() / dst_channels {
72            return Err(CmsError::LaneSizeMismatch);
73        }
74        if src.len() % src_channels != 0 {
75            return Err(CmsError::LaneMultipleOfChannels);
76        }
77        if dst.len() % dst_channels != 0 {
78            return Err(CmsError::LaneMultipleOfChannels);
79        }
80
81        let t = self.profile.adaptation_matrix.transpose();
82
83        let scale = (self.gamma_lut - 1) as f32;
84        let max_colors: T = ((1 << self.bit_depth) - 1).as_();
85
86        unsafe {
87            let m0 = _mm256_setr_ps(
88                t.v[0][0], t.v[0][1], t.v[0][2], 0., t.v[0][0], t.v[0][1], t.v[0][2], 0.,
89            );
90            let m1 = _mm256_setr_ps(
91                t.v[1][0], t.v[1][1], t.v[1][2], 0., t.v[1][0], t.v[1][1], t.v[1][2], 0.,
92            );
93            let m2 = _mm256_setr_ps(
94                t.v[2][0], t.v[2][1], t.v[2][2], 0., t.v[2][0], t.v[2][1], t.v[2][2], 0.,
95            );
96
97            let zeros = _mm_setzero_ps();
98
99            let v_scale = _mm256_set1_ps(scale);
100
101            let mut src = src;
102            let mut dst = dst;
103
104            let src_iter = src.chunks_exact(src_channels * 2);
105            let dst_iter = dst.chunks_exact_mut(dst_channels * 2);
106
107            let (mut r0, mut g0, mut b0, mut a0);
108            let (mut r1, mut g1, mut b1, mut a1);
109
110            for (src, dst) in src_iter.zip(dst_iter) {
111                r0 = _mm_broadcast_ss(&self.profile.r_linear[src[src_cn.r_i()]._as_usize()]);
112                g0 = _mm_broadcast_ss(&self.profile.g_linear[src[src_cn.g_i()]._as_usize()]);
113                b0 = _mm_broadcast_ss(&self.profile.b_linear[src[src_cn.b_i()]._as_usize()]);
114                r1 = _mm_broadcast_ss(
115                    &self.profile.r_linear[src[src_cn.r_i() + src_channels]._as_usize()],
116                );
117                g1 = _mm_broadcast_ss(
118                    &self.profile.g_linear[src[src_cn.g_i() + src_channels]._as_usize()],
119                );
120                b1 = _mm_broadcast_ss(
121                    &self.profile.b_linear[src[src_cn.b_i() + src_channels]._as_usize()],
122                );
123                a0 = if src_channels == 4 {
124                    src[src_cn.a_i()]
125                } else {
126                    max_colors
127                };
128                a1 = if src_channels == 4 {
129                    src[src_cn.a_i() + src_channels]
130                } else {
131                    max_colors
132                };
133
134                let r = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(r0), r1);
135                let g = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(g0), g1);
136                let b = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(b0), b1);
137
138                let mut v = if FMA {
139                    let v0 = _mm256_mul_ps(r, m0);
140                    let v1 = _mm256_fmadd_ps(g, m1, v0);
141                    _mm256_fmadd_ps(b, m2, v1)
142                } else {
143                    let v0 = _mm256_mul_ps(r, m0);
144                    let v1 = _mm256_mul_ps(g, m1);
145                    let v2 = _mm256_mul_ps(b, m2);
146
147                    _mm256_add_ps(_mm256_add_ps(v0, v1), v2)
148                };
149
150                v = _mm256_max_ps(v, _mm256_setzero_ps());
151                v = _mm256_mul_ps(v, v_scale);
152                v = _mm256_min_ps(v, v_scale);
153
154                let zx = _mm256_cvtps_epi32(v);
155                _mm256_store_si256(temporary0.0.as_mut_ptr() as *mut _, zx);
156
157                dst[dst_cn.r_i()] = self.profile.r_gamma[temporary0.0[0] as usize];
158                dst[dst_cn.g_i()] = self.profile.g_gamma[temporary0.0[2] as usize];
159                dst[dst_cn.b_i()] = self.profile.b_gamma[temporary0.0[4] as usize];
160                if dst_channels == 4 {
161                    dst[dst_cn.a_i()] = a0;
162                }
163
164                dst[dst_cn.r_i() + dst_channels] = self.profile.r_gamma[temporary0.0[8] as usize];
165                dst[dst_cn.g_i() + dst_channels] = self.profile.g_gamma[temporary0.0[10] as usize];
166                dst[dst_cn.b_i() + dst_channels] = self.profile.b_gamma[temporary0.0[12] as usize];
167                if dst_channels == 4 {
168                    dst[dst_cn.a_i() + dst_channels] = a1;
169                }
170            }
171
172            src = src.chunks_exact(src_channels * 2).remainder();
173            dst = dst.chunks_exact_mut(dst_channels * 2).into_remainder();
174
175            for (src, dst) in src
176                .chunks_exact(src_channels)
177                .zip(dst.chunks_exact_mut(dst_channels))
178            {
179                let r = _mm_broadcast_ss(&self.profile.r_linear[src[src_cn.r_i()]._as_usize()]);
180                let g = _mm_broadcast_ss(&self.profile.g_linear[src[src_cn.g_i()]._as_usize()]);
181                let b = _mm_broadcast_ss(&self.profile.b_linear[src[src_cn.b_i()]._as_usize()]);
182                let a = if src_channels == 4 {
183                    src[src_cn.a_i()]
184                } else {
185                    max_colors
186                };
187
188                let mut v = if FMA {
189                    let v0 = _mm_mul_ps(r, _mm256_castps256_ps128(m0));
190                    let v1 = _mm_fmadd_ps(g, _mm256_castps256_ps128(m1), v0);
191                    _mm_fmadd_ps(b, _mm256_castps256_ps128(m2), v1)
192                } else {
193                    let v0 = _mm_mul_ps(r, _mm256_castps256_ps128(m0));
194                    let v1 = _mm_mul_ps(g, _mm256_castps256_ps128(m1));
195                    let v2 = _mm_mul_ps(b, _mm256_castps256_ps128(m2));
196
197                    _mm_add_ps(_mm_add_ps(v0, v1), v2)
198                };
199
200                v = _mm_max_ps(v, zeros);
201                v = _mm_mul_ps(v, _mm256_castps256_ps128(v_scale));
202                v = _mm_min_ps(v, _mm256_castps256_ps128(v_scale));
203
204                let zx = _mm_cvtps_epi32(v);
205                _mm_store_si128(temporary0.0.as_mut_ptr() as *mut _, zx);
206
207                dst[dst_cn.r_i()] = self.profile.r_gamma[temporary0.0[0] as usize];
208                dst[dst_cn.g_i()] = self.profile.g_gamma[temporary0.0[2] as usize];
209                dst[dst_cn.b_i()] = self.profile.b_gamma[temporary0.0[4] as usize];
210                if dst_channels == 4 {
211                    dst[dst_cn.a_i()] = a;
212                }
213            }
214        }
215
216        Ok(())
217    }
218}
219
220impl<
221    T: Clone + Copy + 'static + PointeeSizeExpressible + Default,
222    const SRC_LAYOUT: u8,
223    const DST_LAYOUT: u8,
224    const LINEAR_CAP: usize,
225> TransformExecutor<T> for TransformShaperRgbAvx<T, SRC_LAYOUT, DST_LAYOUT, LINEAR_CAP>
226where
227    u32: AsPrimitive<T>,
228{
229    fn transform(&self, src: &[T], dst: &mut [T]) -> Result<(), CmsError> {
230        unsafe {
231            assert!(std::arch::is_x86_feature_detected!("fma"));
232            self.transform_impl::<true>(src, dst)
233        }
234    }
235}
moxcms/conversions/avx/rgb_xyz.rs

moxcms/conversions/avx/
rgb_xyz.rs
