Skip to main content

rustfft/sse/
sse_prime_butterflies.rs

1use core::arch::x86_64::{__m128, __m128d};
2use std::any::TypeId;
3use std::sync::Arc;
4use num_complex::Complex;
5
6use crate::{common::FftNum, FftDirection};
7
8use crate::array_utils::DoubleBuf;
9use crate::twiddles;
10use crate::{Direction, Fft, Length};
11
12use super::sse_common::{assert_f32, assert_f64};
13use super::sse_utils::*;
14use super::sse_vector::*;
15
16/* 
17This file contains autogenerated butterfly algorithms for small prime-sized FFTs using the SSE instruction set.
18
19NOTE: All of the code in this file was **autogenerated** using the following command in the project root:
20    cargo run --manifest-path ./tools/gen_simd_butterflies/Cargo.toml -- sse 7 11 13 17 19 23 29 31
21    
22Do not make changes directly to this file. Instead, update the autogeneration script 
23(Located at tools/gen_simd_butterflies/src/main.rs) and then re-run the script to generate the new code.
24
25For these sizes, we use a variant of the naive DFT algorithm. Even though this algorithm is O(n^2),
26we're able to factor out some redundant computations, and they up being faster than the fancier algorithms.
27
28To generate more or fewer butterfly sizes, simply add or remove numbers from the command above and re-run.
29The code generation script will even handle adding or removing sizes from the planner, all you need to do is run the script.
30*/
31
32pub const fn prime_butterfly_lens() -> &'static [usize] {
33    &[7, 11, 13, 17, 19, 23, 29, 31, ]
34}
35
36/// Safety: The current machine must support the sse4.1 target feature
37#[target_feature(enable = "sse4.1")]
38pub unsafe fn construct_prime_butterfly<T: FftNum>(len: usize, direction: FftDirection) -> Arc<dyn Fft<T>> {
39    let id_f32 = TypeId::of::<f32>();
40    let id_f64 = TypeId::of::<f64>();
41    let id_t = TypeId::of::<T>();
42    if id_t == id_f32 {
43        match len {
44            7 => Arc::new(SseF32Butterfly7::new(direction)) as Arc<dyn Fft<T>>,
45            11 => Arc::new(SseF32Butterfly11::new(direction)) as Arc<dyn Fft<T>>,
46            13 => Arc::new(SseF32Butterfly13::new(direction)) as Arc<dyn Fft<T>>,
47            17 => Arc::new(SseF32Butterfly17::new(direction)) as Arc<dyn Fft<T>>,
48            19 => Arc::new(SseF32Butterfly19::new(direction)) as Arc<dyn Fft<T>>,
49            23 => Arc::new(SseF32Butterfly23::new(direction)) as Arc<dyn Fft<T>>,
50            29 => Arc::new(SseF32Butterfly29::new(direction)) as Arc<dyn Fft<T>>,
51            31 => Arc::new(SseF32Butterfly31::new(direction)) as Arc<dyn Fft<T>>,
52            _ => unimplemented!("Invalid SSE prime butterfly length: {len}"),
53        }
54    } else if id_t == id_f64 {
55        match len {
56            7 => Arc::new(SseF64Butterfly7::new(direction)) as Arc<dyn Fft<T>>,
57            11 => Arc::new(SseF64Butterfly11::new(direction)) as Arc<dyn Fft<T>>,
58            13 => Arc::new(SseF64Butterfly13::new(direction)) as Arc<dyn Fft<T>>,
59            17 => Arc::new(SseF64Butterfly17::new(direction)) as Arc<dyn Fft<T>>,
60            19 => Arc::new(SseF64Butterfly19::new(direction)) as Arc<dyn Fft<T>>,
61            23 => Arc::new(SseF64Butterfly23::new(direction)) as Arc<dyn Fft<T>>,
62            29 => Arc::new(SseF64Butterfly29::new(direction)) as Arc<dyn Fft<T>>,
63            31 => Arc::new(SseF64Butterfly31::new(direction)) as Arc<dyn Fft<T>>,
64            _ => unimplemented!("Invalid SSE prime butterfly length: {len}"),
65        }
66    } else {
67        unimplemented!("Not f32 or f64");
68    }
69}
70
71#[inline(always)]
72fn make_twiddles<const TW: usize, T: FftNum>(len: usize, direction: FftDirection) -> [Complex<T>; TW] {
73    let mut i = 1;
74    [(); TW].map(|_| {
75        let twiddle = twiddles::compute_twiddle(i, len, direction);
76        i += 1;
77        twiddle
78    })
79}
80
81struct SseF32Butterfly7<T> {
82    direction: FftDirection,
83    twiddles_re: [__m128; 3],
84    twiddles_im: [__m128; 3],
85    _phantom: std::marker::PhantomData<T>,
86}
87
88boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly7, 7, |this: &SseF32Butterfly7<_>| this.direction);
89impl<T: FftNum> SseF32Butterfly7<T> {
90    /// Safety: The current machine must support the sse4.1 instruction set
91    #[target_feature(enable = "sse4.1")]
92    unsafe fn new(direction: FftDirection) -> Self {
93        assert_f32::<T>();
94        let twiddles = make_twiddles(7, direction);
95        Self {
96            direction,
97            twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
98            twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
99            _phantom: std::marker::PhantomData,
100        }
101    }
102
103    #[inline(always)]
104    pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
105        let values = read_partial1_complex_to_array!(buffer, { 0,1,2,3,4,5,6 });
106
107        let out = self.perform_parallel_fft_direct(values);
108        
109        write_partial_lo_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6 } ); 
110    }
111
112    #[inline(always)]
113    pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
114        let input_packed = read_complex_to_array!(buffer, { 0,2,4,6,8,10,12 });
115
116        let values = [
117            extract_lo_hi_f32(input_packed[0], input_packed[3]),
118            extract_hi_lo_f32(input_packed[0], input_packed[4]),
119            extract_lo_hi_f32(input_packed[1], input_packed[4]),
120            extract_hi_lo_f32(input_packed[1], input_packed[5]),
121            extract_lo_hi_f32(input_packed[2], input_packed[5]),
122            extract_hi_lo_f32(input_packed[2], input_packed[6]),
123            extract_lo_hi_f32(input_packed[3], input_packed[6]),
124        ];
125
126        let out = self.perform_parallel_fft_direct(values);
127
128        let out_packed = [
129            extract_lo_lo_f32(out[0], out[1]),
130            extract_lo_lo_f32(out[2], out[3]),
131            extract_lo_lo_f32(out[4], out[5]),
132            extract_lo_hi_f32(out[6], out[0]),
133            extract_hi_hi_f32(out[1], out[2]),
134            extract_hi_hi_f32(out[3], out[4]),
135            extract_hi_hi_f32(out[5], out[6]),
136        ];
137
138        write_complex_to_array_strided!(out_packed, buffer, 2, { 0,1,2,3,4,5,6 });
139    }
140
141    #[inline(always)]
142    pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 7]) -> [__m128; 7] {
143        let rotate = SseVector::make_rotate90(FftDirection::Inverse);
144
145        let y00 = values[0];
146        let [x1p6, x1m6] =  SseVector::column_butterfly2([values[1], values[6]]);
147        let x1m6 = SseVector::apply_rotate90(rotate, x1m6);
148        let y00 = SseVector::add(y00, x1p6);
149        let [x2p5, x2m5] =  SseVector::column_butterfly2([values[2], values[5]]);
150        let x2m5 = SseVector::apply_rotate90(rotate, x2m5);
151        let y00 = SseVector::add(y00, x2p5);
152        let [x3p4, x3m4] =  SseVector::column_butterfly2([values[3], values[4]]);
153        let x3m4 = SseVector::apply_rotate90(rotate, x3m4);
154        let y00 = SseVector::add(y00, x3p4);
155
156        let m0106a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p6);
157        let m0106a = SseVector::fmadd(m0106a, self.twiddles_re[1], x2p5);
158        let m0106a = SseVector::fmadd(m0106a, self.twiddles_re[2], x3p4);
159        let m0106b = SseVector::mul(self.twiddles_im[0], x1m6);
160        let m0106b = SseVector::fmadd(m0106b, self.twiddles_im[1], x2m5);
161        let m0106b = SseVector::fmadd(m0106b, self.twiddles_im[2], x3m4);
162        let [y01, y06] = SseVector::column_butterfly2([m0106a, m0106b]);
163
164        let m0205a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p6);
165        let m0205a = SseVector::fmadd(m0205a, self.twiddles_re[2], x2p5);
166        let m0205a = SseVector::fmadd(m0205a, self.twiddles_re[0], x3p4);
167        let m0205b = SseVector::mul(self.twiddles_im[1], x1m6);
168        let m0205b = SseVector::nmadd(m0205b, self.twiddles_im[2], x2m5);
169        let m0205b = SseVector::nmadd(m0205b, self.twiddles_im[0], x3m4);
170        let [y02, y05] = SseVector::column_butterfly2([m0205a, m0205b]);
171
172        let m0304a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p6);
173        let m0304a = SseVector::fmadd(m0304a, self.twiddles_re[0], x2p5);
174        let m0304a = SseVector::fmadd(m0304a, self.twiddles_re[1], x3p4);
175        let m0304b = SseVector::mul(self.twiddles_im[2], x1m6);
176        let m0304b = SseVector::nmadd(m0304b, self.twiddles_im[0], x2m5);
177        let m0304b = SseVector::fmadd(m0304b, self.twiddles_im[1], x3m4);
178        let [y03, y04] = SseVector::column_butterfly2([m0304a, m0304b]);
179
180
181        [y00, y01, y02, y03, y04, y05, y06]
182    }
183}
184
185struct SseF64Butterfly7<T> {
186    direction: FftDirection,
187    twiddles_re: [__m128d; 3],
188    twiddles_im: [__m128d; 3],
189    _phantom: std::marker::PhantomData<T>,
190}
191
192boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly7, 7, |this: &SseF64Butterfly7<_>| this.direction);
193impl<T: FftNum> SseF64Butterfly7<T> {
194    /// Safety: The current machine must support the sse4.1 instruction set
195    #[target_feature(enable = "sse4.1")]
196    unsafe fn new(direction: FftDirection) -> Self {
197        assert_f64::<T>();
198        let twiddles = make_twiddles(7, direction);
199        unsafe {Self {
200            direction,
201            twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
202            twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
203            _phantom: std::marker::PhantomData,
204        }}
205    }
206
207    #[inline(always)]
208    pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
209        let values = read_complex_to_array!(buffer, { 0,1,2,3,4,5,6 });
210
211        let out = self.perform_fft_direct(values);
212
213        write_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6 });   
214    }
215
216    #[inline(always)]
217    pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 7]) -> [__m128d; 7] {
218        let rotate = SseVector::make_rotate90(FftDirection::Inverse);
219
220        let y00 = values[0];
221        let [x1p6, x1m6] =  SseVector::column_butterfly2([values[1], values[6]]);
222        let x1m6 = SseVector::apply_rotate90(rotate, x1m6);
223        let y00 = SseVector::add(y00, x1p6);
224        let [x2p5, x2m5] =  SseVector::column_butterfly2([values[2], values[5]]);
225        let x2m5 = SseVector::apply_rotate90(rotate, x2m5);
226        let y00 = SseVector::add(y00, x2p5);
227        let [x3p4, x3m4] =  SseVector::column_butterfly2([values[3], values[4]]);
228        let x3m4 = SseVector::apply_rotate90(rotate, x3m4);
229        let y00 = SseVector::add(y00, x3p4);
230
231        let m0106a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p6);
232        let m0106a = SseVector::fmadd(m0106a, self.twiddles_re[1], x2p5);
233        let m0106a = SseVector::fmadd(m0106a, self.twiddles_re[2], x3p4);
234        let m0106b = SseVector::mul(self.twiddles_im[0], x1m6);
235        let m0106b = SseVector::fmadd(m0106b, self.twiddles_im[1], x2m5);
236        let m0106b = SseVector::fmadd(m0106b, self.twiddles_im[2], x3m4);
237        let [y01, y06] = SseVector::column_butterfly2([m0106a, m0106b]);
238
239        let m0205a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p6);
240        let m0205a = SseVector::fmadd(m0205a, self.twiddles_re[2], x2p5);
241        let m0205a = SseVector::fmadd(m0205a, self.twiddles_re[0], x3p4);
242        let m0205b = SseVector::mul(self.twiddles_im[1], x1m6);
243        let m0205b = SseVector::nmadd(m0205b, self.twiddles_im[2], x2m5);
244        let m0205b = SseVector::nmadd(m0205b, self.twiddles_im[0], x3m4);
245        let [y02, y05] = SseVector::column_butterfly2([m0205a, m0205b]);
246
247        let m0304a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p6);
248        let m0304a = SseVector::fmadd(m0304a, self.twiddles_re[0], x2p5);
249        let m0304a = SseVector::fmadd(m0304a, self.twiddles_re[1], x3p4);
250        let m0304b = SseVector::mul(self.twiddles_im[2], x1m6);
251        let m0304b = SseVector::nmadd(m0304b, self.twiddles_im[0], x2m5);
252        let m0304b = SseVector::fmadd(m0304b, self.twiddles_im[1], x3m4);
253        let [y03, y04] = SseVector::column_butterfly2([m0304a, m0304b]);
254
255
256        [y00, y01, y02, y03, y04, y05, y06]
257    }
258}
259
260struct SseF32Butterfly11<T> {
261    direction: FftDirection,
262    twiddles_re: [__m128; 5],
263    twiddles_im: [__m128; 5],
264    _phantom: std::marker::PhantomData<T>,
265}
266
267boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly11, 11, |this: &SseF32Butterfly11<_>| this.direction);
268impl<T: FftNum> SseF32Butterfly11<T> {
269    /// Safety: The current machine must support the sse4.1 instruction set
270    #[target_feature(enable = "sse4.1")]
271    unsafe fn new(direction: FftDirection) -> Self {
272        assert_f32::<T>();
273        let twiddles = make_twiddles(11, direction);
274        Self {
275            direction,
276            twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
277            twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
278            _phantom: std::marker::PhantomData,
279        }
280    }
281
282    #[inline(always)]
283    pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
284        let values = read_partial1_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10 });
285
286        let out = self.perform_parallel_fft_direct(values);
287        
288        write_partial_lo_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10 } ); 
289    }
290
291    #[inline(always)]
292    pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
293        let input_packed = read_complex_to_array!(buffer, { 0,2,4,6,8,10,12,14,16,18,20 });
294
295        let values = [
296            extract_lo_hi_f32(input_packed[0], input_packed[5]),
297            extract_hi_lo_f32(input_packed[0], input_packed[6]),
298            extract_lo_hi_f32(input_packed[1], input_packed[6]),
299            extract_hi_lo_f32(input_packed[1], input_packed[7]),
300            extract_lo_hi_f32(input_packed[2], input_packed[7]),
301            extract_hi_lo_f32(input_packed[2], input_packed[8]),
302            extract_lo_hi_f32(input_packed[3], input_packed[8]),
303            extract_hi_lo_f32(input_packed[3], input_packed[9]),
304            extract_lo_hi_f32(input_packed[4], input_packed[9]),
305            extract_hi_lo_f32(input_packed[4], input_packed[10]),
306            extract_lo_hi_f32(input_packed[5], input_packed[10]),
307        ];
308
309        let out = self.perform_parallel_fft_direct(values);
310
311        let out_packed = [
312            extract_lo_lo_f32(out[0], out[1]),
313            extract_lo_lo_f32(out[2], out[3]),
314            extract_lo_lo_f32(out[4], out[5]),
315            extract_lo_lo_f32(out[6], out[7]),
316            extract_lo_lo_f32(out[8], out[9]),
317            extract_lo_hi_f32(out[10], out[0]),
318            extract_hi_hi_f32(out[1], out[2]),
319            extract_hi_hi_f32(out[3], out[4]),
320            extract_hi_hi_f32(out[5], out[6]),
321            extract_hi_hi_f32(out[7], out[8]),
322            extract_hi_hi_f32(out[9], out[10]),
323        ];
324
325        write_complex_to_array_strided!(out_packed, buffer, 2, { 0,1,2,3,4,5,6,7,8,9,10 });
326    }
327
328    #[inline(always)]
329    pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 11]) -> [__m128; 11] {
330        let rotate = SseVector::make_rotate90(FftDirection::Inverse);
331
332        let y00 = values[0];
333        let [x1p10, x1m10] =  SseVector::column_butterfly2([values[1], values[10]]);
334        let x1m10 = SseVector::apply_rotate90(rotate, x1m10);
335        let y00 = SseVector::add(y00, x1p10);
336        let [x2p9, x2m9] =  SseVector::column_butterfly2([values[2], values[9]]);
337        let x2m9 = SseVector::apply_rotate90(rotate, x2m9);
338        let y00 = SseVector::add(y00, x2p9);
339        let [x3p8, x3m8] =  SseVector::column_butterfly2([values[3], values[8]]);
340        let x3m8 = SseVector::apply_rotate90(rotate, x3m8);
341        let y00 = SseVector::add(y00, x3p8);
342        let [x4p7, x4m7] =  SseVector::column_butterfly2([values[4], values[7]]);
343        let x4m7 = SseVector::apply_rotate90(rotate, x4m7);
344        let y00 = SseVector::add(y00, x4p7);
345        let [x5p6, x5m6] =  SseVector::column_butterfly2([values[5], values[6]]);
346        let x5m6 = SseVector::apply_rotate90(rotate, x5m6);
347        let y00 = SseVector::add(y00, x5p6);
348
349        let m0110a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p10);
350        let m0110a = SseVector::fmadd(m0110a, self.twiddles_re[1], x2p9);
351        let m0110a = SseVector::fmadd(m0110a, self.twiddles_re[2], x3p8);
352        let m0110a = SseVector::fmadd(m0110a, self.twiddles_re[3], x4p7);
353        let m0110a = SseVector::fmadd(m0110a, self.twiddles_re[4], x5p6);
354        let m0110b = SseVector::mul(self.twiddles_im[0], x1m10);
355        let m0110b = SseVector::fmadd(m0110b, self.twiddles_im[1], x2m9);
356        let m0110b = SseVector::fmadd(m0110b, self.twiddles_im[2], x3m8);
357        let m0110b = SseVector::fmadd(m0110b, self.twiddles_im[3], x4m7);
358        let m0110b = SseVector::fmadd(m0110b, self.twiddles_im[4], x5m6);
359        let [y01, y10] = SseVector::column_butterfly2([m0110a, m0110b]);
360
361        let m0209a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p10);
362        let m0209a = SseVector::fmadd(m0209a, self.twiddles_re[3], x2p9);
363        let m0209a = SseVector::fmadd(m0209a, self.twiddles_re[4], x3p8);
364        let m0209a = SseVector::fmadd(m0209a, self.twiddles_re[2], x4p7);
365        let m0209a = SseVector::fmadd(m0209a, self.twiddles_re[0], x5p6);
366        let m0209b = SseVector::mul(self.twiddles_im[1], x1m10);
367        let m0209b = SseVector::fmadd(m0209b, self.twiddles_im[3], x2m9);
368        let m0209b = SseVector::nmadd(m0209b, self.twiddles_im[4], x3m8);
369        let m0209b = SseVector::nmadd(m0209b, self.twiddles_im[2], x4m7);
370        let m0209b = SseVector::nmadd(m0209b, self.twiddles_im[0], x5m6);
371        let [y02, y09] = SseVector::column_butterfly2([m0209a, m0209b]);
372
373        let m0308a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p10);
374        let m0308a = SseVector::fmadd(m0308a, self.twiddles_re[4], x2p9);
375        let m0308a = SseVector::fmadd(m0308a, self.twiddles_re[1], x3p8);
376        let m0308a = SseVector::fmadd(m0308a, self.twiddles_re[0], x4p7);
377        let m0308a = SseVector::fmadd(m0308a, self.twiddles_re[3], x5p6);
378        let m0308b = SseVector::mul(self.twiddles_im[2], x1m10);
379        let m0308b = SseVector::nmadd(m0308b, self.twiddles_im[4], x2m9);
380        let m0308b = SseVector::nmadd(m0308b, self.twiddles_im[1], x3m8);
381        let m0308b = SseVector::fmadd(m0308b, self.twiddles_im[0], x4m7);
382        let m0308b = SseVector::fmadd(m0308b, self.twiddles_im[3], x5m6);
383        let [y03, y08] = SseVector::column_butterfly2([m0308a, m0308b]);
384
385        let m0407a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p10);
386        let m0407a = SseVector::fmadd(m0407a, self.twiddles_re[2], x2p9);
387        let m0407a = SseVector::fmadd(m0407a, self.twiddles_re[0], x3p8);
388        let m0407a = SseVector::fmadd(m0407a, self.twiddles_re[4], x4p7);
389        let m0407a = SseVector::fmadd(m0407a, self.twiddles_re[1], x5p6);
390        let m0407b = SseVector::mul(self.twiddles_im[3], x1m10);
391        let m0407b = SseVector::nmadd(m0407b, self.twiddles_im[2], x2m9);
392        let m0407b = SseVector::fmadd(m0407b, self.twiddles_im[0], x3m8);
393        let m0407b = SseVector::fmadd(m0407b, self.twiddles_im[4], x4m7);
394        let m0407b = SseVector::nmadd(m0407b, self.twiddles_im[1], x5m6);
395        let [y04, y07] = SseVector::column_butterfly2([m0407a, m0407b]);
396
397        let m0506a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p10);
398        let m0506a = SseVector::fmadd(m0506a, self.twiddles_re[0], x2p9);
399        let m0506a = SseVector::fmadd(m0506a, self.twiddles_re[3], x3p8);
400        let m0506a = SseVector::fmadd(m0506a, self.twiddles_re[1], x4p7);
401        let m0506a = SseVector::fmadd(m0506a, self.twiddles_re[2], x5p6);
402        let m0506b = SseVector::mul(self.twiddles_im[4], x1m10);
403        let m0506b = SseVector::nmadd(m0506b, self.twiddles_im[0], x2m9);
404        let m0506b = SseVector::fmadd(m0506b, self.twiddles_im[3], x3m8);
405        let m0506b = SseVector::nmadd(m0506b, self.twiddles_im[1], x4m7);
406        let m0506b = SseVector::fmadd(m0506b, self.twiddles_im[2], x5m6);
407        let [y05, y06] = SseVector::column_butterfly2([m0506a, m0506b]);
408
409
410        [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10]
411    }
412}
413
414struct SseF64Butterfly11<T> {
415    direction: FftDirection,
416    twiddles_re: [__m128d; 5],
417    twiddles_im: [__m128d; 5],
418    _phantom: std::marker::PhantomData<T>,
419}
420
421boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly11, 11, |this: &SseF64Butterfly11<_>| this.direction);
422impl<T: FftNum> SseF64Butterfly11<T> {
423    /// Safety: The current machine must support the sse4.1 instruction set
424    #[target_feature(enable = "sse4.1")]
425    unsafe fn new(direction: FftDirection) -> Self {
426        assert_f64::<T>();
427        let twiddles = make_twiddles(11, direction);
428        unsafe {Self {
429            direction,
430            twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
431            twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
432            _phantom: std::marker::PhantomData,
433        }}
434    }
435
436    #[inline(always)]
437    pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
438        let values = read_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10 });
439
440        let out = self.perform_fft_direct(values);
441
442        write_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10 });   
443    }
444
445    #[inline(always)]
446    pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 11]) -> [__m128d; 11] {
447        let rotate = SseVector::make_rotate90(FftDirection::Inverse);
448
449        let y00 = values[0];
450        let [x1p10, x1m10] =  SseVector::column_butterfly2([values[1], values[10]]);
451        let x1m10 = SseVector::apply_rotate90(rotate, x1m10);
452        let y00 = SseVector::add(y00, x1p10);
453        let [x2p9, x2m9] =  SseVector::column_butterfly2([values[2], values[9]]);
454        let x2m9 = SseVector::apply_rotate90(rotate, x2m9);
455        let y00 = SseVector::add(y00, x2p9);
456        let [x3p8, x3m8] =  SseVector::column_butterfly2([values[3], values[8]]);
457        let x3m8 = SseVector::apply_rotate90(rotate, x3m8);
458        let y00 = SseVector::add(y00, x3p8);
459        let [x4p7, x4m7] =  SseVector::column_butterfly2([values[4], values[7]]);
460        let x4m7 = SseVector::apply_rotate90(rotate, x4m7);
461        let y00 = SseVector::add(y00, x4p7);
462        let [x5p6, x5m6] =  SseVector::column_butterfly2([values[5], values[6]]);
463        let x5m6 = SseVector::apply_rotate90(rotate, x5m6);
464        let y00 = SseVector::add(y00, x5p6);
465
466        let m0110a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p10);
467        let m0110a = SseVector::fmadd(m0110a, self.twiddles_re[1], x2p9);
468        let m0110a = SseVector::fmadd(m0110a, self.twiddles_re[2], x3p8);
469        let m0110a = SseVector::fmadd(m0110a, self.twiddles_re[3], x4p7);
470        let m0110a = SseVector::fmadd(m0110a, self.twiddles_re[4], x5p6);
471        let m0110b = SseVector::mul(self.twiddles_im[0], x1m10);
472        let m0110b = SseVector::fmadd(m0110b, self.twiddles_im[1], x2m9);
473        let m0110b = SseVector::fmadd(m0110b, self.twiddles_im[2], x3m8);
474        let m0110b = SseVector::fmadd(m0110b, self.twiddles_im[3], x4m7);
475        let m0110b = SseVector::fmadd(m0110b, self.twiddles_im[4], x5m6);
476        let [y01, y10] = SseVector::column_butterfly2([m0110a, m0110b]);
477
478        let m0209a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p10);
479        let m0209a = SseVector::fmadd(m0209a, self.twiddles_re[3], x2p9);
480        let m0209a = SseVector::fmadd(m0209a, self.twiddles_re[4], x3p8);
481        let m0209a = SseVector::fmadd(m0209a, self.twiddles_re[2], x4p7);
482        let m0209a = SseVector::fmadd(m0209a, self.twiddles_re[0], x5p6);
483        let m0209b = SseVector::mul(self.twiddles_im[1], x1m10);
484        let m0209b = SseVector::fmadd(m0209b, self.twiddles_im[3], x2m9);
485        let m0209b = SseVector::nmadd(m0209b, self.twiddles_im[4], x3m8);
486        let m0209b = SseVector::nmadd(m0209b, self.twiddles_im[2], x4m7);
487        let m0209b = SseVector::nmadd(m0209b, self.twiddles_im[0], x5m6);
488        let [y02, y09] = SseVector::column_butterfly2([m0209a, m0209b]);
489
490        let m0308a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p10);
491        let m0308a = SseVector::fmadd(m0308a, self.twiddles_re[4], x2p9);
492        let m0308a = SseVector::fmadd(m0308a, self.twiddles_re[1], x3p8);
493        let m0308a = SseVector::fmadd(m0308a, self.twiddles_re[0], x4p7);
494        let m0308a = SseVector::fmadd(m0308a, self.twiddles_re[3], x5p6);
495        let m0308b = SseVector::mul(self.twiddles_im[2], x1m10);
496        let m0308b = SseVector::nmadd(m0308b, self.twiddles_im[4], x2m9);
497        let m0308b = SseVector::nmadd(m0308b, self.twiddles_im[1], x3m8);
498        let m0308b = SseVector::fmadd(m0308b, self.twiddles_im[0], x4m7);
499        let m0308b = SseVector::fmadd(m0308b, self.twiddles_im[3], x5m6);
500        let [y03, y08] = SseVector::column_butterfly2([m0308a, m0308b]);
501
502        let m0407a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p10);
503        let m0407a = SseVector::fmadd(m0407a, self.twiddles_re[2], x2p9);
504        let m0407a = SseVector::fmadd(m0407a, self.twiddles_re[0], x3p8);
505        let m0407a = SseVector::fmadd(m0407a, self.twiddles_re[4], x4p7);
506        let m0407a = SseVector::fmadd(m0407a, self.twiddles_re[1], x5p6);
507        let m0407b = SseVector::mul(self.twiddles_im[3], x1m10);
508        let m0407b = SseVector::nmadd(m0407b, self.twiddles_im[2], x2m9);
509        let m0407b = SseVector::fmadd(m0407b, self.twiddles_im[0], x3m8);
510        let m0407b = SseVector::fmadd(m0407b, self.twiddles_im[4], x4m7);
511        let m0407b = SseVector::nmadd(m0407b, self.twiddles_im[1], x5m6);
512        let [y04, y07] = SseVector::column_butterfly2([m0407a, m0407b]);
513
514        let m0506a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p10);
515        let m0506a = SseVector::fmadd(m0506a, self.twiddles_re[0], x2p9);
516        let m0506a = SseVector::fmadd(m0506a, self.twiddles_re[3], x3p8);
517        let m0506a = SseVector::fmadd(m0506a, self.twiddles_re[1], x4p7);
518        let m0506a = SseVector::fmadd(m0506a, self.twiddles_re[2], x5p6);
519        let m0506b = SseVector::mul(self.twiddles_im[4], x1m10);
520        let m0506b = SseVector::nmadd(m0506b, self.twiddles_im[0], x2m9);
521        let m0506b = SseVector::fmadd(m0506b, self.twiddles_im[3], x3m8);
522        let m0506b = SseVector::nmadd(m0506b, self.twiddles_im[1], x4m7);
523        let m0506b = SseVector::fmadd(m0506b, self.twiddles_im[2], x5m6);
524        let [y05, y06] = SseVector::column_butterfly2([m0506a, m0506b]);
525
526
527        [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10]
528    }
529}
530
531struct SseF32Butterfly13<T> {
532    direction: FftDirection,
533    twiddles_re: [__m128; 6],
534    twiddles_im: [__m128; 6],
535    _phantom: std::marker::PhantomData<T>,
536}
537
538boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly13, 13, |this: &SseF32Butterfly13<_>| this.direction);
539impl<T: FftNum> SseF32Butterfly13<T> {
540    /// Safety: The current machine must support the sse4.1 instruction set
541    #[target_feature(enable = "sse4.1")]
542    unsafe fn new(direction: FftDirection) -> Self {
543        assert_f32::<T>();
544        let twiddles = make_twiddles(13, direction);
545        Self {
546            direction,
547            twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
548            twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
549            _phantom: std::marker::PhantomData,
550        }
551    }
552
553    #[inline(always)]
554    pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
555        let values = read_partial1_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12 });
556
557        let out = self.perform_parallel_fft_direct(values);
558        
559        write_partial_lo_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12 } ); 
560    }
561
562    #[inline(always)]
563    pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
564        let input_packed = read_complex_to_array!(buffer, { 0,2,4,6,8,10,12,14,16,18,20,22,24 });
565
566        let values = [
567            extract_lo_hi_f32(input_packed[0], input_packed[6]),
568            extract_hi_lo_f32(input_packed[0], input_packed[7]),
569            extract_lo_hi_f32(input_packed[1], input_packed[7]),
570            extract_hi_lo_f32(input_packed[1], input_packed[8]),
571            extract_lo_hi_f32(input_packed[2], input_packed[8]),
572            extract_hi_lo_f32(input_packed[2], input_packed[9]),
573            extract_lo_hi_f32(input_packed[3], input_packed[9]),
574            extract_hi_lo_f32(input_packed[3], input_packed[10]),
575            extract_lo_hi_f32(input_packed[4], input_packed[10]),
576            extract_hi_lo_f32(input_packed[4], input_packed[11]),
577            extract_lo_hi_f32(input_packed[5], input_packed[11]),
578            extract_hi_lo_f32(input_packed[5], input_packed[12]),
579            extract_lo_hi_f32(input_packed[6], input_packed[12]),
580        ];
581
582        let out = self.perform_parallel_fft_direct(values);
583
584        let out_packed = [
585            extract_lo_lo_f32(out[0], out[1]),
586            extract_lo_lo_f32(out[2], out[3]),
587            extract_lo_lo_f32(out[4], out[5]),
588            extract_lo_lo_f32(out[6], out[7]),
589            extract_lo_lo_f32(out[8], out[9]),
590            extract_lo_lo_f32(out[10], out[11]),
591            extract_lo_hi_f32(out[12], out[0]),
592            extract_hi_hi_f32(out[1], out[2]),
593            extract_hi_hi_f32(out[3], out[4]),
594            extract_hi_hi_f32(out[5], out[6]),
595            extract_hi_hi_f32(out[7], out[8]),
596            extract_hi_hi_f32(out[9], out[10]),
597            extract_hi_hi_f32(out[11], out[12]),
598        ];
599
600        write_complex_to_array_strided!(out_packed, buffer, 2, { 0,1,2,3,4,5,6,7,8,9,10,11,12 });
601    }
602
603    #[inline(always)]
604    pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 13]) -> [__m128; 13] {
605        let rotate = SseVector::make_rotate90(FftDirection::Inverse);
606
607        let y00 = values[0];
608        let [x1p12, x1m12] =  SseVector::column_butterfly2([values[1], values[12]]);
609        let x1m12 = SseVector::apply_rotate90(rotate, x1m12);
610        let y00 = SseVector::add(y00, x1p12);
611        let [x2p11, x2m11] =  SseVector::column_butterfly2([values[2], values[11]]);
612        let x2m11 = SseVector::apply_rotate90(rotate, x2m11);
613        let y00 = SseVector::add(y00, x2p11);
614        let [x3p10, x3m10] =  SseVector::column_butterfly2([values[3], values[10]]);
615        let x3m10 = SseVector::apply_rotate90(rotate, x3m10);
616        let y00 = SseVector::add(y00, x3p10);
617        let [x4p9, x4m9] =  SseVector::column_butterfly2([values[4], values[9]]);
618        let x4m9 = SseVector::apply_rotate90(rotate, x4m9);
619        let y00 = SseVector::add(y00, x4p9);
620        let [x5p8, x5m8] =  SseVector::column_butterfly2([values[5], values[8]]);
621        let x5m8 = SseVector::apply_rotate90(rotate, x5m8);
622        let y00 = SseVector::add(y00, x5p8);
623        let [x6p7, x6m7] =  SseVector::column_butterfly2([values[6], values[7]]);
624        let x6m7 = SseVector::apply_rotate90(rotate, x6m7);
625        let y00 = SseVector::add(y00, x6p7);
626
627        let m0112a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p12);
628        let m0112a = SseVector::fmadd(m0112a, self.twiddles_re[1], x2p11);
629        let m0112a = SseVector::fmadd(m0112a, self.twiddles_re[2], x3p10);
630        let m0112a = SseVector::fmadd(m0112a, self.twiddles_re[3], x4p9);
631        let m0112a = SseVector::fmadd(m0112a, self.twiddles_re[4], x5p8);
632        let m0112a = SseVector::fmadd(m0112a, self.twiddles_re[5], x6p7);
633        let m0112b = SseVector::mul(self.twiddles_im[0], x1m12);
634        let m0112b = SseVector::fmadd(m0112b, self.twiddles_im[1], x2m11);
635        let m0112b = SseVector::fmadd(m0112b, self.twiddles_im[2], x3m10);
636        let m0112b = SseVector::fmadd(m0112b, self.twiddles_im[3], x4m9);
637        let m0112b = SseVector::fmadd(m0112b, self.twiddles_im[4], x5m8);
638        let m0112b = SseVector::fmadd(m0112b, self.twiddles_im[5], x6m7);
639        let [y01, y12] = SseVector::column_butterfly2([m0112a, m0112b]);
640
641        let m0211a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p12);
642        let m0211a = SseVector::fmadd(m0211a, self.twiddles_re[3], x2p11);
643        let m0211a = SseVector::fmadd(m0211a, self.twiddles_re[5], x3p10);
644        let m0211a = SseVector::fmadd(m0211a, self.twiddles_re[4], x4p9);
645        let m0211a = SseVector::fmadd(m0211a, self.twiddles_re[2], x5p8);
646        let m0211a = SseVector::fmadd(m0211a, self.twiddles_re[0], x6p7);
647        let m0211b = SseVector::mul(self.twiddles_im[1], x1m12);
648        let m0211b = SseVector::fmadd(m0211b, self.twiddles_im[3], x2m11);
649        let m0211b = SseVector::fmadd(m0211b, self.twiddles_im[5], x3m10);
650        let m0211b = SseVector::nmadd(m0211b, self.twiddles_im[4], x4m9);
651        let m0211b = SseVector::nmadd(m0211b, self.twiddles_im[2], x5m8);
652        let m0211b = SseVector::nmadd(m0211b, self.twiddles_im[0], x6m7);
653        let [y02, y11] = SseVector::column_butterfly2([m0211a, m0211b]);
654
655        let m0310a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p12);
656        let m0310a = SseVector::fmadd(m0310a, self.twiddles_re[5], x2p11);
657        let m0310a = SseVector::fmadd(m0310a, self.twiddles_re[3], x3p10);
658        let m0310a = SseVector::fmadd(m0310a, self.twiddles_re[0], x4p9);
659        let m0310a = SseVector::fmadd(m0310a, self.twiddles_re[1], x5p8);
660        let m0310a = SseVector::fmadd(m0310a, self.twiddles_re[4], x6p7);
661        let m0310b = SseVector::mul(self.twiddles_im[2], x1m12);
662        let m0310b = SseVector::fmadd(m0310b, self.twiddles_im[5], x2m11);
663        let m0310b = SseVector::nmadd(m0310b, self.twiddles_im[3], x3m10);
664        let m0310b = SseVector::nmadd(m0310b, self.twiddles_im[0], x4m9);
665        let m0310b = SseVector::fmadd(m0310b, self.twiddles_im[1], x5m8);
666        let m0310b = SseVector::fmadd(m0310b, self.twiddles_im[4], x6m7);
667        let [y03, y10] = SseVector::column_butterfly2([m0310a, m0310b]);
668
669        let m0409a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p12);
670        let m0409a = SseVector::fmadd(m0409a, self.twiddles_re[4], x2p11);
671        let m0409a = SseVector::fmadd(m0409a, self.twiddles_re[0], x3p10);
672        let m0409a = SseVector::fmadd(m0409a, self.twiddles_re[2], x4p9);
673        let m0409a = SseVector::fmadd(m0409a, self.twiddles_re[5], x5p8);
674        let m0409a = SseVector::fmadd(m0409a, self.twiddles_re[1], x6p7);
675        let m0409b = SseVector::mul(self.twiddles_im[3], x1m12);
676        let m0409b = SseVector::nmadd(m0409b, self.twiddles_im[4], x2m11);
677        let m0409b = SseVector::nmadd(m0409b, self.twiddles_im[0], x3m10);
678        let m0409b = SseVector::fmadd(m0409b, self.twiddles_im[2], x4m9);
679        let m0409b = SseVector::nmadd(m0409b, self.twiddles_im[5], x5m8);
680        let m0409b = SseVector::nmadd(m0409b, self.twiddles_im[1], x6m7);
681        let [y04, y09] = SseVector::column_butterfly2([m0409a, m0409b]);
682
683        let m0508a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p12);
684        let m0508a = SseVector::fmadd(m0508a, self.twiddles_re[2], x2p11);
685        let m0508a = SseVector::fmadd(m0508a, self.twiddles_re[1], x3p10);
686        let m0508a = SseVector::fmadd(m0508a, self.twiddles_re[5], x4p9);
687        let m0508a = SseVector::fmadd(m0508a, self.twiddles_re[0], x5p8);
688        let m0508a = SseVector::fmadd(m0508a, self.twiddles_re[3], x6p7);
689        let m0508b = SseVector::mul(self.twiddles_im[4], x1m12);
690        let m0508b = SseVector::nmadd(m0508b, self.twiddles_im[2], x2m11);
691        let m0508b = SseVector::fmadd(m0508b, self.twiddles_im[1], x3m10);
692        let m0508b = SseVector::nmadd(m0508b, self.twiddles_im[5], x4m9);
693        let m0508b = SseVector::nmadd(m0508b, self.twiddles_im[0], x5m8);
694        let m0508b = SseVector::fmadd(m0508b, self.twiddles_im[3], x6m7);
695        let [y05, y08] = SseVector::column_butterfly2([m0508a, m0508b]);
696
697        let m0607a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p12);
698        let m0607a = SseVector::fmadd(m0607a, self.twiddles_re[0], x2p11);
699        let m0607a = SseVector::fmadd(m0607a, self.twiddles_re[4], x3p10);
700        let m0607a = SseVector::fmadd(m0607a, self.twiddles_re[1], x4p9);
701        let m0607a = SseVector::fmadd(m0607a, self.twiddles_re[3], x5p8);
702        let m0607a = SseVector::fmadd(m0607a, self.twiddles_re[2], x6p7);
703        let m0607b = SseVector::mul(self.twiddles_im[5], x1m12);
704        let m0607b = SseVector::nmadd(m0607b, self.twiddles_im[0], x2m11);
705        let m0607b = SseVector::fmadd(m0607b, self.twiddles_im[4], x3m10);
706        let m0607b = SseVector::nmadd(m0607b, self.twiddles_im[1], x4m9);
707        let m0607b = SseVector::fmadd(m0607b, self.twiddles_im[3], x5m8);
708        let m0607b = SseVector::nmadd(m0607b, self.twiddles_im[2], x6m7);
709        let [y06, y07] = SseVector::column_butterfly2([m0607a, m0607b]);
710
711
712        [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12]
713    }
714}
715
716struct SseF64Butterfly13<T> {
717    direction: FftDirection,
718    twiddles_re: [__m128d; 6],
719    twiddles_im: [__m128d; 6],
720    _phantom: std::marker::PhantomData<T>,
721}
722
723boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly13, 13, |this: &SseF64Butterfly13<_>| this.direction);
724impl<T: FftNum> SseF64Butterfly13<T> {
725    /// Safety: The current machine must support the sse4.1 instruction set
726    #[target_feature(enable = "sse4.1")]
727    unsafe fn new(direction: FftDirection) -> Self {
728        assert_f64::<T>();
729        let twiddles = make_twiddles(13, direction);
730        unsafe {Self {
731            direction,
732            twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
733            twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
734            _phantom: std::marker::PhantomData,
735        }}
736    }
737
738    #[inline(always)]
739    pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
740        let values = read_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12 });
741
742        let out = self.perform_fft_direct(values);
743
744        write_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12 });   
745    }
746
747    #[inline(always)]
748    pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 13]) -> [__m128d; 13] {
749        let rotate = SseVector::make_rotate90(FftDirection::Inverse);
750
751        let y00 = values[0];
752        let [x1p12, x1m12] =  SseVector::column_butterfly2([values[1], values[12]]);
753        let x1m12 = SseVector::apply_rotate90(rotate, x1m12);
754        let y00 = SseVector::add(y00, x1p12);
755        let [x2p11, x2m11] =  SseVector::column_butterfly2([values[2], values[11]]);
756        let x2m11 = SseVector::apply_rotate90(rotate, x2m11);
757        let y00 = SseVector::add(y00, x2p11);
758        let [x3p10, x3m10] =  SseVector::column_butterfly2([values[3], values[10]]);
759        let x3m10 = SseVector::apply_rotate90(rotate, x3m10);
760        let y00 = SseVector::add(y00, x3p10);
761        let [x4p9, x4m9] =  SseVector::column_butterfly2([values[4], values[9]]);
762        let x4m9 = SseVector::apply_rotate90(rotate, x4m9);
763        let y00 = SseVector::add(y00, x4p9);
764        let [x5p8, x5m8] =  SseVector::column_butterfly2([values[5], values[8]]);
765        let x5m8 = SseVector::apply_rotate90(rotate, x5m8);
766        let y00 = SseVector::add(y00, x5p8);
767        let [x6p7, x6m7] =  SseVector::column_butterfly2([values[6], values[7]]);
768        let x6m7 = SseVector::apply_rotate90(rotate, x6m7);
769        let y00 = SseVector::add(y00, x6p7);
770
771        let m0112a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p12);
772        let m0112a = SseVector::fmadd(m0112a, self.twiddles_re[1], x2p11);
773        let m0112a = SseVector::fmadd(m0112a, self.twiddles_re[2], x3p10);
774        let m0112a = SseVector::fmadd(m0112a, self.twiddles_re[3], x4p9);
775        let m0112a = SseVector::fmadd(m0112a, self.twiddles_re[4], x5p8);
776        let m0112a = SseVector::fmadd(m0112a, self.twiddles_re[5], x6p7);
777        let m0112b = SseVector::mul(self.twiddles_im[0], x1m12);
778        let m0112b = SseVector::fmadd(m0112b, self.twiddles_im[1], x2m11);
779        let m0112b = SseVector::fmadd(m0112b, self.twiddles_im[2], x3m10);
780        let m0112b = SseVector::fmadd(m0112b, self.twiddles_im[3], x4m9);
781        let m0112b = SseVector::fmadd(m0112b, self.twiddles_im[4], x5m8);
782        let m0112b = SseVector::fmadd(m0112b, self.twiddles_im[5], x6m7);
783        let [y01, y12] = SseVector::column_butterfly2([m0112a, m0112b]);
784
785        let m0211a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p12);
786        let m0211a = SseVector::fmadd(m0211a, self.twiddles_re[3], x2p11);
787        let m0211a = SseVector::fmadd(m0211a, self.twiddles_re[5], x3p10);
788        let m0211a = SseVector::fmadd(m0211a, self.twiddles_re[4], x4p9);
789        let m0211a = SseVector::fmadd(m0211a, self.twiddles_re[2], x5p8);
790        let m0211a = SseVector::fmadd(m0211a, self.twiddles_re[0], x6p7);
791        let m0211b = SseVector::mul(self.twiddles_im[1], x1m12);
792        let m0211b = SseVector::fmadd(m0211b, self.twiddles_im[3], x2m11);
793        let m0211b = SseVector::fmadd(m0211b, self.twiddles_im[5], x3m10);
794        let m0211b = SseVector::nmadd(m0211b, self.twiddles_im[4], x4m9);
795        let m0211b = SseVector::nmadd(m0211b, self.twiddles_im[2], x5m8);
796        let m0211b = SseVector::nmadd(m0211b, self.twiddles_im[0], x6m7);
797        let [y02, y11] = SseVector::column_butterfly2([m0211a, m0211b]);
798
799        let m0310a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p12);
800        let m0310a = SseVector::fmadd(m0310a, self.twiddles_re[5], x2p11);
801        let m0310a = SseVector::fmadd(m0310a, self.twiddles_re[3], x3p10);
802        let m0310a = SseVector::fmadd(m0310a, self.twiddles_re[0], x4p9);
803        let m0310a = SseVector::fmadd(m0310a, self.twiddles_re[1], x5p8);
804        let m0310a = SseVector::fmadd(m0310a, self.twiddles_re[4], x6p7);
805        let m0310b = SseVector::mul(self.twiddles_im[2], x1m12);
806        let m0310b = SseVector::fmadd(m0310b, self.twiddles_im[5], x2m11);
807        let m0310b = SseVector::nmadd(m0310b, self.twiddles_im[3], x3m10);
808        let m0310b = SseVector::nmadd(m0310b, self.twiddles_im[0], x4m9);
809        let m0310b = SseVector::fmadd(m0310b, self.twiddles_im[1], x5m8);
810        let m0310b = SseVector::fmadd(m0310b, self.twiddles_im[4], x6m7);
811        let [y03, y10] = SseVector::column_butterfly2([m0310a, m0310b]);
812
813        let m0409a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p12);
814        let m0409a = SseVector::fmadd(m0409a, self.twiddles_re[4], x2p11);
815        let m0409a = SseVector::fmadd(m0409a, self.twiddles_re[0], x3p10);
816        let m0409a = SseVector::fmadd(m0409a, self.twiddles_re[2], x4p9);
817        let m0409a = SseVector::fmadd(m0409a, self.twiddles_re[5], x5p8);
818        let m0409a = SseVector::fmadd(m0409a, self.twiddles_re[1], x6p7);
819        let m0409b = SseVector::mul(self.twiddles_im[3], x1m12);
820        let m0409b = SseVector::nmadd(m0409b, self.twiddles_im[4], x2m11);
821        let m0409b = SseVector::nmadd(m0409b, self.twiddles_im[0], x3m10);
822        let m0409b = SseVector::fmadd(m0409b, self.twiddles_im[2], x4m9);
823        let m0409b = SseVector::nmadd(m0409b, self.twiddles_im[5], x5m8);
824        let m0409b = SseVector::nmadd(m0409b, self.twiddles_im[1], x6m7);
825        let [y04, y09] = SseVector::column_butterfly2([m0409a, m0409b]);
826
827        let m0508a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p12);
828        let m0508a = SseVector::fmadd(m0508a, self.twiddles_re[2], x2p11);
829        let m0508a = SseVector::fmadd(m0508a, self.twiddles_re[1], x3p10);
830        let m0508a = SseVector::fmadd(m0508a, self.twiddles_re[5], x4p9);
831        let m0508a = SseVector::fmadd(m0508a, self.twiddles_re[0], x5p8);
832        let m0508a = SseVector::fmadd(m0508a, self.twiddles_re[3], x6p7);
833        let m0508b = SseVector::mul(self.twiddles_im[4], x1m12);
834        let m0508b = SseVector::nmadd(m0508b, self.twiddles_im[2], x2m11);
835        let m0508b = SseVector::fmadd(m0508b, self.twiddles_im[1], x3m10);
836        let m0508b = SseVector::nmadd(m0508b, self.twiddles_im[5], x4m9);
837        let m0508b = SseVector::nmadd(m0508b, self.twiddles_im[0], x5m8);
838        let m0508b = SseVector::fmadd(m0508b, self.twiddles_im[3], x6m7);
839        let [y05, y08] = SseVector::column_butterfly2([m0508a, m0508b]);
840
841        let m0607a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p12);
842        let m0607a = SseVector::fmadd(m0607a, self.twiddles_re[0], x2p11);
843        let m0607a = SseVector::fmadd(m0607a, self.twiddles_re[4], x3p10);
844        let m0607a = SseVector::fmadd(m0607a, self.twiddles_re[1], x4p9);
845        let m0607a = SseVector::fmadd(m0607a, self.twiddles_re[3], x5p8);
846        let m0607a = SseVector::fmadd(m0607a, self.twiddles_re[2], x6p7);
847        let m0607b = SseVector::mul(self.twiddles_im[5], x1m12);
848        let m0607b = SseVector::nmadd(m0607b, self.twiddles_im[0], x2m11);
849        let m0607b = SseVector::fmadd(m0607b, self.twiddles_im[4], x3m10);
850        let m0607b = SseVector::nmadd(m0607b, self.twiddles_im[1], x4m9);
851        let m0607b = SseVector::fmadd(m0607b, self.twiddles_im[3], x5m8);
852        let m0607b = SseVector::nmadd(m0607b, self.twiddles_im[2], x6m7);
853        let [y06, y07] = SseVector::column_butterfly2([m0607a, m0607b]);
854
855
856        [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12]
857    }
858}
859
860struct SseF32Butterfly17<T> {
861    direction: FftDirection,
862    twiddles_re: [__m128; 8],
863    twiddles_im: [__m128; 8],
864    _phantom: std::marker::PhantomData<T>,
865}
866
867boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly17, 17, |this: &SseF32Butterfly17<_>| this.direction);
868impl<T: FftNum> SseF32Butterfly17<T> {
869    /// Safety: The current machine must support the sse4.1 instruction set
870    #[target_feature(enable = "sse4.1")]
871    unsafe fn new(direction: FftDirection) -> Self {
872        assert_f32::<T>();
873        let twiddles = make_twiddles(17, direction);
874        Self {
875            direction,
876            twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
877            twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
878            _phantom: std::marker::PhantomData,
879        }
880    }
881
882    #[inline(always)]
883    pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
884        let values = read_partial1_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16 });
885
886        let out = self.perform_parallel_fft_direct(values);
887        
888        write_partial_lo_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16 } ); 
889    }
890
891    #[inline(always)]
892    pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
893        let input_packed = read_complex_to_array!(buffer, { 0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32 });
894
895        let values = [
896            extract_lo_hi_f32(input_packed[0], input_packed[8]),
897            extract_hi_lo_f32(input_packed[0], input_packed[9]),
898            extract_lo_hi_f32(input_packed[1], input_packed[9]),
899            extract_hi_lo_f32(input_packed[1], input_packed[10]),
900            extract_lo_hi_f32(input_packed[2], input_packed[10]),
901            extract_hi_lo_f32(input_packed[2], input_packed[11]),
902            extract_lo_hi_f32(input_packed[3], input_packed[11]),
903            extract_hi_lo_f32(input_packed[3], input_packed[12]),
904            extract_lo_hi_f32(input_packed[4], input_packed[12]),
905            extract_hi_lo_f32(input_packed[4], input_packed[13]),
906            extract_lo_hi_f32(input_packed[5], input_packed[13]),
907            extract_hi_lo_f32(input_packed[5], input_packed[14]),
908            extract_lo_hi_f32(input_packed[6], input_packed[14]),
909            extract_hi_lo_f32(input_packed[6], input_packed[15]),
910            extract_lo_hi_f32(input_packed[7], input_packed[15]),
911            extract_hi_lo_f32(input_packed[7], input_packed[16]),
912            extract_lo_hi_f32(input_packed[8], input_packed[16]),
913        ];
914
915        let out = self.perform_parallel_fft_direct(values);
916
917        let out_packed = [
918            extract_lo_lo_f32(out[0], out[1]),
919            extract_lo_lo_f32(out[2], out[3]),
920            extract_lo_lo_f32(out[4], out[5]),
921            extract_lo_lo_f32(out[6], out[7]),
922            extract_lo_lo_f32(out[8], out[9]),
923            extract_lo_lo_f32(out[10], out[11]),
924            extract_lo_lo_f32(out[12], out[13]),
925            extract_lo_lo_f32(out[14], out[15]),
926            extract_lo_hi_f32(out[16], out[0]),
927            extract_hi_hi_f32(out[1], out[2]),
928            extract_hi_hi_f32(out[3], out[4]),
929            extract_hi_hi_f32(out[5], out[6]),
930            extract_hi_hi_f32(out[7], out[8]),
931            extract_hi_hi_f32(out[9], out[10]),
932            extract_hi_hi_f32(out[11], out[12]),
933            extract_hi_hi_f32(out[13], out[14]),
934            extract_hi_hi_f32(out[15], out[16]),
935        ];
936
937        write_complex_to_array_strided!(out_packed, buffer, 2, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16 });
938    }
939
940    #[inline(always)]
941    pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 17]) -> [__m128; 17] {
942        let rotate = SseVector::make_rotate90(FftDirection::Inverse);
943
944        let y00 = values[0];
945        let [x1p16, x1m16] =  SseVector::column_butterfly2([values[1], values[16]]);
946        let x1m16 = SseVector::apply_rotate90(rotate, x1m16);
947        let y00 = SseVector::add(y00, x1p16);
948        let [x2p15, x2m15] =  SseVector::column_butterfly2([values[2], values[15]]);
949        let x2m15 = SseVector::apply_rotate90(rotate, x2m15);
950        let y00 = SseVector::add(y00, x2p15);
951        let [x3p14, x3m14] =  SseVector::column_butterfly2([values[3], values[14]]);
952        let x3m14 = SseVector::apply_rotate90(rotate, x3m14);
953        let y00 = SseVector::add(y00, x3p14);
954        let [x4p13, x4m13] =  SseVector::column_butterfly2([values[4], values[13]]);
955        let x4m13 = SseVector::apply_rotate90(rotate, x4m13);
956        let y00 = SseVector::add(y00, x4p13);
957        let [x5p12, x5m12] =  SseVector::column_butterfly2([values[5], values[12]]);
958        let x5m12 = SseVector::apply_rotate90(rotate, x5m12);
959        let y00 = SseVector::add(y00, x5p12);
960        let [x6p11, x6m11] =  SseVector::column_butterfly2([values[6], values[11]]);
961        let x6m11 = SseVector::apply_rotate90(rotate, x6m11);
962        let y00 = SseVector::add(y00, x6p11);
963        let [x7p10, x7m10] =  SseVector::column_butterfly2([values[7], values[10]]);
964        let x7m10 = SseVector::apply_rotate90(rotate, x7m10);
965        let y00 = SseVector::add(y00, x7p10);
966        let [x8p9, x8m9] =  SseVector::column_butterfly2([values[8], values[9]]);
967        let x8m9 = SseVector::apply_rotate90(rotate, x8m9);
968        let y00 = SseVector::add(y00, x8p9);
969
970        let m0116a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p16);
971        let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[1], x2p15);
972        let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[2], x3p14);
973        let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[3], x4p13);
974        let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[4], x5p12);
975        let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[5], x6p11);
976        let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[6], x7p10);
977        let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[7], x8p9);
978        let m0116b = SseVector::mul(self.twiddles_im[0], x1m16);
979        let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[1], x2m15);
980        let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[2], x3m14);
981        let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[3], x4m13);
982        let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[4], x5m12);
983        let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[5], x6m11);
984        let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[6], x7m10);
985        let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[7], x8m9);
986        let [y01, y16] = SseVector::column_butterfly2([m0116a, m0116b]);
987
988        let m0215a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p16);
989        let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[3], x2p15);
990        let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[5], x3p14);
991        let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[7], x4p13);
992        let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[6], x5p12);
993        let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[4], x6p11);
994        let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[2], x7p10);
995        let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[0], x8p9);
996        let m0215b = SseVector::mul(self.twiddles_im[1], x1m16);
997        let m0215b = SseVector::fmadd(m0215b, self.twiddles_im[3], x2m15);
998        let m0215b = SseVector::fmadd(m0215b, self.twiddles_im[5], x3m14);
999        let m0215b = SseVector::fmadd(m0215b, self.twiddles_im[7], x4m13);
1000        let m0215b = SseVector::nmadd(m0215b, self.twiddles_im[6], x5m12);
1001        let m0215b = SseVector::nmadd(m0215b, self.twiddles_im[4], x6m11);
1002        let m0215b = SseVector::nmadd(m0215b, self.twiddles_im[2], x7m10);
1003        let m0215b = SseVector::nmadd(m0215b, self.twiddles_im[0], x8m9);
1004        let [y02, y15] = SseVector::column_butterfly2([m0215a, m0215b]);
1005
1006        let m0314a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p16);
1007        let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[5], x2p15);
1008        let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[7], x3p14);
1009        let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[4], x4p13);
1010        let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[1], x5p12);
1011        let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[0], x6p11);
1012        let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[3], x7p10);
1013        let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[6], x8p9);
1014        let m0314b = SseVector::mul(self.twiddles_im[2], x1m16);
1015        let m0314b = SseVector::fmadd(m0314b, self.twiddles_im[5], x2m15);
1016        let m0314b = SseVector::nmadd(m0314b, self.twiddles_im[7], x3m14);
1017        let m0314b = SseVector::nmadd(m0314b, self.twiddles_im[4], x4m13);
1018        let m0314b = SseVector::nmadd(m0314b, self.twiddles_im[1], x5m12);
1019        let m0314b = SseVector::fmadd(m0314b, self.twiddles_im[0], x6m11);
1020        let m0314b = SseVector::fmadd(m0314b, self.twiddles_im[3], x7m10);
1021        let m0314b = SseVector::fmadd(m0314b, self.twiddles_im[6], x8m9);
1022        let [y03, y14] = SseVector::column_butterfly2([m0314a, m0314b]);
1023
1024        let m0413a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p16);
1025        let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[7], x2p15);
1026        let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[4], x3p14);
1027        let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[0], x4p13);
1028        let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[2], x5p12);
1029        let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[6], x6p11);
1030        let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[5], x7p10);
1031        let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[1], x8p9);
1032        let m0413b = SseVector::mul(self.twiddles_im[3], x1m16);
1033        let m0413b = SseVector::fmadd(m0413b, self.twiddles_im[7], x2m15);
1034        let m0413b = SseVector::nmadd(m0413b, self.twiddles_im[4], x3m14);
1035        let m0413b = SseVector::nmadd(m0413b, self.twiddles_im[0], x4m13);
1036        let m0413b = SseVector::fmadd(m0413b, self.twiddles_im[2], x5m12);
1037        let m0413b = SseVector::fmadd(m0413b, self.twiddles_im[6], x6m11);
1038        let m0413b = SseVector::nmadd(m0413b, self.twiddles_im[5], x7m10);
1039        let m0413b = SseVector::nmadd(m0413b, self.twiddles_im[1], x8m9);
1040        let [y04, y13] = SseVector::column_butterfly2([m0413a, m0413b]);
1041
1042        let m0512a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p16);
1043        let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[6], x2p15);
1044        let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[1], x3p14);
1045        let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[2], x4p13);
1046        let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[7], x5p12);
1047        let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[3], x6p11);
1048        let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[0], x7p10);
1049        let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[5], x8p9);
1050        let m0512b = SseVector::mul(self.twiddles_im[4], x1m16);
1051        let m0512b = SseVector::nmadd(m0512b, self.twiddles_im[6], x2m15);
1052        let m0512b = SseVector::nmadd(m0512b, self.twiddles_im[1], x3m14);
1053        let m0512b = SseVector::fmadd(m0512b, self.twiddles_im[2], x4m13);
1054        let m0512b = SseVector::fmadd(m0512b, self.twiddles_im[7], x5m12);
1055        let m0512b = SseVector::nmadd(m0512b, self.twiddles_im[3], x6m11);
1056        let m0512b = SseVector::fmadd(m0512b, self.twiddles_im[0], x7m10);
1057        let m0512b = SseVector::fmadd(m0512b, self.twiddles_im[5], x8m9);
1058        let [y05, y12] = SseVector::column_butterfly2([m0512a, m0512b]);
1059
1060        let m0611a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p16);
1061        let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[4], x2p15);
1062        let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[0], x3p14);
1063        let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[6], x4p13);
1064        let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[3], x5p12);
1065        let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[1], x6p11);
1066        let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[7], x7p10);
1067        let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[2], x8p9);
1068        let m0611b = SseVector::mul(self.twiddles_im[5], x1m16);
1069        let m0611b = SseVector::nmadd(m0611b, self.twiddles_im[4], x2m15);
1070        let m0611b = SseVector::fmadd(m0611b, self.twiddles_im[0], x3m14);
1071        let m0611b = SseVector::fmadd(m0611b, self.twiddles_im[6], x4m13);
1072        let m0611b = SseVector::nmadd(m0611b, self.twiddles_im[3], x5m12);
1073        let m0611b = SseVector::fmadd(m0611b, self.twiddles_im[1], x6m11);
1074        let m0611b = SseVector::fmadd(m0611b, self.twiddles_im[7], x7m10);
1075        let m0611b = SseVector::nmadd(m0611b, self.twiddles_im[2], x8m9);
1076        let [y06, y11] = SseVector::column_butterfly2([m0611a, m0611b]);
1077
1078        let m0710a = SseVector::fmadd(values[0], self.twiddles_re[6], x1p16);
1079        let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[2], x2p15);
1080        let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[3], x3p14);
1081        let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[5], x4p13);
1082        let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[0], x5p12);
1083        let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[7], x6p11);
1084        let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[1], x7p10);
1085        let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[4], x8p9);
1086        let m0710b = SseVector::mul(self.twiddles_im[6], x1m16);
1087        let m0710b = SseVector::nmadd(m0710b, self.twiddles_im[2], x2m15);
1088        let m0710b = SseVector::fmadd(m0710b, self.twiddles_im[3], x3m14);
1089        let m0710b = SseVector::nmadd(m0710b, self.twiddles_im[5], x4m13);
1090        let m0710b = SseVector::fmadd(m0710b, self.twiddles_im[0], x5m12);
1091        let m0710b = SseVector::fmadd(m0710b, self.twiddles_im[7], x6m11);
1092        let m0710b = SseVector::nmadd(m0710b, self.twiddles_im[1], x7m10);
1093        let m0710b = SseVector::fmadd(m0710b, self.twiddles_im[4], x8m9);
1094        let [y07, y10] = SseVector::column_butterfly2([m0710a, m0710b]);
1095
1096        let m0809a = SseVector::fmadd(values[0], self.twiddles_re[7], x1p16);
1097        let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[0], x2p15);
1098        let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[6], x3p14);
1099        let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[1], x4p13);
1100        let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[5], x5p12);
1101        let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[2], x6p11);
1102        let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[4], x7p10);
1103        let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[3], x8p9);
1104        let m0809b = SseVector::mul(self.twiddles_im[7], x1m16);
1105        let m0809b = SseVector::nmadd(m0809b, self.twiddles_im[0], x2m15);
1106        let m0809b = SseVector::fmadd(m0809b, self.twiddles_im[6], x3m14);
1107        let m0809b = SseVector::nmadd(m0809b, self.twiddles_im[1], x4m13);
1108        let m0809b = SseVector::fmadd(m0809b, self.twiddles_im[5], x5m12);
1109        let m0809b = SseVector::nmadd(m0809b, self.twiddles_im[2], x6m11);
1110        let m0809b = SseVector::fmadd(m0809b, self.twiddles_im[4], x7m10);
1111        let m0809b = SseVector::nmadd(m0809b, self.twiddles_im[3], x8m9);
1112        let [y08, y09] = SseVector::column_butterfly2([m0809a, m0809b]);
1113
1114
1115        [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12, y13, y14, y15, y16]
1116    }
1117}
1118
1119struct SseF64Butterfly17<T> {
1120    direction: FftDirection,
1121    twiddles_re: [__m128d; 8],
1122    twiddles_im: [__m128d; 8],
1123    _phantom: std::marker::PhantomData<T>,
1124}
1125
1126boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly17, 17, |this: &SseF64Butterfly17<_>| this.direction);
1127impl<T: FftNum> SseF64Butterfly17<T> {
1128    /// Safety: The current machine must support the sse4.1 instruction set
1129    #[target_feature(enable = "sse4.1")]
1130    unsafe fn new(direction: FftDirection) -> Self {
1131        assert_f64::<T>();
1132        let twiddles = make_twiddles(17, direction);
1133        unsafe {Self {
1134            direction,
1135            twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
1136            twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
1137            _phantom: std::marker::PhantomData,
1138        }}
1139    }
1140
1141    #[inline(always)]
1142    pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
1143        let values = read_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16 });
1144
1145        let out = self.perform_fft_direct(values);
1146
1147        write_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16 });   
1148    }
1149
1150    #[inline(always)]
1151    pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 17]) -> [__m128d; 17] {
1152        let rotate = SseVector::make_rotate90(FftDirection::Inverse);
1153
1154        let y00 = values[0];
1155        let [x1p16, x1m16] =  SseVector::column_butterfly2([values[1], values[16]]);
1156        let x1m16 = SseVector::apply_rotate90(rotate, x1m16);
1157        let y00 = SseVector::add(y00, x1p16);
1158        let [x2p15, x2m15] =  SseVector::column_butterfly2([values[2], values[15]]);
1159        let x2m15 = SseVector::apply_rotate90(rotate, x2m15);
1160        let y00 = SseVector::add(y00, x2p15);
1161        let [x3p14, x3m14] =  SseVector::column_butterfly2([values[3], values[14]]);
1162        let x3m14 = SseVector::apply_rotate90(rotate, x3m14);
1163        let y00 = SseVector::add(y00, x3p14);
1164        let [x4p13, x4m13] =  SseVector::column_butterfly2([values[4], values[13]]);
1165        let x4m13 = SseVector::apply_rotate90(rotate, x4m13);
1166        let y00 = SseVector::add(y00, x4p13);
1167        let [x5p12, x5m12] =  SseVector::column_butterfly2([values[5], values[12]]);
1168        let x5m12 = SseVector::apply_rotate90(rotate, x5m12);
1169        let y00 = SseVector::add(y00, x5p12);
1170        let [x6p11, x6m11] =  SseVector::column_butterfly2([values[6], values[11]]);
1171        let x6m11 = SseVector::apply_rotate90(rotate, x6m11);
1172        let y00 = SseVector::add(y00, x6p11);
1173        let [x7p10, x7m10] =  SseVector::column_butterfly2([values[7], values[10]]);
1174        let x7m10 = SseVector::apply_rotate90(rotate, x7m10);
1175        let y00 = SseVector::add(y00, x7p10);
1176        let [x8p9, x8m9] =  SseVector::column_butterfly2([values[8], values[9]]);
1177        let x8m9 = SseVector::apply_rotate90(rotate, x8m9);
1178        let y00 = SseVector::add(y00, x8p9);
1179
1180        let m0116a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p16);
1181        let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[1], x2p15);
1182        let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[2], x3p14);
1183        let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[3], x4p13);
1184        let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[4], x5p12);
1185        let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[5], x6p11);
1186        let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[6], x7p10);
1187        let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[7], x8p9);
1188        let m0116b = SseVector::mul(self.twiddles_im[0], x1m16);
1189        let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[1], x2m15);
1190        let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[2], x3m14);
1191        let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[3], x4m13);
1192        let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[4], x5m12);
1193        let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[5], x6m11);
1194        let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[6], x7m10);
1195        let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[7], x8m9);
1196        let [y01, y16] = SseVector::column_butterfly2([m0116a, m0116b]);
1197
1198        let m0215a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p16);
1199        let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[3], x2p15);
1200        let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[5], x3p14);
1201        let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[7], x4p13);
1202        let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[6], x5p12);
1203        let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[4], x6p11);
1204        let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[2], x7p10);
1205        let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[0], x8p9);
1206        let m0215b = SseVector::mul(self.twiddles_im[1], x1m16);
1207        let m0215b = SseVector::fmadd(m0215b, self.twiddles_im[3], x2m15);
1208        let m0215b = SseVector::fmadd(m0215b, self.twiddles_im[5], x3m14);
1209        let m0215b = SseVector::fmadd(m0215b, self.twiddles_im[7], x4m13);
1210        let m0215b = SseVector::nmadd(m0215b, self.twiddles_im[6], x5m12);
1211        let m0215b = SseVector::nmadd(m0215b, self.twiddles_im[4], x6m11);
1212        let m0215b = SseVector::nmadd(m0215b, self.twiddles_im[2], x7m10);
1213        let m0215b = SseVector::nmadd(m0215b, self.twiddles_im[0], x8m9);
1214        let [y02, y15] = SseVector::column_butterfly2([m0215a, m0215b]);
1215
1216        let m0314a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p16);
1217        let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[5], x2p15);
1218        let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[7], x3p14);
1219        let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[4], x4p13);
1220        let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[1], x5p12);
1221        let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[0], x6p11);
1222        let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[3], x7p10);
1223        let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[6], x8p9);
1224        let m0314b = SseVector::mul(self.twiddles_im[2], x1m16);
1225        let m0314b = SseVector::fmadd(m0314b, self.twiddles_im[5], x2m15);
1226        let m0314b = SseVector::nmadd(m0314b, self.twiddles_im[7], x3m14);
1227        let m0314b = SseVector::nmadd(m0314b, self.twiddles_im[4], x4m13);
1228        let m0314b = SseVector::nmadd(m0314b, self.twiddles_im[1], x5m12);
1229        let m0314b = SseVector::fmadd(m0314b, self.twiddles_im[0], x6m11);
1230        let m0314b = SseVector::fmadd(m0314b, self.twiddles_im[3], x7m10);
1231        let m0314b = SseVector::fmadd(m0314b, self.twiddles_im[6], x8m9);
1232        let [y03, y14] = SseVector::column_butterfly2([m0314a, m0314b]);
1233
1234        let m0413a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p16);
1235        let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[7], x2p15);
1236        let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[4], x3p14);
1237        let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[0], x4p13);
1238        let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[2], x5p12);
1239        let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[6], x6p11);
1240        let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[5], x7p10);
1241        let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[1], x8p9);
1242        let m0413b = SseVector::mul(self.twiddles_im[3], x1m16);
1243        let m0413b = SseVector::fmadd(m0413b, self.twiddles_im[7], x2m15);
1244        let m0413b = SseVector::nmadd(m0413b, self.twiddles_im[4], x3m14);
1245        let m0413b = SseVector::nmadd(m0413b, self.twiddles_im[0], x4m13);
1246        let m0413b = SseVector::fmadd(m0413b, self.twiddles_im[2], x5m12);
1247        let m0413b = SseVector::fmadd(m0413b, self.twiddles_im[6], x6m11);
1248        let m0413b = SseVector::nmadd(m0413b, self.twiddles_im[5], x7m10);
1249        let m0413b = SseVector::nmadd(m0413b, self.twiddles_im[1], x8m9);
1250        let [y04, y13] = SseVector::column_butterfly2([m0413a, m0413b]);
1251
1252        let m0512a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p16);
1253        let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[6], x2p15);
1254        let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[1], x3p14);
1255        let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[2], x4p13);
1256        let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[7], x5p12);
1257        let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[3], x6p11);
1258        let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[0], x7p10);
1259        let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[5], x8p9);
1260        let m0512b = SseVector::mul(self.twiddles_im[4], x1m16);
1261        let m0512b = SseVector::nmadd(m0512b, self.twiddles_im[6], x2m15);
1262        let m0512b = SseVector::nmadd(m0512b, self.twiddles_im[1], x3m14);
1263        let m0512b = SseVector::fmadd(m0512b, self.twiddles_im[2], x4m13);
1264        let m0512b = SseVector::fmadd(m0512b, self.twiddles_im[7], x5m12);
1265        let m0512b = SseVector::nmadd(m0512b, self.twiddles_im[3], x6m11);
1266        let m0512b = SseVector::fmadd(m0512b, self.twiddles_im[0], x7m10);
1267        let m0512b = SseVector::fmadd(m0512b, self.twiddles_im[5], x8m9);
1268        let [y05, y12] = SseVector::column_butterfly2([m0512a, m0512b]);
1269
1270        let m0611a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p16);
1271        let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[4], x2p15);
1272        let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[0], x3p14);
1273        let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[6], x4p13);
1274        let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[3], x5p12);
1275        let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[1], x6p11);
1276        let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[7], x7p10);
1277        let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[2], x8p9);
1278        let m0611b = SseVector::mul(self.twiddles_im[5], x1m16);
1279        let m0611b = SseVector::nmadd(m0611b, self.twiddles_im[4], x2m15);
1280        let m0611b = SseVector::fmadd(m0611b, self.twiddles_im[0], x3m14);
1281        let m0611b = SseVector::fmadd(m0611b, self.twiddles_im[6], x4m13);
1282        let m0611b = SseVector::nmadd(m0611b, self.twiddles_im[3], x5m12);
1283        let m0611b = SseVector::fmadd(m0611b, self.twiddles_im[1], x6m11);
1284        let m0611b = SseVector::fmadd(m0611b, self.twiddles_im[7], x7m10);
1285        let m0611b = SseVector::nmadd(m0611b, self.twiddles_im[2], x8m9);
1286        let [y06, y11] = SseVector::column_butterfly2([m0611a, m0611b]);
1287
1288        let m0710a = SseVector::fmadd(values[0], self.twiddles_re[6], x1p16);
1289        let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[2], x2p15);
1290        let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[3], x3p14);
1291        let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[5], x4p13);
1292        let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[0], x5p12);
1293        let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[7], x6p11);
1294        let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[1], x7p10);
1295        let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[4], x8p9);
1296        let m0710b = SseVector::mul(self.twiddles_im[6], x1m16);
1297        let m0710b = SseVector::nmadd(m0710b, self.twiddles_im[2], x2m15);
1298        let m0710b = SseVector::fmadd(m0710b, self.twiddles_im[3], x3m14);
1299        let m0710b = SseVector::nmadd(m0710b, self.twiddles_im[5], x4m13);
1300        let m0710b = SseVector::fmadd(m0710b, self.twiddles_im[0], x5m12);
1301        let m0710b = SseVector::fmadd(m0710b, self.twiddles_im[7], x6m11);
1302        let m0710b = SseVector::nmadd(m0710b, self.twiddles_im[1], x7m10);
1303        let m0710b = SseVector::fmadd(m0710b, self.twiddles_im[4], x8m9);
1304        let [y07, y10] = SseVector::column_butterfly2([m0710a, m0710b]);
1305
1306        let m0809a = SseVector::fmadd(values[0], self.twiddles_re[7], x1p16);
1307        let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[0], x2p15);
1308        let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[6], x3p14);
1309        let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[1], x4p13);
1310        let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[5], x5p12);
1311        let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[2], x6p11);
1312        let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[4], x7p10);
1313        let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[3], x8p9);
1314        let m0809b = SseVector::mul(self.twiddles_im[7], x1m16);
1315        let m0809b = SseVector::nmadd(m0809b, self.twiddles_im[0], x2m15);
1316        let m0809b = SseVector::fmadd(m0809b, self.twiddles_im[6], x3m14);
1317        let m0809b = SseVector::nmadd(m0809b, self.twiddles_im[1], x4m13);
1318        let m0809b = SseVector::fmadd(m0809b, self.twiddles_im[5], x5m12);
1319        let m0809b = SseVector::nmadd(m0809b, self.twiddles_im[2], x6m11);
1320        let m0809b = SseVector::fmadd(m0809b, self.twiddles_im[4], x7m10);
1321        let m0809b = SseVector::nmadd(m0809b, self.twiddles_im[3], x8m9);
1322        let [y08, y09] = SseVector::column_butterfly2([m0809a, m0809b]);
1323
1324
1325        [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12, y13, y14, y15, y16]
1326    }
1327}
1328
1329struct SseF32Butterfly19<T> {
1330    direction: FftDirection,
1331    twiddles_re: [__m128; 9],
1332    twiddles_im: [__m128; 9],
1333    _phantom: std::marker::PhantomData<T>,
1334}
1335
1336boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly19, 19, |this: &SseF32Butterfly19<_>| this.direction);
1337impl<T: FftNum> SseF32Butterfly19<T> {
1338    /// Safety: The current machine must support the sse4.1 instruction set
1339    #[target_feature(enable = "sse4.1")]
1340    unsafe fn new(direction: FftDirection) -> Self {
1341        assert_f32::<T>();
1342        let twiddles = make_twiddles(19, direction);
1343        Self {
1344            direction,
1345            twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
1346            twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
1347            _phantom: std::marker::PhantomData,
1348        }
1349    }
1350
1351    #[inline(always)]
1352    pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
1353        let values = read_partial1_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18 });
1354
1355        let out = self.perform_parallel_fft_direct(values);
1356        
1357        write_partial_lo_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18 } ); 
1358    }
1359
1360    #[inline(always)]
1361    pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
1362        let input_packed = read_complex_to_array!(buffer, { 0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36 });
1363
1364        let values = [
1365            extract_lo_hi_f32(input_packed[0], input_packed[9]),
1366            extract_hi_lo_f32(input_packed[0], input_packed[10]),
1367            extract_lo_hi_f32(input_packed[1], input_packed[10]),
1368            extract_hi_lo_f32(input_packed[1], input_packed[11]),
1369            extract_lo_hi_f32(input_packed[2], input_packed[11]),
1370            extract_hi_lo_f32(input_packed[2], input_packed[12]),
1371            extract_lo_hi_f32(input_packed[3], input_packed[12]),
1372            extract_hi_lo_f32(input_packed[3], input_packed[13]),
1373            extract_lo_hi_f32(input_packed[4], input_packed[13]),
1374            extract_hi_lo_f32(input_packed[4], input_packed[14]),
1375            extract_lo_hi_f32(input_packed[5], input_packed[14]),
1376            extract_hi_lo_f32(input_packed[5], input_packed[15]),
1377            extract_lo_hi_f32(input_packed[6], input_packed[15]),
1378            extract_hi_lo_f32(input_packed[6], input_packed[16]),
1379            extract_lo_hi_f32(input_packed[7], input_packed[16]),
1380            extract_hi_lo_f32(input_packed[7], input_packed[17]),
1381            extract_lo_hi_f32(input_packed[8], input_packed[17]),
1382            extract_hi_lo_f32(input_packed[8], input_packed[18]),
1383            extract_lo_hi_f32(input_packed[9], input_packed[18]),
1384        ];
1385
1386        let out = self.perform_parallel_fft_direct(values);
1387
1388        let out_packed = [
1389            extract_lo_lo_f32(out[0], out[1]),
1390            extract_lo_lo_f32(out[2], out[3]),
1391            extract_lo_lo_f32(out[4], out[5]),
1392            extract_lo_lo_f32(out[6], out[7]),
1393            extract_lo_lo_f32(out[8], out[9]),
1394            extract_lo_lo_f32(out[10], out[11]),
1395            extract_lo_lo_f32(out[12], out[13]),
1396            extract_lo_lo_f32(out[14], out[15]),
1397            extract_lo_lo_f32(out[16], out[17]),
1398            extract_lo_hi_f32(out[18], out[0]),
1399            extract_hi_hi_f32(out[1], out[2]),
1400            extract_hi_hi_f32(out[3], out[4]),
1401            extract_hi_hi_f32(out[5], out[6]),
1402            extract_hi_hi_f32(out[7], out[8]),
1403            extract_hi_hi_f32(out[9], out[10]),
1404            extract_hi_hi_f32(out[11], out[12]),
1405            extract_hi_hi_f32(out[13], out[14]),
1406            extract_hi_hi_f32(out[15], out[16]),
1407            extract_hi_hi_f32(out[17], out[18]),
1408        ];
1409
1410        write_complex_to_array_strided!(out_packed, buffer, 2, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18 });
1411    }
1412
1413    #[inline(always)]
1414    pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 19]) -> [__m128; 19] {
1415        let rotate = SseVector::make_rotate90(FftDirection::Inverse);
1416
1417        let y00 = values[0];
1418        let [x1p18, x1m18] =  SseVector::column_butterfly2([values[1], values[18]]);
1419        let x1m18 = SseVector::apply_rotate90(rotate, x1m18);
1420        let y00 = SseVector::add(y00, x1p18);
1421        let [x2p17, x2m17] =  SseVector::column_butterfly2([values[2], values[17]]);
1422        let x2m17 = SseVector::apply_rotate90(rotate, x2m17);
1423        let y00 = SseVector::add(y00, x2p17);
1424        let [x3p16, x3m16] =  SseVector::column_butterfly2([values[3], values[16]]);
1425        let x3m16 = SseVector::apply_rotate90(rotate, x3m16);
1426        let y00 = SseVector::add(y00, x3p16);
1427        let [x4p15, x4m15] =  SseVector::column_butterfly2([values[4], values[15]]);
1428        let x4m15 = SseVector::apply_rotate90(rotate, x4m15);
1429        let y00 = SseVector::add(y00, x4p15);
1430        let [x5p14, x5m14] =  SseVector::column_butterfly2([values[5], values[14]]);
1431        let x5m14 = SseVector::apply_rotate90(rotate, x5m14);
1432        let y00 = SseVector::add(y00, x5p14);
1433        let [x6p13, x6m13] =  SseVector::column_butterfly2([values[6], values[13]]);
1434        let x6m13 = SseVector::apply_rotate90(rotate, x6m13);
1435        let y00 = SseVector::add(y00, x6p13);
1436        let [x7p12, x7m12] =  SseVector::column_butterfly2([values[7], values[12]]);
1437        let x7m12 = SseVector::apply_rotate90(rotate, x7m12);
1438        let y00 = SseVector::add(y00, x7p12);
1439        let [x8p11, x8m11] =  SseVector::column_butterfly2([values[8], values[11]]);
1440        let x8m11 = SseVector::apply_rotate90(rotate, x8m11);
1441        let y00 = SseVector::add(y00, x8p11);
1442        let [x9p10, x9m10] =  SseVector::column_butterfly2([values[9], values[10]]);
1443        let x9m10 = SseVector::apply_rotate90(rotate, x9m10);
1444        let y00 = SseVector::add(y00, x9p10);
1445
1446        let m0118a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p18);
1447        let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[1], x2p17);
1448        let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[2], x3p16);
1449        let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[3], x4p15);
1450        let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[4], x5p14);
1451        let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[5], x6p13);
1452        let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[6], x7p12);
1453        let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[7], x8p11);
1454        let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[8], x9p10);
1455        let m0118b = SseVector::mul(self.twiddles_im[0], x1m18);
1456        let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[1], x2m17);
1457        let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[2], x3m16);
1458        let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[3], x4m15);
1459        let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[4], x5m14);
1460        let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[5], x6m13);
1461        let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[6], x7m12);
1462        let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[7], x8m11);
1463        let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[8], x9m10);
1464        let [y01, y18] = SseVector::column_butterfly2([m0118a, m0118b]);
1465
1466        let m0217a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p18);
1467        let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[3], x2p17);
1468        let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[5], x3p16);
1469        let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[7], x4p15);
1470        let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[8], x5p14);
1471        let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[6], x6p13);
1472        let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[4], x7p12);
1473        let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[2], x8p11);
1474        let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[0], x9p10);
1475        let m0217b = SseVector::mul(self.twiddles_im[1], x1m18);
1476        let m0217b = SseVector::fmadd(m0217b, self.twiddles_im[3], x2m17);
1477        let m0217b = SseVector::fmadd(m0217b, self.twiddles_im[5], x3m16);
1478        let m0217b = SseVector::fmadd(m0217b, self.twiddles_im[7], x4m15);
1479        let m0217b = SseVector::nmadd(m0217b, self.twiddles_im[8], x5m14);
1480        let m0217b = SseVector::nmadd(m0217b, self.twiddles_im[6], x6m13);
1481        let m0217b = SseVector::nmadd(m0217b, self.twiddles_im[4], x7m12);
1482        let m0217b = SseVector::nmadd(m0217b, self.twiddles_im[2], x8m11);
1483        let m0217b = SseVector::nmadd(m0217b, self.twiddles_im[0], x9m10);
1484        let [y02, y17] = SseVector::column_butterfly2([m0217a, m0217b]);
1485
1486        let m0316a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p18);
1487        let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[5], x2p17);
1488        let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[8], x3p16);
1489        let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[6], x4p15);
1490        let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[3], x5p14);
1491        let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[0], x6p13);
1492        let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[1], x7p12);
1493        let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[4], x8p11);
1494        let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[7], x9p10);
1495        let m0316b = SseVector::mul(self.twiddles_im[2], x1m18);
1496        let m0316b = SseVector::fmadd(m0316b, self.twiddles_im[5], x2m17);
1497        let m0316b = SseVector::fmadd(m0316b, self.twiddles_im[8], x3m16);
1498        let m0316b = SseVector::nmadd(m0316b, self.twiddles_im[6], x4m15);
1499        let m0316b = SseVector::nmadd(m0316b, self.twiddles_im[3], x5m14);
1500        let m0316b = SseVector::nmadd(m0316b, self.twiddles_im[0], x6m13);
1501        let m0316b = SseVector::fmadd(m0316b, self.twiddles_im[1], x7m12);
1502        let m0316b = SseVector::fmadd(m0316b, self.twiddles_im[4], x8m11);
1503        let m0316b = SseVector::fmadd(m0316b, self.twiddles_im[7], x9m10);
1504        let [y03, y16] = SseVector::column_butterfly2([m0316a, m0316b]);
1505
1506        let m0415a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p18);
1507        let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[7], x2p17);
1508        let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[6], x3p16);
1509        let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[2], x4p15);
1510        let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[0], x5p14);
1511        let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[4], x6p13);
1512        let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[8], x7p12);
1513        let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[5], x8p11);
1514        let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[1], x9p10);
1515        let m0415b = SseVector::mul(self.twiddles_im[3], x1m18);
1516        let m0415b = SseVector::fmadd(m0415b, self.twiddles_im[7], x2m17);
1517        let m0415b = SseVector::nmadd(m0415b, self.twiddles_im[6], x3m16);
1518        let m0415b = SseVector::nmadd(m0415b, self.twiddles_im[2], x4m15);
1519        let m0415b = SseVector::fmadd(m0415b, self.twiddles_im[0], x5m14);
1520        let m0415b = SseVector::fmadd(m0415b, self.twiddles_im[4], x6m13);
1521        let m0415b = SseVector::fmadd(m0415b, self.twiddles_im[8], x7m12);
1522        let m0415b = SseVector::nmadd(m0415b, self.twiddles_im[5], x8m11);
1523        let m0415b = SseVector::nmadd(m0415b, self.twiddles_im[1], x9m10);
1524        let [y04, y15] = SseVector::column_butterfly2([m0415a, m0415b]);
1525
1526        let m0514a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p18);
1527        let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[8], x2p17);
1528        let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[3], x3p16);
1529        let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[0], x4p15);
1530        let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[5], x5p14);
1531        let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[7], x6p13);
1532        let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[2], x7p12);
1533        let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[1], x8p11);
1534        let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[6], x9p10);
1535        let m0514b = SseVector::mul(self.twiddles_im[4], x1m18);
1536        let m0514b = SseVector::nmadd(m0514b, self.twiddles_im[8], x2m17);
1537        let m0514b = SseVector::nmadd(m0514b, self.twiddles_im[3], x3m16);
1538        let m0514b = SseVector::fmadd(m0514b, self.twiddles_im[0], x4m15);
1539        let m0514b = SseVector::fmadd(m0514b, self.twiddles_im[5], x5m14);
1540        let m0514b = SseVector::nmadd(m0514b, self.twiddles_im[7], x6m13);
1541        let m0514b = SseVector::nmadd(m0514b, self.twiddles_im[2], x7m12);
1542        let m0514b = SseVector::fmadd(m0514b, self.twiddles_im[1], x8m11);
1543        let m0514b = SseVector::fmadd(m0514b, self.twiddles_im[6], x9m10);
1544        let [y05, y14] = SseVector::column_butterfly2([m0514a, m0514b]);
1545
1546        let m0613a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p18);
1547        let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[6], x2p17);
1548        let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[0], x3p16);
1549        let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[4], x4p15);
1550        let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[7], x5p14);
1551        let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[1], x6p13);
1552        let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[3], x7p12);
1553        let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[8], x8p11);
1554        let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[2], x9p10);
1555        let m0613b = SseVector::mul(self.twiddles_im[5], x1m18);
1556        let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[6], x2m17);
1557        let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[0], x3m16);
1558        let m0613b = SseVector::fmadd(m0613b, self.twiddles_im[4], x4m15);
1559        let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[7], x5m14);
1560        let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[1], x6m13);
1561        let m0613b = SseVector::fmadd(m0613b, self.twiddles_im[3], x7m12);
1562        let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[8], x8m11);
1563        let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[2], x9m10);
1564        let [y06, y13] = SseVector::column_butterfly2([m0613a, m0613b]);
1565
1566        let m0712a = SseVector::fmadd(values[0], self.twiddles_re[6], x1p18);
1567        let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[4], x2p17);
1568        let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[1], x3p16);
1569        let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[8], x4p15);
1570        let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[2], x5p14);
1571        let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[3], x6p13);
1572        let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[7], x7p12);
1573        let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[0], x8p11);
1574        let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[5], x9p10);
1575        let m0712b = SseVector::mul(self.twiddles_im[6], x1m18);
1576        let m0712b = SseVector::nmadd(m0712b, self.twiddles_im[4], x2m17);
1577        let m0712b = SseVector::fmadd(m0712b, self.twiddles_im[1], x3m16);
1578        let m0712b = SseVector::fmadd(m0712b, self.twiddles_im[8], x4m15);
1579        let m0712b = SseVector::nmadd(m0712b, self.twiddles_im[2], x5m14);
1580        let m0712b = SseVector::fmadd(m0712b, self.twiddles_im[3], x6m13);
1581        let m0712b = SseVector::nmadd(m0712b, self.twiddles_im[7], x7m12);
1582        let m0712b = SseVector::nmadd(m0712b, self.twiddles_im[0], x8m11);
1583        let m0712b = SseVector::fmadd(m0712b, self.twiddles_im[5], x9m10);
1584        let [y07, y12] = SseVector::column_butterfly2([m0712a, m0712b]);
1585
1586        let m0811a = SseVector::fmadd(values[0], self.twiddles_re[7], x1p18);
1587        let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[2], x2p17);
1588        let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[4], x3p16);
1589        let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[5], x4p15);
1590        let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[1], x5p14);
1591        let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[8], x6p13);
1592        let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[0], x7p12);
1593        let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[6], x8p11);
1594        let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[3], x9p10);
1595        let m0811b = SseVector::mul(self.twiddles_im[7], x1m18);
1596        let m0811b = SseVector::nmadd(m0811b, self.twiddles_im[2], x2m17);
1597        let m0811b = SseVector::fmadd(m0811b, self.twiddles_im[4], x3m16);
1598        let m0811b = SseVector::nmadd(m0811b, self.twiddles_im[5], x4m15);
1599        let m0811b = SseVector::fmadd(m0811b, self.twiddles_im[1], x5m14);
1600        let m0811b = SseVector::nmadd(m0811b, self.twiddles_im[8], x6m13);
1601        let m0811b = SseVector::nmadd(m0811b, self.twiddles_im[0], x7m12);
1602        let m0811b = SseVector::fmadd(m0811b, self.twiddles_im[6], x8m11);
1603        let m0811b = SseVector::nmadd(m0811b, self.twiddles_im[3], x9m10);
1604        let [y08, y11] = SseVector::column_butterfly2([m0811a, m0811b]);
1605
1606        let m0910a = SseVector::fmadd(values[0], self.twiddles_re[8], x1p18);
1607        let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[0], x2p17);
1608        let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[7], x3p16);
1609        let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[1], x4p15);
1610        let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[6], x5p14);
1611        let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[2], x6p13);
1612        let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[5], x7p12);
1613        let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[3], x8p11);
1614        let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[4], x9p10);
1615        let m0910b = SseVector::mul(self.twiddles_im[8], x1m18);
1616        let m0910b = SseVector::nmadd(m0910b, self.twiddles_im[0], x2m17);
1617        let m0910b = SseVector::fmadd(m0910b, self.twiddles_im[7], x3m16);
1618        let m0910b = SseVector::nmadd(m0910b, self.twiddles_im[1], x4m15);
1619        let m0910b = SseVector::fmadd(m0910b, self.twiddles_im[6], x5m14);
1620        let m0910b = SseVector::nmadd(m0910b, self.twiddles_im[2], x6m13);
1621        let m0910b = SseVector::fmadd(m0910b, self.twiddles_im[5], x7m12);
1622        let m0910b = SseVector::nmadd(m0910b, self.twiddles_im[3], x8m11);
1623        let m0910b = SseVector::fmadd(m0910b, self.twiddles_im[4], x9m10);
1624        let [y09, y10] = SseVector::column_butterfly2([m0910a, m0910b]);
1625
1626
1627        [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12, y13, y14, y15, y16, y17, y18]
1628    }
1629}
1630
1631struct SseF64Butterfly19<T> {
1632    direction: FftDirection,
1633    twiddles_re: [__m128d; 9],
1634    twiddles_im: [__m128d; 9],
1635    _phantom: std::marker::PhantomData<T>,
1636}
1637
1638boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly19, 19, |this: &SseF64Butterfly19<_>| this.direction);
1639impl<T: FftNum> SseF64Butterfly19<T> {
1640    /// Safety: The current machine must support the sse4.1 instruction set
1641    #[target_feature(enable = "sse4.1")]
1642    unsafe fn new(direction: FftDirection) -> Self {
1643        assert_f64::<T>();
1644        let twiddles = make_twiddles(19, direction);
1645        unsafe {Self {
1646            direction,
1647            twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
1648            twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
1649            _phantom: std::marker::PhantomData,
1650        }}
1651    }
1652
1653    #[inline(always)]
1654    pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
1655        let values = read_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18 });
1656
1657        let out = self.perform_fft_direct(values);
1658
1659        write_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18 });   
1660    }
1661
1662    #[inline(always)]
1663    pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 19]) -> [__m128d; 19] {
1664        let rotate = SseVector::make_rotate90(FftDirection::Inverse);
1665
1666        let y00 = values[0];
1667        let [x1p18, x1m18] =  SseVector::column_butterfly2([values[1], values[18]]);
1668        let x1m18 = SseVector::apply_rotate90(rotate, x1m18);
1669        let y00 = SseVector::add(y00, x1p18);
1670        let [x2p17, x2m17] =  SseVector::column_butterfly2([values[2], values[17]]);
1671        let x2m17 = SseVector::apply_rotate90(rotate, x2m17);
1672        let y00 = SseVector::add(y00, x2p17);
1673        let [x3p16, x3m16] =  SseVector::column_butterfly2([values[3], values[16]]);
1674        let x3m16 = SseVector::apply_rotate90(rotate, x3m16);
1675        let y00 = SseVector::add(y00, x3p16);
1676        let [x4p15, x4m15] =  SseVector::column_butterfly2([values[4], values[15]]);
1677        let x4m15 = SseVector::apply_rotate90(rotate, x4m15);
1678        let y00 = SseVector::add(y00, x4p15);
1679        let [x5p14, x5m14] =  SseVector::column_butterfly2([values[5], values[14]]);
1680        let x5m14 = SseVector::apply_rotate90(rotate, x5m14);
1681        let y00 = SseVector::add(y00, x5p14);
1682        let [x6p13, x6m13] =  SseVector::column_butterfly2([values[6], values[13]]);
1683        let x6m13 = SseVector::apply_rotate90(rotate, x6m13);
1684        let y00 = SseVector::add(y00, x6p13);
1685        let [x7p12, x7m12] =  SseVector::column_butterfly2([values[7], values[12]]);
1686        let x7m12 = SseVector::apply_rotate90(rotate, x7m12);
1687        let y00 = SseVector::add(y00, x7p12);
1688        let [x8p11, x8m11] =  SseVector::column_butterfly2([values[8], values[11]]);
1689        let x8m11 = SseVector::apply_rotate90(rotate, x8m11);
1690        let y00 = SseVector::add(y00, x8p11);
1691        let [x9p10, x9m10] =  SseVector::column_butterfly2([values[9], values[10]]);
1692        let x9m10 = SseVector::apply_rotate90(rotate, x9m10);
1693        let y00 = SseVector::add(y00, x9p10);
1694
1695        let m0118a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p18);
1696        let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[1], x2p17);
1697        let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[2], x3p16);
1698        let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[3], x4p15);
1699        let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[4], x5p14);
1700        let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[5], x6p13);
1701        let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[6], x7p12);
1702        let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[7], x8p11);
1703        let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[8], x9p10);
1704        let m0118b = SseVector::mul(self.twiddles_im[0], x1m18);
1705        let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[1], x2m17);
1706        let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[2], x3m16);
1707        let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[3], x4m15);
1708        let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[4], x5m14);
1709        let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[5], x6m13);
1710        let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[6], x7m12);
1711        let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[7], x8m11);
1712        let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[8], x9m10);
1713        let [y01, y18] = SseVector::column_butterfly2([m0118a, m0118b]);
1714
1715        let m0217a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p18);
1716        let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[3], x2p17);
1717        let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[5], x3p16);
1718        let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[7], x4p15);
1719        let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[8], x5p14);
1720        let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[6], x6p13);
1721        let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[4], x7p12);
1722        let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[2], x8p11);
1723        let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[0], x9p10);
1724        let m0217b = SseVector::mul(self.twiddles_im[1], x1m18);
1725        let m0217b = SseVector::fmadd(m0217b, self.twiddles_im[3], x2m17);
1726        let m0217b = SseVector::fmadd(m0217b, self.twiddles_im[5], x3m16);
1727        let m0217b = SseVector::fmadd(m0217b, self.twiddles_im[7], x4m15);
1728        let m0217b = SseVector::nmadd(m0217b, self.twiddles_im[8], x5m14);
1729        let m0217b = SseVector::nmadd(m0217b, self.twiddles_im[6], x6m13);
1730        let m0217b = SseVector::nmadd(m0217b, self.twiddles_im[4], x7m12);
1731        let m0217b = SseVector::nmadd(m0217b, self.twiddles_im[2], x8m11);
1732        let m0217b = SseVector::nmadd(m0217b, self.twiddles_im[0], x9m10);
1733        let [y02, y17] = SseVector::column_butterfly2([m0217a, m0217b]);
1734
1735        let m0316a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p18);
1736        let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[5], x2p17);
1737        let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[8], x3p16);
1738        let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[6], x4p15);
1739        let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[3], x5p14);
1740        let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[0], x6p13);
1741        let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[1], x7p12);
1742        let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[4], x8p11);
1743        let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[7], x9p10);
1744        let m0316b = SseVector::mul(self.twiddles_im[2], x1m18);
1745        let m0316b = SseVector::fmadd(m0316b, self.twiddles_im[5], x2m17);
1746        let m0316b = SseVector::fmadd(m0316b, self.twiddles_im[8], x3m16);
1747        let m0316b = SseVector::nmadd(m0316b, self.twiddles_im[6], x4m15);
1748        let m0316b = SseVector::nmadd(m0316b, self.twiddles_im[3], x5m14);
1749        let m0316b = SseVector::nmadd(m0316b, self.twiddles_im[0], x6m13);
1750        let m0316b = SseVector::fmadd(m0316b, self.twiddles_im[1], x7m12);
1751        let m0316b = SseVector::fmadd(m0316b, self.twiddles_im[4], x8m11);
1752        let m0316b = SseVector::fmadd(m0316b, self.twiddles_im[7], x9m10);
1753        let [y03, y16] = SseVector::column_butterfly2([m0316a, m0316b]);
1754
1755        let m0415a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p18);
1756        let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[7], x2p17);
1757        let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[6], x3p16);
1758        let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[2], x4p15);
1759        let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[0], x5p14);
1760        let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[4], x6p13);
1761        let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[8], x7p12);
1762        let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[5], x8p11);
1763        let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[1], x9p10);
1764        let m0415b = SseVector::mul(self.twiddles_im[3], x1m18);
1765        let m0415b = SseVector::fmadd(m0415b, self.twiddles_im[7], x2m17);
1766        let m0415b = SseVector::nmadd(m0415b, self.twiddles_im[6], x3m16);
1767        let m0415b = SseVector::nmadd(m0415b, self.twiddles_im[2], x4m15);
1768        let m0415b = SseVector::fmadd(m0415b, self.twiddles_im[0], x5m14);
1769        let m0415b = SseVector::fmadd(m0415b, self.twiddles_im[4], x6m13);
1770        let m0415b = SseVector::fmadd(m0415b, self.twiddles_im[8], x7m12);
1771        let m0415b = SseVector::nmadd(m0415b, self.twiddles_im[5], x8m11);
1772        let m0415b = SseVector::nmadd(m0415b, self.twiddles_im[1], x9m10);
1773        let [y04, y15] = SseVector::column_butterfly2([m0415a, m0415b]);
1774
1775        let m0514a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p18);
1776        let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[8], x2p17);
1777        let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[3], x3p16);
1778        let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[0], x4p15);
1779        let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[5], x5p14);
1780        let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[7], x6p13);
1781        let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[2], x7p12);
1782        let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[1], x8p11);
1783        let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[6], x9p10);
1784        let m0514b = SseVector::mul(self.twiddles_im[4], x1m18);
1785        let m0514b = SseVector::nmadd(m0514b, self.twiddles_im[8], x2m17);
1786        let m0514b = SseVector::nmadd(m0514b, self.twiddles_im[3], x3m16);
1787        let m0514b = SseVector::fmadd(m0514b, self.twiddles_im[0], x4m15);
1788        let m0514b = SseVector::fmadd(m0514b, self.twiddles_im[5], x5m14);
1789        let m0514b = SseVector::nmadd(m0514b, self.twiddles_im[7], x6m13);
1790        let m0514b = SseVector::nmadd(m0514b, self.twiddles_im[2], x7m12);
1791        let m0514b = SseVector::fmadd(m0514b, self.twiddles_im[1], x8m11);
1792        let m0514b = SseVector::fmadd(m0514b, self.twiddles_im[6], x9m10);
1793        let [y05, y14] = SseVector::column_butterfly2([m0514a, m0514b]);
1794
1795        let m0613a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p18);
1796        let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[6], x2p17);
1797        let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[0], x3p16);
1798        let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[4], x4p15);
1799        let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[7], x5p14);
1800        let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[1], x6p13);
1801        let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[3], x7p12);
1802        let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[8], x8p11);
1803        let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[2], x9p10);
1804        let m0613b = SseVector::mul(self.twiddles_im[5], x1m18);
1805        let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[6], x2m17);
1806        let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[0], x3m16);
1807        let m0613b = SseVector::fmadd(m0613b, self.twiddles_im[4], x4m15);
1808        let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[7], x5m14);
1809        let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[1], x6m13);
1810        let m0613b = SseVector::fmadd(m0613b, self.twiddles_im[3], x7m12);
1811        let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[8], x8m11);
1812        let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[2], x9m10);
1813        let [y06, y13] = SseVector::column_butterfly2([m0613a, m0613b]);
1814
1815        let m0712a = SseVector::fmadd(values[0], self.twiddles_re[6], x1p18);
1816        let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[4], x2p17);
1817        let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[1], x3p16);
1818        let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[8], x4p15);
1819        let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[2], x5p14);
1820        let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[3], x6p13);
1821        let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[7], x7p12);
1822        let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[0], x8p11);
1823        let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[5], x9p10);
1824        let m0712b = SseVector::mul(self.twiddles_im[6], x1m18);
1825        let m0712b = SseVector::nmadd(m0712b, self.twiddles_im[4], x2m17);
1826        let m0712b = SseVector::fmadd(m0712b, self.twiddles_im[1], x3m16);
1827        let m0712b = SseVector::fmadd(m0712b, self.twiddles_im[8], x4m15);
1828        let m0712b = SseVector::nmadd(m0712b, self.twiddles_im[2], x5m14);
1829        let m0712b = SseVector::fmadd(m0712b, self.twiddles_im[3], x6m13);
1830        let m0712b = SseVector::nmadd(m0712b, self.twiddles_im[7], x7m12);
1831        let m0712b = SseVector::nmadd(m0712b, self.twiddles_im[0], x8m11);
1832        let m0712b = SseVector::fmadd(m0712b, self.twiddles_im[5], x9m10);
1833        let [y07, y12] = SseVector::column_butterfly2([m0712a, m0712b]);
1834
1835        let m0811a = SseVector::fmadd(values[0], self.twiddles_re[7], x1p18);
1836        let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[2], x2p17);
1837        let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[4], x3p16);
1838        let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[5], x4p15);
1839        let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[1], x5p14);
1840        let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[8], x6p13);
1841        let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[0], x7p12);
1842        let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[6], x8p11);
1843        let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[3], x9p10);
1844        let m0811b = SseVector::mul(self.twiddles_im[7], x1m18);
1845        let m0811b = SseVector::nmadd(m0811b, self.twiddles_im[2], x2m17);
1846        let m0811b = SseVector::fmadd(m0811b, self.twiddles_im[4], x3m16);
1847        let m0811b = SseVector::nmadd(m0811b, self.twiddles_im[5], x4m15);
1848        let m0811b = SseVector::fmadd(m0811b, self.twiddles_im[1], x5m14);
1849        let m0811b = SseVector::nmadd(m0811b, self.twiddles_im[8], x6m13);
1850        let m0811b = SseVector::nmadd(m0811b, self.twiddles_im[0], x7m12);
1851        let m0811b = SseVector::fmadd(m0811b, self.twiddles_im[6], x8m11);
1852        let m0811b = SseVector::nmadd(m0811b, self.twiddles_im[3], x9m10);
1853        let [y08, y11] = SseVector::column_butterfly2([m0811a, m0811b]);
1854
1855        let m0910a = SseVector::fmadd(values[0], self.twiddles_re[8], x1p18);
1856        let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[0], x2p17);
1857        let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[7], x3p16);
1858        let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[1], x4p15);
1859        let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[6], x5p14);
1860        let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[2], x6p13);
1861        let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[5], x7p12);
1862        let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[3], x8p11);
1863        let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[4], x9p10);
1864        let m0910b = SseVector::mul(self.twiddles_im[8], x1m18);
1865        let m0910b = SseVector::nmadd(m0910b, self.twiddles_im[0], x2m17);
1866        let m0910b = SseVector::fmadd(m0910b, self.twiddles_im[7], x3m16);
1867        let m0910b = SseVector::nmadd(m0910b, self.twiddles_im[1], x4m15);
1868        let m0910b = SseVector::fmadd(m0910b, self.twiddles_im[6], x5m14);
1869        let m0910b = SseVector::nmadd(m0910b, self.twiddles_im[2], x6m13);
1870        let m0910b = SseVector::fmadd(m0910b, self.twiddles_im[5], x7m12);
1871        let m0910b = SseVector::nmadd(m0910b, self.twiddles_im[3], x8m11);
1872        let m0910b = SseVector::fmadd(m0910b, self.twiddles_im[4], x9m10);
1873        let [y09, y10] = SseVector::column_butterfly2([m0910a, m0910b]);
1874
1875
1876        [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12, y13, y14, y15, y16, y17, y18]
1877    }
1878}
1879
1880struct SseF32Butterfly23<T> {
1881    direction: FftDirection,
1882    twiddles_re: [__m128; 11],
1883    twiddles_im: [__m128; 11],
1884    _phantom: std::marker::PhantomData<T>,
1885}
1886
1887boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly23, 23, |this: &SseF32Butterfly23<_>| this.direction);
1888impl<T: FftNum> SseF32Butterfly23<T> {
1889    /// Safety: The current machine must support the sse4.1 instruction set
1890    #[target_feature(enable = "sse4.1")]
1891    unsafe fn new(direction: FftDirection) -> Self {
1892        assert_f32::<T>();
1893        let twiddles = make_twiddles(23, direction);
1894        Self {
1895            direction,
1896            twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
1897            twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
1898            _phantom: std::marker::PhantomData,
1899        }
1900    }
1901
1902    #[inline(always)]
1903    pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
1904        let values = read_partial1_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22 });
1905
1906        let out = self.perform_parallel_fft_direct(values);
1907        
1908        write_partial_lo_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22 } ); 
1909    }
1910
1911    #[inline(always)]
1912    pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
1913        let input_packed = read_complex_to_array!(buffer, { 0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44 });
1914
1915        let values = [
1916            extract_lo_hi_f32(input_packed[0], input_packed[11]),
1917            extract_hi_lo_f32(input_packed[0], input_packed[12]),
1918            extract_lo_hi_f32(input_packed[1], input_packed[12]),
1919            extract_hi_lo_f32(input_packed[1], input_packed[13]),
1920            extract_lo_hi_f32(input_packed[2], input_packed[13]),
1921            extract_hi_lo_f32(input_packed[2], input_packed[14]),
1922            extract_lo_hi_f32(input_packed[3], input_packed[14]),
1923            extract_hi_lo_f32(input_packed[3], input_packed[15]),
1924            extract_lo_hi_f32(input_packed[4], input_packed[15]),
1925            extract_hi_lo_f32(input_packed[4], input_packed[16]),
1926            extract_lo_hi_f32(input_packed[5], input_packed[16]),
1927            extract_hi_lo_f32(input_packed[5], input_packed[17]),
1928            extract_lo_hi_f32(input_packed[6], input_packed[17]),
1929            extract_hi_lo_f32(input_packed[6], input_packed[18]),
1930            extract_lo_hi_f32(input_packed[7], input_packed[18]),
1931            extract_hi_lo_f32(input_packed[7], input_packed[19]),
1932            extract_lo_hi_f32(input_packed[8], input_packed[19]),
1933            extract_hi_lo_f32(input_packed[8], input_packed[20]),
1934            extract_lo_hi_f32(input_packed[9], input_packed[20]),
1935            extract_hi_lo_f32(input_packed[9], input_packed[21]),
1936            extract_lo_hi_f32(input_packed[10], input_packed[21]),
1937            extract_hi_lo_f32(input_packed[10], input_packed[22]),
1938            extract_lo_hi_f32(input_packed[11], input_packed[22]),
1939        ];
1940
1941        let out = self.perform_parallel_fft_direct(values);
1942
1943        let out_packed = [
1944            extract_lo_lo_f32(out[0], out[1]),
1945            extract_lo_lo_f32(out[2], out[3]),
1946            extract_lo_lo_f32(out[4], out[5]),
1947            extract_lo_lo_f32(out[6], out[7]),
1948            extract_lo_lo_f32(out[8], out[9]),
1949            extract_lo_lo_f32(out[10], out[11]),
1950            extract_lo_lo_f32(out[12], out[13]),
1951            extract_lo_lo_f32(out[14], out[15]),
1952            extract_lo_lo_f32(out[16], out[17]),
1953            extract_lo_lo_f32(out[18], out[19]),
1954            extract_lo_lo_f32(out[20], out[21]),
1955            extract_lo_hi_f32(out[22], out[0]),
1956            extract_hi_hi_f32(out[1], out[2]),
1957            extract_hi_hi_f32(out[3], out[4]),
1958            extract_hi_hi_f32(out[5], out[6]),
1959            extract_hi_hi_f32(out[7], out[8]),
1960            extract_hi_hi_f32(out[9], out[10]),
1961            extract_hi_hi_f32(out[11], out[12]),
1962            extract_hi_hi_f32(out[13], out[14]),
1963            extract_hi_hi_f32(out[15], out[16]),
1964            extract_hi_hi_f32(out[17], out[18]),
1965            extract_hi_hi_f32(out[19], out[20]),
1966            extract_hi_hi_f32(out[21], out[22]),
1967        ];
1968
1969        write_complex_to_array_strided!(out_packed, buffer, 2, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22 });
1970    }
1971
1972    #[inline(always)]
1973    pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 23]) -> [__m128; 23] {
1974        let rotate = SseVector::make_rotate90(FftDirection::Inverse);
1975
1976        let y00 = values[0];
1977        let [x1p22, x1m22] =  SseVector::column_butterfly2([values[1], values[22]]);
1978        let x1m22 = SseVector::apply_rotate90(rotate, x1m22);
1979        let y00 = SseVector::add(y00, x1p22);
1980        let [x2p21, x2m21] =  SseVector::column_butterfly2([values[2], values[21]]);
1981        let x2m21 = SseVector::apply_rotate90(rotate, x2m21);
1982        let y00 = SseVector::add(y00, x2p21);
1983        let [x3p20, x3m20] =  SseVector::column_butterfly2([values[3], values[20]]);
1984        let x3m20 = SseVector::apply_rotate90(rotate, x3m20);
1985        let y00 = SseVector::add(y00, x3p20);
1986        let [x4p19, x4m19] =  SseVector::column_butterfly2([values[4], values[19]]);
1987        let x4m19 = SseVector::apply_rotate90(rotate, x4m19);
1988        let y00 = SseVector::add(y00, x4p19);
1989        let [x5p18, x5m18] =  SseVector::column_butterfly2([values[5], values[18]]);
1990        let x5m18 = SseVector::apply_rotate90(rotate, x5m18);
1991        let y00 = SseVector::add(y00, x5p18);
1992        let [x6p17, x6m17] =  SseVector::column_butterfly2([values[6], values[17]]);
1993        let x6m17 = SseVector::apply_rotate90(rotate, x6m17);
1994        let y00 = SseVector::add(y00, x6p17);
1995        let [x7p16, x7m16] =  SseVector::column_butterfly2([values[7], values[16]]);
1996        let x7m16 = SseVector::apply_rotate90(rotate, x7m16);
1997        let y00 = SseVector::add(y00, x7p16);
1998        let [x8p15, x8m15] =  SseVector::column_butterfly2([values[8], values[15]]);
1999        let x8m15 = SseVector::apply_rotate90(rotate, x8m15);
2000        let y00 = SseVector::add(y00, x8p15);
2001        let [x9p14, x9m14] =  SseVector::column_butterfly2([values[9], values[14]]);
2002        let x9m14 = SseVector::apply_rotate90(rotate, x9m14);
2003        let y00 = SseVector::add(y00, x9p14);
2004        let [x10p13, x10m13] =  SseVector::column_butterfly2([values[10], values[13]]);
2005        let x10m13 = SseVector::apply_rotate90(rotate, x10m13);
2006        let y00 = SseVector::add(y00, x10p13);
2007        let [x11p12, x11m12] =  SseVector::column_butterfly2([values[11], values[12]]);
2008        let x11m12 = SseVector::apply_rotate90(rotate, x11m12);
2009        let y00 = SseVector::add(y00, x11p12);
2010
2011        let m0122a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p22);
2012        let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[1], x2p21);
2013        let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[2], x3p20);
2014        let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[3], x4p19);
2015        let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[4], x5p18);
2016        let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[5], x6p17);
2017        let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[6], x7p16);
2018        let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[7], x8p15);
2019        let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[8], x9p14);
2020        let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[9], x10p13);
2021        let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[10], x11p12);
2022        let m0122b = SseVector::mul(self.twiddles_im[0], x1m22);
2023        let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[1], x2m21);
2024        let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[2], x3m20);
2025        let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[3], x4m19);
2026        let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[4], x5m18);
2027        let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[5], x6m17);
2028        let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[6], x7m16);
2029        let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[7], x8m15);
2030        let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[8], x9m14);
2031        let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[9], x10m13);
2032        let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[10], x11m12);
2033        let [y01, y22] = SseVector::column_butterfly2([m0122a, m0122b]);
2034
2035        let m0221a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p22);
2036        let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[3], x2p21);
2037        let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[5], x3p20);
2038        let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[7], x4p19);
2039        let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[9], x5p18);
2040        let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[10], x6p17);
2041        let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[8], x7p16);
2042        let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[6], x8p15);
2043        let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[4], x9p14);
2044        let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[2], x10p13);
2045        let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[0], x11p12);
2046        let m0221b = SseVector::mul(self.twiddles_im[1], x1m22);
2047        let m0221b = SseVector::fmadd(m0221b, self.twiddles_im[3], x2m21);
2048        let m0221b = SseVector::fmadd(m0221b, self.twiddles_im[5], x3m20);
2049        let m0221b = SseVector::fmadd(m0221b, self.twiddles_im[7], x4m19);
2050        let m0221b = SseVector::fmadd(m0221b, self.twiddles_im[9], x5m18);
2051        let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[10], x6m17);
2052        let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[8], x7m16);
2053        let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[6], x8m15);
2054        let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[4], x9m14);
2055        let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[2], x10m13);
2056        let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[0], x11m12);
2057        let [y02, y21] = SseVector::column_butterfly2([m0221a, m0221b]);
2058
2059        let m0320a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p22);
2060        let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[5], x2p21);
2061        let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[8], x3p20);
2062        let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[10], x4p19);
2063        let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[7], x5p18);
2064        let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[4], x6p17);
2065        let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[1], x7p16);
2066        let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[0], x8p15);
2067        let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[3], x9p14);
2068        let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[6], x10p13);
2069        let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[9], x11p12);
2070        let m0320b = SseVector::mul(self.twiddles_im[2], x1m22);
2071        let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[5], x2m21);
2072        let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[8], x3m20);
2073        let m0320b = SseVector::nmadd(m0320b, self.twiddles_im[10], x4m19);
2074        let m0320b = SseVector::nmadd(m0320b, self.twiddles_im[7], x5m18);
2075        let m0320b = SseVector::nmadd(m0320b, self.twiddles_im[4], x6m17);
2076        let m0320b = SseVector::nmadd(m0320b, self.twiddles_im[1], x7m16);
2077        let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[0], x8m15);
2078        let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[3], x9m14);
2079        let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[6], x10m13);
2080        let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[9], x11m12);
2081        let [y03, y20] = SseVector::column_butterfly2([m0320a, m0320b]);
2082
2083        let m0419a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p22);
2084        let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[7], x2p21);
2085        let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[10], x3p20);
2086        let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[6], x4p19);
2087        let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[2], x5p18);
2088        let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[0], x6p17);
2089        let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[4], x7p16);
2090        let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[8], x8p15);
2091        let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[9], x9p14);
2092        let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[5], x10p13);
2093        let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[1], x11p12);
2094        let m0419b = SseVector::mul(self.twiddles_im[3], x1m22);
2095        let m0419b = SseVector::fmadd(m0419b, self.twiddles_im[7], x2m21);
2096        let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[10], x3m20);
2097        let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[6], x4m19);
2098        let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[2], x5m18);
2099        let m0419b = SseVector::fmadd(m0419b, self.twiddles_im[0], x6m17);
2100        let m0419b = SseVector::fmadd(m0419b, self.twiddles_im[4], x7m16);
2101        let m0419b = SseVector::fmadd(m0419b, self.twiddles_im[8], x8m15);
2102        let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[9], x9m14);
2103        let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[5], x10m13);
2104        let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[1], x11m12);
2105        let [y04, y19] = SseVector::column_butterfly2([m0419a, m0419b]);
2106
2107        let m0518a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p22);
2108        let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[9], x2p21);
2109        let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[7], x3p20);
2110        let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[2], x4p19);
2111        let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[1], x5p18);
2112        let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[6], x6p17);
2113        let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[10], x7p16);
2114        let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[5], x8p15);
2115        let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[0], x9p14);
2116        let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[3], x10p13);
2117        let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[8], x11p12);
2118        let m0518b = SseVector::mul(self.twiddles_im[4], x1m22);
2119        let m0518b = SseVector::fmadd(m0518b, self.twiddles_im[9], x2m21);
2120        let m0518b = SseVector::nmadd(m0518b, self.twiddles_im[7], x3m20);
2121        let m0518b = SseVector::nmadd(m0518b, self.twiddles_im[2], x4m19);
2122        let m0518b = SseVector::fmadd(m0518b, self.twiddles_im[1], x5m18);
2123        let m0518b = SseVector::fmadd(m0518b, self.twiddles_im[6], x6m17);
2124        let m0518b = SseVector::nmadd(m0518b, self.twiddles_im[10], x7m16);
2125        let m0518b = SseVector::nmadd(m0518b, self.twiddles_im[5], x8m15);
2126        let m0518b = SseVector::nmadd(m0518b, self.twiddles_im[0], x9m14);
2127        let m0518b = SseVector::fmadd(m0518b, self.twiddles_im[3], x10m13);
2128        let m0518b = SseVector::fmadd(m0518b, self.twiddles_im[8], x11m12);
2129        let [y05, y18] = SseVector::column_butterfly2([m0518a, m0518b]);
2130
2131        let m0617a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p22);
2132        let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[10], x2p21);
2133        let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[4], x3p20);
2134        let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[0], x4p19);
2135        let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[6], x5p18);
2136        let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[9], x6p17);
2137        let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[3], x7p16);
2138        let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[1], x8p15);
2139        let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[7], x9p14);
2140        let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[8], x10p13);
2141        let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[2], x11p12);
2142        let m0617b = SseVector::mul(self.twiddles_im[5], x1m22);
2143        let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[10], x2m21);
2144        let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[4], x3m20);
2145        let m0617b = SseVector::fmadd(m0617b, self.twiddles_im[0], x4m19);
2146        let m0617b = SseVector::fmadd(m0617b, self.twiddles_im[6], x5m18);
2147        let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[9], x6m17);
2148        let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[3], x7m16);
2149        let m0617b = SseVector::fmadd(m0617b, self.twiddles_im[1], x8m15);
2150        let m0617b = SseVector::fmadd(m0617b, self.twiddles_im[7], x9m14);
2151        let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[8], x10m13);
2152        let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[2], x11m12);
2153        let [y06, y17] = SseVector::column_butterfly2([m0617a, m0617b]);
2154
2155        let m0716a = SseVector::fmadd(values[0], self.twiddles_re[6], x1p22);
2156        let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[8], x2p21);
2157        let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[1], x3p20);
2158        let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[4], x4p19);
2159        let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[10], x5p18);
2160        let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[3], x6p17);
2161        let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[2], x7p16);
2162        let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[9], x8p15);
2163        let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[5], x9p14);
2164        let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[0], x10p13);
2165        let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[7], x11p12);
2166        let m0716b = SseVector::mul(self.twiddles_im[6], x1m22);
2167        let m0716b = SseVector::nmadd(m0716b, self.twiddles_im[8], x2m21);
2168        let m0716b = SseVector::nmadd(m0716b, self.twiddles_im[1], x3m20);
2169        let m0716b = SseVector::fmadd(m0716b, self.twiddles_im[4], x4m19);
2170        let m0716b = SseVector::nmadd(m0716b, self.twiddles_im[10], x5m18);
2171        let m0716b = SseVector::nmadd(m0716b, self.twiddles_im[3], x6m17);
2172        let m0716b = SseVector::fmadd(m0716b, self.twiddles_im[2], x7m16);
2173        let m0716b = SseVector::fmadd(m0716b, self.twiddles_im[9], x8m15);
2174        let m0716b = SseVector::nmadd(m0716b, self.twiddles_im[5], x9m14);
2175        let m0716b = SseVector::fmadd(m0716b, self.twiddles_im[0], x10m13);
2176        let m0716b = SseVector::fmadd(m0716b, self.twiddles_im[7], x11m12);
2177        let [y07, y16] = SseVector::column_butterfly2([m0716a, m0716b]);
2178
2179        let m0815a = SseVector::fmadd(values[0], self.twiddles_re[7], x1p22);
2180        let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[6], x2p21);
2181        let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[0], x3p20);
2182        let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[8], x4p19);
2183        let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[5], x5p18);
2184        let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[1], x6p17);
2185        let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[9], x7p16);
2186        let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[4], x8p15);
2187        let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[2], x9p14);
2188        let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[10], x10p13);
2189        let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[3], x11p12);
2190        let m0815b = SseVector::mul(self.twiddles_im[7], x1m22);
2191        let m0815b = SseVector::nmadd(m0815b, self.twiddles_im[6], x2m21);
2192        let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[0], x3m20);
2193        let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[8], x4m19);
2194        let m0815b = SseVector::nmadd(m0815b, self.twiddles_im[5], x5m18);
2195        let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[1], x6m17);
2196        let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[9], x7m16);
2197        let m0815b = SseVector::nmadd(m0815b, self.twiddles_im[4], x8m15);
2198        let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[2], x9m14);
2199        let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[10], x10m13);
2200        let m0815b = SseVector::nmadd(m0815b, self.twiddles_im[3], x11m12);
2201        let [y08, y15] = SseVector::column_butterfly2([m0815a, m0815b]);
2202
2203        let m0914a = SseVector::fmadd(values[0], self.twiddles_re[8], x1p22);
2204        let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[4], x2p21);
2205        let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[3], x3p20);
2206        let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[9], x4p19);
2207        let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[0], x5p18);
2208        let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[7], x6p17);
2209        let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[5], x7p16);
2210        let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[2], x8p15);
2211        let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[10], x9p14);
2212        let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[1], x10p13);
2213        let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[6], x11p12);
2214        let m0914b = SseVector::mul(self.twiddles_im[8], x1m22);
2215        let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[4], x2m21);
2216        let m0914b = SseVector::fmadd(m0914b, self.twiddles_im[3], x3m20);
2217        let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[9], x4m19);
2218        let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[0], x5m18);
2219        let m0914b = SseVector::fmadd(m0914b, self.twiddles_im[7], x6m17);
2220        let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[5], x7m16);
2221        let m0914b = SseVector::fmadd(m0914b, self.twiddles_im[2], x8m15);
2222        let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[10], x9m14);
2223        let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[1], x10m13);
2224        let m0914b = SseVector::fmadd(m0914b, self.twiddles_im[6], x11m12);
2225        let [y09, y14] = SseVector::column_butterfly2([m0914a, m0914b]);
2226
2227        let m1013a = SseVector::fmadd(values[0], self.twiddles_re[9], x1p22);
2228        let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[2], x2p21);
2229        let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[6], x3p20);
2230        let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[5], x4p19);
2231        let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[3], x5p18);
2232        let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[8], x6p17);
2233        let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[0], x7p16);
2234        let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[10], x8p15);
2235        let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[1], x9p14);
2236        let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[7], x10p13);
2237        let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[4], x11p12);
2238        let m1013b = SseVector::mul(self.twiddles_im[9], x1m22);
2239        let m1013b = SseVector::nmadd(m1013b, self.twiddles_im[2], x2m21);
2240        let m1013b = SseVector::fmadd(m1013b, self.twiddles_im[6], x3m20);
2241        let m1013b = SseVector::nmadd(m1013b, self.twiddles_im[5], x4m19);
2242        let m1013b = SseVector::fmadd(m1013b, self.twiddles_im[3], x5m18);
2243        let m1013b = SseVector::nmadd(m1013b, self.twiddles_im[8], x6m17);
2244        let m1013b = SseVector::fmadd(m1013b, self.twiddles_im[0], x7m16);
2245        let m1013b = SseVector::fmadd(m1013b, self.twiddles_im[10], x8m15);
2246        let m1013b = SseVector::nmadd(m1013b, self.twiddles_im[1], x9m14);
2247        let m1013b = SseVector::fmadd(m1013b, self.twiddles_im[7], x10m13);
2248        let m1013b = SseVector::nmadd(m1013b, self.twiddles_im[4], x11m12);
2249        let [y10, y13] = SseVector::column_butterfly2([m1013a, m1013b]);
2250
2251        let m1112a = SseVector::fmadd(values[0], self.twiddles_re[10], x1p22);
2252        let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[0], x2p21);
2253        let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[9], x3p20);
2254        let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[1], x4p19);
2255        let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[8], x5p18);
2256        let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[2], x6p17);
2257        let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[7], x7p16);
2258        let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[3], x8p15);
2259        let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[6], x9p14);
2260        let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[4], x10p13);
2261        let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[5], x11p12);
2262        let m1112b = SseVector::mul(self.twiddles_im[10], x1m22);
2263        let m1112b = SseVector::nmadd(m1112b, self.twiddles_im[0], x2m21);
2264        let m1112b = SseVector::fmadd(m1112b, self.twiddles_im[9], x3m20);
2265        let m1112b = SseVector::nmadd(m1112b, self.twiddles_im[1], x4m19);
2266        let m1112b = SseVector::fmadd(m1112b, self.twiddles_im[8], x5m18);
2267        let m1112b = SseVector::nmadd(m1112b, self.twiddles_im[2], x6m17);
2268        let m1112b = SseVector::fmadd(m1112b, self.twiddles_im[7], x7m16);
2269        let m1112b = SseVector::nmadd(m1112b, self.twiddles_im[3], x8m15);
2270        let m1112b = SseVector::fmadd(m1112b, self.twiddles_im[6], x9m14);
2271        let m1112b = SseVector::nmadd(m1112b, self.twiddles_im[4], x10m13);
2272        let m1112b = SseVector::fmadd(m1112b, self.twiddles_im[5], x11m12);
2273        let [y11, y12] = SseVector::column_butterfly2([m1112a, m1112b]);
2274
2275
2276        [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12, y13, y14, y15, y16, y17, y18, y19, y20, y21, y22]
2277    }
2278}
2279
2280struct SseF64Butterfly23<T> {
2281    direction: FftDirection,
2282    twiddles_re: [__m128d; 11],
2283    twiddles_im: [__m128d; 11],
2284    _phantom: std::marker::PhantomData<T>,
2285}
2286
2287boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly23, 23, |this: &SseF64Butterfly23<_>| this.direction);
2288impl<T: FftNum> SseF64Butterfly23<T> {
2289    /// Safety: The current machine must support the sse4.1 instruction set
2290    #[target_feature(enable = "sse4.1")]
2291    unsafe fn new(direction: FftDirection) -> Self {
2292        assert_f64::<T>();
2293        let twiddles = make_twiddles(23, direction);
2294        unsafe {Self {
2295            direction,
2296            twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
2297            twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
2298            _phantom: std::marker::PhantomData,
2299        }}
2300    }
2301
2302    #[inline(always)]
2303    pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
2304        let values = read_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22 });
2305
2306        let out = self.perform_fft_direct(values);
2307
2308        write_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22 });   
2309    }
2310
2311    #[inline(always)]
2312    pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 23]) -> [__m128d; 23] {
2313        let rotate = SseVector::make_rotate90(FftDirection::Inverse);
2314
2315        let y00 = values[0];
2316        let [x1p22, x1m22] =  SseVector::column_butterfly2([values[1], values[22]]);
2317        let x1m22 = SseVector::apply_rotate90(rotate, x1m22);
2318        let y00 = SseVector::add(y00, x1p22);
2319        let [x2p21, x2m21] =  SseVector::column_butterfly2([values[2], values[21]]);
2320        let x2m21 = SseVector::apply_rotate90(rotate, x2m21);
2321        let y00 = SseVector::add(y00, x2p21);
2322        let [x3p20, x3m20] =  SseVector::column_butterfly2([values[3], values[20]]);
2323        let x3m20 = SseVector::apply_rotate90(rotate, x3m20);
2324        let y00 = SseVector::add(y00, x3p20);
2325        let [x4p19, x4m19] =  SseVector::column_butterfly2([values[4], values[19]]);
2326        let x4m19 = SseVector::apply_rotate90(rotate, x4m19);
2327        let y00 = SseVector::add(y00, x4p19);
2328        let [x5p18, x5m18] =  SseVector::column_butterfly2([values[5], values[18]]);
2329        let x5m18 = SseVector::apply_rotate90(rotate, x5m18);
2330        let y00 = SseVector::add(y00, x5p18);
2331        let [x6p17, x6m17] =  SseVector::column_butterfly2([values[6], values[17]]);
2332        let x6m17 = SseVector::apply_rotate90(rotate, x6m17);
2333        let y00 = SseVector::add(y00, x6p17);
2334        let [x7p16, x7m16] =  SseVector::column_butterfly2([values[7], values[16]]);
2335        let x7m16 = SseVector::apply_rotate90(rotate, x7m16);
2336        let y00 = SseVector::add(y00, x7p16);
2337        let [x8p15, x8m15] =  SseVector::column_butterfly2([values[8], values[15]]);
2338        let x8m15 = SseVector::apply_rotate90(rotate, x8m15);
2339        let y00 = SseVector::add(y00, x8p15);
2340        let [x9p14, x9m14] =  SseVector::column_butterfly2([values[9], values[14]]);
2341        let x9m14 = SseVector::apply_rotate90(rotate, x9m14);
2342        let y00 = SseVector::add(y00, x9p14);
2343        let [x10p13, x10m13] =  SseVector::column_butterfly2([values[10], values[13]]);
2344        let x10m13 = SseVector::apply_rotate90(rotate, x10m13);
2345        let y00 = SseVector::add(y00, x10p13);
2346        let [x11p12, x11m12] =  SseVector::column_butterfly2([values[11], values[12]]);
2347        let x11m12 = SseVector::apply_rotate90(rotate, x11m12);
2348        let y00 = SseVector::add(y00, x11p12);
2349
2350        let m0122a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p22);
2351        let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[1], x2p21);
2352        let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[2], x3p20);
2353        let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[3], x4p19);
2354        let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[4], x5p18);
2355        let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[5], x6p17);
2356        let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[6], x7p16);
2357        let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[7], x8p15);
2358        let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[8], x9p14);
2359        let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[9], x10p13);
2360        let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[10], x11p12);
2361        let m0122b = SseVector::mul(self.twiddles_im[0], x1m22);
2362        let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[1], x2m21);
2363        let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[2], x3m20);
2364        let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[3], x4m19);
2365        let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[4], x5m18);
2366        let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[5], x6m17);
2367        let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[6], x7m16);
2368        let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[7], x8m15);
2369        let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[8], x9m14);
2370        let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[9], x10m13);
2371        let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[10], x11m12);
2372        let [y01, y22] = SseVector::column_butterfly2([m0122a, m0122b]);
2373
2374        let m0221a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p22);
2375        let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[3], x2p21);
2376        let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[5], x3p20);
2377        let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[7], x4p19);
2378        let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[9], x5p18);
2379        let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[10], x6p17);
2380        let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[8], x7p16);
2381        let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[6], x8p15);
2382        let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[4], x9p14);
2383        let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[2], x10p13);
2384        let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[0], x11p12);
2385        let m0221b = SseVector::mul(self.twiddles_im[1], x1m22);
2386        let m0221b = SseVector::fmadd(m0221b, self.twiddles_im[3], x2m21);
2387        let m0221b = SseVector::fmadd(m0221b, self.twiddles_im[5], x3m20);
2388        let m0221b = SseVector::fmadd(m0221b, self.twiddles_im[7], x4m19);
2389        let m0221b = SseVector::fmadd(m0221b, self.twiddles_im[9], x5m18);
2390        let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[10], x6m17);
2391        let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[8], x7m16);
2392        let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[6], x8m15);
2393        let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[4], x9m14);
2394        let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[2], x10m13);
2395        let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[0], x11m12);
2396        let [y02, y21] = SseVector::column_butterfly2([m0221a, m0221b]);
2397
2398        let m0320a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p22);
2399        let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[5], x2p21);
2400        let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[8], x3p20);
2401        let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[10], x4p19);
2402        let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[7], x5p18);
2403        let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[4], x6p17);
2404        let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[1], x7p16);
2405        let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[0], x8p15);
2406        let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[3], x9p14);
2407        let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[6], x10p13);
2408        let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[9], x11p12);
2409        let m0320b = SseVector::mul(self.twiddles_im[2], x1m22);
2410        let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[5], x2m21);
2411        let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[8], x3m20);
2412        let m0320b = SseVector::nmadd(m0320b, self.twiddles_im[10], x4m19);
2413        let m0320b = SseVector::nmadd(m0320b, self.twiddles_im[7], x5m18);
2414        let m0320b = SseVector::nmadd(m0320b, self.twiddles_im[4], x6m17);
2415        let m0320b = SseVector::nmadd(m0320b, self.twiddles_im[1], x7m16);
2416        let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[0], x8m15);
2417        let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[3], x9m14);
2418        let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[6], x10m13);
2419        let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[9], x11m12);
2420        let [y03, y20] = SseVector::column_butterfly2([m0320a, m0320b]);
2421
2422        let m0419a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p22);
2423        let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[7], x2p21);
2424        let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[10], x3p20);
2425        let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[6], x4p19);
2426        let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[2], x5p18);
2427        let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[0], x6p17);
2428        let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[4], x7p16);
2429        let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[8], x8p15);
2430        let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[9], x9p14);
2431        let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[5], x10p13);
2432        let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[1], x11p12);
2433        let m0419b = SseVector::mul(self.twiddles_im[3], x1m22);
2434        let m0419b = SseVector::fmadd(m0419b, self.twiddles_im[7], x2m21);
2435        let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[10], x3m20);
2436        let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[6], x4m19);
2437        let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[2], x5m18);
2438        let m0419b = SseVector::fmadd(m0419b, self.twiddles_im[0], x6m17);
2439        let m0419b = SseVector::fmadd(m0419b, self.twiddles_im[4], x7m16);
2440        let m0419b = SseVector::fmadd(m0419b, self.twiddles_im[8], x8m15);
2441        let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[9], x9m14);
2442        let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[5], x10m13);
2443        let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[1], x11m12);
2444        let [y04, y19] = SseVector::column_butterfly2([m0419a, m0419b]);
2445
2446        let m0518a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p22);
2447        let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[9], x2p21);
2448        let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[7], x3p20);
2449        let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[2], x4p19);
2450        let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[1], x5p18);
2451        let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[6], x6p17);
2452        let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[10], x7p16);
2453        let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[5], x8p15);
2454        let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[0], x9p14);
2455        let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[3], x10p13);
2456        let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[8], x11p12);
2457        let m0518b = SseVector::mul(self.twiddles_im[4], x1m22);
2458        let m0518b = SseVector::fmadd(m0518b, self.twiddles_im[9], x2m21);
2459        let m0518b = SseVector::nmadd(m0518b, self.twiddles_im[7], x3m20);
2460        let m0518b = SseVector::nmadd(m0518b, self.twiddles_im[2], x4m19);
2461        let m0518b = SseVector::fmadd(m0518b, self.twiddles_im[1], x5m18);
2462        let m0518b = SseVector::fmadd(m0518b, self.twiddles_im[6], x6m17);
2463        let m0518b = SseVector::nmadd(m0518b, self.twiddles_im[10], x7m16);
2464        let m0518b = SseVector::nmadd(m0518b, self.twiddles_im[5], x8m15);
2465        let m0518b = SseVector::nmadd(m0518b, self.twiddles_im[0], x9m14);
2466        let m0518b = SseVector::fmadd(m0518b, self.twiddles_im[3], x10m13);
2467        let m0518b = SseVector::fmadd(m0518b, self.twiddles_im[8], x11m12);
2468        let [y05, y18] = SseVector::column_butterfly2([m0518a, m0518b]);
2469
2470        let m0617a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p22);
2471        let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[10], x2p21);
2472        let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[4], x3p20);
2473        let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[0], x4p19);
2474        let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[6], x5p18);
2475        let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[9], x6p17);
2476        let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[3], x7p16);
2477        let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[1], x8p15);
2478        let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[7], x9p14);
2479        let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[8], x10p13);
2480        let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[2], x11p12);
2481        let m0617b = SseVector::mul(self.twiddles_im[5], x1m22);
2482        let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[10], x2m21);
2483        let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[4], x3m20);
2484        let m0617b = SseVector::fmadd(m0617b, self.twiddles_im[0], x4m19);
2485        let m0617b = SseVector::fmadd(m0617b, self.twiddles_im[6], x5m18);
2486        let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[9], x6m17);
2487        let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[3], x7m16);
2488        let m0617b = SseVector::fmadd(m0617b, self.twiddles_im[1], x8m15);
2489        let m0617b = SseVector::fmadd(m0617b, self.twiddles_im[7], x9m14);
2490        let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[8], x10m13);
2491        let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[2], x11m12);
2492        let [y06, y17] = SseVector::column_butterfly2([m0617a, m0617b]);
2493
2494        let m0716a = SseVector::fmadd(values[0], self.twiddles_re[6], x1p22);
2495        let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[8], x2p21);
2496        let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[1], x3p20);
2497        let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[4], x4p19);
2498        let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[10], x5p18);
2499        let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[3], x6p17);
2500        let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[2], x7p16);
2501        let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[9], x8p15);
2502        let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[5], x9p14);
2503        let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[0], x10p13);
2504        let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[7], x11p12);
2505        let m0716b = SseVector::mul(self.twiddles_im[6], x1m22);
2506        let m0716b = SseVector::nmadd(m0716b, self.twiddles_im[8], x2m21);
2507        let m0716b = SseVector::nmadd(m0716b, self.twiddles_im[1], x3m20);
2508        let m0716b = SseVector::fmadd(m0716b, self.twiddles_im[4], x4m19);
2509        let m0716b = SseVector::nmadd(m0716b, self.twiddles_im[10], x5m18);
2510        let m0716b = SseVector::nmadd(m0716b, self.twiddles_im[3], x6m17);
2511        let m0716b = SseVector::fmadd(m0716b, self.twiddles_im[2], x7m16);
2512        let m0716b = SseVector::fmadd(m0716b, self.twiddles_im[9], x8m15);
2513        let m0716b = SseVector::nmadd(m0716b, self.twiddles_im[5], x9m14);
2514        let m0716b = SseVector::fmadd(m0716b, self.twiddles_im[0], x10m13);
2515        let m0716b = SseVector::fmadd(m0716b, self.twiddles_im[7], x11m12);
2516        let [y07, y16] = SseVector::column_butterfly2([m0716a, m0716b]);
2517
2518        let m0815a = SseVector::fmadd(values[0], self.twiddles_re[7], x1p22);
2519        let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[6], x2p21);
2520        let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[0], x3p20);
2521        let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[8], x4p19);
2522        let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[5], x5p18);
2523        let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[1], x6p17);
2524        let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[9], x7p16);
2525        let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[4], x8p15);
2526        let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[2], x9p14);
2527        let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[10], x10p13);
2528        let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[3], x11p12);
2529        let m0815b = SseVector::mul(self.twiddles_im[7], x1m22);
2530        let m0815b = SseVector::nmadd(m0815b, self.twiddles_im[6], x2m21);
2531        let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[0], x3m20);
2532        let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[8], x4m19);
2533        let m0815b = SseVector::nmadd(m0815b, self.twiddles_im[5], x5m18);
2534        let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[1], x6m17);
2535        let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[9], x7m16);
2536        let m0815b = SseVector::nmadd(m0815b, self.twiddles_im[4], x8m15);
2537        let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[2], x9m14);
2538        let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[10], x10m13);
2539        let m0815b = SseVector::nmadd(m0815b, self.twiddles_im[3], x11m12);
2540        let [y08, y15] = SseVector::column_butterfly2([m0815a, m0815b]);
2541
2542        let m0914a = SseVector::fmadd(values[0], self.twiddles_re[8], x1p22);
2543        let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[4], x2p21);
2544        let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[3], x3p20);
2545        let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[9], x4p19);
2546        let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[0], x5p18);
2547        let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[7], x6p17);
2548        let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[5], x7p16);
2549        let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[2], x8p15);
2550        let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[10], x9p14);
2551        let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[1], x10p13);
2552        let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[6], x11p12);
2553        let m0914b = SseVector::mul(self.twiddles_im[8], x1m22);
2554        let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[4], x2m21);
2555        let m0914b = SseVector::fmadd(m0914b, self.twiddles_im[3], x3m20);
2556        let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[9], x4m19);
2557        let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[0], x5m18);
2558        let m0914b = SseVector::fmadd(m0914b, self.twiddles_im[7], x6m17);
2559        let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[5], x7m16);
2560        let m0914b = SseVector::fmadd(m0914b, self.twiddles_im[2], x8m15);
2561        let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[10], x9m14);
2562        let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[1], x10m13);
2563        let m0914b = SseVector::fmadd(m0914b, self.twiddles_im[6], x11m12);
2564        let [y09, y14] = SseVector::column_butterfly2([m0914a, m0914b]);
2565
2566        let m1013a = SseVector::fmadd(values[0], self.twiddles_re[9], x1p22);
2567        let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[2], x2p21);
2568        let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[6], x3p20);
2569        let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[5], x4p19);
2570        let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[3], x5p18);
2571        let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[8], x6p17);
2572        let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[0], x7p16);
2573        let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[10], x8p15);
2574        let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[1], x9p14);
2575        let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[7], x10p13);
2576        let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[4], x11p12);
2577        let m1013b = SseVector::mul(self.twiddles_im[9], x1m22);
2578        let m1013b = SseVector::nmadd(m1013b, self.twiddles_im[2], x2m21);
2579        let m1013b = SseVector::fmadd(m1013b, self.twiddles_im[6], x3m20);
2580        let m1013b = SseVector::nmadd(m1013b, self.twiddles_im[5], x4m19);
2581        let m1013b = SseVector::fmadd(m1013b, self.twiddles_im[3], x5m18);
2582        let m1013b = SseVector::nmadd(m1013b, self.twiddles_im[8], x6m17);
2583        let m1013b = SseVector::fmadd(m1013b, self.twiddles_im[0], x7m16);
2584        let m1013b = SseVector::fmadd(m1013b, self.twiddles_im[10], x8m15);
2585        let m1013b = SseVector::nmadd(m1013b, self.twiddles_im[1], x9m14);
2586        let m1013b = SseVector::fmadd(m1013b, self.twiddles_im[7], x10m13);
2587        let m1013b = SseVector::nmadd(m1013b, self.twiddles_im[4], x11m12);
2588        let [y10, y13] = SseVector::column_butterfly2([m1013a, m1013b]);
2589
2590        let m1112a = SseVector::fmadd(values[0], self.twiddles_re[10], x1p22);
2591        let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[0], x2p21);
2592        let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[9], x3p20);
2593        let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[1], x4p19);
2594        let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[8], x5p18);
2595        let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[2], x6p17);
2596        let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[7], x7p16);
2597        let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[3], x8p15);
2598        let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[6], x9p14);
2599        let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[4], x10p13);
2600        let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[5], x11p12);
2601        let m1112b = SseVector::mul(self.twiddles_im[10], x1m22);
2602        let m1112b = SseVector::nmadd(m1112b, self.twiddles_im[0], x2m21);
2603        let m1112b = SseVector::fmadd(m1112b, self.twiddles_im[9], x3m20);
2604        let m1112b = SseVector::nmadd(m1112b, self.twiddles_im[1], x4m19);
2605        let m1112b = SseVector::fmadd(m1112b, self.twiddles_im[8], x5m18);
2606        let m1112b = SseVector::nmadd(m1112b, self.twiddles_im[2], x6m17);
2607        let m1112b = SseVector::fmadd(m1112b, self.twiddles_im[7], x7m16);
2608        let m1112b = SseVector::nmadd(m1112b, self.twiddles_im[3], x8m15);
2609        let m1112b = SseVector::fmadd(m1112b, self.twiddles_im[6], x9m14);
2610        let m1112b = SseVector::nmadd(m1112b, self.twiddles_im[4], x10m13);
2611        let m1112b = SseVector::fmadd(m1112b, self.twiddles_im[5], x11m12);
2612        let [y11, y12] = SseVector::column_butterfly2([m1112a, m1112b]);
2613
2614
2615        [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12, y13, y14, y15, y16, y17, y18, y19, y20, y21, y22]
2616    }
2617}
2618
2619struct SseF32Butterfly29<T> {
2620    direction: FftDirection,
2621    twiddles_re: [__m128; 14],
2622    twiddles_im: [__m128; 14],
2623    _phantom: std::marker::PhantomData<T>,
2624}
2625
2626boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly29, 29, |this: &SseF32Butterfly29<_>| this.direction);
2627impl<T: FftNum> SseF32Butterfly29<T> {
2628    /// Safety: The current machine must support the sse4.1 instruction set
2629    #[target_feature(enable = "sse4.1")]
2630    unsafe fn new(direction: FftDirection) -> Self {
2631        assert_f32::<T>();
2632        let twiddles = make_twiddles(29, direction);
2633        Self {
2634            direction,
2635            twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
2636            twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
2637            _phantom: std::marker::PhantomData,
2638        }
2639    }
2640
2641    #[inline(always)]
2642    pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
2643        let values = read_partial1_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28 });
2644
2645        let out = self.perform_parallel_fft_direct(values);
2646        
2647        write_partial_lo_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28 } ); 
2648    }
2649
2650    #[inline(always)]
2651    pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
2652        let input_packed = read_complex_to_array!(buffer, { 0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56 });
2653
2654        let values = [
2655            extract_lo_hi_f32(input_packed[0], input_packed[14]),
2656            extract_hi_lo_f32(input_packed[0], input_packed[15]),
2657            extract_lo_hi_f32(input_packed[1], input_packed[15]),
2658            extract_hi_lo_f32(input_packed[1], input_packed[16]),
2659            extract_lo_hi_f32(input_packed[2], input_packed[16]),
2660            extract_hi_lo_f32(input_packed[2], input_packed[17]),
2661            extract_lo_hi_f32(input_packed[3], input_packed[17]),
2662            extract_hi_lo_f32(input_packed[3], input_packed[18]),
2663            extract_lo_hi_f32(input_packed[4], input_packed[18]),
2664            extract_hi_lo_f32(input_packed[4], input_packed[19]),
2665            extract_lo_hi_f32(input_packed[5], input_packed[19]),
2666            extract_hi_lo_f32(input_packed[5], input_packed[20]),
2667            extract_lo_hi_f32(input_packed[6], input_packed[20]),
2668            extract_hi_lo_f32(input_packed[6], input_packed[21]),
2669            extract_lo_hi_f32(input_packed[7], input_packed[21]),
2670            extract_hi_lo_f32(input_packed[7], input_packed[22]),
2671            extract_lo_hi_f32(input_packed[8], input_packed[22]),
2672            extract_hi_lo_f32(input_packed[8], input_packed[23]),
2673            extract_lo_hi_f32(input_packed[9], input_packed[23]),
2674            extract_hi_lo_f32(input_packed[9], input_packed[24]),
2675            extract_lo_hi_f32(input_packed[10], input_packed[24]),
2676            extract_hi_lo_f32(input_packed[10], input_packed[25]),
2677            extract_lo_hi_f32(input_packed[11], input_packed[25]),
2678            extract_hi_lo_f32(input_packed[11], input_packed[26]),
2679            extract_lo_hi_f32(input_packed[12], input_packed[26]),
2680            extract_hi_lo_f32(input_packed[12], input_packed[27]),
2681            extract_lo_hi_f32(input_packed[13], input_packed[27]),
2682            extract_hi_lo_f32(input_packed[13], input_packed[28]),
2683            extract_lo_hi_f32(input_packed[14], input_packed[28]),
2684        ];
2685
2686        let out = self.perform_parallel_fft_direct(values);
2687
2688        let out_packed = [
2689            extract_lo_lo_f32(out[0], out[1]),
2690            extract_lo_lo_f32(out[2], out[3]),
2691            extract_lo_lo_f32(out[4], out[5]),
2692            extract_lo_lo_f32(out[6], out[7]),
2693            extract_lo_lo_f32(out[8], out[9]),
2694            extract_lo_lo_f32(out[10], out[11]),
2695            extract_lo_lo_f32(out[12], out[13]),
2696            extract_lo_lo_f32(out[14], out[15]),
2697            extract_lo_lo_f32(out[16], out[17]),
2698            extract_lo_lo_f32(out[18], out[19]),
2699            extract_lo_lo_f32(out[20], out[21]),
2700            extract_lo_lo_f32(out[22], out[23]),
2701            extract_lo_lo_f32(out[24], out[25]),
2702            extract_lo_lo_f32(out[26], out[27]),
2703            extract_lo_hi_f32(out[28], out[0]),
2704            extract_hi_hi_f32(out[1], out[2]),
2705            extract_hi_hi_f32(out[3], out[4]),
2706            extract_hi_hi_f32(out[5], out[6]),
2707            extract_hi_hi_f32(out[7], out[8]),
2708            extract_hi_hi_f32(out[9], out[10]),
2709            extract_hi_hi_f32(out[11], out[12]),
2710            extract_hi_hi_f32(out[13], out[14]),
2711            extract_hi_hi_f32(out[15], out[16]),
2712            extract_hi_hi_f32(out[17], out[18]),
2713            extract_hi_hi_f32(out[19], out[20]),
2714            extract_hi_hi_f32(out[21], out[22]),
2715            extract_hi_hi_f32(out[23], out[24]),
2716            extract_hi_hi_f32(out[25], out[26]),
2717            extract_hi_hi_f32(out[27], out[28]),
2718        ];
2719
2720        write_complex_to_array_strided!(out_packed, buffer, 2, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28 });
2721    }
2722
2723    #[inline(always)]
2724    pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 29]) -> [__m128; 29] {
2725        let rotate = SseVector::make_rotate90(FftDirection::Inverse);
2726
2727        let y00 = values[0];
2728        let [x1p28, x1m28] =  SseVector::column_butterfly2([values[1], values[28]]);
2729        let x1m28 = SseVector::apply_rotate90(rotate, x1m28);
2730        let y00 = SseVector::add(y00, x1p28);
2731        let [x2p27, x2m27] =  SseVector::column_butterfly2([values[2], values[27]]);
2732        let x2m27 = SseVector::apply_rotate90(rotate, x2m27);
2733        let y00 = SseVector::add(y00, x2p27);
2734        let [x3p26, x3m26] =  SseVector::column_butterfly2([values[3], values[26]]);
2735        let x3m26 = SseVector::apply_rotate90(rotate, x3m26);
2736        let y00 = SseVector::add(y00, x3p26);
2737        let [x4p25, x4m25] =  SseVector::column_butterfly2([values[4], values[25]]);
2738        let x4m25 = SseVector::apply_rotate90(rotate, x4m25);
2739        let y00 = SseVector::add(y00, x4p25);
2740        let [x5p24, x5m24] =  SseVector::column_butterfly2([values[5], values[24]]);
2741        let x5m24 = SseVector::apply_rotate90(rotate, x5m24);
2742        let y00 = SseVector::add(y00, x5p24);
2743        let [x6p23, x6m23] =  SseVector::column_butterfly2([values[6], values[23]]);
2744        let x6m23 = SseVector::apply_rotate90(rotate, x6m23);
2745        let y00 = SseVector::add(y00, x6p23);
2746        let [x7p22, x7m22] =  SseVector::column_butterfly2([values[7], values[22]]);
2747        let x7m22 = SseVector::apply_rotate90(rotate, x7m22);
2748        let y00 = SseVector::add(y00, x7p22);
2749        let [x8p21, x8m21] =  SseVector::column_butterfly2([values[8], values[21]]);
2750        let x8m21 = SseVector::apply_rotate90(rotate, x8m21);
2751        let y00 = SseVector::add(y00, x8p21);
2752        let [x9p20, x9m20] =  SseVector::column_butterfly2([values[9], values[20]]);
2753        let x9m20 = SseVector::apply_rotate90(rotate, x9m20);
2754        let y00 = SseVector::add(y00, x9p20);
2755        let [x10p19, x10m19] =  SseVector::column_butterfly2([values[10], values[19]]);
2756        let x10m19 = SseVector::apply_rotate90(rotate, x10m19);
2757        let y00 = SseVector::add(y00, x10p19);
2758        let [x11p18, x11m18] =  SseVector::column_butterfly2([values[11], values[18]]);
2759        let x11m18 = SseVector::apply_rotate90(rotate, x11m18);
2760        let y00 = SseVector::add(y00, x11p18);
2761        let [x12p17, x12m17] =  SseVector::column_butterfly2([values[12], values[17]]);
2762        let x12m17 = SseVector::apply_rotate90(rotate, x12m17);
2763        let y00 = SseVector::add(y00, x12p17);
2764        let [x13p16, x13m16] =  SseVector::column_butterfly2([values[13], values[16]]);
2765        let x13m16 = SseVector::apply_rotate90(rotate, x13m16);
2766        let y00 = SseVector::add(y00, x13p16);
2767        let [x14p15, x14m15] =  SseVector::column_butterfly2([values[14], values[15]]);
2768        let x14m15 = SseVector::apply_rotate90(rotate, x14m15);
2769        let y00 = SseVector::add(y00, x14p15);
2770
2771        let m0128a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p28);
2772        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[1], x2p27);
2773        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[2], x3p26);
2774        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[3], x4p25);
2775        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[4], x5p24);
2776        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[5], x6p23);
2777        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[6], x7p22);
2778        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[7], x8p21);
2779        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[8], x9p20);
2780        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[9], x10p19);
2781        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[10], x11p18);
2782        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[11], x12p17);
2783        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[12], x13p16);
2784        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[13], x14p15);
2785        let m0128b = SseVector::mul(self.twiddles_im[0], x1m28);
2786        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[1], x2m27);
2787        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[2], x3m26);
2788        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[3], x4m25);
2789        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[4], x5m24);
2790        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[5], x6m23);
2791        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[6], x7m22);
2792        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[7], x8m21);
2793        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[8], x9m20);
2794        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[9], x10m19);
2795        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[10], x11m18);
2796        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[11], x12m17);
2797        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[12], x13m16);
2798        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[13], x14m15);
2799        let [y01, y28] = SseVector::column_butterfly2([m0128a, m0128b]);
2800
2801        let m0227a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p28);
2802        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[3], x2p27);
2803        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[5], x3p26);
2804        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[7], x4p25);
2805        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[9], x5p24);
2806        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[11], x6p23);
2807        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[13], x7p22);
2808        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[12], x8p21);
2809        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[10], x9p20);
2810        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[8], x10p19);
2811        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[6], x11p18);
2812        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[4], x12p17);
2813        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[2], x13p16);
2814        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[0], x14p15);
2815        let m0227b = SseVector::mul(self.twiddles_im[1], x1m28);
2816        let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[3], x2m27);
2817        let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[5], x3m26);
2818        let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[7], x4m25);
2819        let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[9], x5m24);
2820        let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[11], x6m23);
2821        let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[13], x7m22);
2822        let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[12], x8m21);
2823        let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[10], x9m20);
2824        let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[8], x10m19);
2825        let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[6], x11m18);
2826        let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[4], x12m17);
2827        let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[2], x13m16);
2828        let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[0], x14m15);
2829        let [y02, y27] = SseVector::column_butterfly2([m0227a, m0227b]);
2830
2831        let m0326a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p28);
2832        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[5], x2p27);
2833        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[8], x3p26);
2834        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[11], x4p25);
2835        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[13], x5p24);
2836        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[10], x6p23);
2837        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[7], x7p22);
2838        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[4], x8p21);
2839        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[1], x9p20);
2840        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[0], x10p19);
2841        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[3], x11p18);
2842        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[6], x12p17);
2843        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[9], x13p16);
2844        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[12], x14p15);
2845        let m0326b = SseVector::mul(self.twiddles_im[2], x1m28);
2846        let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[5], x2m27);
2847        let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[8], x3m26);
2848        let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[11], x4m25);
2849        let m0326b = SseVector::nmadd(m0326b, self.twiddles_im[13], x5m24);
2850        let m0326b = SseVector::nmadd(m0326b, self.twiddles_im[10], x6m23);
2851        let m0326b = SseVector::nmadd(m0326b, self.twiddles_im[7], x7m22);
2852        let m0326b = SseVector::nmadd(m0326b, self.twiddles_im[4], x8m21);
2853        let m0326b = SseVector::nmadd(m0326b, self.twiddles_im[1], x9m20);
2854        let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[0], x10m19);
2855        let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[3], x11m18);
2856        let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[6], x12m17);
2857        let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[9], x13m16);
2858        let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[12], x14m15);
2859        let [y03, y26] = SseVector::column_butterfly2([m0326a, m0326b]);
2860
2861        let m0425a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p28);
2862        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[7], x2p27);
2863        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[11], x3p26);
2864        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[12], x4p25);
2865        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[8], x5p24);
2866        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[4], x6p23);
2867        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[0], x7p22);
2868        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[2], x8p21);
2869        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[6], x9p20);
2870        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[10], x10p19);
2871        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[13], x11p18);
2872        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[9], x12p17);
2873        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[5], x13p16);
2874        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[1], x14p15);
2875        let m0425b = SseVector::mul(self.twiddles_im[3], x1m28);
2876        let m0425b = SseVector::fmadd(m0425b, self.twiddles_im[7], x2m27);
2877        let m0425b = SseVector::fmadd(m0425b, self.twiddles_im[11], x3m26);
2878        let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[12], x4m25);
2879        let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[8], x5m24);
2880        let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[4], x6m23);
2881        let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[0], x7m22);
2882        let m0425b = SseVector::fmadd(m0425b, self.twiddles_im[2], x8m21);
2883        let m0425b = SseVector::fmadd(m0425b, self.twiddles_im[6], x9m20);
2884        let m0425b = SseVector::fmadd(m0425b, self.twiddles_im[10], x10m19);
2885        let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[13], x11m18);
2886        let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[9], x12m17);
2887        let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[5], x13m16);
2888        let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[1], x14m15);
2889        let [y04, y25] = SseVector::column_butterfly2([m0425a, m0425b]);
2890
2891        let m0524a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p28);
2892        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[9], x2p27);
2893        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[13], x3p26);
2894        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[8], x4p25);
2895        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[3], x5p24);
2896        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[0], x6p23);
2897        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[5], x7p22);
2898        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[10], x8p21);
2899        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[12], x9p20);
2900        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[7], x10p19);
2901        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[2], x11p18);
2902        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[1], x12p17);
2903        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[6], x13p16);
2904        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[11], x14p15);
2905        let m0524b = SseVector::mul(self.twiddles_im[4], x1m28);
2906        let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[9], x2m27);
2907        let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[13], x3m26);
2908        let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[8], x4m25);
2909        let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[3], x5m24);
2910        let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[0], x6m23);
2911        let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[5], x7m22);
2912        let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[10], x8m21);
2913        let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[12], x9m20);
2914        let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[7], x10m19);
2915        let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[2], x11m18);
2916        let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[1], x12m17);
2917        let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[6], x13m16);
2918        let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[11], x14m15);
2919        let [y05, y24] = SseVector::column_butterfly2([m0524a, m0524b]);
2920
2921        let m0623a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p28);
2922        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[11], x2p27);
2923        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[10], x3p26);
2924        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[4], x4p25);
2925        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[0], x5p24);
2926        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[6], x6p23);
2927        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[12], x7p22);
2928        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[9], x8p21);
2929        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[3], x9p20);
2930        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[1], x10p19);
2931        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[7], x11p18);
2932        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[13], x12p17);
2933        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[8], x13p16);
2934        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[2], x14p15);
2935        let m0623b = SseVector::mul(self.twiddles_im[5], x1m28);
2936        let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[11], x2m27);
2937        let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[10], x3m26);
2938        let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[4], x4m25);
2939        let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[0], x5m24);
2940        let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[6], x6m23);
2941        let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[12], x7m22);
2942        let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[9], x8m21);
2943        let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[3], x9m20);
2944        let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[1], x10m19);
2945        let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[7], x11m18);
2946        let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[13], x12m17);
2947        let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[8], x13m16);
2948        let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[2], x14m15);
2949        let [y06, y23] = SseVector::column_butterfly2([m0623a, m0623b]);
2950
2951        let m0722a = SseVector::fmadd(values[0], self.twiddles_re[6], x1p28);
2952        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[13], x2p27);
2953        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[7], x3p26);
2954        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[0], x4p25);
2955        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[5], x5p24);
2956        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[12], x6p23);
2957        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[8], x7p22);
2958        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[1], x8p21);
2959        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[4], x9p20);
2960        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[11], x10p19);
2961        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[9], x11p18);
2962        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[2], x12p17);
2963        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[3], x13p16);
2964        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[10], x14p15);
2965        let m0722b = SseVector::mul(self.twiddles_im[6], x1m28);
2966        let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[13], x2m27);
2967        let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[7], x3m26);
2968        let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[0], x4m25);
2969        let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[5], x5m24);
2970        let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[12], x6m23);
2971        let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[8], x7m22);
2972        let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[1], x8m21);
2973        let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[4], x9m20);
2974        let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[11], x10m19);
2975        let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[9], x11m18);
2976        let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[2], x12m17);
2977        let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[3], x13m16);
2978        let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[10], x14m15);
2979        let [y07, y22] = SseVector::column_butterfly2([m0722a, m0722b]);
2980
2981        let m0821a = SseVector::fmadd(values[0], self.twiddles_re[7], x1p28);
2982        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[12], x2p27);
2983        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[4], x3p26);
2984        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[2], x4p25);
2985        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[10], x5p24);
2986        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[9], x6p23);
2987        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[1], x7p22);
2988        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[5], x8p21);
2989        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[13], x9p20);
2990        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[6], x10p19);
2991        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[0], x11p18);
2992        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[8], x12p17);
2993        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[11], x13p16);
2994        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[3], x14p15);
2995        let m0821b = SseVector::mul(self.twiddles_im[7], x1m28);
2996        let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[12], x2m27);
2997        let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[4], x3m26);
2998        let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[2], x4m25);
2999        let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[10], x5m24);
3000        let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[9], x6m23);
3001        let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[1], x7m22);
3002        let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[5], x8m21);
3003        let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[13], x9m20);
3004        let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[6], x10m19);
3005        let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[0], x11m18);
3006        let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[8], x12m17);
3007        let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[11], x13m16);
3008        let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[3], x14m15);
3009        let [y08, y21] = SseVector::column_butterfly2([m0821a, m0821b]);
3010
3011        let m0920a = SseVector::fmadd(values[0], self.twiddles_re[8], x1p28);
3012        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[10], x2p27);
3013        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[1], x3p26);
3014        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[6], x4p25);
3015        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[12], x5p24);
3016        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[3], x6p23);
3017        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[4], x7p22);
3018        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[13], x8p21);
3019        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[5], x9p20);
3020        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[2], x10p19);
3021        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[11], x11p18);
3022        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[7], x12p17);
3023        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[0], x13p16);
3024        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[9], x14p15);
3025        let m0920b = SseVector::mul(self.twiddles_im[8], x1m28);
3026        let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[10], x2m27);
3027        let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[1], x3m26);
3028        let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[6], x4m25);
3029        let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[12], x5m24);
3030        let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[3], x6m23);
3031        let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[4], x7m22);
3032        let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[13], x8m21);
3033        let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[5], x9m20);
3034        let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[2], x10m19);
3035        let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[11], x11m18);
3036        let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[7], x12m17);
3037        let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[0], x13m16);
3038        let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[9], x14m15);
3039        let [y09, y20] = SseVector::column_butterfly2([m0920a, m0920b]);
3040
3041        let m1019a = SseVector::fmadd(values[0], self.twiddles_re[9], x1p28);
3042        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[8], x2p27);
3043        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[0], x3p26);
3044        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[10], x4p25);
3045        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[7], x5p24);
3046        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[1], x6p23);
3047        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[11], x7p22);
3048        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[6], x8p21);
3049        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[2], x9p20);
3050        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[12], x10p19);
3051        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[5], x11p18);
3052        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[3], x12p17);
3053        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[13], x13p16);
3054        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[4], x14p15);
3055        let m1019b = SseVector::mul(self.twiddles_im[9], x1m28);
3056        let m1019b = SseVector::nmadd(m1019b, self.twiddles_im[8], x2m27);
3057        let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[0], x3m26);
3058        let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[10], x4m25);
3059        let m1019b = SseVector::nmadd(m1019b, self.twiddles_im[7], x5m24);
3060        let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[1], x6m23);
3061        let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[11], x7m22);
3062        let m1019b = SseVector::nmadd(m1019b, self.twiddles_im[6], x8m21);
3063        let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[2], x9m20);
3064        let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[12], x10m19);
3065        let m1019b = SseVector::nmadd(m1019b, self.twiddles_im[5], x11m18);
3066        let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[3], x12m17);
3067        let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[13], x13m16);
3068        let m1019b = SseVector::nmadd(m1019b, self.twiddles_im[4], x14m15);
3069        let [y10, y19] = SseVector::column_butterfly2([m1019a, m1019b]);
3070
3071        let m1118a = SseVector::fmadd(values[0], self.twiddles_re[10], x1p28);
3072        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[6], x2p27);
3073        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[3], x3p26);
3074        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[13], x4p25);
3075        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[2], x5p24);
3076        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[7], x6p23);
3077        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[9], x7p22);
3078        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[0], x8p21);
3079        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[11], x9p20);
3080        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[5], x10p19);
3081        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[4], x11p18);
3082        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[12], x12p17);
3083        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[1], x13p16);
3084        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[8], x14p15);
3085        let m1118b = SseVector::mul(self.twiddles_im[10], x1m28);
3086        let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[6], x2m27);
3087        let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[3], x3m26);
3088        let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[13], x4m25);
3089        let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[2], x5m24);
3090        let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[7], x6m23);
3091        let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[9], x7m22);
3092        let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[0], x8m21);
3093        let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[11], x9m20);
3094        let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[5], x10m19);
3095        let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[4], x11m18);
3096        let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[12], x12m17);
3097        let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[1], x13m16);
3098        let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[8], x14m15);
3099        let [y11, y18] = SseVector::column_butterfly2([m1118a, m1118b]);
3100
3101        let m1217a = SseVector::fmadd(values[0], self.twiddles_re[11], x1p28);
3102        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[4], x2p27);
3103        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[6], x3p26);
3104        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[9], x4p25);
3105        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[1], x5p24);
3106        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[13], x6p23);
3107        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[2], x7p22);
3108        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[8], x8p21);
3109        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[7], x9p20);
3110        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[3], x10p19);
3111        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[12], x11p18);
3112        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[0], x12p17);
3113        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[10], x13p16);
3114        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[5], x14p15);
3115        let m1217b = SseVector::mul(self.twiddles_im[11], x1m28);
3116        let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[4], x2m27);
3117        let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[6], x3m26);
3118        let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[9], x4m25);
3119        let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[1], x5m24);
3120        let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[13], x6m23);
3121        let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[2], x7m22);
3122        let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[8], x8m21);
3123        let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[7], x9m20);
3124        let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[3], x10m19);
3125        let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[12], x11m18);
3126        let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[0], x12m17);
3127        let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[10], x13m16);
3128        let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[5], x14m15);
3129        let [y12, y17] = SseVector::column_butterfly2([m1217a, m1217b]);
3130
3131        let m1316a = SseVector::fmadd(values[0], self.twiddles_re[12], x1p28);
3132        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[2], x2p27);
3133        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[9], x3p26);
3134        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[5], x4p25);
3135        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[6], x5p24);
3136        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[8], x6p23);
3137        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[3], x7p22);
3138        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[11], x8p21);
3139        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[0], x9p20);
3140        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[13], x10p19);
3141        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[1], x11p18);
3142        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[10], x12p17);
3143        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[4], x13p16);
3144        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[7], x14p15);
3145        let m1316b = SseVector::mul(self.twiddles_im[12], x1m28);
3146        let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[2], x2m27);
3147        let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[9], x3m26);
3148        let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[5], x4m25);
3149        let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[6], x5m24);
3150        let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[8], x6m23);
3151        let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[3], x7m22);
3152        let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[11], x8m21);
3153        let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[0], x9m20);
3154        let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[13], x10m19);
3155        let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[1], x11m18);
3156        let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[10], x12m17);
3157        let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[4], x13m16);
3158        let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[7], x14m15);
3159        let [y13, y16] = SseVector::column_butterfly2([m1316a, m1316b]);
3160
3161        let m1415a = SseVector::fmadd(values[0], self.twiddles_re[13], x1p28);
3162        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[0], x2p27);
3163        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[12], x3p26);
3164        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[1], x4p25);
3165        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[11], x5p24);
3166        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[2], x6p23);
3167        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[10], x7p22);
3168        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[3], x8p21);
3169        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[9], x9p20);
3170        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[4], x10p19);
3171        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[8], x11p18);
3172        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[5], x12p17);
3173        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[7], x13p16);
3174        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[6], x14p15);
3175        let m1415b = SseVector::mul(self.twiddles_im[13], x1m28);
3176        let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[0], x2m27);
3177        let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[12], x3m26);
3178        let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[1], x4m25);
3179        let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[11], x5m24);
3180        let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[2], x6m23);
3181        let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[10], x7m22);
3182        let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[3], x8m21);
3183        let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[9], x9m20);
3184        let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[4], x10m19);
3185        let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[8], x11m18);
3186        let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[5], x12m17);
3187        let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[7], x13m16);
3188        let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[6], x14m15);
3189        let [y14, y15] = SseVector::column_butterfly2([m1415a, m1415b]);
3190
3191
3192        [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12, y13, y14, y15, y16, y17, y18, y19, y20, y21, y22, y23, y24, y25, y26, y27, y28]
3193    }
3194}
3195
3196struct SseF64Butterfly29<T> {
3197    direction: FftDirection,
3198    twiddles_re: [__m128d; 14],
3199    twiddles_im: [__m128d; 14],
3200    _phantom: std::marker::PhantomData<T>,
3201}
3202
3203boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly29, 29, |this: &SseF64Butterfly29<_>| this.direction);
3204impl<T: FftNum> SseF64Butterfly29<T> {
3205    /// Safety: The current machine must support the sse4.1 instruction set
3206    #[target_feature(enable = "sse4.1")]
3207    unsafe fn new(direction: FftDirection) -> Self {
3208        assert_f64::<T>();
3209        let twiddles = make_twiddles(29, direction);
3210        unsafe {Self {
3211            direction,
3212            twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
3213            twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
3214            _phantom: std::marker::PhantomData,
3215        }}
3216    }
3217
3218    #[inline(always)]
3219    pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
3220        let values = read_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28 });
3221
3222        let out = self.perform_fft_direct(values);
3223
3224        write_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28 });   
3225    }
3226
3227    #[inline(always)]
3228    pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 29]) -> [__m128d; 29] {
3229        let rotate = SseVector::make_rotate90(FftDirection::Inverse);
3230
3231        let y00 = values[0];
3232        let [x1p28, x1m28] =  SseVector::column_butterfly2([values[1], values[28]]);
3233        let x1m28 = SseVector::apply_rotate90(rotate, x1m28);
3234        let y00 = SseVector::add(y00, x1p28);
3235        let [x2p27, x2m27] =  SseVector::column_butterfly2([values[2], values[27]]);
3236        let x2m27 = SseVector::apply_rotate90(rotate, x2m27);
3237        let y00 = SseVector::add(y00, x2p27);
3238        let [x3p26, x3m26] =  SseVector::column_butterfly2([values[3], values[26]]);
3239        let x3m26 = SseVector::apply_rotate90(rotate, x3m26);
3240        let y00 = SseVector::add(y00, x3p26);
3241        let [x4p25, x4m25] =  SseVector::column_butterfly2([values[4], values[25]]);
3242        let x4m25 = SseVector::apply_rotate90(rotate, x4m25);
3243        let y00 = SseVector::add(y00, x4p25);
3244        let [x5p24, x5m24] =  SseVector::column_butterfly2([values[5], values[24]]);
3245        let x5m24 = SseVector::apply_rotate90(rotate, x5m24);
3246        let y00 = SseVector::add(y00, x5p24);
3247        let [x6p23, x6m23] =  SseVector::column_butterfly2([values[6], values[23]]);
3248        let x6m23 = SseVector::apply_rotate90(rotate, x6m23);
3249        let y00 = SseVector::add(y00, x6p23);
3250        let [x7p22, x7m22] =  SseVector::column_butterfly2([values[7], values[22]]);
3251        let x7m22 = SseVector::apply_rotate90(rotate, x7m22);
3252        let y00 = SseVector::add(y00, x7p22);
3253        let [x8p21, x8m21] =  SseVector::column_butterfly2([values[8], values[21]]);
3254        let x8m21 = SseVector::apply_rotate90(rotate, x8m21);
3255        let y00 = SseVector::add(y00, x8p21);
3256        let [x9p20, x9m20] =  SseVector::column_butterfly2([values[9], values[20]]);
3257        let x9m20 = SseVector::apply_rotate90(rotate, x9m20);
3258        let y00 = SseVector::add(y00, x9p20);
3259        let [x10p19, x10m19] =  SseVector::column_butterfly2([values[10], values[19]]);
3260        let x10m19 = SseVector::apply_rotate90(rotate, x10m19);
3261        let y00 = SseVector::add(y00, x10p19);
3262        let [x11p18, x11m18] =  SseVector::column_butterfly2([values[11], values[18]]);
3263        let x11m18 = SseVector::apply_rotate90(rotate, x11m18);
3264        let y00 = SseVector::add(y00, x11p18);
3265        let [x12p17, x12m17] =  SseVector::column_butterfly2([values[12], values[17]]);
3266        let x12m17 = SseVector::apply_rotate90(rotate, x12m17);
3267        let y00 = SseVector::add(y00, x12p17);
3268        let [x13p16, x13m16] =  SseVector::column_butterfly2([values[13], values[16]]);
3269        let x13m16 = SseVector::apply_rotate90(rotate, x13m16);
3270        let y00 = SseVector::add(y00, x13p16);
3271        let [x14p15, x14m15] =  SseVector::column_butterfly2([values[14], values[15]]);
3272        let x14m15 = SseVector::apply_rotate90(rotate, x14m15);
3273        let y00 = SseVector::add(y00, x14p15);
3274
3275        let m0128a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p28);
3276        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[1], x2p27);
3277        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[2], x3p26);
3278        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[3], x4p25);
3279        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[4], x5p24);
3280        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[5], x6p23);
3281        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[6], x7p22);
3282        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[7], x8p21);
3283        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[8], x9p20);
3284        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[9], x10p19);
3285        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[10], x11p18);
3286        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[11], x12p17);
3287        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[12], x13p16);
3288        let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[13], x14p15);
3289        let m0128b = SseVector::mul(self.twiddles_im[0], x1m28);
3290        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[1], x2m27);
3291        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[2], x3m26);
3292        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[3], x4m25);
3293        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[4], x5m24);
3294        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[5], x6m23);
3295        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[6], x7m22);
3296        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[7], x8m21);
3297        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[8], x9m20);
3298        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[9], x10m19);
3299        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[10], x11m18);
3300        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[11], x12m17);
3301        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[12], x13m16);
3302        let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[13], x14m15);
3303        let [y01, y28] = SseVector::column_butterfly2([m0128a, m0128b]);
3304
3305        let m0227a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p28);
3306        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[3], x2p27);
3307        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[5], x3p26);
3308        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[7], x4p25);
3309        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[9], x5p24);
3310        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[11], x6p23);
3311        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[13], x7p22);
3312        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[12], x8p21);
3313        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[10], x9p20);
3314        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[8], x10p19);
3315        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[6], x11p18);
3316        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[4], x12p17);
3317        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[2], x13p16);
3318        let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[0], x14p15);
3319        let m0227b = SseVector::mul(self.twiddles_im[1], x1m28);
3320        let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[3], x2m27);
3321        let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[5], x3m26);
3322        let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[7], x4m25);
3323        let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[9], x5m24);
3324        let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[11], x6m23);
3325        let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[13], x7m22);
3326        let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[12], x8m21);
3327        let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[10], x9m20);
3328        let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[8], x10m19);
3329        let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[6], x11m18);
3330        let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[4], x12m17);
3331        let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[2], x13m16);
3332        let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[0], x14m15);
3333        let [y02, y27] = SseVector::column_butterfly2([m0227a, m0227b]);
3334
3335        let m0326a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p28);
3336        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[5], x2p27);
3337        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[8], x3p26);
3338        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[11], x4p25);
3339        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[13], x5p24);
3340        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[10], x6p23);
3341        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[7], x7p22);
3342        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[4], x8p21);
3343        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[1], x9p20);
3344        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[0], x10p19);
3345        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[3], x11p18);
3346        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[6], x12p17);
3347        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[9], x13p16);
3348        let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[12], x14p15);
3349        let m0326b = SseVector::mul(self.twiddles_im[2], x1m28);
3350        let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[5], x2m27);
3351        let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[8], x3m26);
3352        let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[11], x4m25);
3353        let m0326b = SseVector::nmadd(m0326b, self.twiddles_im[13], x5m24);
3354        let m0326b = SseVector::nmadd(m0326b, self.twiddles_im[10], x6m23);
3355        let m0326b = SseVector::nmadd(m0326b, self.twiddles_im[7], x7m22);
3356        let m0326b = SseVector::nmadd(m0326b, self.twiddles_im[4], x8m21);
3357        let m0326b = SseVector::nmadd(m0326b, self.twiddles_im[1], x9m20);
3358        let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[0], x10m19);
3359        let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[3], x11m18);
3360        let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[6], x12m17);
3361        let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[9], x13m16);
3362        let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[12], x14m15);
3363        let [y03, y26] = SseVector::column_butterfly2([m0326a, m0326b]);
3364
3365        let m0425a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p28);
3366        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[7], x2p27);
3367        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[11], x3p26);
3368        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[12], x4p25);
3369        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[8], x5p24);
3370        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[4], x6p23);
3371        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[0], x7p22);
3372        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[2], x8p21);
3373        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[6], x9p20);
3374        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[10], x10p19);
3375        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[13], x11p18);
3376        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[9], x12p17);
3377        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[5], x13p16);
3378        let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[1], x14p15);
3379        let m0425b = SseVector::mul(self.twiddles_im[3], x1m28);
3380        let m0425b = SseVector::fmadd(m0425b, self.twiddles_im[7], x2m27);
3381        let m0425b = SseVector::fmadd(m0425b, self.twiddles_im[11], x3m26);
3382        let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[12], x4m25);
3383        let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[8], x5m24);
3384        let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[4], x6m23);
3385        let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[0], x7m22);
3386        let m0425b = SseVector::fmadd(m0425b, self.twiddles_im[2], x8m21);
3387        let m0425b = SseVector::fmadd(m0425b, self.twiddles_im[6], x9m20);
3388        let m0425b = SseVector::fmadd(m0425b, self.twiddles_im[10], x10m19);
3389        let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[13], x11m18);
3390        let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[9], x12m17);
3391        let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[5], x13m16);
3392        let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[1], x14m15);
3393        let [y04, y25] = SseVector::column_butterfly2([m0425a, m0425b]);
3394
3395        let m0524a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p28);
3396        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[9], x2p27);
3397        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[13], x3p26);
3398        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[8], x4p25);
3399        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[3], x5p24);
3400        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[0], x6p23);
3401        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[5], x7p22);
3402        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[10], x8p21);
3403        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[12], x9p20);
3404        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[7], x10p19);
3405        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[2], x11p18);
3406        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[1], x12p17);
3407        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[6], x13p16);
3408        let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[11], x14p15);
3409        let m0524b = SseVector::mul(self.twiddles_im[4], x1m28);
3410        let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[9], x2m27);
3411        let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[13], x3m26);
3412        let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[8], x4m25);
3413        let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[3], x5m24);
3414        let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[0], x6m23);
3415        let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[5], x7m22);
3416        let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[10], x8m21);
3417        let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[12], x9m20);
3418        let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[7], x10m19);
3419        let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[2], x11m18);
3420        let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[1], x12m17);
3421        let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[6], x13m16);
3422        let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[11], x14m15);
3423        let [y05, y24] = SseVector::column_butterfly2([m0524a, m0524b]);
3424
3425        let m0623a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p28);
3426        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[11], x2p27);
3427        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[10], x3p26);
3428        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[4], x4p25);
3429        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[0], x5p24);
3430        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[6], x6p23);
3431        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[12], x7p22);
3432        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[9], x8p21);
3433        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[3], x9p20);
3434        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[1], x10p19);
3435        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[7], x11p18);
3436        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[13], x12p17);
3437        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[8], x13p16);
3438        let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[2], x14p15);
3439        let m0623b = SseVector::mul(self.twiddles_im[5], x1m28);
3440        let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[11], x2m27);
3441        let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[10], x3m26);
3442        let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[4], x4m25);
3443        let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[0], x5m24);
3444        let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[6], x6m23);
3445        let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[12], x7m22);
3446        let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[9], x8m21);
3447        let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[3], x9m20);
3448        let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[1], x10m19);
3449        let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[7], x11m18);
3450        let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[13], x12m17);
3451        let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[8], x13m16);
3452        let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[2], x14m15);
3453        let [y06, y23] = SseVector::column_butterfly2([m0623a, m0623b]);
3454
3455        let m0722a = SseVector::fmadd(values[0], self.twiddles_re[6], x1p28);
3456        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[13], x2p27);
3457        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[7], x3p26);
3458        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[0], x4p25);
3459        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[5], x5p24);
3460        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[12], x6p23);
3461        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[8], x7p22);
3462        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[1], x8p21);
3463        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[4], x9p20);
3464        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[11], x10p19);
3465        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[9], x11p18);
3466        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[2], x12p17);
3467        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[3], x13p16);
3468        let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[10], x14p15);
3469        let m0722b = SseVector::mul(self.twiddles_im[6], x1m28);
3470        let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[13], x2m27);
3471        let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[7], x3m26);
3472        let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[0], x4m25);
3473        let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[5], x5m24);
3474        let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[12], x6m23);
3475        let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[8], x7m22);
3476        let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[1], x8m21);
3477        let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[4], x9m20);
3478        let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[11], x10m19);
3479        let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[9], x11m18);
3480        let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[2], x12m17);
3481        let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[3], x13m16);
3482        let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[10], x14m15);
3483        let [y07, y22] = SseVector::column_butterfly2([m0722a, m0722b]);
3484
3485        let m0821a = SseVector::fmadd(values[0], self.twiddles_re[7], x1p28);
3486        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[12], x2p27);
3487        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[4], x3p26);
3488        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[2], x4p25);
3489        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[10], x5p24);
3490        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[9], x6p23);
3491        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[1], x7p22);
3492        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[5], x8p21);
3493        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[13], x9p20);
3494        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[6], x10p19);
3495        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[0], x11p18);
3496        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[8], x12p17);
3497        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[11], x13p16);
3498        let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[3], x14p15);
3499        let m0821b = SseVector::mul(self.twiddles_im[7], x1m28);
3500        let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[12], x2m27);
3501        let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[4], x3m26);
3502        let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[2], x4m25);
3503        let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[10], x5m24);
3504        let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[9], x6m23);
3505        let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[1], x7m22);
3506        let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[5], x8m21);
3507        let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[13], x9m20);
3508        let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[6], x10m19);
3509        let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[0], x11m18);
3510        let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[8], x12m17);
3511        let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[11], x13m16);
3512        let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[3], x14m15);
3513        let [y08, y21] = SseVector::column_butterfly2([m0821a, m0821b]);
3514
3515        let m0920a = SseVector::fmadd(values[0], self.twiddles_re[8], x1p28);
3516        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[10], x2p27);
3517        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[1], x3p26);
3518        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[6], x4p25);
3519        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[12], x5p24);
3520        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[3], x6p23);
3521        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[4], x7p22);
3522        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[13], x8p21);
3523        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[5], x9p20);
3524        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[2], x10p19);
3525        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[11], x11p18);
3526        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[7], x12p17);
3527        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[0], x13p16);
3528        let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[9], x14p15);
3529        let m0920b = SseVector::mul(self.twiddles_im[8], x1m28);
3530        let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[10], x2m27);
3531        let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[1], x3m26);
3532        let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[6], x4m25);
3533        let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[12], x5m24);
3534        let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[3], x6m23);
3535        let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[4], x7m22);
3536        let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[13], x8m21);
3537        let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[5], x9m20);
3538        let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[2], x10m19);
3539        let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[11], x11m18);
3540        let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[7], x12m17);
3541        let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[0], x13m16);
3542        let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[9], x14m15);
3543        let [y09, y20] = SseVector::column_butterfly2([m0920a, m0920b]);
3544
3545        let m1019a = SseVector::fmadd(values[0], self.twiddles_re[9], x1p28);
3546        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[8], x2p27);
3547        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[0], x3p26);
3548        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[10], x4p25);
3549        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[7], x5p24);
3550        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[1], x6p23);
3551        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[11], x7p22);
3552        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[6], x8p21);
3553        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[2], x9p20);
3554        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[12], x10p19);
3555        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[5], x11p18);
3556        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[3], x12p17);
3557        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[13], x13p16);
3558        let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[4], x14p15);
3559        let m1019b = SseVector::mul(self.twiddles_im[9], x1m28);
3560        let m1019b = SseVector::nmadd(m1019b, self.twiddles_im[8], x2m27);
3561        let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[0], x3m26);
3562        let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[10], x4m25);
3563        let m1019b = SseVector::nmadd(m1019b, self.twiddles_im[7], x5m24);
3564        let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[1], x6m23);
3565        let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[11], x7m22);
3566        let m1019b = SseVector::nmadd(m1019b, self.twiddles_im[6], x8m21);
3567        let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[2], x9m20);
3568        let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[12], x10m19);
3569        let m1019b = SseVector::nmadd(m1019b, self.twiddles_im[5], x11m18);
3570        let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[3], x12m17);
3571        let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[13], x13m16);
3572        let m1019b = SseVector::nmadd(m1019b, self.twiddles_im[4], x14m15);
3573        let [y10, y19] = SseVector::column_butterfly2([m1019a, m1019b]);
3574
3575        let m1118a = SseVector::fmadd(values[0], self.twiddles_re[10], x1p28);
3576        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[6], x2p27);
3577        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[3], x3p26);
3578        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[13], x4p25);
3579        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[2], x5p24);
3580        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[7], x6p23);
3581        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[9], x7p22);
3582        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[0], x8p21);
3583        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[11], x9p20);
3584        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[5], x10p19);
3585        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[4], x11p18);
3586        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[12], x12p17);
3587        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[1], x13p16);
3588        let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[8], x14p15);
3589        let m1118b = SseVector::mul(self.twiddles_im[10], x1m28);
3590        let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[6], x2m27);
3591        let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[3], x3m26);
3592        let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[13], x4m25);
3593        let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[2], x5m24);
3594        let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[7], x6m23);
3595        let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[9], x7m22);
3596        let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[0], x8m21);
3597        let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[11], x9m20);
3598        let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[5], x10m19);
3599        let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[4], x11m18);
3600        let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[12], x12m17);
3601        let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[1], x13m16);
3602        let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[8], x14m15);
3603        let [y11, y18] = SseVector::column_butterfly2([m1118a, m1118b]);
3604
3605        let m1217a = SseVector::fmadd(values[0], self.twiddles_re[11], x1p28);
3606        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[4], x2p27);
3607        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[6], x3p26);
3608        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[9], x4p25);
3609        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[1], x5p24);
3610        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[13], x6p23);
3611        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[2], x7p22);
3612        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[8], x8p21);
3613        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[7], x9p20);
3614        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[3], x10p19);
3615        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[12], x11p18);
3616        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[0], x12p17);
3617        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[10], x13p16);
3618        let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[5], x14p15);
3619        let m1217b = SseVector::mul(self.twiddles_im[11], x1m28);
3620        let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[4], x2m27);
3621        let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[6], x3m26);
3622        let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[9], x4m25);
3623        let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[1], x5m24);
3624        let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[13], x6m23);
3625        let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[2], x7m22);
3626        let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[8], x8m21);
3627        let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[7], x9m20);
3628        let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[3], x10m19);
3629        let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[12], x11m18);
3630        let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[0], x12m17);
3631        let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[10], x13m16);
3632        let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[5], x14m15);
3633        let [y12, y17] = SseVector::column_butterfly2([m1217a, m1217b]);
3634
3635        let m1316a = SseVector::fmadd(values[0], self.twiddles_re[12], x1p28);
3636        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[2], x2p27);
3637        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[9], x3p26);
3638        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[5], x4p25);
3639        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[6], x5p24);
3640        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[8], x6p23);
3641        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[3], x7p22);
3642        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[11], x8p21);
3643        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[0], x9p20);
3644        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[13], x10p19);
3645        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[1], x11p18);
3646        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[10], x12p17);
3647        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[4], x13p16);
3648        let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[7], x14p15);
3649        let m1316b = SseVector::mul(self.twiddles_im[12], x1m28);
3650        let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[2], x2m27);
3651        let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[9], x3m26);
3652        let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[5], x4m25);
3653        let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[6], x5m24);
3654        let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[8], x6m23);
3655        let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[3], x7m22);
3656        let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[11], x8m21);
3657        let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[0], x9m20);
3658        let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[13], x10m19);
3659        let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[1], x11m18);
3660        let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[10], x12m17);
3661        let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[4], x13m16);
3662        let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[7], x14m15);
3663        let [y13, y16] = SseVector::column_butterfly2([m1316a, m1316b]);
3664
3665        let m1415a = SseVector::fmadd(values[0], self.twiddles_re[13], x1p28);
3666        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[0], x2p27);
3667        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[12], x3p26);
3668        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[1], x4p25);
3669        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[11], x5p24);
3670        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[2], x6p23);
3671        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[10], x7p22);
3672        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[3], x8p21);
3673        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[9], x9p20);
3674        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[4], x10p19);
3675        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[8], x11p18);
3676        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[5], x12p17);
3677        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[7], x13p16);
3678        let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[6], x14p15);
3679        let m1415b = SseVector::mul(self.twiddles_im[13], x1m28);
3680        let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[0], x2m27);
3681        let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[12], x3m26);
3682        let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[1], x4m25);
3683        let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[11], x5m24);
3684        let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[2], x6m23);
3685        let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[10], x7m22);
3686        let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[3], x8m21);
3687        let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[9], x9m20);
3688        let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[4], x10m19);
3689        let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[8], x11m18);
3690        let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[5], x12m17);
3691        let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[7], x13m16);
3692        let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[6], x14m15);
3693        let [y14, y15] = SseVector::column_butterfly2([m1415a, m1415b]);
3694
3695
3696        [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12, y13, y14, y15, y16, y17, y18, y19, y20, y21, y22, y23, y24, y25, y26, y27, y28]
3697    }
3698}
3699
3700struct SseF32Butterfly31<T> {
3701    direction: FftDirection,
3702    twiddles_re: [__m128; 15],
3703    twiddles_im: [__m128; 15],
3704    _phantom: std::marker::PhantomData<T>,
3705}
3706
3707boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly31, 31, |this: &SseF32Butterfly31<_>| this.direction);
3708impl<T: FftNum> SseF32Butterfly31<T> {
3709    /// Safety: The current machine must support the sse4.1 instruction set
3710    #[target_feature(enable = "sse4.1")]
3711    unsafe fn new(direction: FftDirection) -> Self {
3712        assert_f32::<T>();
3713        let twiddles = make_twiddles(31, direction);
3714        Self {
3715            direction,
3716            twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
3717            twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
3718            _phantom: std::marker::PhantomData,
3719        }
3720    }
3721
3722    #[inline(always)]
3723    pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
3724        let values = read_partial1_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 });
3725
3726        let out = self.perform_parallel_fft_direct(values);
3727        
3728        write_partial_lo_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 } ); 
3729    }
3730
3731    #[inline(always)]
3732    pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
3733        let input_packed = read_complex_to_array!(buffer, { 0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60 });
3734
3735        let values = [
3736            extract_lo_hi_f32(input_packed[0], input_packed[15]),
3737            extract_hi_lo_f32(input_packed[0], input_packed[16]),
3738            extract_lo_hi_f32(input_packed[1], input_packed[16]),
3739            extract_hi_lo_f32(input_packed[1], input_packed[17]),
3740            extract_lo_hi_f32(input_packed[2], input_packed[17]),
3741            extract_hi_lo_f32(input_packed[2], input_packed[18]),
3742            extract_lo_hi_f32(input_packed[3], input_packed[18]),
3743            extract_hi_lo_f32(input_packed[3], input_packed[19]),
3744            extract_lo_hi_f32(input_packed[4], input_packed[19]),
3745            extract_hi_lo_f32(input_packed[4], input_packed[20]),
3746            extract_lo_hi_f32(input_packed[5], input_packed[20]),
3747            extract_hi_lo_f32(input_packed[5], input_packed[21]),
3748            extract_lo_hi_f32(input_packed[6], input_packed[21]),
3749            extract_hi_lo_f32(input_packed[6], input_packed[22]),
3750            extract_lo_hi_f32(input_packed[7], input_packed[22]),
3751            extract_hi_lo_f32(input_packed[7], input_packed[23]),
3752            extract_lo_hi_f32(input_packed[8], input_packed[23]),
3753            extract_hi_lo_f32(input_packed[8], input_packed[24]),
3754            extract_lo_hi_f32(input_packed[9], input_packed[24]),
3755            extract_hi_lo_f32(input_packed[9], input_packed[25]),
3756            extract_lo_hi_f32(input_packed[10], input_packed[25]),
3757            extract_hi_lo_f32(input_packed[10], input_packed[26]),
3758            extract_lo_hi_f32(input_packed[11], input_packed[26]),
3759            extract_hi_lo_f32(input_packed[11], input_packed[27]),
3760            extract_lo_hi_f32(input_packed[12], input_packed[27]),
3761            extract_hi_lo_f32(input_packed[12], input_packed[28]),
3762            extract_lo_hi_f32(input_packed[13], input_packed[28]),
3763            extract_hi_lo_f32(input_packed[13], input_packed[29]),
3764            extract_lo_hi_f32(input_packed[14], input_packed[29]),
3765            extract_hi_lo_f32(input_packed[14], input_packed[30]),
3766            extract_lo_hi_f32(input_packed[15], input_packed[30]),
3767        ];
3768
3769        let out = self.perform_parallel_fft_direct(values);
3770
3771        let out_packed = [
3772            extract_lo_lo_f32(out[0], out[1]),
3773            extract_lo_lo_f32(out[2], out[3]),
3774            extract_lo_lo_f32(out[4], out[5]),
3775            extract_lo_lo_f32(out[6], out[7]),
3776            extract_lo_lo_f32(out[8], out[9]),
3777            extract_lo_lo_f32(out[10], out[11]),
3778            extract_lo_lo_f32(out[12], out[13]),
3779            extract_lo_lo_f32(out[14], out[15]),
3780            extract_lo_lo_f32(out[16], out[17]),
3781            extract_lo_lo_f32(out[18], out[19]),
3782            extract_lo_lo_f32(out[20], out[21]),
3783            extract_lo_lo_f32(out[22], out[23]),
3784            extract_lo_lo_f32(out[24], out[25]),
3785            extract_lo_lo_f32(out[26], out[27]),
3786            extract_lo_lo_f32(out[28], out[29]),
3787            extract_lo_hi_f32(out[30], out[0]),
3788            extract_hi_hi_f32(out[1], out[2]),
3789            extract_hi_hi_f32(out[3], out[4]),
3790            extract_hi_hi_f32(out[5], out[6]),
3791            extract_hi_hi_f32(out[7], out[8]),
3792            extract_hi_hi_f32(out[9], out[10]),
3793            extract_hi_hi_f32(out[11], out[12]),
3794            extract_hi_hi_f32(out[13], out[14]),
3795            extract_hi_hi_f32(out[15], out[16]),
3796            extract_hi_hi_f32(out[17], out[18]),
3797            extract_hi_hi_f32(out[19], out[20]),
3798            extract_hi_hi_f32(out[21], out[22]),
3799            extract_hi_hi_f32(out[23], out[24]),
3800            extract_hi_hi_f32(out[25], out[26]),
3801            extract_hi_hi_f32(out[27], out[28]),
3802            extract_hi_hi_f32(out[29], out[30]),
3803        ];
3804
3805        write_complex_to_array_strided!(out_packed, buffer, 2, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 });
3806    }
3807
3808    #[inline(always)]
3809    pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 31]) -> [__m128; 31] {
3810        let rotate = SseVector::make_rotate90(FftDirection::Inverse);
3811
3812        let y00 = values[0];
3813        let [x1p30, x1m30] =  SseVector::column_butterfly2([values[1], values[30]]);
3814        let x1m30 = SseVector::apply_rotate90(rotate, x1m30);
3815        let y00 = SseVector::add(y00, x1p30);
3816        let [x2p29, x2m29] =  SseVector::column_butterfly2([values[2], values[29]]);
3817        let x2m29 = SseVector::apply_rotate90(rotate, x2m29);
3818        let y00 = SseVector::add(y00, x2p29);
3819        let [x3p28, x3m28] =  SseVector::column_butterfly2([values[3], values[28]]);
3820        let x3m28 = SseVector::apply_rotate90(rotate, x3m28);
3821        let y00 = SseVector::add(y00, x3p28);
3822        let [x4p27, x4m27] =  SseVector::column_butterfly2([values[4], values[27]]);
3823        let x4m27 = SseVector::apply_rotate90(rotate, x4m27);
3824        let y00 = SseVector::add(y00, x4p27);
3825        let [x5p26, x5m26] =  SseVector::column_butterfly2([values[5], values[26]]);
3826        let x5m26 = SseVector::apply_rotate90(rotate, x5m26);
3827        let y00 = SseVector::add(y00, x5p26);
3828        let [x6p25, x6m25] =  SseVector::column_butterfly2([values[6], values[25]]);
3829        let x6m25 = SseVector::apply_rotate90(rotate, x6m25);
3830        let y00 = SseVector::add(y00, x6p25);
3831        let [x7p24, x7m24] =  SseVector::column_butterfly2([values[7], values[24]]);
3832        let x7m24 = SseVector::apply_rotate90(rotate, x7m24);
3833        let y00 = SseVector::add(y00, x7p24);
3834        let [x8p23, x8m23] =  SseVector::column_butterfly2([values[8], values[23]]);
3835        let x8m23 = SseVector::apply_rotate90(rotate, x8m23);
3836        let y00 = SseVector::add(y00, x8p23);
3837        let [x9p22, x9m22] =  SseVector::column_butterfly2([values[9], values[22]]);
3838        let x9m22 = SseVector::apply_rotate90(rotate, x9m22);
3839        let y00 = SseVector::add(y00, x9p22);
3840        let [x10p21, x10m21] =  SseVector::column_butterfly2([values[10], values[21]]);
3841        let x10m21 = SseVector::apply_rotate90(rotate, x10m21);
3842        let y00 = SseVector::add(y00, x10p21);
3843        let [x11p20, x11m20] =  SseVector::column_butterfly2([values[11], values[20]]);
3844        let x11m20 = SseVector::apply_rotate90(rotate, x11m20);
3845        let y00 = SseVector::add(y00, x11p20);
3846        let [x12p19, x12m19] =  SseVector::column_butterfly2([values[12], values[19]]);
3847        let x12m19 = SseVector::apply_rotate90(rotate, x12m19);
3848        let y00 = SseVector::add(y00, x12p19);
3849        let [x13p18, x13m18] =  SseVector::column_butterfly2([values[13], values[18]]);
3850        let x13m18 = SseVector::apply_rotate90(rotate, x13m18);
3851        let y00 = SseVector::add(y00, x13p18);
3852        let [x14p17, x14m17] =  SseVector::column_butterfly2([values[14], values[17]]);
3853        let x14m17 = SseVector::apply_rotate90(rotate, x14m17);
3854        let y00 = SseVector::add(y00, x14p17);
3855        let [x15p16, x15m16] =  SseVector::column_butterfly2([values[15], values[16]]);
3856        let x15m16 = SseVector::apply_rotate90(rotate, x15m16);
3857        let y00 = SseVector::add(y00, x15p16);
3858
3859        let m0130a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p30);
3860        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[1], x2p29);
3861        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[2], x3p28);
3862        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[3], x4p27);
3863        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[4], x5p26);
3864        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[5], x6p25);
3865        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[6], x7p24);
3866        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[7], x8p23);
3867        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[8], x9p22);
3868        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[9], x10p21);
3869        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[10], x11p20);
3870        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[11], x12p19);
3871        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[12], x13p18);
3872        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[13], x14p17);
3873        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[14], x15p16);
3874        let m0130b = SseVector::mul(self.twiddles_im[0], x1m30);
3875        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[1], x2m29);
3876        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[2], x3m28);
3877        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[3], x4m27);
3878        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[4], x5m26);
3879        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[5], x6m25);
3880        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[6], x7m24);
3881        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[7], x8m23);
3882        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[8], x9m22);
3883        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[9], x10m21);
3884        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[10], x11m20);
3885        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[11], x12m19);
3886        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[12], x13m18);
3887        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[13], x14m17);
3888        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[14], x15m16);
3889        let [y01, y30] = SseVector::column_butterfly2([m0130a, m0130b]);
3890
3891        let m0229a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p30);
3892        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[3], x2p29);
3893        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[5], x3p28);
3894        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[7], x4p27);
3895        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[9], x5p26);
3896        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[11], x6p25);
3897        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[13], x7p24);
3898        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[14], x8p23);
3899        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[12], x9p22);
3900        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[10], x10p21);
3901        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[8], x11p20);
3902        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[6], x12p19);
3903        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[4], x13p18);
3904        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[2], x14p17);
3905        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[0], x15p16);
3906        let m0229b = SseVector::mul(self.twiddles_im[1], x1m30);
3907        let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[3], x2m29);
3908        let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[5], x3m28);
3909        let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[7], x4m27);
3910        let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[9], x5m26);
3911        let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[11], x6m25);
3912        let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[13], x7m24);
3913        let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[14], x8m23);
3914        let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[12], x9m22);
3915        let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[10], x10m21);
3916        let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[8], x11m20);
3917        let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[6], x12m19);
3918        let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[4], x13m18);
3919        let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[2], x14m17);
3920        let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[0], x15m16);
3921        let [y02, y29] = SseVector::column_butterfly2([m0229a, m0229b]);
3922
3923        let m0328a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p30);
3924        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[5], x2p29);
3925        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[8], x3p28);
3926        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[11], x4p27);
3927        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[14], x5p26);
3928        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[12], x6p25);
3929        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[9], x7p24);
3930        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[6], x8p23);
3931        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[3], x9p22);
3932        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[0], x10p21);
3933        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[1], x11p20);
3934        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[4], x12p19);
3935        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[7], x13p18);
3936        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[10], x14p17);
3937        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[13], x15p16);
3938        let m0328b = SseVector::mul(self.twiddles_im[2], x1m30);
3939        let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[5], x2m29);
3940        let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[8], x3m28);
3941        let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[11], x4m27);
3942        let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[14], x5m26);
3943        let m0328b = SseVector::nmadd(m0328b, self.twiddles_im[12], x6m25);
3944        let m0328b = SseVector::nmadd(m0328b, self.twiddles_im[9], x7m24);
3945        let m0328b = SseVector::nmadd(m0328b, self.twiddles_im[6], x8m23);
3946        let m0328b = SseVector::nmadd(m0328b, self.twiddles_im[3], x9m22);
3947        let m0328b = SseVector::nmadd(m0328b, self.twiddles_im[0], x10m21);
3948        let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[1], x11m20);
3949        let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[4], x12m19);
3950        let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[7], x13m18);
3951        let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[10], x14m17);
3952        let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[13], x15m16);
3953        let [y03, y28] = SseVector::column_butterfly2([m0328a, m0328b]);
3954
3955        let m0427a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p30);
3956        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[7], x2p29);
3957        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[11], x3p28);
3958        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[14], x4p27);
3959        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[10], x5p26);
3960        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[6], x6p25);
3961        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[2], x7p24);
3962        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[0], x8p23);
3963        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[4], x9p22);
3964        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[8], x10p21);
3965        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[12], x11p20);
3966        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[13], x12p19);
3967        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[9], x13p18);
3968        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[5], x14p17);
3969        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[1], x15p16);
3970        let m0427b = SseVector::mul(self.twiddles_im[3], x1m30);
3971        let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[7], x2m29);
3972        let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[11], x3m28);
3973        let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[14], x4m27);
3974        let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[10], x5m26);
3975        let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[6], x6m25);
3976        let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[2], x7m24);
3977        let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[0], x8m23);
3978        let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[4], x9m22);
3979        let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[8], x10m21);
3980        let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[12], x11m20);
3981        let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[13], x12m19);
3982        let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[9], x13m18);
3983        let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[5], x14m17);
3984        let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[1], x15m16);
3985        let [y04, y27] = SseVector::column_butterfly2([m0427a, m0427b]);
3986
3987        let m0526a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p30);
3988        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[9], x2p29);
3989        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[14], x3p28);
3990        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[10], x4p27);
3991        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[5], x5p26);
3992        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[0], x6p25);
3993        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[3], x7p24);
3994        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[8], x8p23);
3995        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[13], x9p22);
3996        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[11], x10p21);
3997        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[6], x11p20);
3998        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[1], x12p19);
3999        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[2], x13p18);
4000        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[7], x14p17);
4001        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[12], x15p16);
4002        let m0526b = SseVector::mul(self.twiddles_im[4], x1m30);
4003        let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[9], x2m29);
4004        let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[14], x3m28);
4005        let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[10], x4m27);
4006        let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[5], x5m26);
4007        let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[0], x6m25);
4008        let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[3], x7m24);
4009        let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[8], x8m23);
4010        let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[13], x9m22);
4011        let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[11], x10m21);
4012        let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[6], x11m20);
4013        let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[1], x12m19);
4014        let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[2], x13m18);
4015        let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[7], x14m17);
4016        let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[12], x15m16);
4017        let [y05, y26] = SseVector::column_butterfly2([m0526a, m0526b]);
4018
4019        let m0625a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p30);
4020        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[11], x2p29);
4021        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[12], x3p28);
4022        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[6], x4p27);
4023        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[0], x5p26);
4024        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[4], x6p25);
4025        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[10], x7p24);
4026        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[13], x8p23);
4027        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[7], x9p22);
4028        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[1], x10p21);
4029        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[3], x11p20);
4030        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[9], x12p19);
4031        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[14], x13p18);
4032        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[8], x14p17);
4033        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[2], x15p16);
4034        let m0625b = SseVector::mul(self.twiddles_im[5], x1m30);
4035        let m0625b = SseVector::fmadd(m0625b, self.twiddles_im[11], x2m29);
4036        let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[12], x3m28);
4037        let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[6], x4m27);
4038        let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[0], x5m26);
4039        let m0625b = SseVector::fmadd(m0625b, self.twiddles_im[4], x6m25);
4040        let m0625b = SseVector::fmadd(m0625b, self.twiddles_im[10], x7m24);
4041        let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[13], x8m23);
4042        let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[7], x9m22);
4043        let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[1], x10m21);
4044        let m0625b = SseVector::fmadd(m0625b, self.twiddles_im[3], x11m20);
4045        let m0625b = SseVector::fmadd(m0625b, self.twiddles_im[9], x12m19);
4046        let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[14], x13m18);
4047        let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[8], x14m17);
4048        let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[2], x15m16);
4049        let [y06, y25] = SseVector::column_butterfly2([m0625a, m0625b]);
4050
4051        let m0724a = SseVector::fmadd(values[0], self.twiddles_re[6], x1p30);
4052        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[13], x2p29);
4053        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[9], x3p28);
4054        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[2], x4p27);
4055        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[3], x5p26);
4056        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[10], x6p25);
4057        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[12], x7p24);
4058        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[5], x8p23);
4059        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[0], x9p22);
4060        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[7], x10p21);
4061        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[14], x11p20);
4062        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[8], x12p19);
4063        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[1], x13p18);
4064        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[4], x14p17);
4065        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[11], x15p16);
4066        let m0724b = SseVector::mul(self.twiddles_im[6], x1m30);
4067        let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[13], x2m29);
4068        let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[9], x3m28);
4069        let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[2], x4m27);
4070        let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[3], x5m26);
4071        let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[10], x6m25);
4072        let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[12], x7m24);
4073        let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[5], x8m23);
4074        let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[0], x9m22);
4075        let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[7], x10m21);
4076        let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[14], x11m20);
4077        let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[8], x12m19);
4078        let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[1], x13m18);
4079        let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[4], x14m17);
4080        let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[11], x15m16);
4081        let [y07, y24] = SseVector::column_butterfly2([m0724a, m0724b]);
4082
4083        let m0823a = SseVector::fmadd(values[0], self.twiddles_re[7], x1p30);
4084        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[14], x2p29);
4085        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[6], x3p28);
4086        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[0], x4p27);
4087        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[8], x5p26);
4088        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[13], x6p25);
4089        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[5], x7p24);
4090        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[1], x8p23);
4091        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[9], x9p22);
4092        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[12], x10p21);
4093        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[4], x11p20);
4094        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[2], x12p19);
4095        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[10], x13p18);
4096        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[11], x14p17);
4097        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[3], x15p16);
4098        let m0823b = SseVector::mul(self.twiddles_im[7], x1m30);
4099        let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[14], x2m29);
4100        let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[6], x3m28);
4101        let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[0], x4m27);
4102        let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[8], x5m26);
4103        let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[13], x6m25);
4104        let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[5], x7m24);
4105        let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[1], x8m23);
4106        let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[9], x9m22);
4107        let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[12], x10m21);
4108        let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[4], x11m20);
4109        let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[2], x12m19);
4110        let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[10], x13m18);
4111        let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[11], x14m17);
4112        let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[3], x15m16);
4113        let [y08, y23] = SseVector::column_butterfly2([m0823a, m0823b]);
4114
4115        let m0922a = SseVector::fmadd(values[0], self.twiddles_re[8], x1p30);
4116        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[12], x2p29);
4117        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[3], x3p28);
4118        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[4], x4p27);
4119        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[13], x5p26);
4120        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[7], x6p25);
4121        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[0], x7p24);
4122        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[9], x8p23);
4123        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[11], x9p22);
4124        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[2], x10p21);
4125        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[5], x11p20);
4126        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[14], x12p19);
4127        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[6], x13p18);
4128        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[1], x14p17);
4129        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[10], x15p16);
4130        let m0922b = SseVector::mul(self.twiddles_im[8], x1m30);
4131        let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[12], x2m29);
4132        let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[3], x3m28);
4133        let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[4], x4m27);
4134        let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[13], x5m26);
4135        let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[7], x6m25);
4136        let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[0], x7m24);
4137        let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[9], x8m23);
4138        let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[11], x9m22);
4139        let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[2], x10m21);
4140        let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[5], x11m20);
4141        let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[14], x12m19);
4142        let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[6], x13m18);
4143        let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[1], x14m17);
4144        let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[10], x15m16);
4145        let [y09, y22] = SseVector::column_butterfly2([m0922a, m0922b]);
4146
4147        let m1021a = SseVector::fmadd(values[0], self.twiddles_re[9], x1p30);
4148        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[10], x2p29);
4149        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[0], x3p28);
4150        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[8], x4p27);
4151        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[11], x5p26);
4152        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[1], x6p25);
4153        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[7], x7p24);
4154        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[12], x8p23);
4155        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[2], x9p22);
4156        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[6], x10p21);
4157        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[13], x11p20);
4158        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[3], x12p19);
4159        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[5], x13p18);
4160        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[14], x14p17);
4161        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[4], x15p16);
4162        let m1021b = SseVector::mul(self.twiddles_im[9], x1m30);
4163        let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[10], x2m29);
4164        let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[0], x3m28);
4165        let m1021b = SseVector::fmadd(m1021b, self.twiddles_im[8], x4m27);
4166        let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[11], x5m26);
4167        let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[1], x6m25);
4168        let m1021b = SseVector::fmadd(m1021b, self.twiddles_im[7], x7m24);
4169        let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[12], x8m23);
4170        let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[2], x9m22);
4171        let m1021b = SseVector::fmadd(m1021b, self.twiddles_im[6], x10m21);
4172        let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[13], x11m20);
4173        let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[3], x12m19);
4174        let m1021b = SseVector::fmadd(m1021b, self.twiddles_im[5], x13m18);
4175        let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[14], x14m17);
4176        let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[4], x15m16);
4177        let [y10, y21] = SseVector::column_butterfly2([m1021a, m1021b]);
4178
4179        let m1120a = SseVector::fmadd(values[0], self.twiddles_re[10], x1p30);
4180        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[8], x2p29);
4181        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[1], x3p28);
4182        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[12], x4p27);
4183        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[6], x5p26);
4184        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[3], x6p25);
4185        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[14], x7p24);
4186        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[4], x8p23);
4187        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[5], x9p22);
4188        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[13], x10p21);
4189        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[2], x11p20);
4190        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[7], x12p19);
4191        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[11], x13p18);
4192        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[0], x14p17);
4193        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[9], x15p16);
4194        let m1120b = SseVector::mul(self.twiddles_im[10], x1m30);
4195        let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[8], x2m29);
4196        let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[1], x3m28);
4197        let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[12], x4m27);
4198        let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[6], x5m26);
4199        let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[3], x6m25);
4200        let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[14], x7m24);
4201        let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[4], x8m23);
4202        let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[5], x9m22);
4203        let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[13], x10m21);
4204        let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[2], x11m20);
4205        let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[7], x12m19);
4206        let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[11], x13m18);
4207        let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[0], x14m17);
4208        let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[9], x15m16);
4209        let [y11, y20] = SseVector::column_butterfly2([m1120a, m1120b]);
4210
4211        let m1219a = SseVector::fmadd(values[0], self.twiddles_re[11], x1p30);
4212        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[6], x2p29);
4213        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[4], x3p28);
4214        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[13], x4p27);
4215        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[1], x5p26);
4216        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[9], x6p25);
4217        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[8], x7p24);
4218        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[2], x8p23);
4219        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[14], x9p22);
4220        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[3], x10p21);
4221        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[7], x11p20);
4222        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[10], x12p19);
4223        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[0], x13p18);
4224        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[12], x14p17);
4225        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[5], x15p16);
4226        let m1219b = SseVector::mul(self.twiddles_im[11], x1m30);
4227        let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[6], x2m29);
4228        let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[4], x3m28);
4229        let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[13], x4m27);
4230        let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[1], x5m26);
4231        let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[9], x6m25);
4232        let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[8], x7m24);
4233        let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[2], x8m23);
4234        let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[14], x9m22);
4235        let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[3], x10m21);
4236        let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[7], x11m20);
4237        let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[10], x12m19);
4238        let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[0], x13m18);
4239        let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[12], x14m17);
4240        let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[5], x15m16);
4241        let [y12, y19] = SseVector::column_butterfly2([m1219a, m1219b]);
4242
4243        let m1318a = SseVector::fmadd(values[0], self.twiddles_re[12], x1p30);
4244        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[4], x2p29);
4245        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[7], x3p28);
4246        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[9], x4p27);
4247        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[2], x5p26);
4248        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[14], x6p25);
4249        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[1], x7p24);
4250        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[10], x8p23);
4251        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[6], x9p22);
4252        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[5], x10p21);
4253        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[11], x11p20);
4254        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[0], x12p19);
4255        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[13], x13p18);
4256        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[3], x14p17);
4257        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[8], x15p16);
4258        let m1318b = SseVector::mul(self.twiddles_im[12], x1m30);
4259        let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[4], x2m29);
4260        let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[7], x3m28);
4261        let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[9], x4m27);
4262        let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[2], x5m26);
4263        let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[14], x6m25);
4264        let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[1], x7m24);
4265        let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[10], x8m23);
4266        let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[6], x9m22);
4267        let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[5], x10m21);
4268        let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[11], x11m20);
4269        let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[0], x12m19);
4270        let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[13], x13m18);
4271        let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[3], x14m17);
4272        let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[8], x15m16);
4273        let [y13, y18] = SseVector::column_butterfly2([m1318a, m1318b]);
4274
4275        let m1417a = SseVector::fmadd(values[0], self.twiddles_re[13], x1p30);
4276        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[2], x2p29);
4277        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[10], x3p28);
4278        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[5], x4p27);
4279        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[7], x5p26);
4280        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[8], x6p25);
4281        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[4], x7p24);
4282        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[11], x8p23);
4283        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[1], x9p22);
4284        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[14], x10p21);
4285        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[0], x11p20);
4286        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[12], x12p19);
4287        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[3], x13p18);
4288        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[9], x14p17);
4289        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[6], x15p16);
4290        let m1417b = SseVector::mul(self.twiddles_im[13], x1m30);
4291        let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[2], x2m29);
4292        let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[10], x3m28);
4293        let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[5], x4m27);
4294        let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[7], x5m26);
4295        let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[8], x6m25);
4296        let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[4], x7m24);
4297        let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[11], x8m23);
4298        let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[1], x9m22);
4299        let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[14], x10m21);
4300        let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[0], x11m20);
4301        let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[12], x12m19);
4302        let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[3], x13m18);
4303        let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[9], x14m17);
4304        let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[6], x15m16);
4305        let [y14, y17] = SseVector::column_butterfly2([m1417a, m1417b]);
4306
4307        let m1516a = SseVector::fmadd(values[0], self.twiddles_re[14], x1p30);
4308        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[0], x2p29);
4309        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[13], x3p28);
4310        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[1], x4p27);
4311        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[12], x5p26);
4312        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[2], x6p25);
4313        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[11], x7p24);
4314        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[3], x8p23);
4315        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[10], x9p22);
4316        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[4], x10p21);
4317        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[9], x11p20);
4318        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[5], x12p19);
4319        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[8], x13p18);
4320        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[6], x14p17);
4321        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[7], x15p16);
4322        let m1516b = SseVector::mul(self.twiddles_im[14], x1m30);
4323        let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[0], x2m29);
4324        let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[13], x3m28);
4325        let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[1], x4m27);
4326        let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[12], x5m26);
4327        let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[2], x6m25);
4328        let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[11], x7m24);
4329        let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[3], x8m23);
4330        let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[10], x9m22);
4331        let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[4], x10m21);
4332        let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[9], x11m20);
4333        let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[5], x12m19);
4334        let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[8], x13m18);
4335        let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[6], x14m17);
4336        let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[7], x15m16);
4337        let [y15, y16] = SseVector::column_butterfly2([m1516a, m1516b]);
4338
4339
4340        [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12, y13, y14, y15, y16, y17, y18, y19, y20, y21, y22, y23, y24, y25, y26, y27, y28, y29, y30]
4341    }
4342}
4343
4344struct SseF64Butterfly31<T> {
4345    direction: FftDirection,
4346    twiddles_re: [__m128d; 15],
4347    twiddles_im: [__m128d; 15],
4348    _phantom: std::marker::PhantomData<T>,
4349}
4350
4351boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly31, 31, |this: &SseF64Butterfly31<_>| this.direction);
4352impl<T: FftNum> SseF64Butterfly31<T> {
4353    /// Safety: The current machine must support the sse4.1 instruction set
4354    #[target_feature(enable = "sse4.1")]
4355    unsafe fn new(direction: FftDirection) -> Self {
4356        assert_f64::<T>();
4357        let twiddles = make_twiddles(31, direction);
4358        unsafe {Self {
4359            direction,
4360            twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
4361            twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
4362            _phantom: std::marker::PhantomData,
4363        }}
4364    }
4365
4366    #[inline(always)]
4367    pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
4368        let values = read_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 });
4369
4370        let out = self.perform_fft_direct(values);
4371
4372        write_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 });   
4373    }
4374
4375    #[inline(always)]
4376    pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 31]) -> [__m128d; 31] {
4377        let rotate = SseVector::make_rotate90(FftDirection::Inverse);
4378
4379        let y00 = values[0];
4380        let [x1p30, x1m30] =  SseVector::column_butterfly2([values[1], values[30]]);
4381        let x1m30 = SseVector::apply_rotate90(rotate, x1m30);
4382        let y00 = SseVector::add(y00, x1p30);
4383        let [x2p29, x2m29] =  SseVector::column_butterfly2([values[2], values[29]]);
4384        let x2m29 = SseVector::apply_rotate90(rotate, x2m29);
4385        let y00 = SseVector::add(y00, x2p29);
4386        let [x3p28, x3m28] =  SseVector::column_butterfly2([values[3], values[28]]);
4387        let x3m28 = SseVector::apply_rotate90(rotate, x3m28);
4388        let y00 = SseVector::add(y00, x3p28);
4389        let [x4p27, x4m27] =  SseVector::column_butterfly2([values[4], values[27]]);
4390        let x4m27 = SseVector::apply_rotate90(rotate, x4m27);
4391        let y00 = SseVector::add(y00, x4p27);
4392        let [x5p26, x5m26] =  SseVector::column_butterfly2([values[5], values[26]]);
4393        let x5m26 = SseVector::apply_rotate90(rotate, x5m26);
4394        let y00 = SseVector::add(y00, x5p26);
4395        let [x6p25, x6m25] =  SseVector::column_butterfly2([values[6], values[25]]);
4396        let x6m25 = SseVector::apply_rotate90(rotate, x6m25);
4397        let y00 = SseVector::add(y00, x6p25);
4398        let [x7p24, x7m24] =  SseVector::column_butterfly2([values[7], values[24]]);
4399        let x7m24 = SseVector::apply_rotate90(rotate, x7m24);
4400        let y00 = SseVector::add(y00, x7p24);
4401        let [x8p23, x8m23] =  SseVector::column_butterfly2([values[8], values[23]]);
4402        let x8m23 = SseVector::apply_rotate90(rotate, x8m23);
4403        let y00 = SseVector::add(y00, x8p23);
4404        let [x9p22, x9m22] =  SseVector::column_butterfly2([values[9], values[22]]);
4405        let x9m22 = SseVector::apply_rotate90(rotate, x9m22);
4406        let y00 = SseVector::add(y00, x9p22);
4407        let [x10p21, x10m21] =  SseVector::column_butterfly2([values[10], values[21]]);
4408        let x10m21 = SseVector::apply_rotate90(rotate, x10m21);
4409        let y00 = SseVector::add(y00, x10p21);
4410        let [x11p20, x11m20] =  SseVector::column_butterfly2([values[11], values[20]]);
4411        let x11m20 = SseVector::apply_rotate90(rotate, x11m20);
4412        let y00 = SseVector::add(y00, x11p20);
4413        let [x12p19, x12m19] =  SseVector::column_butterfly2([values[12], values[19]]);
4414        let x12m19 = SseVector::apply_rotate90(rotate, x12m19);
4415        let y00 = SseVector::add(y00, x12p19);
4416        let [x13p18, x13m18] =  SseVector::column_butterfly2([values[13], values[18]]);
4417        let x13m18 = SseVector::apply_rotate90(rotate, x13m18);
4418        let y00 = SseVector::add(y00, x13p18);
4419        let [x14p17, x14m17] =  SseVector::column_butterfly2([values[14], values[17]]);
4420        let x14m17 = SseVector::apply_rotate90(rotate, x14m17);
4421        let y00 = SseVector::add(y00, x14p17);
4422        let [x15p16, x15m16] =  SseVector::column_butterfly2([values[15], values[16]]);
4423        let x15m16 = SseVector::apply_rotate90(rotate, x15m16);
4424        let y00 = SseVector::add(y00, x15p16);
4425
4426        let m0130a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p30);
4427        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[1], x2p29);
4428        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[2], x3p28);
4429        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[3], x4p27);
4430        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[4], x5p26);
4431        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[5], x6p25);
4432        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[6], x7p24);
4433        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[7], x8p23);
4434        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[8], x9p22);
4435        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[9], x10p21);
4436        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[10], x11p20);
4437        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[11], x12p19);
4438        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[12], x13p18);
4439        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[13], x14p17);
4440        let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[14], x15p16);
4441        let m0130b = SseVector::mul(self.twiddles_im[0], x1m30);
4442        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[1], x2m29);
4443        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[2], x3m28);
4444        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[3], x4m27);
4445        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[4], x5m26);
4446        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[5], x6m25);
4447        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[6], x7m24);
4448        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[7], x8m23);
4449        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[8], x9m22);
4450        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[9], x10m21);
4451        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[10], x11m20);
4452        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[11], x12m19);
4453        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[12], x13m18);
4454        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[13], x14m17);
4455        let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[14], x15m16);
4456        let [y01, y30] = SseVector::column_butterfly2([m0130a, m0130b]);
4457
4458        let m0229a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p30);
4459        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[3], x2p29);
4460        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[5], x3p28);
4461        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[7], x4p27);
4462        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[9], x5p26);
4463        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[11], x6p25);
4464        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[13], x7p24);
4465        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[14], x8p23);
4466        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[12], x9p22);
4467        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[10], x10p21);
4468        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[8], x11p20);
4469        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[6], x12p19);
4470        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[4], x13p18);
4471        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[2], x14p17);
4472        let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[0], x15p16);
4473        let m0229b = SseVector::mul(self.twiddles_im[1], x1m30);
4474        let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[3], x2m29);
4475        let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[5], x3m28);
4476        let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[7], x4m27);
4477        let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[9], x5m26);
4478        let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[11], x6m25);
4479        let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[13], x7m24);
4480        let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[14], x8m23);
4481        let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[12], x9m22);
4482        let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[10], x10m21);
4483        let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[8], x11m20);
4484        let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[6], x12m19);
4485        let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[4], x13m18);
4486        let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[2], x14m17);
4487        let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[0], x15m16);
4488        let [y02, y29] = SseVector::column_butterfly2([m0229a, m0229b]);
4489
4490        let m0328a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p30);
4491        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[5], x2p29);
4492        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[8], x3p28);
4493        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[11], x4p27);
4494        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[14], x5p26);
4495        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[12], x6p25);
4496        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[9], x7p24);
4497        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[6], x8p23);
4498        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[3], x9p22);
4499        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[0], x10p21);
4500        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[1], x11p20);
4501        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[4], x12p19);
4502        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[7], x13p18);
4503        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[10], x14p17);
4504        let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[13], x15p16);
4505        let m0328b = SseVector::mul(self.twiddles_im[2], x1m30);
4506        let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[5], x2m29);
4507        let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[8], x3m28);
4508        let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[11], x4m27);
4509        let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[14], x5m26);
4510        let m0328b = SseVector::nmadd(m0328b, self.twiddles_im[12], x6m25);
4511        let m0328b = SseVector::nmadd(m0328b, self.twiddles_im[9], x7m24);
4512        let m0328b = SseVector::nmadd(m0328b, self.twiddles_im[6], x8m23);
4513        let m0328b = SseVector::nmadd(m0328b, self.twiddles_im[3], x9m22);
4514        let m0328b = SseVector::nmadd(m0328b, self.twiddles_im[0], x10m21);
4515        let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[1], x11m20);
4516        let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[4], x12m19);
4517        let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[7], x13m18);
4518        let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[10], x14m17);
4519        let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[13], x15m16);
4520        let [y03, y28] = SseVector::column_butterfly2([m0328a, m0328b]);
4521
4522        let m0427a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p30);
4523        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[7], x2p29);
4524        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[11], x3p28);
4525        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[14], x4p27);
4526        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[10], x5p26);
4527        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[6], x6p25);
4528        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[2], x7p24);
4529        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[0], x8p23);
4530        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[4], x9p22);
4531        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[8], x10p21);
4532        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[12], x11p20);
4533        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[13], x12p19);
4534        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[9], x13p18);
4535        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[5], x14p17);
4536        let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[1], x15p16);
4537        let m0427b = SseVector::mul(self.twiddles_im[3], x1m30);
4538        let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[7], x2m29);
4539        let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[11], x3m28);
4540        let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[14], x4m27);
4541        let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[10], x5m26);
4542        let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[6], x6m25);
4543        let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[2], x7m24);
4544        let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[0], x8m23);
4545        let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[4], x9m22);
4546        let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[8], x10m21);
4547        let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[12], x11m20);
4548        let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[13], x12m19);
4549        let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[9], x13m18);
4550        let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[5], x14m17);
4551        let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[1], x15m16);
4552        let [y04, y27] = SseVector::column_butterfly2([m0427a, m0427b]);
4553
4554        let m0526a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p30);
4555        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[9], x2p29);
4556        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[14], x3p28);
4557        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[10], x4p27);
4558        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[5], x5p26);
4559        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[0], x6p25);
4560        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[3], x7p24);
4561        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[8], x8p23);
4562        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[13], x9p22);
4563        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[11], x10p21);
4564        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[6], x11p20);
4565        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[1], x12p19);
4566        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[2], x13p18);
4567        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[7], x14p17);
4568        let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[12], x15p16);
4569        let m0526b = SseVector::mul(self.twiddles_im[4], x1m30);
4570        let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[9], x2m29);
4571        let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[14], x3m28);
4572        let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[10], x4m27);
4573        let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[5], x5m26);
4574        let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[0], x6m25);
4575        let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[3], x7m24);
4576        let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[8], x8m23);
4577        let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[13], x9m22);
4578        let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[11], x10m21);
4579        let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[6], x11m20);
4580        let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[1], x12m19);
4581        let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[2], x13m18);
4582        let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[7], x14m17);
4583        let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[12], x15m16);
4584        let [y05, y26] = SseVector::column_butterfly2([m0526a, m0526b]);
4585
4586        let m0625a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p30);
4587        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[11], x2p29);
4588        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[12], x3p28);
4589        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[6], x4p27);
4590        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[0], x5p26);
4591        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[4], x6p25);
4592        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[10], x7p24);
4593        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[13], x8p23);
4594        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[7], x9p22);
4595        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[1], x10p21);
4596        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[3], x11p20);
4597        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[9], x12p19);
4598        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[14], x13p18);
4599        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[8], x14p17);
4600        let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[2], x15p16);
4601        let m0625b = SseVector::mul(self.twiddles_im[5], x1m30);
4602        let m0625b = SseVector::fmadd(m0625b, self.twiddles_im[11], x2m29);
4603        let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[12], x3m28);
4604        let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[6], x4m27);
4605        let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[0], x5m26);
4606        let m0625b = SseVector::fmadd(m0625b, self.twiddles_im[4], x6m25);
4607        let m0625b = SseVector::fmadd(m0625b, self.twiddles_im[10], x7m24);
4608        let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[13], x8m23);
4609        let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[7], x9m22);
4610        let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[1], x10m21);
4611        let m0625b = SseVector::fmadd(m0625b, self.twiddles_im[3], x11m20);
4612        let m0625b = SseVector::fmadd(m0625b, self.twiddles_im[9], x12m19);
4613        let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[14], x13m18);
4614        let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[8], x14m17);
4615        let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[2], x15m16);
4616        let [y06, y25] = SseVector::column_butterfly2([m0625a, m0625b]);
4617
4618        let m0724a = SseVector::fmadd(values[0], self.twiddles_re[6], x1p30);
4619        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[13], x2p29);
4620        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[9], x3p28);
4621        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[2], x4p27);
4622        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[3], x5p26);
4623        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[10], x6p25);
4624        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[12], x7p24);
4625        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[5], x8p23);
4626        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[0], x9p22);
4627        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[7], x10p21);
4628        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[14], x11p20);
4629        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[8], x12p19);
4630        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[1], x13p18);
4631        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[4], x14p17);
4632        let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[11], x15p16);
4633        let m0724b = SseVector::mul(self.twiddles_im[6], x1m30);
4634        let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[13], x2m29);
4635        let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[9], x3m28);
4636        let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[2], x4m27);
4637        let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[3], x5m26);
4638        let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[10], x6m25);
4639        let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[12], x7m24);
4640        let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[5], x8m23);
4641        let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[0], x9m22);
4642        let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[7], x10m21);
4643        let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[14], x11m20);
4644        let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[8], x12m19);
4645        let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[1], x13m18);
4646        let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[4], x14m17);
4647        let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[11], x15m16);
4648        let [y07, y24] = SseVector::column_butterfly2([m0724a, m0724b]);
4649
4650        let m0823a = SseVector::fmadd(values[0], self.twiddles_re[7], x1p30);
4651        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[14], x2p29);
4652        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[6], x3p28);
4653        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[0], x4p27);
4654        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[8], x5p26);
4655        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[13], x6p25);
4656        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[5], x7p24);
4657        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[1], x8p23);
4658        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[9], x9p22);
4659        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[12], x10p21);
4660        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[4], x11p20);
4661        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[2], x12p19);
4662        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[10], x13p18);
4663        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[11], x14p17);
4664        let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[3], x15p16);
4665        let m0823b = SseVector::mul(self.twiddles_im[7], x1m30);
4666        let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[14], x2m29);
4667        let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[6], x3m28);
4668        let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[0], x4m27);
4669        let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[8], x5m26);
4670        let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[13], x6m25);
4671        let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[5], x7m24);
4672        let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[1], x8m23);
4673        let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[9], x9m22);
4674        let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[12], x10m21);
4675        let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[4], x11m20);
4676        let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[2], x12m19);
4677        let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[10], x13m18);
4678        let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[11], x14m17);
4679        let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[3], x15m16);
4680        let [y08, y23] = SseVector::column_butterfly2([m0823a, m0823b]);
4681
4682        let m0922a = SseVector::fmadd(values[0], self.twiddles_re[8], x1p30);
4683        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[12], x2p29);
4684        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[3], x3p28);
4685        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[4], x4p27);
4686        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[13], x5p26);
4687        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[7], x6p25);
4688        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[0], x7p24);
4689        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[9], x8p23);
4690        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[11], x9p22);
4691        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[2], x10p21);
4692        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[5], x11p20);
4693        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[14], x12p19);
4694        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[6], x13p18);
4695        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[1], x14p17);
4696        let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[10], x15p16);
4697        let m0922b = SseVector::mul(self.twiddles_im[8], x1m30);
4698        let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[12], x2m29);
4699        let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[3], x3m28);
4700        let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[4], x4m27);
4701        let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[13], x5m26);
4702        let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[7], x6m25);
4703        let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[0], x7m24);
4704        let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[9], x8m23);
4705        let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[11], x9m22);
4706        let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[2], x10m21);
4707        let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[5], x11m20);
4708        let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[14], x12m19);
4709        let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[6], x13m18);
4710        let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[1], x14m17);
4711        let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[10], x15m16);
4712        let [y09, y22] = SseVector::column_butterfly2([m0922a, m0922b]);
4713
4714        let m1021a = SseVector::fmadd(values[0], self.twiddles_re[9], x1p30);
4715        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[10], x2p29);
4716        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[0], x3p28);
4717        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[8], x4p27);
4718        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[11], x5p26);
4719        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[1], x6p25);
4720        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[7], x7p24);
4721        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[12], x8p23);
4722        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[2], x9p22);
4723        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[6], x10p21);
4724        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[13], x11p20);
4725        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[3], x12p19);
4726        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[5], x13p18);
4727        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[14], x14p17);
4728        let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[4], x15p16);
4729        let m1021b = SseVector::mul(self.twiddles_im[9], x1m30);
4730        let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[10], x2m29);
4731        let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[0], x3m28);
4732        let m1021b = SseVector::fmadd(m1021b, self.twiddles_im[8], x4m27);
4733        let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[11], x5m26);
4734        let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[1], x6m25);
4735        let m1021b = SseVector::fmadd(m1021b, self.twiddles_im[7], x7m24);
4736        let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[12], x8m23);
4737        let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[2], x9m22);
4738        let m1021b = SseVector::fmadd(m1021b, self.twiddles_im[6], x10m21);
4739        let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[13], x11m20);
4740        let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[3], x12m19);
4741        let m1021b = SseVector::fmadd(m1021b, self.twiddles_im[5], x13m18);
4742        let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[14], x14m17);
4743        let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[4], x15m16);
4744        let [y10, y21] = SseVector::column_butterfly2([m1021a, m1021b]);
4745
4746        let m1120a = SseVector::fmadd(values[0], self.twiddles_re[10], x1p30);
4747        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[8], x2p29);
4748        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[1], x3p28);
4749        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[12], x4p27);
4750        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[6], x5p26);
4751        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[3], x6p25);
4752        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[14], x7p24);
4753        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[4], x8p23);
4754        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[5], x9p22);
4755        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[13], x10p21);
4756        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[2], x11p20);
4757        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[7], x12p19);
4758        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[11], x13p18);
4759        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[0], x14p17);
4760        let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[9], x15p16);
4761        let m1120b = SseVector::mul(self.twiddles_im[10], x1m30);
4762        let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[8], x2m29);
4763        let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[1], x3m28);
4764        let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[12], x4m27);
4765        let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[6], x5m26);
4766        let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[3], x6m25);
4767        let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[14], x7m24);
4768        let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[4], x8m23);
4769        let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[5], x9m22);
4770        let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[13], x10m21);
4771        let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[2], x11m20);
4772        let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[7], x12m19);
4773        let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[11], x13m18);
4774        let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[0], x14m17);
4775        let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[9], x15m16);
4776        let [y11, y20] = SseVector::column_butterfly2([m1120a, m1120b]);
4777
4778        let m1219a = SseVector::fmadd(values[0], self.twiddles_re[11], x1p30);
4779        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[6], x2p29);
4780        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[4], x3p28);
4781        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[13], x4p27);
4782        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[1], x5p26);
4783        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[9], x6p25);
4784        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[8], x7p24);
4785        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[2], x8p23);
4786        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[14], x9p22);
4787        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[3], x10p21);
4788        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[7], x11p20);
4789        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[10], x12p19);
4790        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[0], x13p18);
4791        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[12], x14p17);
4792        let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[5], x15p16);
4793        let m1219b = SseVector::mul(self.twiddles_im[11], x1m30);
4794        let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[6], x2m29);
4795        let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[4], x3m28);
4796        let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[13], x4m27);
4797        let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[1], x5m26);
4798        let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[9], x6m25);
4799        let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[8], x7m24);
4800        let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[2], x8m23);
4801        let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[14], x9m22);
4802        let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[3], x10m21);
4803        let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[7], x11m20);
4804        let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[10], x12m19);
4805        let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[0], x13m18);
4806        let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[12], x14m17);
4807        let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[5], x15m16);
4808        let [y12, y19] = SseVector::column_butterfly2([m1219a, m1219b]);
4809
4810        let m1318a = SseVector::fmadd(values[0], self.twiddles_re[12], x1p30);
4811        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[4], x2p29);
4812        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[7], x3p28);
4813        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[9], x4p27);
4814        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[2], x5p26);
4815        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[14], x6p25);
4816        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[1], x7p24);
4817        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[10], x8p23);
4818        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[6], x9p22);
4819        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[5], x10p21);
4820        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[11], x11p20);
4821        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[0], x12p19);
4822        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[13], x13p18);
4823        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[3], x14p17);
4824        let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[8], x15p16);
4825        let m1318b = SseVector::mul(self.twiddles_im[12], x1m30);
4826        let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[4], x2m29);
4827        let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[7], x3m28);
4828        let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[9], x4m27);
4829        let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[2], x5m26);
4830        let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[14], x6m25);
4831        let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[1], x7m24);
4832        let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[10], x8m23);
4833        let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[6], x9m22);
4834        let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[5], x10m21);
4835        let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[11], x11m20);
4836        let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[0], x12m19);
4837        let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[13], x13m18);
4838        let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[3], x14m17);
4839        let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[8], x15m16);
4840        let [y13, y18] = SseVector::column_butterfly2([m1318a, m1318b]);
4841
4842        let m1417a = SseVector::fmadd(values[0], self.twiddles_re[13], x1p30);
4843        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[2], x2p29);
4844        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[10], x3p28);
4845        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[5], x4p27);
4846        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[7], x5p26);
4847        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[8], x6p25);
4848        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[4], x7p24);
4849        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[11], x8p23);
4850        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[1], x9p22);
4851        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[14], x10p21);
4852        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[0], x11p20);
4853        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[12], x12p19);
4854        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[3], x13p18);
4855        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[9], x14p17);
4856        let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[6], x15p16);
4857        let m1417b = SseVector::mul(self.twiddles_im[13], x1m30);
4858        let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[2], x2m29);
4859        let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[10], x3m28);
4860        let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[5], x4m27);
4861        let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[7], x5m26);
4862        let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[8], x6m25);
4863        let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[4], x7m24);
4864        let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[11], x8m23);
4865        let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[1], x9m22);
4866        let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[14], x10m21);
4867        let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[0], x11m20);
4868        let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[12], x12m19);
4869        let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[3], x13m18);
4870        let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[9], x14m17);
4871        let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[6], x15m16);
4872        let [y14, y17] = SseVector::column_butterfly2([m1417a, m1417b]);
4873
4874        let m1516a = SseVector::fmadd(values[0], self.twiddles_re[14], x1p30);
4875        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[0], x2p29);
4876        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[13], x3p28);
4877        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[1], x4p27);
4878        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[12], x5p26);
4879        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[2], x6p25);
4880        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[11], x7p24);
4881        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[3], x8p23);
4882        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[10], x9p22);
4883        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[4], x10p21);
4884        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[9], x11p20);
4885        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[5], x12p19);
4886        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[8], x13p18);
4887        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[6], x14p17);
4888        let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[7], x15p16);
4889        let m1516b = SseVector::mul(self.twiddles_im[14], x1m30);
4890        let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[0], x2m29);
4891        let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[13], x3m28);
4892        let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[1], x4m27);
4893        let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[12], x5m26);
4894        let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[2], x6m25);
4895        let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[11], x7m24);
4896        let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[3], x8m23);
4897        let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[10], x9m22);
4898        let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[4], x10m21);
4899        let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[9], x11m20);
4900        let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[5], x12m19);
4901        let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[8], x13m18);
4902        let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[6], x14m17);
4903        let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[7], x15m16);
4904        let [y15, y16] = SseVector::column_butterfly2([m1516a, m1516b]);
4905
4906
4907        [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12, y13, y14, y15, y16, y17, y18, y19, y20, y21, y22, y23, y24, y25, y26, y27, y28, y29, y30]
4908    }
4909}
4910
4911
4912#[cfg(test)]
4913mod unit_tests {
4914    use super::*;
4915    use crate::test_utils::check_fft_algorithm;
4916
4917    macro_rules! test_butterfly_32_func {
4918        ($test_name:ident, $struct_name:ident, $size:expr) => {
4919            #[test]
4920            fn $test_name() {
4921                assert!(std::arch::is_x86_feature_detected!("sse4.1"));
4922
4923                let fwd = unsafe { $struct_name::new(FftDirection::Forward) };
4924                check_fft_algorithm::<f32>(&fwd, $size, FftDirection::Forward);
4925
4926                let inv = unsafe { $struct_name::new(FftDirection::Inverse) }; 
4927                check_fft_algorithm::<f32>(&inv, $size, FftDirection::Inverse);
4928            }
4929        };
4930    }
4931    macro_rules! test_butterfly_64_func {
4932        ($test_name:ident, $struct_name:ident, $size:expr) => {
4933            #[test]
4934            fn $test_name() {
4935                assert!(std::arch::is_x86_feature_detected!("sse4.1"));
4936
4937                let fwd = unsafe { $struct_name::new(FftDirection::Forward) };
4938                check_fft_algorithm::<f64>(&fwd, $size, FftDirection::Forward);
4939
4940                let inv = unsafe { $struct_name::new(FftDirection::Inverse) };
4941                check_fft_algorithm::<f64>(&inv, $size, FftDirection::Inverse);
4942            }
4943        };
4944    }
4945    test_butterfly_32_func!(test_ssef32_butterfly7, SseF32Butterfly7, 7);
4946    test_butterfly_32_func!(test_ssef32_butterfly11, SseF32Butterfly11, 11);
4947    test_butterfly_32_func!(test_ssef32_butterfly13, SseF32Butterfly13, 13);
4948    test_butterfly_32_func!(test_ssef32_butterfly17, SseF32Butterfly17, 17);
4949    test_butterfly_32_func!(test_ssef32_butterfly19, SseF32Butterfly19, 19);
4950    test_butterfly_32_func!(test_ssef32_butterfly23, SseF32Butterfly23, 23);
4951    test_butterfly_32_func!(test_ssef32_butterfly29, SseF32Butterfly29, 29);
4952    test_butterfly_32_func!(test_ssef32_butterfly31, SseF32Butterfly31, 31);
4953    test_butterfly_64_func!(test_ssef64_butterfly7, SseF64Butterfly7, 7);
4954    test_butterfly_64_func!(test_ssef64_butterfly11, SseF64Butterfly11, 11);
4955    test_butterfly_64_func!(test_ssef64_butterfly13, SseF64Butterfly13, 13);
4956    test_butterfly_64_func!(test_ssef64_butterfly17, SseF64Butterfly17, 17);
4957    test_butterfly_64_func!(test_ssef64_butterfly19, SseF64Butterfly19, 19);
4958    test_butterfly_64_func!(test_ssef64_butterfly23, SseF64Butterfly23, 23);
4959    test_butterfly_64_func!(test_ssef64_butterfly29, SseF64Butterfly29, 29);
4960    test_butterfly_64_func!(test_ssef64_butterfly31, SseF64Butterfly31, 31);
4961}
4962