1use core::arch::x86_64::{__m128, __m128d};
2use std::any::TypeId;
3use std::sync::Arc;
4use num_complex::Complex;
5
6use crate::{common::FftNum, FftDirection};
7
8use crate::array_utils::DoubleBuf;
9use crate::twiddles;
10use crate::{Direction, Fft, Length};
11
12use super::sse_common::{assert_f32, assert_f64};
13use super::sse_utils::*;
14use super::sse_vector::*;
15
16pub const fn prime_butterfly_lens() -> &'static [usize] {
33 &[7, 11, 13, 17, 19, 23, 29, 31, ]
34}
35
36#[target_feature(enable = "sse4.1")]
38pub unsafe fn construct_prime_butterfly<T: FftNum>(len: usize, direction: FftDirection) -> Arc<dyn Fft<T>> {
39 let id_f32 = TypeId::of::<f32>();
40 let id_f64 = TypeId::of::<f64>();
41 let id_t = TypeId::of::<T>();
42 if id_t == id_f32 {
43 match len {
44 7 => Arc::new(SseF32Butterfly7::new(direction)) as Arc<dyn Fft<T>>,
45 11 => Arc::new(SseF32Butterfly11::new(direction)) as Arc<dyn Fft<T>>,
46 13 => Arc::new(SseF32Butterfly13::new(direction)) as Arc<dyn Fft<T>>,
47 17 => Arc::new(SseF32Butterfly17::new(direction)) as Arc<dyn Fft<T>>,
48 19 => Arc::new(SseF32Butterfly19::new(direction)) as Arc<dyn Fft<T>>,
49 23 => Arc::new(SseF32Butterfly23::new(direction)) as Arc<dyn Fft<T>>,
50 29 => Arc::new(SseF32Butterfly29::new(direction)) as Arc<dyn Fft<T>>,
51 31 => Arc::new(SseF32Butterfly31::new(direction)) as Arc<dyn Fft<T>>,
52 _ => unimplemented!("Invalid SSE prime butterfly length: {len}"),
53 }
54 } else if id_t == id_f64 {
55 match len {
56 7 => Arc::new(SseF64Butterfly7::new(direction)) as Arc<dyn Fft<T>>,
57 11 => Arc::new(SseF64Butterfly11::new(direction)) as Arc<dyn Fft<T>>,
58 13 => Arc::new(SseF64Butterfly13::new(direction)) as Arc<dyn Fft<T>>,
59 17 => Arc::new(SseF64Butterfly17::new(direction)) as Arc<dyn Fft<T>>,
60 19 => Arc::new(SseF64Butterfly19::new(direction)) as Arc<dyn Fft<T>>,
61 23 => Arc::new(SseF64Butterfly23::new(direction)) as Arc<dyn Fft<T>>,
62 29 => Arc::new(SseF64Butterfly29::new(direction)) as Arc<dyn Fft<T>>,
63 31 => Arc::new(SseF64Butterfly31::new(direction)) as Arc<dyn Fft<T>>,
64 _ => unimplemented!("Invalid SSE prime butterfly length: {len}"),
65 }
66 } else {
67 unimplemented!("Not f32 or f64");
68 }
69}
70
71#[inline(always)]
72fn make_twiddles<const TW: usize, T: FftNum>(len: usize, direction: FftDirection) -> [Complex<T>; TW] {
73 let mut i = 1;
74 [(); TW].map(|_| {
75 let twiddle = twiddles::compute_twiddle(i, len, direction);
76 i += 1;
77 twiddle
78 })
79}
80
81struct SseF32Butterfly7<T> {
82 direction: FftDirection,
83 twiddles_re: [__m128; 3],
84 twiddles_im: [__m128; 3],
85 _phantom: std::marker::PhantomData<T>,
86}
87
88boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly7, 7, |this: &SseF32Butterfly7<_>| this.direction);
89impl<T: FftNum> SseF32Butterfly7<T> {
90 #[target_feature(enable = "sse4.1")]
92 unsafe fn new(direction: FftDirection) -> Self {
93 assert_f32::<T>();
94 let twiddles = make_twiddles(7, direction);
95 Self {
96 direction,
97 twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
98 twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
99 _phantom: std::marker::PhantomData,
100 }
101 }
102
103 #[inline(always)]
104 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
105 let values = read_partial1_complex_to_array!(buffer, { 0,1,2,3,4,5,6 });
106
107 let out = self.perform_parallel_fft_direct(values);
108
109 write_partial_lo_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6 } );
110 }
111
112 #[inline(always)]
113 pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
114 let input_packed = read_complex_to_array!(buffer, { 0,2,4,6,8,10,12 });
115
116 let values = [
117 extract_lo_hi_f32(input_packed[0], input_packed[3]),
118 extract_hi_lo_f32(input_packed[0], input_packed[4]),
119 extract_lo_hi_f32(input_packed[1], input_packed[4]),
120 extract_hi_lo_f32(input_packed[1], input_packed[5]),
121 extract_lo_hi_f32(input_packed[2], input_packed[5]),
122 extract_hi_lo_f32(input_packed[2], input_packed[6]),
123 extract_lo_hi_f32(input_packed[3], input_packed[6]),
124 ];
125
126 let out = self.perform_parallel_fft_direct(values);
127
128 let out_packed = [
129 extract_lo_lo_f32(out[0], out[1]),
130 extract_lo_lo_f32(out[2], out[3]),
131 extract_lo_lo_f32(out[4], out[5]),
132 extract_lo_hi_f32(out[6], out[0]),
133 extract_hi_hi_f32(out[1], out[2]),
134 extract_hi_hi_f32(out[3], out[4]),
135 extract_hi_hi_f32(out[5], out[6]),
136 ];
137
138 write_complex_to_array_strided!(out_packed, buffer, 2, { 0,1,2,3,4,5,6 });
139 }
140
141 #[inline(always)]
142 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 7]) -> [__m128; 7] {
143 let rotate = SseVector::make_rotate90(FftDirection::Inverse);
144
145 let y00 = values[0];
146 let [x1p6, x1m6] = SseVector::column_butterfly2([values[1], values[6]]);
147 let x1m6 = SseVector::apply_rotate90(rotate, x1m6);
148 let y00 = SseVector::add(y00, x1p6);
149 let [x2p5, x2m5] = SseVector::column_butterfly2([values[2], values[5]]);
150 let x2m5 = SseVector::apply_rotate90(rotate, x2m5);
151 let y00 = SseVector::add(y00, x2p5);
152 let [x3p4, x3m4] = SseVector::column_butterfly2([values[3], values[4]]);
153 let x3m4 = SseVector::apply_rotate90(rotate, x3m4);
154 let y00 = SseVector::add(y00, x3p4);
155
156 let m0106a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p6);
157 let m0106a = SseVector::fmadd(m0106a, self.twiddles_re[1], x2p5);
158 let m0106a = SseVector::fmadd(m0106a, self.twiddles_re[2], x3p4);
159 let m0106b = SseVector::mul(self.twiddles_im[0], x1m6);
160 let m0106b = SseVector::fmadd(m0106b, self.twiddles_im[1], x2m5);
161 let m0106b = SseVector::fmadd(m0106b, self.twiddles_im[2], x3m4);
162 let [y01, y06] = SseVector::column_butterfly2([m0106a, m0106b]);
163
164 let m0205a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p6);
165 let m0205a = SseVector::fmadd(m0205a, self.twiddles_re[2], x2p5);
166 let m0205a = SseVector::fmadd(m0205a, self.twiddles_re[0], x3p4);
167 let m0205b = SseVector::mul(self.twiddles_im[1], x1m6);
168 let m0205b = SseVector::nmadd(m0205b, self.twiddles_im[2], x2m5);
169 let m0205b = SseVector::nmadd(m0205b, self.twiddles_im[0], x3m4);
170 let [y02, y05] = SseVector::column_butterfly2([m0205a, m0205b]);
171
172 let m0304a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p6);
173 let m0304a = SseVector::fmadd(m0304a, self.twiddles_re[0], x2p5);
174 let m0304a = SseVector::fmadd(m0304a, self.twiddles_re[1], x3p4);
175 let m0304b = SseVector::mul(self.twiddles_im[2], x1m6);
176 let m0304b = SseVector::nmadd(m0304b, self.twiddles_im[0], x2m5);
177 let m0304b = SseVector::fmadd(m0304b, self.twiddles_im[1], x3m4);
178 let [y03, y04] = SseVector::column_butterfly2([m0304a, m0304b]);
179
180
181 [y00, y01, y02, y03, y04, y05, y06]
182 }
183}
184
185struct SseF64Butterfly7<T> {
186 direction: FftDirection,
187 twiddles_re: [__m128d; 3],
188 twiddles_im: [__m128d; 3],
189 _phantom: std::marker::PhantomData<T>,
190}
191
192boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly7, 7, |this: &SseF64Butterfly7<_>| this.direction);
193impl<T: FftNum> SseF64Butterfly7<T> {
194 #[target_feature(enable = "sse4.1")]
196 unsafe fn new(direction: FftDirection) -> Self {
197 assert_f64::<T>();
198 let twiddles = make_twiddles(7, direction);
199 unsafe {Self {
200 direction,
201 twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
202 twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
203 _phantom: std::marker::PhantomData,
204 }}
205 }
206
207 #[inline(always)]
208 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
209 let values = read_complex_to_array!(buffer, { 0,1,2,3,4,5,6 });
210
211 let out = self.perform_fft_direct(values);
212
213 write_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6 });
214 }
215
216 #[inline(always)]
217 pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 7]) -> [__m128d; 7] {
218 let rotate = SseVector::make_rotate90(FftDirection::Inverse);
219
220 let y00 = values[0];
221 let [x1p6, x1m6] = SseVector::column_butterfly2([values[1], values[6]]);
222 let x1m6 = SseVector::apply_rotate90(rotate, x1m6);
223 let y00 = SseVector::add(y00, x1p6);
224 let [x2p5, x2m5] = SseVector::column_butterfly2([values[2], values[5]]);
225 let x2m5 = SseVector::apply_rotate90(rotate, x2m5);
226 let y00 = SseVector::add(y00, x2p5);
227 let [x3p4, x3m4] = SseVector::column_butterfly2([values[3], values[4]]);
228 let x3m4 = SseVector::apply_rotate90(rotate, x3m4);
229 let y00 = SseVector::add(y00, x3p4);
230
231 let m0106a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p6);
232 let m0106a = SseVector::fmadd(m0106a, self.twiddles_re[1], x2p5);
233 let m0106a = SseVector::fmadd(m0106a, self.twiddles_re[2], x3p4);
234 let m0106b = SseVector::mul(self.twiddles_im[0], x1m6);
235 let m0106b = SseVector::fmadd(m0106b, self.twiddles_im[1], x2m5);
236 let m0106b = SseVector::fmadd(m0106b, self.twiddles_im[2], x3m4);
237 let [y01, y06] = SseVector::column_butterfly2([m0106a, m0106b]);
238
239 let m0205a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p6);
240 let m0205a = SseVector::fmadd(m0205a, self.twiddles_re[2], x2p5);
241 let m0205a = SseVector::fmadd(m0205a, self.twiddles_re[0], x3p4);
242 let m0205b = SseVector::mul(self.twiddles_im[1], x1m6);
243 let m0205b = SseVector::nmadd(m0205b, self.twiddles_im[2], x2m5);
244 let m0205b = SseVector::nmadd(m0205b, self.twiddles_im[0], x3m4);
245 let [y02, y05] = SseVector::column_butterfly2([m0205a, m0205b]);
246
247 let m0304a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p6);
248 let m0304a = SseVector::fmadd(m0304a, self.twiddles_re[0], x2p5);
249 let m0304a = SseVector::fmadd(m0304a, self.twiddles_re[1], x3p4);
250 let m0304b = SseVector::mul(self.twiddles_im[2], x1m6);
251 let m0304b = SseVector::nmadd(m0304b, self.twiddles_im[0], x2m5);
252 let m0304b = SseVector::fmadd(m0304b, self.twiddles_im[1], x3m4);
253 let [y03, y04] = SseVector::column_butterfly2([m0304a, m0304b]);
254
255
256 [y00, y01, y02, y03, y04, y05, y06]
257 }
258}
259
260struct SseF32Butterfly11<T> {
261 direction: FftDirection,
262 twiddles_re: [__m128; 5],
263 twiddles_im: [__m128; 5],
264 _phantom: std::marker::PhantomData<T>,
265}
266
267boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly11, 11, |this: &SseF32Butterfly11<_>| this.direction);
268impl<T: FftNum> SseF32Butterfly11<T> {
269 #[target_feature(enable = "sse4.1")]
271 unsafe fn new(direction: FftDirection) -> Self {
272 assert_f32::<T>();
273 let twiddles = make_twiddles(11, direction);
274 Self {
275 direction,
276 twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
277 twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
278 _phantom: std::marker::PhantomData,
279 }
280 }
281
282 #[inline(always)]
283 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
284 let values = read_partial1_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10 });
285
286 let out = self.perform_parallel_fft_direct(values);
287
288 write_partial_lo_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10 } );
289 }
290
291 #[inline(always)]
292 pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
293 let input_packed = read_complex_to_array!(buffer, { 0,2,4,6,8,10,12,14,16,18,20 });
294
295 let values = [
296 extract_lo_hi_f32(input_packed[0], input_packed[5]),
297 extract_hi_lo_f32(input_packed[0], input_packed[6]),
298 extract_lo_hi_f32(input_packed[1], input_packed[6]),
299 extract_hi_lo_f32(input_packed[1], input_packed[7]),
300 extract_lo_hi_f32(input_packed[2], input_packed[7]),
301 extract_hi_lo_f32(input_packed[2], input_packed[8]),
302 extract_lo_hi_f32(input_packed[3], input_packed[8]),
303 extract_hi_lo_f32(input_packed[3], input_packed[9]),
304 extract_lo_hi_f32(input_packed[4], input_packed[9]),
305 extract_hi_lo_f32(input_packed[4], input_packed[10]),
306 extract_lo_hi_f32(input_packed[5], input_packed[10]),
307 ];
308
309 let out = self.perform_parallel_fft_direct(values);
310
311 let out_packed = [
312 extract_lo_lo_f32(out[0], out[1]),
313 extract_lo_lo_f32(out[2], out[3]),
314 extract_lo_lo_f32(out[4], out[5]),
315 extract_lo_lo_f32(out[6], out[7]),
316 extract_lo_lo_f32(out[8], out[9]),
317 extract_lo_hi_f32(out[10], out[0]),
318 extract_hi_hi_f32(out[1], out[2]),
319 extract_hi_hi_f32(out[3], out[4]),
320 extract_hi_hi_f32(out[5], out[6]),
321 extract_hi_hi_f32(out[7], out[8]),
322 extract_hi_hi_f32(out[9], out[10]),
323 ];
324
325 write_complex_to_array_strided!(out_packed, buffer, 2, { 0,1,2,3,4,5,6,7,8,9,10 });
326 }
327
328 #[inline(always)]
329 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 11]) -> [__m128; 11] {
330 let rotate = SseVector::make_rotate90(FftDirection::Inverse);
331
332 let y00 = values[0];
333 let [x1p10, x1m10] = SseVector::column_butterfly2([values[1], values[10]]);
334 let x1m10 = SseVector::apply_rotate90(rotate, x1m10);
335 let y00 = SseVector::add(y00, x1p10);
336 let [x2p9, x2m9] = SseVector::column_butterfly2([values[2], values[9]]);
337 let x2m9 = SseVector::apply_rotate90(rotate, x2m9);
338 let y00 = SseVector::add(y00, x2p9);
339 let [x3p8, x3m8] = SseVector::column_butterfly2([values[3], values[8]]);
340 let x3m8 = SseVector::apply_rotate90(rotate, x3m8);
341 let y00 = SseVector::add(y00, x3p8);
342 let [x4p7, x4m7] = SseVector::column_butterfly2([values[4], values[7]]);
343 let x4m7 = SseVector::apply_rotate90(rotate, x4m7);
344 let y00 = SseVector::add(y00, x4p7);
345 let [x5p6, x5m6] = SseVector::column_butterfly2([values[5], values[6]]);
346 let x5m6 = SseVector::apply_rotate90(rotate, x5m6);
347 let y00 = SseVector::add(y00, x5p6);
348
349 let m0110a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p10);
350 let m0110a = SseVector::fmadd(m0110a, self.twiddles_re[1], x2p9);
351 let m0110a = SseVector::fmadd(m0110a, self.twiddles_re[2], x3p8);
352 let m0110a = SseVector::fmadd(m0110a, self.twiddles_re[3], x4p7);
353 let m0110a = SseVector::fmadd(m0110a, self.twiddles_re[4], x5p6);
354 let m0110b = SseVector::mul(self.twiddles_im[0], x1m10);
355 let m0110b = SseVector::fmadd(m0110b, self.twiddles_im[1], x2m9);
356 let m0110b = SseVector::fmadd(m0110b, self.twiddles_im[2], x3m8);
357 let m0110b = SseVector::fmadd(m0110b, self.twiddles_im[3], x4m7);
358 let m0110b = SseVector::fmadd(m0110b, self.twiddles_im[4], x5m6);
359 let [y01, y10] = SseVector::column_butterfly2([m0110a, m0110b]);
360
361 let m0209a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p10);
362 let m0209a = SseVector::fmadd(m0209a, self.twiddles_re[3], x2p9);
363 let m0209a = SseVector::fmadd(m0209a, self.twiddles_re[4], x3p8);
364 let m0209a = SseVector::fmadd(m0209a, self.twiddles_re[2], x4p7);
365 let m0209a = SseVector::fmadd(m0209a, self.twiddles_re[0], x5p6);
366 let m0209b = SseVector::mul(self.twiddles_im[1], x1m10);
367 let m0209b = SseVector::fmadd(m0209b, self.twiddles_im[3], x2m9);
368 let m0209b = SseVector::nmadd(m0209b, self.twiddles_im[4], x3m8);
369 let m0209b = SseVector::nmadd(m0209b, self.twiddles_im[2], x4m7);
370 let m0209b = SseVector::nmadd(m0209b, self.twiddles_im[0], x5m6);
371 let [y02, y09] = SseVector::column_butterfly2([m0209a, m0209b]);
372
373 let m0308a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p10);
374 let m0308a = SseVector::fmadd(m0308a, self.twiddles_re[4], x2p9);
375 let m0308a = SseVector::fmadd(m0308a, self.twiddles_re[1], x3p8);
376 let m0308a = SseVector::fmadd(m0308a, self.twiddles_re[0], x4p7);
377 let m0308a = SseVector::fmadd(m0308a, self.twiddles_re[3], x5p6);
378 let m0308b = SseVector::mul(self.twiddles_im[2], x1m10);
379 let m0308b = SseVector::nmadd(m0308b, self.twiddles_im[4], x2m9);
380 let m0308b = SseVector::nmadd(m0308b, self.twiddles_im[1], x3m8);
381 let m0308b = SseVector::fmadd(m0308b, self.twiddles_im[0], x4m7);
382 let m0308b = SseVector::fmadd(m0308b, self.twiddles_im[3], x5m6);
383 let [y03, y08] = SseVector::column_butterfly2([m0308a, m0308b]);
384
385 let m0407a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p10);
386 let m0407a = SseVector::fmadd(m0407a, self.twiddles_re[2], x2p9);
387 let m0407a = SseVector::fmadd(m0407a, self.twiddles_re[0], x3p8);
388 let m0407a = SseVector::fmadd(m0407a, self.twiddles_re[4], x4p7);
389 let m0407a = SseVector::fmadd(m0407a, self.twiddles_re[1], x5p6);
390 let m0407b = SseVector::mul(self.twiddles_im[3], x1m10);
391 let m0407b = SseVector::nmadd(m0407b, self.twiddles_im[2], x2m9);
392 let m0407b = SseVector::fmadd(m0407b, self.twiddles_im[0], x3m8);
393 let m0407b = SseVector::fmadd(m0407b, self.twiddles_im[4], x4m7);
394 let m0407b = SseVector::nmadd(m0407b, self.twiddles_im[1], x5m6);
395 let [y04, y07] = SseVector::column_butterfly2([m0407a, m0407b]);
396
397 let m0506a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p10);
398 let m0506a = SseVector::fmadd(m0506a, self.twiddles_re[0], x2p9);
399 let m0506a = SseVector::fmadd(m0506a, self.twiddles_re[3], x3p8);
400 let m0506a = SseVector::fmadd(m0506a, self.twiddles_re[1], x4p7);
401 let m0506a = SseVector::fmadd(m0506a, self.twiddles_re[2], x5p6);
402 let m0506b = SseVector::mul(self.twiddles_im[4], x1m10);
403 let m0506b = SseVector::nmadd(m0506b, self.twiddles_im[0], x2m9);
404 let m0506b = SseVector::fmadd(m0506b, self.twiddles_im[3], x3m8);
405 let m0506b = SseVector::nmadd(m0506b, self.twiddles_im[1], x4m7);
406 let m0506b = SseVector::fmadd(m0506b, self.twiddles_im[2], x5m6);
407 let [y05, y06] = SseVector::column_butterfly2([m0506a, m0506b]);
408
409
410 [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10]
411 }
412}
413
414struct SseF64Butterfly11<T> {
415 direction: FftDirection,
416 twiddles_re: [__m128d; 5],
417 twiddles_im: [__m128d; 5],
418 _phantom: std::marker::PhantomData<T>,
419}
420
421boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly11, 11, |this: &SseF64Butterfly11<_>| this.direction);
422impl<T: FftNum> SseF64Butterfly11<T> {
423 #[target_feature(enable = "sse4.1")]
425 unsafe fn new(direction: FftDirection) -> Self {
426 assert_f64::<T>();
427 let twiddles = make_twiddles(11, direction);
428 unsafe {Self {
429 direction,
430 twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
431 twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
432 _phantom: std::marker::PhantomData,
433 }}
434 }
435
436 #[inline(always)]
437 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
438 let values = read_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10 });
439
440 let out = self.perform_fft_direct(values);
441
442 write_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10 });
443 }
444
445 #[inline(always)]
446 pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 11]) -> [__m128d; 11] {
447 let rotate = SseVector::make_rotate90(FftDirection::Inverse);
448
449 let y00 = values[0];
450 let [x1p10, x1m10] = SseVector::column_butterfly2([values[1], values[10]]);
451 let x1m10 = SseVector::apply_rotate90(rotate, x1m10);
452 let y00 = SseVector::add(y00, x1p10);
453 let [x2p9, x2m9] = SseVector::column_butterfly2([values[2], values[9]]);
454 let x2m9 = SseVector::apply_rotate90(rotate, x2m9);
455 let y00 = SseVector::add(y00, x2p9);
456 let [x3p8, x3m8] = SseVector::column_butterfly2([values[3], values[8]]);
457 let x3m8 = SseVector::apply_rotate90(rotate, x3m8);
458 let y00 = SseVector::add(y00, x3p8);
459 let [x4p7, x4m7] = SseVector::column_butterfly2([values[4], values[7]]);
460 let x4m7 = SseVector::apply_rotate90(rotate, x4m7);
461 let y00 = SseVector::add(y00, x4p7);
462 let [x5p6, x5m6] = SseVector::column_butterfly2([values[5], values[6]]);
463 let x5m6 = SseVector::apply_rotate90(rotate, x5m6);
464 let y00 = SseVector::add(y00, x5p6);
465
466 let m0110a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p10);
467 let m0110a = SseVector::fmadd(m0110a, self.twiddles_re[1], x2p9);
468 let m0110a = SseVector::fmadd(m0110a, self.twiddles_re[2], x3p8);
469 let m0110a = SseVector::fmadd(m0110a, self.twiddles_re[3], x4p7);
470 let m0110a = SseVector::fmadd(m0110a, self.twiddles_re[4], x5p6);
471 let m0110b = SseVector::mul(self.twiddles_im[0], x1m10);
472 let m0110b = SseVector::fmadd(m0110b, self.twiddles_im[1], x2m9);
473 let m0110b = SseVector::fmadd(m0110b, self.twiddles_im[2], x3m8);
474 let m0110b = SseVector::fmadd(m0110b, self.twiddles_im[3], x4m7);
475 let m0110b = SseVector::fmadd(m0110b, self.twiddles_im[4], x5m6);
476 let [y01, y10] = SseVector::column_butterfly2([m0110a, m0110b]);
477
478 let m0209a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p10);
479 let m0209a = SseVector::fmadd(m0209a, self.twiddles_re[3], x2p9);
480 let m0209a = SseVector::fmadd(m0209a, self.twiddles_re[4], x3p8);
481 let m0209a = SseVector::fmadd(m0209a, self.twiddles_re[2], x4p7);
482 let m0209a = SseVector::fmadd(m0209a, self.twiddles_re[0], x5p6);
483 let m0209b = SseVector::mul(self.twiddles_im[1], x1m10);
484 let m0209b = SseVector::fmadd(m0209b, self.twiddles_im[3], x2m9);
485 let m0209b = SseVector::nmadd(m0209b, self.twiddles_im[4], x3m8);
486 let m0209b = SseVector::nmadd(m0209b, self.twiddles_im[2], x4m7);
487 let m0209b = SseVector::nmadd(m0209b, self.twiddles_im[0], x5m6);
488 let [y02, y09] = SseVector::column_butterfly2([m0209a, m0209b]);
489
490 let m0308a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p10);
491 let m0308a = SseVector::fmadd(m0308a, self.twiddles_re[4], x2p9);
492 let m0308a = SseVector::fmadd(m0308a, self.twiddles_re[1], x3p8);
493 let m0308a = SseVector::fmadd(m0308a, self.twiddles_re[0], x4p7);
494 let m0308a = SseVector::fmadd(m0308a, self.twiddles_re[3], x5p6);
495 let m0308b = SseVector::mul(self.twiddles_im[2], x1m10);
496 let m0308b = SseVector::nmadd(m0308b, self.twiddles_im[4], x2m9);
497 let m0308b = SseVector::nmadd(m0308b, self.twiddles_im[1], x3m8);
498 let m0308b = SseVector::fmadd(m0308b, self.twiddles_im[0], x4m7);
499 let m0308b = SseVector::fmadd(m0308b, self.twiddles_im[3], x5m6);
500 let [y03, y08] = SseVector::column_butterfly2([m0308a, m0308b]);
501
502 let m0407a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p10);
503 let m0407a = SseVector::fmadd(m0407a, self.twiddles_re[2], x2p9);
504 let m0407a = SseVector::fmadd(m0407a, self.twiddles_re[0], x3p8);
505 let m0407a = SseVector::fmadd(m0407a, self.twiddles_re[4], x4p7);
506 let m0407a = SseVector::fmadd(m0407a, self.twiddles_re[1], x5p6);
507 let m0407b = SseVector::mul(self.twiddles_im[3], x1m10);
508 let m0407b = SseVector::nmadd(m0407b, self.twiddles_im[2], x2m9);
509 let m0407b = SseVector::fmadd(m0407b, self.twiddles_im[0], x3m8);
510 let m0407b = SseVector::fmadd(m0407b, self.twiddles_im[4], x4m7);
511 let m0407b = SseVector::nmadd(m0407b, self.twiddles_im[1], x5m6);
512 let [y04, y07] = SseVector::column_butterfly2([m0407a, m0407b]);
513
514 let m0506a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p10);
515 let m0506a = SseVector::fmadd(m0506a, self.twiddles_re[0], x2p9);
516 let m0506a = SseVector::fmadd(m0506a, self.twiddles_re[3], x3p8);
517 let m0506a = SseVector::fmadd(m0506a, self.twiddles_re[1], x4p7);
518 let m0506a = SseVector::fmadd(m0506a, self.twiddles_re[2], x5p6);
519 let m0506b = SseVector::mul(self.twiddles_im[4], x1m10);
520 let m0506b = SseVector::nmadd(m0506b, self.twiddles_im[0], x2m9);
521 let m0506b = SseVector::fmadd(m0506b, self.twiddles_im[3], x3m8);
522 let m0506b = SseVector::nmadd(m0506b, self.twiddles_im[1], x4m7);
523 let m0506b = SseVector::fmadd(m0506b, self.twiddles_im[2], x5m6);
524 let [y05, y06] = SseVector::column_butterfly2([m0506a, m0506b]);
525
526
527 [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10]
528 }
529}
530
531struct SseF32Butterfly13<T> {
532 direction: FftDirection,
533 twiddles_re: [__m128; 6],
534 twiddles_im: [__m128; 6],
535 _phantom: std::marker::PhantomData<T>,
536}
537
538boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly13, 13, |this: &SseF32Butterfly13<_>| this.direction);
539impl<T: FftNum> SseF32Butterfly13<T> {
540 #[target_feature(enable = "sse4.1")]
542 unsafe fn new(direction: FftDirection) -> Self {
543 assert_f32::<T>();
544 let twiddles = make_twiddles(13, direction);
545 Self {
546 direction,
547 twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
548 twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
549 _phantom: std::marker::PhantomData,
550 }
551 }
552
553 #[inline(always)]
554 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
555 let values = read_partial1_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12 });
556
557 let out = self.perform_parallel_fft_direct(values);
558
559 write_partial_lo_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12 } );
560 }
561
562 #[inline(always)]
563 pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
564 let input_packed = read_complex_to_array!(buffer, { 0,2,4,6,8,10,12,14,16,18,20,22,24 });
565
566 let values = [
567 extract_lo_hi_f32(input_packed[0], input_packed[6]),
568 extract_hi_lo_f32(input_packed[0], input_packed[7]),
569 extract_lo_hi_f32(input_packed[1], input_packed[7]),
570 extract_hi_lo_f32(input_packed[1], input_packed[8]),
571 extract_lo_hi_f32(input_packed[2], input_packed[8]),
572 extract_hi_lo_f32(input_packed[2], input_packed[9]),
573 extract_lo_hi_f32(input_packed[3], input_packed[9]),
574 extract_hi_lo_f32(input_packed[3], input_packed[10]),
575 extract_lo_hi_f32(input_packed[4], input_packed[10]),
576 extract_hi_lo_f32(input_packed[4], input_packed[11]),
577 extract_lo_hi_f32(input_packed[5], input_packed[11]),
578 extract_hi_lo_f32(input_packed[5], input_packed[12]),
579 extract_lo_hi_f32(input_packed[6], input_packed[12]),
580 ];
581
582 let out = self.perform_parallel_fft_direct(values);
583
584 let out_packed = [
585 extract_lo_lo_f32(out[0], out[1]),
586 extract_lo_lo_f32(out[2], out[3]),
587 extract_lo_lo_f32(out[4], out[5]),
588 extract_lo_lo_f32(out[6], out[7]),
589 extract_lo_lo_f32(out[8], out[9]),
590 extract_lo_lo_f32(out[10], out[11]),
591 extract_lo_hi_f32(out[12], out[0]),
592 extract_hi_hi_f32(out[1], out[2]),
593 extract_hi_hi_f32(out[3], out[4]),
594 extract_hi_hi_f32(out[5], out[6]),
595 extract_hi_hi_f32(out[7], out[8]),
596 extract_hi_hi_f32(out[9], out[10]),
597 extract_hi_hi_f32(out[11], out[12]),
598 ];
599
600 write_complex_to_array_strided!(out_packed, buffer, 2, { 0,1,2,3,4,5,6,7,8,9,10,11,12 });
601 }
602
603 #[inline(always)]
604 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 13]) -> [__m128; 13] {
605 let rotate = SseVector::make_rotate90(FftDirection::Inverse);
606
607 let y00 = values[0];
608 let [x1p12, x1m12] = SseVector::column_butterfly2([values[1], values[12]]);
609 let x1m12 = SseVector::apply_rotate90(rotate, x1m12);
610 let y00 = SseVector::add(y00, x1p12);
611 let [x2p11, x2m11] = SseVector::column_butterfly2([values[2], values[11]]);
612 let x2m11 = SseVector::apply_rotate90(rotate, x2m11);
613 let y00 = SseVector::add(y00, x2p11);
614 let [x3p10, x3m10] = SseVector::column_butterfly2([values[3], values[10]]);
615 let x3m10 = SseVector::apply_rotate90(rotate, x3m10);
616 let y00 = SseVector::add(y00, x3p10);
617 let [x4p9, x4m9] = SseVector::column_butterfly2([values[4], values[9]]);
618 let x4m9 = SseVector::apply_rotate90(rotate, x4m9);
619 let y00 = SseVector::add(y00, x4p9);
620 let [x5p8, x5m8] = SseVector::column_butterfly2([values[5], values[8]]);
621 let x5m8 = SseVector::apply_rotate90(rotate, x5m8);
622 let y00 = SseVector::add(y00, x5p8);
623 let [x6p7, x6m7] = SseVector::column_butterfly2([values[6], values[7]]);
624 let x6m7 = SseVector::apply_rotate90(rotate, x6m7);
625 let y00 = SseVector::add(y00, x6p7);
626
627 let m0112a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p12);
628 let m0112a = SseVector::fmadd(m0112a, self.twiddles_re[1], x2p11);
629 let m0112a = SseVector::fmadd(m0112a, self.twiddles_re[2], x3p10);
630 let m0112a = SseVector::fmadd(m0112a, self.twiddles_re[3], x4p9);
631 let m0112a = SseVector::fmadd(m0112a, self.twiddles_re[4], x5p8);
632 let m0112a = SseVector::fmadd(m0112a, self.twiddles_re[5], x6p7);
633 let m0112b = SseVector::mul(self.twiddles_im[0], x1m12);
634 let m0112b = SseVector::fmadd(m0112b, self.twiddles_im[1], x2m11);
635 let m0112b = SseVector::fmadd(m0112b, self.twiddles_im[2], x3m10);
636 let m0112b = SseVector::fmadd(m0112b, self.twiddles_im[3], x4m9);
637 let m0112b = SseVector::fmadd(m0112b, self.twiddles_im[4], x5m8);
638 let m0112b = SseVector::fmadd(m0112b, self.twiddles_im[5], x6m7);
639 let [y01, y12] = SseVector::column_butterfly2([m0112a, m0112b]);
640
641 let m0211a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p12);
642 let m0211a = SseVector::fmadd(m0211a, self.twiddles_re[3], x2p11);
643 let m0211a = SseVector::fmadd(m0211a, self.twiddles_re[5], x3p10);
644 let m0211a = SseVector::fmadd(m0211a, self.twiddles_re[4], x4p9);
645 let m0211a = SseVector::fmadd(m0211a, self.twiddles_re[2], x5p8);
646 let m0211a = SseVector::fmadd(m0211a, self.twiddles_re[0], x6p7);
647 let m0211b = SseVector::mul(self.twiddles_im[1], x1m12);
648 let m0211b = SseVector::fmadd(m0211b, self.twiddles_im[3], x2m11);
649 let m0211b = SseVector::fmadd(m0211b, self.twiddles_im[5], x3m10);
650 let m0211b = SseVector::nmadd(m0211b, self.twiddles_im[4], x4m9);
651 let m0211b = SseVector::nmadd(m0211b, self.twiddles_im[2], x5m8);
652 let m0211b = SseVector::nmadd(m0211b, self.twiddles_im[0], x6m7);
653 let [y02, y11] = SseVector::column_butterfly2([m0211a, m0211b]);
654
655 let m0310a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p12);
656 let m0310a = SseVector::fmadd(m0310a, self.twiddles_re[5], x2p11);
657 let m0310a = SseVector::fmadd(m0310a, self.twiddles_re[3], x3p10);
658 let m0310a = SseVector::fmadd(m0310a, self.twiddles_re[0], x4p9);
659 let m0310a = SseVector::fmadd(m0310a, self.twiddles_re[1], x5p8);
660 let m0310a = SseVector::fmadd(m0310a, self.twiddles_re[4], x6p7);
661 let m0310b = SseVector::mul(self.twiddles_im[2], x1m12);
662 let m0310b = SseVector::fmadd(m0310b, self.twiddles_im[5], x2m11);
663 let m0310b = SseVector::nmadd(m0310b, self.twiddles_im[3], x3m10);
664 let m0310b = SseVector::nmadd(m0310b, self.twiddles_im[0], x4m9);
665 let m0310b = SseVector::fmadd(m0310b, self.twiddles_im[1], x5m8);
666 let m0310b = SseVector::fmadd(m0310b, self.twiddles_im[4], x6m7);
667 let [y03, y10] = SseVector::column_butterfly2([m0310a, m0310b]);
668
669 let m0409a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p12);
670 let m0409a = SseVector::fmadd(m0409a, self.twiddles_re[4], x2p11);
671 let m0409a = SseVector::fmadd(m0409a, self.twiddles_re[0], x3p10);
672 let m0409a = SseVector::fmadd(m0409a, self.twiddles_re[2], x4p9);
673 let m0409a = SseVector::fmadd(m0409a, self.twiddles_re[5], x5p8);
674 let m0409a = SseVector::fmadd(m0409a, self.twiddles_re[1], x6p7);
675 let m0409b = SseVector::mul(self.twiddles_im[3], x1m12);
676 let m0409b = SseVector::nmadd(m0409b, self.twiddles_im[4], x2m11);
677 let m0409b = SseVector::nmadd(m0409b, self.twiddles_im[0], x3m10);
678 let m0409b = SseVector::fmadd(m0409b, self.twiddles_im[2], x4m9);
679 let m0409b = SseVector::nmadd(m0409b, self.twiddles_im[5], x5m8);
680 let m0409b = SseVector::nmadd(m0409b, self.twiddles_im[1], x6m7);
681 let [y04, y09] = SseVector::column_butterfly2([m0409a, m0409b]);
682
683 let m0508a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p12);
684 let m0508a = SseVector::fmadd(m0508a, self.twiddles_re[2], x2p11);
685 let m0508a = SseVector::fmadd(m0508a, self.twiddles_re[1], x3p10);
686 let m0508a = SseVector::fmadd(m0508a, self.twiddles_re[5], x4p9);
687 let m0508a = SseVector::fmadd(m0508a, self.twiddles_re[0], x5p8);
688 let m0508a = SseVector::fmadd(m0508a, self.twiddles_re[3], x6p7);
689 let m0508b = SseVector::mul(self.twiddles_im[4], x1m12);
690 let m0508b = SseVector::nmadd(m0508b, self.twiddles_im[2], x2m11);
691 let m0508b = SseVector::fmadd(m0508b, self.twiddles_im[1], x3m10);
692 let m0508b = SseVector::nmadd(m0508b, self.twiddles_im[5], x4m9);
693 let m0508b = SseVector::nmadd(m0508b, self.twiddles_im[0], x5m8);
694 let m0508b = SseVector::fmadd(m0508b, self.twiddles_im[3], x6m7);
695 let [y05, y08] = SseVector::column_butterfly2([m0508a, m0508b]);
696
697 let m0607a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p12);
698 let m0607a = SseVector::fmadd(m0607a, self.twiddles_re[0], x2p11);
699 let m0607a = SseVector::fmadd(m0607a, self.twiddles_re[4], x3p10);
700 let m0607a = SseVector::fmadd(m0607a, self.twiddles_re[1], x4p9);
701 let m0607a = SseVector::fmadd(m0607a, self.twiddles_re[3], x5p8);
702 let m0607a = SseVector::fmadd(m0607a, self.twiddles_re[2], x6p7);
703 let m0607b = SseVector::mul(self.twiddles_im[5], x1m12);
704 let m0607b = SseVector::nmadd(m0607b, self.twiddles_im[0], x2m11);
705 let m0607b = SseVector::fmadd(m0607b, self.twiddles_im[4], x3m10);
706 let m0607b = SseVector::nmadd(m0607b, self.twiddles_im[1], x4m9);
707 let m0607b = SseVector::fmadd(m0607b, self.twiddles_im[3], x5m8);
708 let m0607b = SseVector::nmadd(m0607b, self.twiddles_im[2], x6m7);
709 let [y06, y07] = SseVector::column_butterfly2([m0607a, m0607b]);
710
711
712 [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12]
713 }
714}
715
716struct SseF64Butterfly13<T> {
717 direction: FftDirection,
718 twiddles_re: [__m128d; 6],
719 twiddles_im: [__m128d; 6],
720 _phantom: std::marker::PhantomData<T>,
721}
722
723boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly13, 13, |this: &SseF64Butterfly13<_>| this.direction);
724impl<T: FftNum> SseF64Butterfly13<T> {
725 #[target_feature(enable = "sse4.1")]
727 unsafe fn new(direction: FftDirection) -> Self {
728 assert_f64::<T>();
729 let twiddles = make_twiddles(13, direction);
730 unsafe {Self {
731 direction,
732 twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
733 twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
734 _phantom: std::marker::PhantomData,
735 }}
736 }
737
738 #[inline(always)]
739 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
740 let values = read_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12 });
741
742 let out = self.perform_fft_direct(values);
743
744 write_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12 });
745 }
746
747 #[inline(always)]
748 pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 13]) -> [__m128d; 13] {
749 let rotate = SseVector::make_rotate90(FftDirection::Inverse);
750
751 let y00 = values[0];
752 let [x1p12, x1m12] = SseVector::column_butterfly2([values[1], values[12]]);
753 let x1m12 = SseVector::apply_rotate90(rotate, x1m12);
754 let y00 = SseVector::add(y00, x1p12);
755 let [x2p11, x2m11] = SseVector::column_butterfly2([values[2], values[11]]);
756 let x2m11 = SseVector::apply_rotate90(rotate, x2m11);
757 let y00 = SseVector::add(y00, x2p11);
758 let [x3p10, x3m10] = SseVector::column_butterfly2([values[3], values[10]]);
759 let x3m10 = SseVector::apply_rotate90(rotate, x3m10);
760 let y00 = SseVector::add(y00, x3p10);
761 let [x4p9, x4m9] = SseVector::column_butterfly2([values[4], values[9]]);
762 let x4m9 = SseVector::apply_rotate90(rotate, x4m9);
763 let y00 = SseVector::add(y00, x4p9);
764 let [x5p8, x5m8] = SseVector::column_butterfly2([values[5], values[8]]);
765 let x5m8 = SseVector::apply_rotate90(rotate, x5m8);
766 let y00 = SseVector::add(y00, x5p8);
767 let [x6p7, x6m7] = SseVector::column_butterfly2([values[6], values[7]]);
768 let x6m7 = SseVector::apply_rotate90(rotate, x6m7);
769 let y00 = SseVector::add(y00, x6p7);
770
771 let m0112a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p12);
772 let m0112a = SseVector::fmadd(m0112a, self.twiddles_re[1], x2p11);
773 let m0112a = SseVector::fmadd(m0112a, self.twiddles_re[2], x3p10);
774 let m0112a = SseVector::fmadd(m0112a, self.twiddles_re[3], x4p9);
775 let m0112a = SseVector::fmadd(m0112a, self.twiddles_re[4], x5p8);
776 let m0112a = SseVector::fmadd(m0112a, self.twiddles_re[5], x6p7);
777 let m0112b = SseVector::mul(self.twiddles_im[0], x1m12);
778 let m0112b = SseVector::fmadd(m0112b, self.twiddles_im[1], x2m11);
779 let m0112b = SseVector::fmadd(m0112b, self.twiddles_im[2], x3m10);
780 let m0112b = SseVector::fmadd(m0112b, self.twiddles_im[3], x4m9);
781 let m0112b = SseVector::fmadd(m0112b, self.twiddles_im[4], x5m8);
782 let m0112b = SseVector::fmadd(m0112b, self.twiddles_im[5], x6m7);
783 let [y01, y12] = SseVector::column_butterfly2([m0112a, m0112b]);
784
785 let m0211a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p12);
786 let m0211a = SseVector::fmadd(m0211a, self.twiddles_re[3], x2p11);
787 let m0211a = SseVector::fmadd(m0211a, self.twiddles_re[5], x3p10);
788 let m0211a = SseVector::fmadd(m0211a, self.twiddles_re[4], x4p9);
789 let m0211a = SseVector::fmadd(m0211a, self.twiddles_re[2], x5p8);
790 let m0211a = SseVector::fmadd(m0211a, self.twiddles_re[0], x6p7);
791 let m0211b = SseVector::mul(self.twiddles_im[1], x1m12);
792 let m0211b = SseVector::fmadd(m0211b, self.twiddles_im[3], x2m11);
793 let m0211b = SseVector::fmadd(m0211b, self.twiddles_im[5], x3m10);
794 let m0211b = SseVector::nmadd(m0211b, self.twiddles_im[4], x4m9);
795 let m0211b = SseVector::nmadd(m0211b, self.twiddles_im[2], x5m8);
796 let m0211b = SseVector::nmadd(m0211b, self.twiddles_im[0], x6m7);
797 let [y02, y11] = SseVector::column_butterfly2([m0211a, m0211b]);
798
799 let m0310a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p12);
800 let m0310a = SseVector::fmadd(m0310a, self.twiddles_re[5], x2p11);
801 let m0310a = SseVector::fmadd(m0310a, self.twiddles_re[3], x3p10);
802 let m0310a = SseVector::fmadd(m0310a, self.twiddles_re[0], x4p9);
803 let m0310a = SseVector::fmadd(m0310a, self.twiddles_re[1], x5p8);
804 let m0310a = SseVector::fmadd(m0310a, self.twiddles_re[4], x6p7);
805 let m0310b = SseVector::mul(self.twiddles_im[2], x1m12);
806 let m0310b = SseVector::fmadd(m0310b, self.twiddles_im[5], x2m11);
807 let m0310b = SseVector::nmadd(m0310b, self.twiddles_im[3], x3m10);
808 let m0310b = SseVector::nmadd(m0310b, self.twiddles_im[0], x4m9);
809 let m0310b = SseVector::fmadd(m0310b, self.twiddles_im[1], x5m8);
810 let m0310b = SseVector::fmadd(m0310b, self.twiddles_im[4], x6m7);
811 let [y03, y10] = SseVector::column_butterfly2([m0310a, m0310b]);
812
813 let m0409a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p12);
814 let m0409a = SseVector::fmadd(m0409a, self.twiddles_re[4], x2p11);
815 let m0409a = SseVector::fmadd(m0409a, self.twiddles_re[0], x3p10);
816 let m0409a = SseVector::fmadd(m0409a, self.twiddles_re[2], x4p9);
817 let m0409a = SseVector::fmadd(m0409a, self.twiddles_re[5], x5p8);
818 let m0409a = SseVector::fmadd(m0409a, self.twiddles_re[1], x6p7);
819 let m0409b = SseVector::mul(self.twiddles_im[3], x1m12);
820 let m0409b = SseVector::nmadd(m0409b, self.twiddles_im[4], x2m11);
821 let m0409b = SseVector::nmadd(m0409b, self.twiddles_im[0], x3m10);
822 let m0409b = SseVector::fmadd(m0409b, self.twiddles_im[2], x4m9);
823 let m0409b = SseVector::nmadd(m0409b, self.twiddles_im[5], x5m8);
824 let m0409b = SseVector::nmadd(m0409b, self.twiddles_im[1], x6m7);
825 let [y04, y09] = SseVector::column_butterfly2([m0409a, m0409b]);
826
827 let m0508a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p12);
828 let m0508a = SseVector::fmadd(m0508a, self.twiddles_re[2], x2p11);
829 let m0508a = SseVector::fmadd(m0508a, self.twiddles_re[1], x3p10);
830 let m0508a = SseVector::fmadd(m0508a, self.twiddles_re[5], x4p9);
831 let m0508a = SseVector::fmadd(m0508a, self.twiddles_re[0], x5p8);
832 let m0508a = SseVector::fmadd(m0508a, self.twiddles_re[3], x6p7);
833 let m0508b = SseVector::mul(self.twiddles_im[4], x1m12);
834 let m0508b = SseVector::nmadd(m0508b, self.twiddles_im[2], x2m11);
835 let m0508b = SseVector::fmadd(m0508b, self.twiddles_im[1], x3m10);
836 let m0508b = SseVector::nmadd(m0508b, self.twiddles_im[5], x4m9);
837 let m0508b = SseVector::nmadd(m0508b, self.twiddles_im[0], x5m8);
838 let m0508b = SseVector::fmadd(m0508b, self.twiddles_im[3], x6m7);
839 let [y05, y08] = SseVector::column_butterfly2([m0508a, m0508b]);
840
841 let m0607a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p12);
842 let m0607a = SseVector::fmadd(m0607a, self.twiddles_re[0], x2p11);
843 let m0607a = SseVector::fmadd(m0607a, self.twiddles_re[4], x3p10);
844 let m0607a = SseVector::fmadd(m0607a, self.twiddles_re[1], x4p9);
845 let m0607a = SseVector::fmadd(m0607a, self.twiddles_re[3], x5p8);
846 let m0607a = SseVector::fmadd(m0607a, self.twiddles_re[2], x6p7);
847 let m0607b = SseVector::mul(self.twiddles_im[5], x1m12);
848 let m0607b = SseVector::nmadd(m0607b, self.twiddles_im[0], x2m11);
849 let m0607b = SseVector::fmadd(m0607b, self.twiddles_im[4], x3m10);
850 let m0607b = SseVector::nmadd(m0607b, self.twiddles_im[1], x4m9);
851 let m0607b = SseVector::fmadd(m0607b, self.twiddles_im[3], x5m8);
852 let m0607b = SseVector::nmadd(m0607b, self.twiddles_im[2], x6m7);
853 let [y06, y07] = SseVector::column_butterfly2([m0607a, m0607b]);
854
855
856 [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12]
857 }
858}
859
860struct SseF32Butterfly17<T> {
861 direction: FftDirection,
862 twiddles_re: [__m128; 8],
863 twiddles_im: [__m128; 8],
864 _phantom: std::marker::PhantomData<T>,
865}
866
867boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly17, 17, |this: &SseF32Butterfly17<_>| this.direction);
868impl<T: FftNum> SseF32Butterfly17<T> {
869 #[target_feature(enable = "sse4.1")]
871 unsafe fn new(direction: FftDirection) -> Self {
872 assert_f32::<T>();
873 let twiddles = make_twiddles(17, direction);
874 Self {
875 direction,
876 twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
877 twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
878 _phantom: std::marker::PhantomData,
879 }
880 }
881
882 #[inline(always)]
883 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
884 let values = read_partial1_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16 });
885
886 let out = self.perform_parallel_fft_direct(values);
887
888 write_partial_lo_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16 } );
889 }
890
891 #[inline(always)]
892 pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
893 let input_packed = read_complex_to_array!(buffer, { 0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32 });
894
895 let values = [
896 extract_lo_hi_f32(input_packed[0], input_packed[8]),
897 extract_hi_lo_f32(input_packed[0], input_packed[9]),
898 extract_lo_hi_f32(input_packed[1], input_packed[9]),
899 extract_hi_lo_f32(input_packed[1], input_packed[10]),
900 extract_lo_hi_f32(input_packed[2], input_packed[10]),
901 extract_hi_lo_f32(input_packed[2], input_packed[11]),
902 extract_lo_hi_f32(input_packed[3], input_packed[11]),
903 extract_hi_lo_f32(input_packed[3], input_packed[12]),
904 extract_lo_hi_f32(input_packed[4], input_packed[12]),
905 extract_hi_lo_f32(input_packed[4], input_packed[13]),
906 extract_lo_hi_f32(input_packed[5], input_packed[13]),
907 extract_hi_lo_f32(input_packed[5], input_packed[14]),
908 extract_lo_hi_f32(input_packed[6], input_packed[14]),
909 extract_hi_lo_f32(input_packed[6], input_packed[15]),
910 extract_lo_hi_f32(input_packed[7], input_packed[15]),
911 extract_hi_lo_f32(input_packed[7], input_packed[16]),
912 extract_lo_hi_f32(input_packed[8], input_packed[16]),
913 ];
914
915 let out = self.perform_parallel_fft_direct(values);
916
917 let out_packed = [
918 extract_lo_lo_f32(out[0], out[1]),
919 extract_lo_lo_f32(out[2], out[3]),
920 extract_lo_lo_f32(out[4], out[5]),
921 extract_lo_lo_f32(out[6], out[7]),
922 extract_lo_lo_f32(out[8], out[9]),
923 extract_lo_lo_f32(out[10], out[11]),
924 extract_lo_lo_f32(out[12], out[13]),
925 extract_lo_lo_f32(out[14], out[15]),
926 extract_lo_hi_f32(out[16], out[0]),
927 extract_hi_hi_f32(out[1], out[2]),
928 extract_hi_hi_f32(out[3], out[4]),
929 extract_hi_hi_f32(out[5], out[6]),
930 extract_hi_hi_f32(out[7], out[8]),
931 extract_hi_hi_f32(out[9], out[10]),
932 extract_hi_hi_f32(out[11], out[12]),
933 extract_hi_hi_f32(out[13], out[14]),
934 extract_hi_hi_f32(out[15], out[16]),
935 ];
936
937 write_complex_to_array_strided!(out_packed, buffer, 2, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16 });
938 }
939
940 #[inline(always)]
941 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 17]) -> [__m128; 17] {
942 let rotate = SseVector::make_rotate90(FftDirection::Inverse);
943
944 let y00 = values[0];
945 let [x1p16, x1m16] = SseVector::column_butterfly2([values[1], values[16]]);
946 let x1m16 = SseVector::apply_rotate90(rotate, x1m16);
947 let y00 = SseVector::add(y00, x1p16);
948 let [x2p15, x2m15] = SseVector::column_butterfly2([values[2], values[15]]);
949 let x2m15 = SseVector::apply_rotate90(rotate, x2m15);
950 let y00 = SseVector::add(y00, x2p15);
951 let [x3p14, x3m14] = SseVector::column_butterfly2([values[3], values[14]]);
952 let x3m14 = SseVector::apply_rotate90(rotate, x3m14);
953 let y00 = SseVector::add(y00, x3p14);
954 let [x4p13, x4m13] = SseVector::column_butterfly2([values[4], values[13]]);
955 let x4m13 = SseVector::apply_rotate90(rotate, x4m13);
956 let y00 = SseVector::add(y00, x4p13);
957 let [x5p12, x5m12] = SseVector::column_butterfly2([values[5], values[12]]);
958 let x5m12 = SseVector::apply_rotate90(rotate, x5m12);
959 let y00 = SseVector::add(y00, x5p12);
960 let [x6p11, x6m11] = SseVector::column_butterfly2([values[6], values[11]]);
961 let x6m11 = SseVector::apply_rotate90(rotate, x6m11);
962 let y00 = SseVector::add(y00, x6p11);
963 let [x7p10, x7m10] = SseVector::column_butterfly2([values[7], values[10]]);
964 let x7m10 = SseVector::apply_rotate90(rotate, x7m10);
965 let y00 = SseVector::add(y00, x7p10);
966 let [x8p9, x8m9] = SseVector::column_butterfly2([values[8], values[9]]);
967 let x8m9 = SseVector::apply_rotate90(rotate, x8m9);
968 let y00 = SseVector::add(y00, x8p9);
969
970 let m0116a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p16);
971 let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[1], x2p15);
972 let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[2], x3p14);
973 let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[3], x4p13);
974 let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[4], x5p12);
975 let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[5], x6p11);
976 let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[6], x7p10);
977 let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[7], x8p9);
978 let m0116b = SseVector::mul(self.twiddles_im[0], x1m16);
979 let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[1], x2m15);
980 let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[2], x3m14);
981 let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[3], x4m13);
982 let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[4], x5m12);
983 let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[5], x6m11);
984 let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[6], x7m10);
985 let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[7], x8m9);
986 let [y01, y16] = SseVector::column_butterfly2([m0116a, m0116b]);
987
988 let m0215a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p16);
989 let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[3], x2p15);
990 let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[5], x3p14);
991 let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[7], x4p13);
992 let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[6], x5p12);
993 let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[4], x6p11);
994 let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[2], x7p10);
995 let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[0], x8p9);
996 let m0215b = SseVector::mul(self.twiddles_im[1], x1m16);
997 let m0215b = SseVector::fmadd(m0215b, self.twiddles_im[3], x2m15);
998 let m0215b = SseVector::fmadd(m0215b, self.twiddles_im[5], x3m14);
999 let m0215b = SseVector::fmadd(m0215b, self.twiddles_im[7], x4m13);
1000 let m0215b = SseVector::nmadd(m0215b, self.twiddles_im[6], x5m12);
1001 let m0215b = SseVector::nmadd(m0215b, self.twiddles_im[4], x6m11);
1002 let m0215b = SseVector::nmadd(m0215b, self.twiddles_im[2], x7m10);
1003 let m0215b = SseVector::nmadd(m0215b, self.twiddles_im[0], x8m9);
1004 let [y02, y15] = SseVector::column_butterfly2([m0215a, m0215b]);
1005
1006 let m0314a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p16);
1007 let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[5], x2p15);
1008 let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[7], x3p14);
1009 let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[4], x4p13);
1010 let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[1], x5p12);
1011 let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[0], x6p11);
1012 let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[3], x7p10);
1013 let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[6], x8p9);
1014 let m0314b = SseVector::mul(self.twiddles_im[2], x1m16);
1015 let m0314b = SseVector::fmadd(m0314b, self.twiddles_im[5], x2m15);
1016 let m0314b = SseVector::nmadd(m0314b, self.twiddles_im[7], x3m14);
1017 let m0314b = SseVector::nmadd(m0314b, self.twiddles_im[4], x4m13);
1018 let m0314b = SseVector::nmadd(m0314b, self.twiddles_im[1], x5m12);
1019 let m0314b = SseVector::fmadd(m0314b, self.twiddles_im[0], x6m11);
1020 let m0314b = SseVector::fmadd(m0314b, self.twiddles_im[3], x7m10);
1021 let m0314b = SseVector::fmadd(m0314b, self.twiddles_im[6], x8m9);
1022 let [y03, y14] = SseVector::column_butterfly2([m0314a, m0314b]);
1023
1024 let m0413a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p16);
1025 let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[7], x2p15);
1026 let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[4], x3p14);
1027 let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[0], x4p13);
1028 let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[2], x5p12);
1029 let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[6], x6p11);
1030 let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[5], x7p10);
1031 let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[1], x8p9);
1032 let m0413b = SseVector::mul(self.twiddles_im[3], x1m16);
1033 let m0413b = SseVector::fmadd(m0413b, self.twiddles_im[7], x2m15);
1034 let m0413b = SseVector::nmadd(m0413b, self.twiddles_im[4], x3m14);
1035 let m0413b = SseVector::nmadd(m0413b, self.twiddles_im[0], x4m13);
1036 let m0413b = SseVector::fmadd(m0413b, self.twiddles_im[2], x5m12);
1037 let m0413b = SseVector::fmadd(m0413b, self.twiddles_im[6], x6m11);
1038 let m0413b = SseVector::nmadd(m0413b, self.twiddles_im[5], x7m10);
1039 let m0413b = SseVector::nmadd(m0413b, self.twiddles_im[1], x8m9);
1040 let [y04, y13] = SseVector::column_butterfly2([m0413a, m0413b]);
1041
1042 let m0512a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p16);
1043 let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[6], x2p15);
1044 let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[1], x3p14);
1045 let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[2], x4p13);
1046 let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[7], x5p12);
1047 let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[3], x6p11);
1048 let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[0], x7p10);
1049 let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[5], x8p9);
1050 let m0512b = SseVector::mul(self.twiddles_im[4], x1m16);
1051 let m0512b = SseVector::nmadd(m0512b, self.twiddles_im[6], x2m15);
1052 let m0512b = SseVector::nmadd(m0512b, self.twiddles_im[1], x3m14);
1053 let m0512b = SseVector::fmadd(m0512b, self.twiddles_im[2], x4m13);
1054 let m0512b = SseVector::fmadd(m0512b, self.twiddles_im[7], x5m12);
1055 let m0512b = SseVector::nmadd(m0512b, self.twiddles_im[3], x6m11);
1056 let m0512b = SseVector::fmadd(m0512b, self.twiddles_im[0], x7m10);
1057 let m0512b = SseVector::fmadd(m0512b, self.twiddles_im[5], x8m9);
1058 let [y05, y12] = SseVector::column_butterfly2([m0512a, m0512b]);
1059
1060 let m0611a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p16);
1061 let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[4], x2p15);
1062 let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[0], x3p14);
1063 let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[6], x4p13);
1064 let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[3], x5p12);
1065 let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[1], x6p11);
1066 let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[7], x7p10);
1067 let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[2], x8p9);
1068 let m0611b = SseVector::mul(self.twiddles_im[5], x1m16);
1069 let m0611b = SseVector::nmadd(m0611b, self.twiddles_im[4], x2m15);
1070 let m0611b = SseVector::fmadd(m0611b, self.twiddles_im[0], x3m14);
1071 let m0611b = SseVector::fmadd(m0611b, self.twiddles_im[6], x4m13);
1072 let m0611b = SseVector::nmadd(m0611b, self.twiddles_im[3], x5m12);
1073 let m0611b = SseVector::fmadd(m0611b, self.twiddles_im[1], x6m11);
1074 let m0611b = SseVector::fmadd(m0611b, self.twiddles_im[7], x7m10);
1075 let m0611b = SseVector::nmadd(m0611b, self.twiddles_im[2], x8m9);
1076 let [y06, y11] = SseVector::column_butterfly2([m0611a, m0611b]);
1077
1078 let m0710a = SseVector::fmadd(values[0], self.twiddles_re[6], x1p16);
1079 let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[2], x2p15);
1080 let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[3], x3p14);
1081 let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[5], x4p13);
1082 let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[0], x5p12);
1083 let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[7], x6p11);
1084 let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[1], x7p10);
1085 let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[4], x8p9);
1086 let m0710b = SseVector::mul(self.twiddles_im[6], x1m16);
1087 let m0710b = SseVector::nmadd(m0710b, self.twiddles_im[2], x2m15);
1088 let m0710b = SseVector::fmadd(m0710b, self.twiddles_im[3], x3m14);
1089 let m0710b = SseVector::nmadd(m0710b, self.twiddles_im[5], x4m13);
1090 let m0710b = SseVector::fmadd(m0710b, self.twiddles_im[0], x5m12);
1091 let m0710b = SseVector::fmadd(m0710b, self.twiddles_im[7], x6m11);
1092 let m0710b = SseVector::nmadd(m0710b, self.twiddles_im[1], x7m10);
1093 let m0710b = SseVector::fmadd(m0710b, self.twiddles_im[4], x8m9);
1094 let [y07, y10] = SseVector::column_butterfly2([m0710a, m0710b]);
1095
1096 let m0809a = SseVector::fmadd(values[0], self.twiddles_re[7], x1p16);
1097 let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[0], x2p15);
1098 let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[6], x3p14);
1099 let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[1], x4p13);
1100 let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[5], x5p12);
1101 let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[2], x6p11);
1102 let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[4], x7p10);
1103 let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[3], x8p9);
1104 let m0809b = SseVector::mul(self.twiddles_im[7], x1m16);
1105 let m0809b = SseVector::nmadd(m0809b, self.twiddles_im[0], x2m15);
1106 let m0809b = SseVector::fmadd(m0809b, self.twiddles_im[6], x3m14);
1107 let m0809b = SseVector::nmadd(m0809b, self.twiddles_im[1], x4m13);
1108 let m0809b = SseVector::fmadd(m0809b, self.twiddles_im[5], x5m12);
1109 let m0809b = SseVector::nmadd(m0809b, self.twiddles_im[2], x6m11);
1110 let m0809b = SseVector::fmadd(m0809b, self.twiddles_im[4], x7m10);
1111 let m0809b = SseVector::nmadd(m0809b, self.twiddles_im[3], x8m9);
1112 let [y08, y09] = SseVector::column_butterfly2([m0809a, m0809b]);
1113
1114
1115 [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12, y13, y14, y15, y16]
1116 }
1117}
1118
1119struct SseF64Butterfly17<T> {
1120 direction: FftDirection,
1121 twiddles_re: [__m128d; 8],
1122 twiddles_im: [__m128d; 8],
1123 _phantom: std::marker::PhantomData<T>,
1124}
1125
1126boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly17, 17, |this: &SseF64Butterfly17<_>| this.direction);
1127impl<T: FftNum> SseF64Butterfly17<T> {
1128 #[target_feature(enable = "sse4.1")]
1130 unsafe fn new(direction: FftDirection) -> Self {
1131 assert_f64::<T>();
1132 let twiddles = make_twiddles(17, direction);
1133 unsafe {Self {
1134 direction,
1135 twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
1136 twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
1137 _phantom: std::marker::PhantomData,
1138 }}
1139 }
1140
1141 #[inline(always)]
1142 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
1143 let values = read_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16 });
1144
1145 let out = self.perform_fft_direct(values);
1146
1147 write_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16 });
1148 }
1149
1150 #[inline(always)]
1151 pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 17]) -> [__m128d; 17] {
1152 let rotate = SseVector::make_rotate90(FftDirection::Inverse);
1153
1154 let y00 = values[0];
1155 let [x1p16, x1m16] = SseVector::column_butterfly2([values[1], values[16]]);
1156 let x1m16 = SseVector::apply_rotate90(rotate, x1m16);
1157 let y00 = SseVector::add(y00, x1p16);
1158 let [x2p15, x2m15] = SseVector::column_butterfly2([values[2], values[15]]);
1159 let x2m15 = SseVector::apply_rotate90(rotate, x2m15);
1160 let y00 = SseVector::add(y00, x2p15);
1161 let [x3p14, x3m14] = SseVector::column_butterfly2([values[3], values[14]]);
1162 let x3m14 = SseVector::apply_rotate90(rotate, x3m14);
1163 let y00 = SseVector::add(y00, x3p14);
1164 let [x4p13, x4m13] = SseVector::column_butterfly2([values[4], values[13]]);
1165 let x4m13 = SseVector::apply_rotate90(rotate, x4m13);
1166 let y00 = SseVector::add(y00, x4p13);
1167 let [x5p12, x5m12] = SseVector::column_butterfly2([values[5], values[12]]);
1168 let x5m12 = SseVector::apply_rotate90(rotate, x5m12);
1169 let y00 = SseVector::add(y00, x5p12);
1170 let [x6p11, x6m11] = SseVector::column_butterfly2([values[6], values[11]]);
1171 let x6m11 = SseVector::apply_rotate90(rotate, x6m11);
1172 let y00 = SseVector::add(y00, x6p11);
1173 let [x7p10, x7m10] = SseVector::column_butterfly2([values[7], values[10]]);
1174 let x7m10 = SseVector::apply_rotate90(rotate, x7m10);
1175 let y00 = SseVector::add(y00, x7p10);
1176 let [x8p9, x8m9] = SseVector::column_butterfly2([values[8], values[9]]);
1177 let x8m9 = SseVector::apply_rotate90(rotate, x8m9);
1178 let y00 = SseVector::add(y00, x8p9);
1179
1180 let m0116a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p16);
1181 let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[1], x2p15);
1182 let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[2], x3p14);
1183 let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[3], x4p13);
1184 let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[4], x5p12);
1185 let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[5], x6p11);
1186 let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[6], x7p10);
1187 let m0116a = SseVector::fmadd(m0116a, self.twiddles_re[7], x8p9);
1188 let m0116b = SseVector::mul(self.twiddles_im[0], x1m16);
1189 let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[1], x2m15);
1190 let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[2], x3m14);
1191 let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[3], x4m13);
1192 let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[4], x5m12);
1193 let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[5], x6m11);
1194 let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[6], x7m10);
1195 let m0116b = SseVector::fmadd(m0116b, self.twiddles_im[7], x8m9);
1196 let [y01, y16] = SseVector::column_butterfly2([m0116a, m0116b]);
1197
1198 let m0215a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p16);
1199 let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[3], x2p15);
1200 let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[5], x3p14);
1201 let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[7], x4p13);
1202 let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[6], x5p12);
1203 let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[4], x6p11);
1204 let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[2], x7p10);
1205 let m0215a = SseVector::fmadd(m0215a, self.twiddles_re[0], x8p9);
1206 let m0215b = SseVector::mul(self.twiddles_im[1], x1m16);
1207 let m0215b = SseVector::fmadd(m0215b, self.twiddles_im[3], x2m15);
1208 let m0215b = SseVector::fmadd(m0215b, self.twiddles_im[5], x3m14);
1209 let m0215b = SseVector::fmadd(m0215b, self.twiddles_im[7], x4m13);
1210 let m0215b = SseVector::nmadd(m0215b, self.twiddles_im[6], x5m12);
1211 let m0215b = SseVector::nmadd(m0215b, self.twiddles_im[4], x6m11);
1212 let m0215b = SseVector::nmadd(m0215b, self.twiddles_im[2], x7m10);
1213 let m0215b = SseVector::nmadd(m0215b, self.twiddles_im[0], x8m9);
1214 let [y02, y15] = SseVector::column_butterfly2([m0215a, m0215b]);
1215
1216 let m0314a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p16);
1217 let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[5], x2p15);
1218 let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[7], x3p14);
1219 let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[4], x4p13);
1220 let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[1], x5p12);
1221 let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[0], x6p11);
1222 let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[3], x7p10);
1223 let m0314a = SseVector::fmadd(m0314a, self.twiddles_re[6], x8p9);
1224 let m0314b = SseVector::mul(self.twiddles_im[2], x1m16);
1225 let m0314b = SseVector::fmadd(m0314b, self.twiddles_im[5], x2m15);
1226 let m0314b = SseVector::nmadd(m0314b, self.twiddles_im[7], x3m14);
1227 let m0314b = SseVector::nmadd(m0314b, self.twiddles_im[4], x4m13);
1228 let m0314b = SseVector::nmadd(m0314b, self.twiddles_im[1], x5m12);
1229 let m0314b = SseVector::fmadd(m0314b, self.twiddles_im[0], x6m11);
1230 let m0314b = SseVector::fmadd(m0314b, self.twiddles_im[3], x7m10);
1231 let m0314b = SseVector::fmadd(m0314b, self.twiddles_im[6], x8m9);
1232 let [y03, y14] = SseVector::column_butterfly2([m0314a, m0314b]);
1233
1234 let m0413a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p16);
1235 let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[7], x2p15);
1236 let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[4], x3p14);
1237 let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[0], x4p13);
1238 let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[2], x5p12);
1239 let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[6], x6p11);
1240 let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[5], x7p10);
1241 let m0413a = SseVector::fmadd(m0413a, self.twiddles_re[1], x8p9);
1242 let m0413b = SseVector::mul(self.twiddles_im[3], x1m16);
1243 let m0413b = SseVector::fmadd(m0413b, self.twiddles_im[7], x2m15);
1244 let m0413b = SseVector::nmadd(m0413b, self.twiddles_im[4], x3m14);
1245 let m0413b = SseVector::nmadd(m0413b, self.twiddles_im[0], x4m13);
1246 let m0413b = SseVector::fmadd(m0413b, self.twiddles_im[2], x5m12);
1247 let m0413b = SseVector::fmadd(m0413b, self.twiddles_im[6], x6m11);
1248 let m0413b = SseVector::nmadd(m0413b, self.twiddles_im[5], x7m10);
1249 let m0413b = SseVector::nmadd(m0413b, self.twiddles_im[1], x8m9);
1250 let [y04, y13] = SseVector::column_butterfly2([m0413a, m0413b]);
1251
1252 let m0512a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p16);
1253 let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[6], x2p15);
1254 let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[1], x3p14);
1255 let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[2], x4p13);
1256 let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[7], x5p12);
1257 let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[3], x6p11);
1258 let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[0], x7p10);
1259 let m0512a = SseVector::fmadd(m0512a, self.twiddles_re[5], x8p9);
1260 let m0512b = SseVector::mul(self.twiddles_im[4], x1m16);
1261 let m0512b = SseVector::nmadd(m0512b, self.twiddles_im[6], x2m15);
1262 let m0512b = SseVector::nmadd(m0512b, self.twiddles_im[1], x3m14);
1263 let m0512b = SseVector::fmadd(m0512b, self.twiddles_im[2], x4m13);
1264 let m0512b = SseVector::fmadd(m0512b, self.twiddles_im[7], x5m12);
1265 let m0512b = SseVector::nmadd(m0512b, self.twiddles_im[3], x6m11);
1266 let m0512b = SseVector::fmadd(m0512b, self.twiddles_im[0], x7m10);
1267 let m0512b = SseVector::fmadd(m0512b, self.twiddles_im[5], x8m9);
1268 let [y05, y12] = SseVector::column_butterfly2([m0512a, m0512b]);
1269
1270 let m0611a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p16);
1271 let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[4], x2p15);
1272 let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[0], x3p14);
1273 let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[6], x4p13);
1274 let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[3], x5p12);
1275 let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[1], x6p11);
1276 let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[7], x7p10);
1277 let m0611a = SseVector::fmadd(m0611a, self.twiddles_re[2], x8p9);
1278 let m0611b = SseVector::mul(self.twiddles_im[5], x1m16);
1279 let m0611b = SseVector::nmadd(m0611b, self.twiddles_im[4], x2m15);
1280 let m0611b = SseVector::fmadd(m0611b, self.twiddles_im[0], x3m14);
1281 let m0611b = SseVector::fmadd(m0611b, self.twiddles_im[6], x4m13);
1282 let m0611b = SseVector::nmadd(m0611b, self.twiddles_im[3], x5m12);
1283 let m0611b = SseVector::fmadd(m0611b, self.twiddles_im[1], x6m11);
1284 let m0611b = SseVector::fmadd(m0611b, self.twiddles_im[7], x7m10);
1285 let m0611b = SseVector::nmadd(m0611b, self.twiddles_im[2], x8m9);
1286 let [y06, y11] = SseVector::column_butterfly2([m0611a, m0611b]);
1287
1288 let m0710a = SseVector::fmadd(values[0], self.twiddles_re[6], x1p16);
1289 let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[2], x2p15);
1290 let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[3], x3p14);
1291 let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[5], x4p13);
1292 let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[0], x5p12);
1293 let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[7], x6p11);
1294 let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[1], x7p10);
1295 let m0710a = SseVector::fmadd(m0710a, self.twiddles_re[4], x8p9);
1296 let m0710b = SseVector::mul(self.twiddles_im[6], x1m16);
1297 let m0710b = SseVector::nmadd(m0710b, self.twiddles_im[2], x2m15);
1298 let m0710b = SseVector::fmadd(m0710b, self.twiddles_im[3], x3m14);
1299 let m0710b = SseVector::nmadd(m0710b, self.twiddles_im[5], x4m13);
1300 let m0710b = SseVector::fmadd(m0710b, self.twiddles_im[0], x5m12);
1301 let m0710b = SseVector::fmadd(m0710b, self.twiddles_im[7], x6m11);
1302 let m0710b = SseVector::nmadd(m0710b, self.twiddles_im[1], x7m10);
1303 let m0710b = SseVector::fmadd(m0710b, self.twiddles_im[4], x8m9);
1304 let [y07, y10] = SseVector::column_butterfly2([m0710a, m0710b]);
1305
1306 let m0809a = SseVector::fmadd(values[0], self.twiddles_re[7], x1p16);
1307 let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[0], x2p15);
1308 let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[6], x3p14);
1309 let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[1], x4p13);
1310 let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[5], x5p12);
1311 let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[2], x6p11);
1312 let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[4], x7p10);
1313 let m0809a = SseVector::fmadd(m0809a, self.twiddles_re[3], x8p9);
1314 let m0809b = SseVector::mul(self.twiddles_im[7], x1m16);
1315 let m0809b = SseVector::nmadd(m0809b, self.twiddles_im[0], x2m15);
1316 let m0809b = SseVector::fmadd(m0809b, self.twiddles_im[6], x3m14);
1317 let m0809b = SseVector::nmadd(m0809b, self.twiddles_im[1], x4m13);
1318 let m0809b = SseVector::fmadd(m0809b, self.twiddles_im[5], x5m12);
1319 let m0809b = SseVector::nmadd(m0809b, self.twiddles_im[2], x6m11);
1320 let m0809b = SseVector::fmadd(m0809b, self.twiddles_im[4], x7m10);
1321 let m0809b = SseVector::nmadd(m0809b, self.twiddles_im[3], x8m9);
1322 let [y08, y09] = SseVector::column_butterfly2([m0809a, m0809b]);
1323
1324
1325 [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12, y13, y14, y15, y16]
1326 }
1327}
1328
1329struct SseF32Butterfly19<T> {
1330 direction: FftDirection,
1331 twiddles_re: [__m128; 9],
1332 twiddles_im: [__m128; 9],
1333 _phantom: std::marker::PhantomData<T>,
1334}
1335
1336boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly19, 19, |this: &SseF32Butterfly19<_>| this.direction);
1337impl<T: FftNum> SseF32Butterfly19<T> {
1338 #[target_feature(enable = "sse4.1")]
1340 unsafe fn new(direction: FftDirection) -> Self {
1341 assert_f32::<T>();
1342 let twiddles = make_twiddles(19, direction);
1343 Self {
1344 direction,
1345 twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
1346 twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
1347 _phantom: std::marker::PhantomData,
1348 }
1349 }
1350
1351 #[inline(always)]
1352 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
1353 let values = read_partial1_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18 });
1354
1355 let out = self.perform_parallel_fft_direct(values);
1356
1357 write_partial_lo_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18 } );
1358 }
1359
1360 #[inline(always)]
1361 pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
1362 let input_packed = read_complex_to_array!(buffer, { 0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36 });
1363
1364 let values = [
1365 extract_lo_hi_f32(input_packed[0], input_packed[9]),
1366 extract_hi_lo_f32(input_packed[0], input_packed[10]),
1367 extract_lo_hi_f32(input_packed[1], input_packed[10]),
1368 extract_hi_lo_f32(input_packed[1], input_packed[11]),
1369 extract_lo_hi_f32(input_packed[2], input_packed[11]),
1370 extract_hi_lo_f32(input_packed[2], input_packed[12]),
1371 extract_lo_hi_f32(input_packed[3], input_packed[12]),
1372 extract_hi_lo_f32(input_packed[3], input_packed[13]),
1373 extract_lo_hi_f32(input_packed[4], input_packed[13]),
1374 extract_hi_lo_f32(input_packed[4], input_packed[14]),
1375 extract_lo_hi_f32(input_packed[5], input_packed[14]),
1376 extract_hi_lo_f32(input_packed[5], input_packed[15]),
1377 extract_lo_hi_f32(input_packed[6], input_packed[15]),
1378 extract_hi_lo_f32(input_packed[6], input_packed[16]),
1379 extract_lo_hi_f32(input_packed[7], input_packed[16]),
1380 extract_hi_lo_f32(input_packed[7], input_packed[17]),
1381 extract_lo_hi_f32(input_packed[8], input_packed[17]),
1382 extract_hi_lo_f32(input_packed[8], input_packed[18]),
1383 extract_lo_hi_f32(input_packed[9], input_packed[18]),
1384 ];
1385
1386 let out = self.perform_parallel_fft_direct(values);
1387
1388 let out_packed = [
1389 extract_lo_lo_f32(out[0], out[1]),
1390 extract_lo_lo_f32(out[2], out[3]),
1391 extract_lo_lo_f32(out[4], out[5]),
1392 extract_lo_lo_f32(out[6], out[7]),
1393 extract_lo_lo_f32(out[8], out[9]),
1394 extract_lo_lo_f32(out[10], out[11]),
1395 extract_lo_lo_f32(out[12], out[13]),
1396 extract_lo_lo_f32(out[14], out[15]),
1397 extract_lo_lo_f32(out[16], out[17]),
1398 extract_lo_hi_f32(out[18], out[0]),
1399 extract_hi_hi_f32(out[1], out[2]),
1400 extract_hi_hi_f32(out[3], out[4]),
1401 extract_hi_hi_f32(out[5], out[6]),
1402 extract_hi_hi_f32(out[7], out[8]),
1403 extract_hi_hi_f32(out[9], out[10]),
1404 extract_hi_hi_f32(out[11], out[12]),
1405 extract_hi_hi_f32(out[13], out[14]),
1406 extract_hi_hi_f32(out[15], out[16]),
1407 extract_hi_hi_f32(out[17], out[18]),
1408 ];
1409
1410 write_complex_to_array_strided!(out_packed, buffer, 2, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18 });
1411 }
1412
1413 #[inline(always)]
1414 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 19]) -> [__m128; 19] {
1415 let rotate = SseVector::make_rotate90(FftDirection::Inverse);
1416
1417 let y00 = values[0];
1418 let [x1p18, x1m18] = SseVector::column_butterfly2([values[1], values[18]]);
1419 let x1m18 = SseVector::apply_rotate90(rotate, x1m18);
1420 let y00 = SseVector::add(y00, x1p18);
1421 let [x2p17, x2m17] = SseVector::column_butterfly2([values[2], values[17]]);
1422 let x2m17 = SseVector::apply_rotate90(rotate, x2m17);
1423 let y00 = SseVector::add(y00, x2p17);
1424 let [x3p16, x3m16] = SseVector::column_butterfly2([values[3], values[16]]);
1425 let x3m16 = SseVector::apply_rotate90(rotate, x3m16);
1426 let y00 = SseVector::add(y00, x3p16);
1427 let [x4p15, x4m15] = SseVector::column_butterfly2([values[4], values[15]]);
1428 let x4m15 = SseVector::apply_rotate90(rotate, x4m15);
1429 let y00 = SseVector::add(y00, x4p15);
1430 let [x5p14, x5m14] = SseVector::column_butterfly2([values[5], values[14]]);
1431 let x5m14 = SseVector::apply_rotate90(rotate, x5m14);
1432 let y00 = SseVector::add(y00, x5p14);
1433 let [x6p13, x6m13] = SseVector::column_butterfly2([values[6], values[13]]);
1434 let x6m13 = SseVector::apply_rotate90(rotate, x6m13);
1435 let y00 = SseVector::add(y00, x6p13);
1436 let [x7p12, x7m12] = SseVector::column_butterfly2([values[7], values[12]]);
1437 let x7m12 = SseVector::apply_rotate90(rotate, x7m12);
1438 let y00 = SseVector::add(y00, x7p12);
1439 let [x8p11, x8m11] = SseVector::column_butterfly2([values[8], values[11]]);
1440 let x8m11 = SseVector::apply_rotate90(rotate, x8m11);
1441 let y00 = SseVector::add(y00, x8p11);
1442 let [x9p10, x9m10] = SseVector::column_butterfly2([values[9], values[10]]);
1443 let x9m10 = SseVector::apply_rotate90(rotate, x9m10);
1444 let y00 = SseVector::add(y00, x9p10);
1445
1446 let m0118a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p18);
1447 let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[1], x2p17);
1448 let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[2], x3p16);
1449 let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[3], x4p15);
1450 let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[4], x5p14);
1451 let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[5], x6p13);
1452 let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[6], x7p12);
1453 let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[7], x8p11);
1454 let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[8], x9p10);
1455 let m0118b = SseVector::mul(self.twiddles_im[0], x1m18);
1456 let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[1], x2m17);
1457 let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[2], x3m16);
1458 let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[3], x4m15);
1459 let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[4], x5m14);
1460 let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[5], x6m13);
1461 let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[6], x7m12);
1462 let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[7], x8m11);
1463 let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[8], x9m10);
1464 let [y01, y18] = SseVector::column_butterfly2([m0118a, m0118b]);
1465
1466 let m0217a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p18);
1467 let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[3], x2p17);
1468 let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[5], x3p16);
1469 let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[7], x4p15);
1470 let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[8], x5p14);
1471 let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[6], x6p13);
1472 let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[4], x7p12);
1473 let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[2], x8p11);
1474 let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[0], x9p10);
1475 let m0217b = SseVector::mul(self.twiddles_im[1], x1m18);
1476 let m0217b = SseVector::fmadd(m0217b, self.twiddles_im[3], x2m17);
1477 let m0217b = SseVector::fmadd(m0217b, self.twiddles_im[5], x3m16);
1478 let m0217b = SseVector::fmadd(m0217b, self.twiddles_im[7], x4m15);
1479 let m0217b = SseVector::nmadd(m0217b, self.twiddles_im[8], x5m14);
1480 let m0217b = SseVector::nmadd(m0217b, self.twiddles_im[6], x6m13);
1481 let m0217b = SseVector::nmadd(m0217b, self.twiddles_im[4], x7m12);
1482 let m0217b = SseVector::nmadd(m0217b, self.twiddles_im[2], x8m11);
1483 let m0217b = SseVector::nmadd(m0217b, self.twiddles_im[0], x9m10);
1484 let [y02, y17] = SseVector::column_butterfly2([m0217a, m0217b]);
1485
1486 let m0316a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p18);
1487 let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[5], x2p17);
1488 let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[8], x3p16);
1489 let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[6], x4p15);
1490 let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[3], x5p14);
1491 let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[0], x6p13);
1492 let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[1], x7p12);
1493 let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[4], x8p11);
1494 let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[7], x9p10);
1495 let m0316b = SseVector::mul(self.twiddles_im[2], x1m18);
1496 let m0316b = SseVector::fmadd(m0316b, self.twiddles_im[5], x2m17);
1497 let m0316b = SseVector::fmadd(m0316b, self.twiddles_im[8], x3m16);
1498 let m0316b = SseVector::nmadd(m0316b, self.twiddles_im[6], x4m15);
1499 let m0316b = SseVector::nmadd(m0316b, self.twiddles_im[3], x5m14);
1500 let m0316b = SseVector::nmadd(m0316b, self.twiddles_im[0], x6m13);
1501 let m0316b = SseVector::fmadd(m0316b, self.twiddles_im[1], x7m12);
1502 let m0316b = SseVector::fmadd(m0316b, self.twiddles_im[4], x8m11);
1503 let m0316b = SseVector::fmadd(m0316b, self.twiddles_im[7], x9m10);
1504 let [y03, y16] = SseVector::column_butterfly2([m0316a, m0316b]);
1505
1506 let m0415a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p18);
1507 let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[7], x2p17);
1508 let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[6], x3p16);
1509 let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[2], x4p15);
1510 let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[0], x5p14);
1511 let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[4], x6p13);
1512 let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[8], x7p12);
1513 let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[5], x8p11);
1514 let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[1], x9p10);
1515 let m0415b = SseVector::mul(self.twiddles_im[3], x1m18);
1516 let m0415b = SseVector::fmadd(m0415b, self.twiddles_im[7], x2m17);
1517 let m0415b = SseVector::nmadd(m0415b, self.twiddles_im[6], x3m16);
1518 let m0415b = SseVector::nmadd(m0415b, self.twiddles_im[2], x4m15);
1519 let m0415b = SseVector::fmadd(m0415b, self.twiddles_im[0], x5m14);
1520 let m0415b = SseVector::fmadd(m0415b, self.twiddles_im[4], x6m13);
1521 let m0415b = SseVector::fmadd(m0415b, self.twiddles_im[8], x7m12);
1522 let m0415b = SseVector::nmadd(m0415b, self.twiddles_im[5], x8m11);
1523 let m0415b = SseVector::nmadd(m0415b, self.twiddles_im[1], x9m10);
1524 let [y04, y15] = SseVector::column_butterfly2([m0415a, m0415b]);
1525
1526 let m0514a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p18);
1527 let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[8], x2p17);
1528 let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[3], x3p16);
1529 let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[0], x4p15);
1530 let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[5], x5p14);
1531 let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[7], x6p13);
1532 let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[2], x7p12);
1533 let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[1], x8p11);
1534 let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[6], x9p10);
1535 let m0514b = SseVector::mul(self.twiddles_im[4], x1m18);
1536 let m0514b = SseVector::nmadd(m0514b, self.twiddles_im[8], x2m17);
1537 let m0514b = SseVector::nmadd(m0514b, self.twiddles_im[3], x3m16);
1538 let m0514b = SseVector::fmadd(m0514b, self.twiddles_im[0], x4m15);
1539 let m0514b = SseVector::fmadd(m0514b, self.twiddles_im[5], x5m14);
1540 let m0514b = SseVector::nmadd(m0514b, self.twiddles_im[7], x6m13);
1541 let m0514b = SseVector::nmadd(m0514b, self.twiddles_im[2], x7m12);
1542 let m0514b = SseVector::fmadd(m0514b, self.twiddles_im[1], x8m11);
1543 let m0514b = SseVector::fmadd(m0514b, self.twiddles_im[6], x9m10);
1544 let [y05, y14] = SseVector::column_butterfly2([m0514a, m0514b]);
1545
1546 let m0613a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p18);
1547 let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[6], x2p17);
1548 let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[0], x3p16);
1549 let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[4], x4p15);
1550 let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[7], x5p14);
1551 let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[1], x6p13);
1552 let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[3], x7p12);
1553 let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[8], x8p11);
1554 let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[2], x9p10);
1555 let m0613b = SseVector::mul(self.twiddles_im[5], x1m18);
1556 let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[6], x2m17);
1557 let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[0], x3m16);
1558 let m0613b = SseVector::fmadd(m0613b, self.twiddles_im[4], x4m15);
1559 let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[7], x5m14);
1560 let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[1], x6m13);
1561 let m0613b = SseVector::fmadd(m0613b, self.twiddles_im[3], x7m12);
1562 let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[8], x8m11);
1563 let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[2], x9m10);
1564 let [y06, y13] = SseVector::column_butterfly2([m0613a, m0613b]);
1565
1566 let m0712a = SseVector::fmadd(values[0], self.twiddles_re[6], x1p18);
1567 let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[4], x2p17);
1568 let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[1], x3p16);
1569 let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[8], x4p15);
1570 let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[2], x5p14);
1571 let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[3], x6p13);
1572 let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[7], x7p12);
1573 let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[0], x8p11);
1574 let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[5], x9p10);
1575 let m0712b = SseVector::mul(self.twiddles_im[6], x1m18);
1576 let m0712b = SseVector::nmadd(m0712b, self.twiddles_im[4], x2m17);
1577 let m0712b = SseVector::fmadd(m0712b, self.twiddles_im[1], x3m16);
1578 let m0712b = SseVector::fmadd(m0712b, self.twiddles_im[8], x4m15);
1579 let m0712b = SseVector::nmadd(m0712b, self.twiddles_im[2], x5m14);
1580 let m0712b = SseVector::fmadd(m0712b, self.twiddles_im[3], x6m13);
1581 let m0712b = SseVector::nmadd(m0712b, self.twiddles_im[7], x7m12);
1582 let m0712b = SseVector::nmadd(m0712b, self.twiddles_im[0], x8m11);
1583 let m0712b = SseVector::fmadd(m0712b, self.twiddles_im[5], x9m10);
1584 let [y07, y12] = SseVector::column_butterfly2([m0712a, m0712b]);
1585
1586 let m0811a = SseVector::fmadd(values[0], self.twiddles_re[7], x1p18);
1587 let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[2], x2p17);
1588 let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[4], x3p16);
1589 let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[5], x4p15);
1590 let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[1], x5p14);
1591 let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[8], x6p13);
1592 let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[0], x7p12);
1593 let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[6], x8p11);
1594 let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[3], x9p10);
1595 let m0811b = SseVector::mul(self.twiddles_im[7], x1m18);
1596 let m0811b = SseVector::nmadd(m0811b, self.twiddles_im[2], x2m17);
1597 let m0811b = SseVector::fmadd(m0811b, self.twiddles_im[4], x3m16);
1598 let m0811b = SseVector::nmadd(m0811b, self.twiddles_im[5], x4m15);
1599 let m0811b = SseVector::fmadd(m0811b, self.twiddles_im[1], x5m14);
1600 let m0811b = SseVector::nmadd(m0811b, self.twiddles_im[8], x6m13);
1601 let m0811b = SseVector::nmadd(m0811b, self.twiddles_im[0], x7m12);
1602 let m0811b = SseVector::fmadd(m0811b, self.twiddles_im[6], x8m11);
1603 let m0811b = SseVector::nmadd(m0811b, self.twiddles_im[3], x9m10);
1604 let [y08, y11] = SseVector::column_butterfly2([m0811a, m0811b]);
1605
1606 let m0910a = SseVector::fmadd(values[0], self.twiddles_re[8], x1p18);
1607 let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[0], x2p17);
1608 let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[7], x3p16);
1609 let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[1], x4p15);
1610 let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[6], x5p14);
1611 let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[2], x6p13);
1612 let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[5], x7p12);
1613 let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[3], x8p11);
1614 let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[4], x9p10);
1615 let m0910b = SseVector::mul(self.twiddles_im[8], x1m18);
1616 let m0910b = SseVector::nmadd(m0910b, self.twiddles_im[0], x2m17);
1617 let m0910b = SseVector::fmadd(m0910b, self.twiddles_im[7], x3m16);
1618 let m0910b = SseVector::nmadd(m0910b, self.twiddles_im[1], x4m15);
1619 let m0910b = SseVector::fmadd(m0910b, self.twiddles_im[6], x5m14);
1620 let m0910b = SseVector::nmadd(m0910b, self.twiddles_im[2], x6m13);
1621 let m0910b = SseVector::fmadd(m0910b, self.twiddles_im[5], x7m12);
1622 let m0910b = SseVector::nmadd(m0910b, self.twiddles_im[3], x8m11);
1623 let m0910b = SseVector::fmadd(m0910b, self.twiddles_im[4], x9m10);
1624 let [y09, y10] = SseVector::column_butterfly2([m0910a, m0910b]);
1625
1626
1627 [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12, y13, y14, y15, y16, y17, y18]
1628 }
1629}
1630
1631struct SseF64Butterfly19<T> {
1632 direction: FftDirection,
1633 twiddles_re: [__m128d; 9],
1634 twiddles_im: [__m128d; 9],
1635 _phantom: std::marker::PhantomData<T>,
1636}
1637
1638boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly19, 19, |this: &SseF64Butterfly19<_>| this.direction);
1639impl<T: FftNum> SseF64Butterfly19<T> {
1640 #[target_feature(enable = "sse4.1")]
1642 unsafe fn new(direction: FftDirection) -> Self {
1643 assert_f64::<T>();
1644 let twiddles = make_twiddles(19, direction);
1645 unsafe {Self {
1646 direction,
1647 twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
1648 twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
1649 _phantom: std::marker::PhantomData,
1650 }}
1651 }
1652
1653 #[inline(always)]
1654 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
1655 let values = read_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18 });
1656
1657 let out = self.perform_fft_direct(values);
1658
1659 write_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18 });
1660 }
1661
1662 #[inline(always)]
1663 pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 19]) -> [__m128d; 19] {
1664 let rotate = SseVector::make_rotate90(FftDirection::Inverse);
1665
1666 let y00 = values[0];
1667 let [x1p18, x1m18] = SseVector::column_butterfly2([values[1], values[18]]);
1668 let x1m18 = SseVector::apply_rotate90(rotate, x1m18);
1669 let y00 = SseVector::add(y00, x1p18);
1670 let [x2p17, x2m17] = SseVector::column_butterfly2([values[2], values[17]]);
1671 let x2m17 = SseVector::apply_rotate90(rotate, x2m17);
1672 let y00 = SseVector::add(y00, x2p17);
1673 let [x3p16, x3m16] = SseVector::column_butterfly2([values[3], values[16]]);
1674 let x3m16 = SseVector::apply_rotate90(rotate, x3m16);
1675 let y00 = SseVector::add(y00, x3p16);
1676 let [x4p15, x4m15] = SseVector::column_butterfly2([values[4], values[15]]);
1677 let x4m15 = SseVector::apply_rotate90(rotate, x4m15);
1678 let y00 = SseVector::add(y00, x4p15);
1679 let [x5p14, x5m14] = SseVector::column_butterfly2([values[5], values[14]]);
1680 let x5m14 = SseVector::apply_rotate90(rotate, x5m14);
1681 let y00 = SseVector::add(y00, x5p14);
1682 let [x6p13, x6m13] = SseVector::column_butterfly2([values[6], values[13]]);
1683 let x6m13 = SseVector::apply_rotate90(rotate, x6m13);
1684 let y00 = SseVector::add(y00, x6p13);
1685 let [x7p12, x7m12] = SseVector::column_butterfly2([values[7], values[12]]);
1686 let x7m12 = SseVector::apply_rotate90(rotate, x7m12);
1687 let y00 = SseVector::add(y00, x7p12);
1688 let [x8p11, x8m11] = SseVector::column_butterfly2([values[8], values[11]]);
1689 let x8m11 = SseVector::apply_rotate90(rotate, x8m11);
1690 let y00 = SseVector::add(y00, x8p11);
1691 let [x9p10, x9m10] = SseVector::column_butterfly2([values[9], values[10]]);
1692 let x9m10 = SseVector::apply_rotate90(rotate, x9m10);
1693 let y00 = SseVector::add(y00, x9p10);
1694
1695 let m0118a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p18);
1696 let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[1], x2p17);
1697 let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[2], x3p16);
1698 let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[3], x4p15);
1699 let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[4], x5p14);
1700 let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[5], x6p13);
1701 let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[6], x7p12);
1702 let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[7], x8p11);
1703 let m0118a = SseVector::fmadd(m0118a, self.twiddles_re[8], x9p10);
1704 let m0118b = SseVector::mul(self.twiddles_im[0], x1m18);
1705 let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[1], x2m17);
1706 let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[2], x3m16);
1707 let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[3], x4m15);
1708 let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[4], x5m14);
1709 let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[5], x6m13);
1710 let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[6], x7m12);
1711 let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[7], x8m11);
1712 let m0118b = SseVector::fmadd(m0118b, self.twiddles_im[8], x9m10);
1713 let [y01, y18] = SseVector::column_butterfly2([m0118a, m0118b]);
1714
1715 let m0217a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p18);
1716 let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[3], x2p17);
1717 let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[5], x3p16);
1718 let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[7], x4p15);
1719 let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[8], x5p14);
1720 let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[6], x6p13);
1721 let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[4], x7p12);
1722 let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[2], x8p11);
1723 let m0217a = SseVector::fmadd(m0217a, self.twiddles_re[0], x9p10);
1724 let m0217b = SseVector::mul(self.twiddles_im[1], x1m18);
1725 let m0217b = SseVector::fmadd(m0217b, self.twiddles_im[3], x2m17);
1726 let m0217b = SseVector::fmadd(m0217b, self.twiddles_im[5], x3m16);
1727 let m0217b = SseVector::fmadd(m0217b, self.twiddles_im[7], x4m15);
1728 let m0217b = SseVector::nmadd(m0217b, self.twiddles_im[8], x5m14);
1729 let m0217b = SseVector::nmadd(m0217b, self.twiddles_im[6], x6m13);
1730 let m0217b = SseVector::nmadd(m0217b, self.twiddles_im[4], x7m12);
1731 let m0217b = SseVector::nmadd(m0217b, self.twiddles_im[2], x8m11);
1732 let m0217b = SseVector::nmadd(m0217b, self.twiddles_im[0], x9m10);
1733 let [y02, y17] = SseVector::column_butterfly2([m0217a, m0217b]);
1734
1735 let m0316a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p18);
1736 let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[5], x2p17);
1737 let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[8], x3p16);
1738 let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[6], x4p15);
1739 let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[3], x5p14);
1740 let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[0], x6p13);
1741 let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[1], x7p12);
1742 let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[4], x8p11);
1743 let m0316a = SseVector::fmadd(m0316a, self.twiddles_re[7], x9p10);
1744 let m0316b = SseVector::mul(self.twiddles_im[2], x1m18);
1745 let m0316b = SseVector::fmadd(m0316b, self.twiddles_im[5], x2m17);
1746 let m0316b = SseVector::fmadd(m0316b, self.twiddles_im[8], x3m16);
1747 let m0316b = SseVector::nmadd(m0316b, self.twiddles_im[6], x4m15);
1748 let m0316b = SseVector::nmadd(m0316b, self.twiddles_im[3], x5m14);
1749 let m0316b = SseVector::nmadd(m0316b, self.twiddles_im[0], x6m13);
1750 let m0316b = SseVector::fmadd(m0316b, self.twiddles_im[1], x7m12);
1751 let m0316b = SseVector::fmadd(m0316b, self.twiddles_im[4], x8m11);
1752 let m0316b = SseVector::fmadd(m0316b, self.twiddles_im[7], x9m10);
1753 let [y03, y16] = SseVector::column_butterfly2([m0316a, m0316b]);
1754
1755 let m0415a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p18);
1756 let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[7], x2p17);
1757 let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[6], x3p16);
1758 let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[2], x4p15);
1759 let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[0], x5p14);
1760 let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[4], x6p13);
1761 let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[8], x7p12);
1762 let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[5], x8p11);
1763 let m0415a = SseVector::fmadd(m0415a, self.twiddles_re[1], x9p10);
1764 let m0415b = SseVector::mul(self.twiddles_im[3], x1m18);
1765 let m0415b = SseVector::fmadd(m0415b, self.twiddles_im[7], x2m17);
1766 let m0415b = SseVector::nmadd(m0415b, self.twiddles_im[6], x3m16);
1767 let m0415b = SseVector::nmadd(m0415b, self.twiddles_im[2], x4m15);
1768 let m0415b = SseVector::fmadd(m0415b, self.twiddles_im[0], x5m14);
1769 let m0415b = SseVector::fmadd(m0415b, self.twiddles_im[4], x6m13);
1770 let m0415b = SseVector::fmadd(m0415b, self.twiddles_im[8], x7m12);
1771 let m0415b = SseVector::nmadd(m0415b, self.twiddles_im[5], x8m11);
1772 let m0415b = SseVector::nmadd(m0415b, self.twiddles_im[1], x9m10);
1773 let [y04, y15] = SseVector::column_butterfly2([m0415a, m0415b]);
1774
1775 let m0514a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p18);
1776 let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[8], x2p17);
1777 let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[3], x3p16);
1778 let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[0], x4p15);
1779 let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[5], x5p14);
1780 let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[7], x6p13);
1781 let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[2], x7p12);
1782 let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[1], x8p11);
1783 let m0514a = SseVector::fmadd(m0514a, self.twiddles_re[6], x9p10);
1784 let m0514b = SseVector::mul(self.twiddles_im[4], x1m18);
1785 let m0514b = SseVector::nmadd(m0514b, self.twiddles_im[8], x2m17);
1786 let m0514b = SseVector::nmadd(m0514b, self.twiddles_im[3], x3m16);
1787 let m0514b = SseVector::fmadd(m0514b, self.twiddles_im[0], x4m15);
1788 let m0514b = SseVector::fmadd(m0514b, self.twiddles_im[5], x5m14);
1789 let m0514b = SseVector::nmadd(m0514b, self.twiddles_im[7], x6m13);
1790 let m0514b = SseVector::nmadd(m0514b, self.twiddles_im[2], x7m12);
1791 let m0514b = SseVector::fmadd(m0514b, self.twiddles_im[1], x8m11);
1792 let m0514b = SseVector::fmadd(m0514b, self.twiddles_im[6], x9m10);
1793 let [y05, y14] = SseVector::column_butterfly2([m0514a, m0514b]);
1794
1795 let m0613a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p18);
1796 let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[6], x2p17);
1797 let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[0], x3p16);
1798 let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[4], x4p15);
1799 let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[7], x5p14);
1800 let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[1], x6p13);
1801 let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[3], x7p12);
1802 let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[8], x8p11);
1803 let m0613a = SseVector::fmadd(m0613a, self.twiddles_re[2], x9p10);
1804 let m0613b = SseVector::mul(self.twiddles_im[5], x1m18);
1805 let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[6], x2m17);
1806 let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[0], x3m16);
1807 let m0613b = SseVector::fmadd(m0613b, self.twiddles_im[4], x4m15);
1808 let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[7], x5m14);
1809 let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[1], x6m13);
1810 let m0613b = SseVector::fmadd(m0613b, self.twiddles_im[3], x7m12);
1811 let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[8], x8m11);
1812 let m0613b = SseVector::nmadd(m0613b, self.twiddles_im[2], x9m10);
1813 let [y06, y13] = SseVector::column_butterfly2([m0613a, m0613b]);
1814
1815 let m0712a = SseVector::fmadd(values[0], self.twiddles_re[6], x1p18);
1816 let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[4], x2p17);
1817 let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[1], x3p16);
1818 let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[8], x4p15);
1819 let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[2], x5p14);
1820 let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[3], x6p13);
1821 let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[7], x7p12);
1822 let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[0], x8p11);
1823 let m0712a = SseVector::fmadd(m0712a, self.twiddles_re[5], x9p10);
1824 let m0712b = SseVector::mul(self.twiddles_im[6], x1m18);
1825 let m0712b = SseVector::nmadd(m0712b, self.twiddles_im[4], x2m17);
1826 let m0712b = SseVector::fmadd(m0712b, self.twiddles_im[1], x3m16);
1827 let m0712b = SseVector::fmadd(m0712b, self.twiddles_im[8], x4m15);
1828 let m0712b = SseVector::nmadd(m0712b, self.twiddles_im[2], x5m14);
1829 let m0712b = SseVector::fmadd(m0712b, self.twiddles_im[3], x6m13);
1830 let m0712b = SseVector::nmadd(m0712b, self.twiddles_im[7], x7m12);
1831 let m0712b = SseVector::nmadd(m0712b, self.twiddles_im[0], x8m11);
1832 let m0712b = SseVector::fmadd(m0712b, self.twiddles_im[5], x9m10);
1833 let [y07, y12] = SseVector::column_butterfly2([m0712a, m0712b]);
1834
1835 let m0811a = SseVector::fmadd(values[0], self.twiddles_re[7], x1p18);
1836 let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[2], x2p17);
1837 let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[4], x3p16);
1838 let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[5], x4p15);
1839 let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[1], x5p14);
1840 let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[8], x6p13);
1841 let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[0], x7p12);
1842 let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[6], x8p11);
1843 let m0811a = SseVector::fmadd(m0811a, self.twiddles_re[3], x9p10);
1844 let m0811b = SseVector::mul(self.twiddles_im[7], x1m18);
1845 let m0811b = SseVector::nmadd(m0811b, self.twiddles_im[2], x2m17);
1846 let m0811b = SseVector::fmadd(m0811b, self.twiddles_im[4], x3m16);
1847 let m0811b = SseVector::nmadd(m0811b, self.twiddles_im[5], x4m15);
1848 let m0811b = SseVector::fmadd(m0811b, self.twiddles_im[1], x5m14);
1849 let m0811b = SseVector::nmadd(m0811b, self.twiddles_im[8], x6m13);
1850 let m0811b = SseVector::nmadd(m0811b, self.twiddles_im[0], x7m12);
1851 let m0811b = SseVector::fmadd(m0811b, self.twiddles_im[6], x8m11);
1852 let m0811b = SseVector::nmadd(m0811b, self.twiddles_im[3], x9m10);
1853 let [y08, y11] = SseVector::column_butterfly2([m0811a, m0811b]);
1854
1855 let m0910a = SseVector::fmadd(values[0], self.twiddles_re[8], x1p18);
1856 let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[0], x2p17);
1857 let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[7], x3p16);
1858 let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[1], x4p15);
1859 let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[6], x5p14);
1860 let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[2], x6p13);
1861 let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[5], x7p12);
1862 let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[3], x8p11);
1863 let m0910a = SseVector::fmadd(m0910a, self.twiddles_re[4], x9p10);
1864 let m0910b = SseVector::mul(self.twiddles_im[8], x1m18);
1865 let m0910b = SseVector::nmadd(m0910b, self.twiddles_im[0], x2m17);
1866 let m0910b = SseVector::fmadd(m0910b, self.twiddles_im[7], x3m16);
1867 let m0910b = SseVector::nmadd(m0910b, self.twiddles_im[1], x4m15);
1868 let m0910b = SseVector::fmadd(m0910b, self.twiddles_im[6], x5m14);
1869 let m0910b = SseVector::nmadd(m0910b, self.twiddles_im[2], x6m13);
1870 let m0910b = SseVector::fmadd(m0910b, self.twiddles_im[5], x7m12);
1871 let m0910b = SseVector::nmadd(m0910b, self.twiddles_im[3], x8m11);
1872 let m0910b = SseVector::fmadd(m0910b, self.twiddles_im[4], x9m10);
1873 let [y09, y10] = SseVector::column_butterfly2([m0910a, m0910b]);
1874
1875
1876 [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12, y13, y14, y15, y16, y17, y18]
1877 }
1878}
1879
1880struct SseF32Butterfly23<T> {
1881 direction: FftDirection,
1882 twiddles_re: [__m128; 11],
1883 twiddles_im: [__m128; 11],
1884 _phantom: std::marker::PhantomData<T>,
1885}
1886
1887boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly23, 23, |this: &SseF32Butterfly23<_>| this.direction);
1888impl<T: FftNum> SseF32Butterfly23<T> {
1889 #[target_feature(enable = "sse4.1")]
1891 unsafe fn new(direction: FftDirection) -> Self {
1892 assert_f32::<T>();
1893 let twiddles = make_twiddles(23, direction);
1894 Self {
1895 direction,
1896 twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
1897 twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
1898 _phantom: std::marker::PhantomData,
1899 }
1900 }
1901
1902 #[inline(always)]
1903 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
1904 let values = read_partial1_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22 });
1905
1906 let out = self.perform_parallel_fft_direct(values);
1907
1908 write_partial_lo_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22 } );
1909 }
1910
1911 #[inline(always)]
1912 pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
1913 let input_packed = read_complex_to_array!(buffer, { 0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44 });
1914
1915 let values = [
1916 extract_lo_hi_f32(input_packed[0], input_packed[11]),
1917 extract_hi_lo_f32(input_packed[0], input_packed[12]),
1918 extract_lo_hi_f32(input_packed[1], input_packed[12]),
1919 extract_hi_lo_f32(input_packed[1], input_packed[13]),
1920 extract_lo_hi_f32(input_packed[2], input_packed[13]),
1921 extract_hi_lo_f32(input_packed[2], input_packed[14]),
1922 extract_lo_hi_f32(input_packed[3], input_packed[14]),
1923 extract_hi_lo_f32(input_packed[3], input_packed[15]),
1924 extract_lo_hi_f32(input_packed[4], input_packed[15]),
1925 extract_hi_lo_f32(input_packed[4], input_packed[16]),
1926 extract_lo_hi_f32(input_packed[5], input_packed[16]),
1927 extract_hi_lo_f32(input_packed[5], input_packed[17]),
1928 extract_lo_hi_f32(input_packed[6], input_packed[17]),
1929 extract_hi_lo_f32(input_packed[6], input_packed[18]),
1930 extract_lo_hi_f32(input_packed[7], input_packed[18]),
1931 extract_hi_lo_f32(input_packed[7], input_packed[19]),
1932 extract_lo_hi_f32(input_packed[8], input_packed[19]),
1933 extract_hi_lo_f32(input_packed[8], input_packed[20]),
1934 extract_lo_hi_f32(input_packed[9], input_packed[20]),
1935 extract_hi_lo_f32(input_packed[9], input_packed[21]),
1936 extract_lo_hi_f32(input_packed[10], input_packed[21]),
1937 extract_hi_lo_f32(input_packed[10], input_packed[22]),
1938 extract_lo_hi_f32(input_packed[11], input_packed[22]),
1939 ];
1940
1941 let out = self.perform_parallel_fft_direct(values);
1942
1943 let out_packed = [
1944 extract_lo_lo_f32(out[0], out[1]),
1945 extract_lo_lo_f32(out[2], out[3]),
1946 extract_lo_lo_f32(out[4], out[5]),
1947 extract_lo_lo_f32(out[6], out[7]),
1948 extract_lo_lo_f32(out[8], out[9]),
1949 extract_lo_lo_f32(out[10], out[11]),
1950 extract_lo_lo_f32(out[12], out[13]),
1951 extract_lo_lo_f32(out[14], out[15]),
1952 extract_lo_lo_f32(out[16], out[17]),
1953 extract_lo_lo_f32(out[18], out[19]),
1954 extract_lo_lo_f32(out[20], out[21]),
1955 extract_lo_hi_f32(out[22], out[0]),
1956 extract_hi_hi_f32(out[1], out[2]),
1957 extract_hi_hi_f32(out[3], out[4]),
1958 extract_hi_hi_f32(out[5], out[6]),
1959 extract_hi_hi_f32(out[7], out[8]),
1960 extract_hi_hi_f32(out[9], out[10]),
1961 extract_hi_hi_f32(out[11], out[12]),
1962 extract_hi_hi_f32(out[13], out[14]),
1963 extract_hi_hi_f32(out[15], out[16]),
1964 extract_hi_hi_f32(out[17], out[18]),
1965 extract_hi_hi_f32(out[19], out[20]),
1966 extract_hi_hi_f32(out[21], out[22]),
1967 ];
1968
1969 write_complex_to_array_strided!(out_packed, buffer, 2, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22 });
1970 }
1971
1972 #[inline(always)]
1973 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 23]) -> [__m128; 23] {
1974 let rotate = SseVector::make_rotate90(FftDirection::Inverse);
1975
1976 let y00 = values[0];
1977 let [x1p22, x1m22] = SseVector::column_butterfly2([values[1], values[22]]);
1978 let x1m22 = SseVector::apply_rotate90(rotate, x1m22);
1979 let y00 = SseVector::add(y00, x1p22);
1980 let [x2p21, x2m21] = SseVector::column_butterfly2([values[2], values[21]]);
1981 let x2m21 = SseVector::apply_rotate90(rotate, x2m21);
1982 let y00 = SseVector::add(y00, x2p21);
1983 let [x3p20, x3m20] = SseVector::column_butterfly2([values[3], values[20]]);
1984 let x3m20 = SseVector::apply_rotate90(rotate, x3m20);
1985 let y00 = SseVector::add(y00, x3p20);
1986 let [x4p19, x4m19] = SseVector::column_butterfly2([values[4], values[19]]);
1987 let x4m19 = SseVector::apply_rotate90(rotate, x4m19);
1988 let y00 = SseVector::add(y00, x4p19);
1989 let [x5p18, x5m18] = SseVector::column_butterfly2([values[5], values[18]]);
1990 let x5m18 = SseVector::apply_rotate90(rotate, x5m18);
1991 let y00 = SseVector::add(y00, x5p18);
1992 let [x6p17, x6m17] = SseVector::column_butterfly2([values[6], values[17]]);
1993 let x6m17 = SseVector::apply_rotate90(rotate, x6m17);
1994 let y00 = SseVector::add(y00, x6p17);
1995 let [x7p16, x7m16] = SseVector::column_butterfly2([values[7], values[16]]);
1996 let x7m16 = SseVector::apply_rotate90(rotate, x7m16);
1997 let y00 = SseVector::add(y00, x7p16);
1998 let [x8p15, x8m15] = SseVector::column_butterfly2([values[8], values[15]]);
1999 let x8m15 = SseVector::apply_rotate90(rotate, x8m15);
2000 let y00 = SseVector::add(y00, x8p15);
2001 let [x9p14, x9m14] = SseVector::column_butterfly2([values[9], values[14]]);
2002 let x9m14 = SseVector::apply_rotate90(rotate, x9m14);
2003 let y00 = SseVector::add(y00, x9p14);
2004 let [x10p13, x10m13] = SseVector::column_butterfly2([values[10], values[13]]);
2005 let x10m13 = SseVector::apply_rotate90(rotate, x10m13);
2006 let y00 = SseVector::add(y00, x10p13);
2007 let [x11p12, x11m12] = SseVector::column_butterfly2([values[11], values[12]]);
2008 let x11m12 = SseVector::apply_rotate90(rotate, x11m12);
2009 let y00 = SseVector::add(y00, x11p12);
2010
2011 let m0122a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p22);
2012 let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[1], x2p21);
2013 let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[2], x3p20);
2014 let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[3], x4p19);
2015 let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[4], x5p18);
2016 let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[5], x6p17);
2017 let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[6], x7p16);
2018 let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[7], x8p15);
2019 let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[8], x9p14);
2020 let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[9], x10p13);
2021 let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[10], x11p12);
2022 let m0122b = SseVector::mul(self.twiddles_im[0], x1m22);
2023 let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[1], x2m21);
2024 let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[2], x3m20);
2025 let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[3], x4m19);
2026 let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[4], x5m18);
2027 let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[5], x6m17);
2028 let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[6], x7m16);
2029 let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[7], x8m15);
2030 let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[8], x9m14);
2031 let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[9], x10m13);
2032 let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[10], x11m12);
2033 let [y01, y22] = SseVector::column_butterfly2([m0122a, m0122b]);
2034
2035 let m0221a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p22);
2036 let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[3], x2p21);
2037 let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[5], x3p20);
2038 let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[7], x4p19);
2039 let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[9], x5p18);
2040 let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[10], x6p17);
2041 let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[8], x7p16);
2042 let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[6], x8p15);
2043 let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[4], x9p14);
2044 let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[2], x10p13);
2045 let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[0], x11p12);
2046 let m0221b = SseVector::mul(self.twiddles_im[1], x1m22);
2047 let m0221b = SseVector::fmadd(m0221b, self.twiddles_im[3], x2m21);
2048 let m0221b = SseVector::fmadd(m0221b, self.twiddles_im[5], x3m20);
2049 let m0221b = SseVector::fmadd(m0221b, self.twiddles_im[7], x4m19);
2050 let m0221b = SseVector::fmadd(m0221b, self.twiddles_im[9], x5m18);
2051 let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[10], x6m17);
2052 let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[8], x7m16);
2053 let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[6], x8m15);
2054 let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[4], x9m14);
2055 let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[2], x10m13);
2056 let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[0], x11m12);
2057 let [y02, y21] = SseVector::column_butterfly2([m0221a, m0221b]);
2058
2059 let m0320a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p22);
2060 let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[5], x2p21);
2061 let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[8], x3p20);
2062 let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[10], x4p19);
2063 let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[7], x5p18);
2064 let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[4], x6p17);
2065 let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[1], x7p16);
2066 let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[0], x8p15);
2067 let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[3], x9p14);
2068 let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[6], x10p13);
2069 let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[9], x11p12);
2070 let m0320b = SseVector::mul(self.twiddles_im[2], x1m22);
2071 let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[5], x2m21);
2072 let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[8], x3m20);
2073 let m0320b = SseVector::nmadd(m0320b, self.twiddles_im[10], x4m19);
2074 let m0320b = SseVector::nmadd(m0320b, self.twiddles_im[7], x5m18);
2075 let m0320b = SseVector::nmadd(m0320b, self.twiddles_im[4], x6m17);
2076 let m0320b = SseVector::nmadd(m0320b, self.twiddles_im[1], x7m16);
2077 let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[0], x8m15);
2078 let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[3], x9m14);
2079 let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[6], x10m13);
2080 let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[9], x11m12);
2081 let [y03, y20] = SseVector::column_butterfly2([m0320a, m0320b]);
2082
2083 let m0419a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p22);
2084 let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[7], x2p21);
2085 let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[10], x3p20);
2086 let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[6], x4p19);
2087 let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[2], x5p18);
2088 let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[0], x6p17);
2089 let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[4], x7p16);
2090 let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[8], x8p15);
2091 let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[9], x9p14);
2092 let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[5], x10p13);
2093 let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[1], x11p12);
2094 let m0419b = SseVector::mul(self.twiddles_im[3], x1m22);
2095 let m0419b = SseVector::fmadd(m0419b, self.twiddles_im[7], x2m21);
2096 let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[10], x3m20);
2097 let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[6], x4m19);
2098 let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[2], x5m18);
2099 let m0419b = SseVector::fmadd(m0419b, self.twiddles_im[0], x6m17);
2100 let m0419b = SseVector::fmadd(m0419b, self.twiddles_im[4], x7m16);
2101 let m0419b = SseVector::fmadd(m0419b, self.twiddles_im[8], x8m15);
2102 let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[9], x9m14);
2103 let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[5], x10m13);
2104 let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[1], x11m12);
2105 let [y04, y19] = SseVector::column_butterfly2([m0419a, m0419b]);
2106
2107 let m0518a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p22);
2108 let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[9], x2p21);
2109 let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[7], x3p20);
2110 let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[2], x4p19);
2111 let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[1], x5p18);
2112 let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[6], x6p17);
2113 let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[10], x7p16);
2114 let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[5], x8p15);
2115 let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[0], x9p14);
2116 let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[3], x10p13);
2117 let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[8], x11p12);
2118 let m0518b = SseVector::mul(self.twiddles_im[4], x1m22);
2119 let m0518b = SseVector::fmadd(m0518b, self.twiddles_im[9], x2m21);
2120 let m0518b = SseVector::nmadd(m0518b, self.twiddles_im[7], x3m20);
2121 let m0518b = SseVector::nmadd(m0518b, self.twiddles_im[2], x4m19);
2122 let m0518b = SseVector::fmadd(m0518b, self.twiddles_im[1], x5m18);
2123 let m0518b = SseVector::fmadd(m0518b, self.twiddles_im[6], x6m17);
2124 let m0518b = SseVector::nmadd(m0518b, self.twiddles_im[10], x7m16);
2125 let m0518b = SseVector::nmadd(m0518b, self.twiddles_im[5], x8m15);
2126 let m0518b = SseVector::nmadd(m0518b, self.twiddles_im[0], x9m14);
2127 let m0518b = SseVector::fmadd(m0518b, self.twiddles_im[3], x10m13);
2128 let m0518b = SseVector::fmadd(m0518b, self.twiddles_im[8], x11m12);
2129 let [y05, y18] = SseVector::column_butterfly2([m0518a, m0518b]);
2130
2131 let m0617a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p22);
2132 let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[10], x2p21);
2133 let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[4], x3p20);
2134 let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[0], x4p19);
2135 let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[6], x5p18);
2136 let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[9], x6p17);
2137 let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[3], x7p16);
2138 let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[1], x8p15);
2139 let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[7], x9p14);
2140 let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[8], x10p13);
2141 let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[2], x11p12);
2142 let m0617b = SseVector::mul(self.twiddles_im[5], x1m22);
2143 let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[10], x2m21);
2144 let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[4], x3m20);
2145 let m0617b = SseVector::fmadd(m0617b, self.twiddles_im[0], x4m19);
2146 let m0617b = SseVector::fmadd(m0617b, self.twiddles_im[6], x5m18);
2147 let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[9], x6m17);
2148 let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[3], x7m16);
2149 let m0617b = SseVector::fmadd(m0617b, self.twiddles_im[1], x8m15);
2150 let m0617b = SseVector::fmadd(m0617b, self.twiddles_im[7], x9m14);
2151 let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[8], x10m13);
2152 let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[2], x11m12);
2153 let [y06, y17] = SseVector::column_butterfly2([m0617a, m0617b]);
2154
2155 let m0716a = SseVector::fmadd(values[0], self.twiddles_re[6], x1p22);
2156 let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[8], x2p21);
2157 let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[1], x3p20);
2158 let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[4], x4p19);
2159 let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[10], x5p18);
2160 let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[3], x6p17);
2161 let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[2], x7p16);
2162 let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[9], x8p15);
2163 let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[5], x9p14);
2164 let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[0], x10p13);
2165 let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[7], x11p12);
2166 let m0716b = SseVector::mul(self.twiddles_im[6], x1m22);
2167 let m0716b = SseVector::nmadd(m0716b, self.twiddles_im[8], x2m21);
2168 let m0716b = SseVector::nmadd(m0716b, self.twiddles_im[1], x3m20);
2169 let m0716b = SseVector::fmadd(m0716b, self.twiddles_im[4], x4m19);
2170 let m0716b = SseVector::nmadd(m0716b, self.twiddles_im[10], x5m18);
2171 let m0716b = SseVector::nmadd(m0716b, self.twiddles_im[3], x6m17);
2172 let m0716b = SseVector::fmadd(m0716b, self.twiddles_im[2], x7m16);
2173 let m0716b = SseVector::fmadd(m0716b, self.twiddles_im[9], x8m15);
2174 let m0716b = SseVector::nmadd(m0716b, self.twiddles_im[5], x9m14);
2175 let m0716b = SseVector::fmadd(m0716b, self.twiddles_im[0], x10m13);
2176 let m0716b = SseVector::fmadd(m0716b, self.twiddles_im[7], x11m12);
2177 let [y07, y16] = SseVector::column_butterfly2([m0716a, m0716b]);
2178
2179 let m0815a = SseVector::fmadd(values[0], self.twiddles_re[7], x1p22);
2180 let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[6], x2p21);
2181 let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[0], x3p20);
2182 let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[8], x4p19);
2183 let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[5], x5p18);
2184 let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[1], x6p17);
2185 let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[9], x7p16);
2186 let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[4], x8p15);
2187 let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[2], x9p14);
2188 let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[10], x10p13);
2189 let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[3], x11p12);
2190 let m0815b = SseVector::mul(self.twiddles_im[7], x1m22);
2191 let m0815b = SseVector::nmadd(m0815b, self.twiddles_im[6], x2m21);
2192 let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[0], x3m20);
2193 let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[8], x4m19);
2194 let m0815b = SseVector::nmadd(m0815b, self.twiddles_im[5], x5m18);
2195 let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[1], x6m17);
2196 let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[9], x7m16);
2197 let m0815b = SseVector::nmadd(m0815b, self.twiddles_im[4], x8m15);
2198 let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[2], x9m14);
2199 let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[10], x10m13);
2200 let m0815b = SseVector::nmadd(m0815b, self.twiddles_im[3], x11m12);
2201 let [y08, y15] = SseVector::column_butterfly2([m0815a, m0815b]);
2202
2203 let m0914a = SseVector::fmadd(values[0], self.twiddles_re[8], x1p22);
2204 let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[4], x2p21);
2205 let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[3], x3p20);
2206 let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[9], x4p19);
2207 let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[0], x5p18);
2208 let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[7], x6p17);
2209 let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[5], x7p16);
2210 let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[2], x8p15);
2211 let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[10], x9p14);
2212 let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[1], x10p13);
2213 let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[6], x11p12);
2214 let m0914b = SseVector::mul(self.twiddles_im[8], x1m22);
2215 let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[4], x2m21);
2216 let m0914b = SseVector::fmadd(m0914b, self.twiddles_im[3], x3m20);
2217 let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[9], x4m19);
2218 let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[0], x5m18);
2219 let m0914b = SseVector::fmadd(m0914b, self.twiddles_im[7], x6m17);
2220 let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[5], x7m16);
2221 let m0914b = SseVector::fmadd(m0914b, self.twiddles_im[2], x8m15);
2222 let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[10], x9m14);
2223 let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[1], x10m13);
2224 let m0914b = SseVector::fmadd(m0914b, self.twiddles_im[6], x11m12);
2225 let [y09, y14] = SseVector::column_butterfly2([m0914a, m0914b]);
2226
2227 let m1013a = SseVector::fmadd(values[0], self.twiddles_re[9], x1p22);
2228 let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[2], x2p21);
2229 let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[6], x3p20);
2230 let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[5], x4p19);
2231 let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[3], x5p18);
2232 let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[8], x6p17);
2233 let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[0], x7p16);
2234 let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[10], x8p15);
2235 let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[1], x9p14);
2236 let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[7], x10p13);
2237 let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[4], x11p12);
2238 let m1013b = SseVector::mul(self.twiddles_im[9], x1m22);
2239 let m1013b = SseVector::nmadd(m1013b, self.twiddles_im[2], x2m21);
2240 let m1013b = SseVector::fmadd(m1013b, self.twiddles_im[6], x3m20);
2241 let m1013b = SseVector::nmadd(m1013b, self.twiddles_im[5], x4m19);
2242 let m1013b = SseVector::fmadd(m1013b, self.twiddles_im[3], x5m18);
2243 let m1013b = SseVector::nmadd(m1013b, self.twiddles_im[8], x6m17);
2244 let m1013b = SseVector::fmadd(m1013b, self.twiddles_im[0], x7m16);
2245 let m1013b = SseVector::fmadd(m1013b, self.twiddles_im[10], x8m15);
2246 let m1013b = SseVector::nmadd(m1013b, self.twiddles_im[1], x9m14);
2247 let m1013b = SseVector::fmadd(m1013b, self.twiddles_im[7], x10m13);
2248 let m1013b = SseVector::nmadd(m1013b, self.twiddles_im[4], x11m12);
2249 let [y10, y13] = SseVector::column_butterfly2([m1013a, m1013b]);
2250
2251 let m1112a = SseVector::fmadd(values[0], self.twiddles_re[10], x1p22);
2252 let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[0], x2p21);
2253 let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[9], x3p20);
2254 let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[1], x4p19);
2255 let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[8], x5p18);
2256 let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[2], x6p17);
2257 let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[7], x7p16);
2258 let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[3], x8p15);
2259 let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[6], x9p14);
2260 let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[4], x10p13);
2261 let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[5], x11p12);
2262 let m1112b = SseVector::mul(self.twiddles_im[10], x1m22);
2263 let m1112b = SseVector::nmadd(m1112b, self.twiddles_im[0], x2m21);
2264 let m1112b = SseVector::fmadd(m1112b, self.twiddles_im[9], x3m20);
2265 let m1112b = SseVector::nmadd(m1112b, self.twiddles_im[1], x4m19);
2266 let m1112b = SseVector::fmadd(m1112b, self.twiddles_im[8], x5m18);
2267 let m1112b = SseVector::nmadd(m1112b, self.twiddles_im[2], x6m17);
2268 let m1112b = SseVector::fmadd(m1112b, self.twiddles_im[7], x7m16);
2269 let m1112b = SseVector::nmadd(m1112b, self.twiddles_im[3], x8m15);
2270 let m1112b = SseVector::fmadd(m1112b, self.twiddles_im[6], x9m14);
2271 let m1112b = SseVector::nmadd(m1112b, self.twiddles_im[4], x10m13);
2272 let m1112b = SseVector::fmadd(m1112b, self.twiddles_im[5], x11m12);
2273 let [y11, y12] = SseVector::column_butterfly2([m1112a, m1112b]);
2274
2275
2276 [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12, y13, y14, y15, y16, y17, y18, y19, y20, y21, y22]
2277 }
2278}
2279
2280struct SseF64Butterfly23<T> {
2281 direction: FftDirection,
2282 twiddles_re: [__m128d; 11],
2283 twiddles_im: [__m128d; 11],
2284 _phantom: std::marker::PhantomData<T>,
2285}
2286
2287boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly23, 23, |this: &SseF64Butterfly23<_>| this.direction);
2288impl<T: FftNum> SseF64Butterfly23<T> {
2289 #[target_feature(enable = "sse4.1")]
2291 unsafe fn new(direction: FftDirection) -> Self {
2292 assert_f64::<T>();
2293 let twiddles = make_twiddles(23, direction);
2294 unsafe {Self {
2295 direction,
2296 twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
2297 twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
2298 _phantom: std::marker::PhantomData,
2299 }}
2300 }
2301
2302 #[inline(always)]
2303 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
2304 let values = read_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22 });
2305
2306 let out = self.perform_fft_direct(values);
2307
2308 write_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22 });
2309 }
2310
2311 #[inline(always)]
2312 pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 23]) -> [__m128d; 23] {
2313 let rotate = SseVector::make_rotate90(FftDirection::Inverse);
2314
2315 let y00 = values[0];
2316 let [x1p22, x1m22] = SseVector::column_butterfly2([values[1], values[22]]);
2317 let x1m22 = SseVector::apply_rotate90(rotate, x1m22);
2318 let y00 = SseVector::add(y00, x1p22);
2319 let [x2p21, x2m21] = SseVector::column_butterfly2([values[2], values[21]]);
2320 let x2m21 = SseVector::apply_rotate90(rotate, x2m21);
2321 let y00 = SseVector::add(y00, x2p21);
2322 let [x3p20, x3m20] = SseVector::column_butterfly2([values[3], values[20]]);
2323 let x3m20 = SseVector::apply_rotate90(rotate, x3m20);
2324 let y00 = SseVector::add(y00, x3p20);
2325 let [x4p19, x4m19] = SseVector::column_butterfly2([values[4], values[19]]);
2326 let x4m19 = SseVector::apply_rotate90(rotate, x4m19);
2327 let y00 = SseVector::add(y00, x4p19);
2328 let [x5p18, x5m18] = SseVector::column_butterfly2([values[5], values[18]]);
2329 let x5m18 = SseVector::apply_rotate90(rotate, x5m18);
2330 let y00 = SseVector::add(y00, x5p18);
2331 let [x6p17, x6m17] = SseVector::column_butterfly2([values[6], values[17]]);
2332 let x6m17 = SseVector::apply_rotate90(rotate, x6m17);
2333 let y00 = SseVector::add(y00, x6p17);
2334 let [x7p16, x7m16] = SseVector::column_butterfly2([values[7], values[16]]);
2335 let x7m16 = SseVector::apply_rotate90(rotate, x7m16);
2336 let y00 = SseVector::add(y00, x7p16);
2337 let [x8p15, x8m15] = SseVector::column_butterfly2([values[8], values[15]]);
2338 let x8m15 = SseVector::apply_rotate90(rotate, x8m15);
2339 let y00 = SseVector::add(y00, x8p15);
2340 let [x9p14, x9m14] = SseVector::column_butterfly2([values[9], values[14]]);
2341 let x9m14 = SseVector::apply_rotate90(rotate, x9m14);
2342 let y00 = SseVector::add(y00, x9p14);
2343 let [x10p13, x10m13] = SseVector::column_butterfly2([values[10], values[13]]);
2344 let x10m13 = SseVector::apply_rotate90(rotate, x10m13);
2345 let y00 = SseVector::add(y00, x10p13);
2346 let [x11p12, x11m12] = SseVector::column_butterfly2([values[11], values[12]]);
2347 let x11m12 = SseVector::apply_rotate90(rotate, x11m12);
2348 let y00 = SseVector::add(y00, x11p12);
2349
2350 let m0122a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p22);
2351 let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[1], x2p21);
2352 let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[2], x3p20);
2353 let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[3], x4p19);
2354 let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[4], x5p18);
2355 let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[5], x6p17);
2356 let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[6], x7p16);
2357 let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[7], x8p15);
2358 let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[8], x9p14);
2359 let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[9], x10p13);
2360 let m0122a = SseVector::fmadd(m0122a, self.twiddles_re[10], x11p12);
2361 let m0122b = SseVector::mul(self.twiddles_im[0], x1m22);
2362 let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[1], x2m21);
2363 let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[2], x3m20);
2364 let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[3], x4m19);
2365 let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[4], x5m18);
2366 let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[5], x6m17);
2367 let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[6], x7m16);
2368 let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[7], x8m15);
2369 let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[8], x9m14);
2370 let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[9], x10m13);
2371 let m0122b = SseVector::fmadd(m0122b, self.twiddles_im[10], x11m12);
2372 let [y01, y22] = SseVector::column_butterfly2([m0122a, m0122b]);
2373
2374 let m0221a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p22);
2375 let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[3], x2p21);
2376 let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[5], x3p20);
2377 let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[7], x4p19);
2378 let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[9], x5p18);
2379 let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[10], x6p17);
2380 let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[8], x7p16);
2381 let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[6], x8p15);
2382 let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[4], x9p14);
2383 let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[2], x10p13);
2384 let m0221a = SseVector::fmadd(m0221a, self.twiddles_re[0], x11p12);
2385 let m0221b = SseVector::mul(self.twiddles_im[1], x1m22);
2386 let m0221b = SseVector::fmadd(m0221b, self.twiddles_im[3], x2m21);
2387 let m0221b = SseVector::fmadd(m0221b, self.twiddles_im[5], x3m20);
2388 let m0221b = SseVector::fmadd(m0221b, self.twiddles_im[7], x4m19);
2389 let m0221b = SseVector::fmadd(m0221b, self.twiddles_im[9], x5m18);
2390 let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[10], x6m17);
2391 let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[8], x7m16);
2392 let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[6], x8m15);
2393 let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[4], x9m14);
2394 let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[2], x10m13);
2395 let m0221b = SseVector::nmadd(m0221b, self.twiddles_im[0], x11m12);
2396 let [y02, y21] = SseVector::column_butterfly2([m0221a, m0221b]);
2397
2398 let m0320a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p22);
2399 let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[5], x2p21);
2400 let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[8], x3p20);
2401 let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[10], x4p19);
2402 let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[7], x5p18);
2403 let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[4], x6p17);
2404 let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[1], x7p16);
2405 let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[0], x8p15);
2406 let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[3], x9p14);
2407 let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[6], x10p13);
2408 let m0320a = SseVector::fmadd(m0320a, self.twiddles_re[9], x11p12);
2409 let m0320b = SseVector::mul(self.twiddles_im[2], x1m22);
2410 let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[5], x2m21);
2411 let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[8], x3m20);
2412 let m0320b = SseVector::nmadd(m0320b, self.twiddles_im[10], x4m19);
2413 let m0320b = SseVector::nmadd(m0320b, self.twiddles_im[7], x5m18);
2414 let m0320b = SseVector::nmadd(m0320b, self.twiddles_im[4], x6m17);
2415 let m0320b = SseVector::nmadd(m0320b, self.twiddles_im[1], x7m16);
2416 let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[0], x8m15);
2417 let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[3], x9m14);
2418 let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[6], x10m13);
2419 let m0320b = SseVector::fmadd(m0320b, self.twiddles_im[9], x11m12);
2420 let [y03, y20] = SseVector::column_butterfly2([m0320a, m0320b]);
2421
2422 let m0419a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p22);
2423 let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[7], x2p21);
2424 let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[10], x3p20);
2425 let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[6], x4p19);
2426 let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[2], x5p18);
2427 let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[0], x6p17);
2428 let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[4], x7p16);
2429 let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[8], x8p15);
2430 let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[9], x9p14);
2431 let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[5], x10p13);
2432 let m0419a = SseVector::fmadd(m0419a, self.twiddles_re[1], x11p12);
2433 let m0419b = SseVector::mul(self.twiddles_im[3], x1m22);
2434 let m0419b = SseVector::fmadd(m0419b, self.twiddles_im[7], x2m21);
2435 let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[10], x3m20);
2436 let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[6], x4m19);
2437 let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[2], x5m18);
2438 let m0419b = SseVector::fmadd(m0419b, self.twiddles_im[0], x6m17);
2439 let m0419b = SseVector::fmadd(m0419b, self.twiddles_im[4], x7m16);
2440 let m0419b = SseVector::fmadd(m0419b, self.twiddles_im[8], x8m15);
2441 let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[9], x9m14);
2442 let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[5], x10m13);
2443 let m0419b = SseVector::nmadd(m0419b, self.twiddles_im[1], x11m12);
2444 let [y04, y19] = SseVector::column_butterfly2([m0419a, m0419b]);
2445
2446 let m0518a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p22);
2447 let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[9], x2p21);
2448 let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[7], x3p20);
2449 let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[2], x4p19);
2450 let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[1], x5p18);
2451 let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[6], x6p17);
2452 let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[10], x7p16);
2453 let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[5], x8p15);
2454 let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[0], x9p14);
2455 let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[3], x10p13);
2456 let m0518a = SseVector::fmadd(m0518a, self.twiddles_re[8], x11p12);
2457 let m0518b = SseVector::mul(self.twiddles_im[4], x1m22);
2458 let m0518b = SseVector::fmadd(m0518b, self.twiddles_im[9], x2m21);
2459 let m0518b = SseVector::nmadd(m0518b, self.twiddles_im[7], x3m20);
2460 let m0518b = SseVector::nmadd(m0518b, self.twiddles_im[2], x4m19);
2461 let m0518b = SseVector::fmadd(m0518b, self.twiddles_im[1], x5m18);
2462 let m0518b = SseVector::fmadd(m0518b, self.twiddles_im[6], x6m17);
2463 let m0518b = SseVector::nmadd(m0518b, self.twiddles_im[10], x7m16);
2464 let m0518b = SseVector::nmadd(m0518b, self.twiddles_im[5], x8m15);
2465 let m0518b = SseVector::nmadd(m0518b, self.twiddles_im[0], x9m14);
2466 let m0518b = SseVector::fmadd(m0518b, self.twiddles_im[3], x10m13);
2467 let m0518b = SseVector::fmadd(m0518b, self.twiddles_im[8], x11m12);
2468 let [y05, y18] = SseVector::column_butterfly2([m0518a, m0518b]);
2469
2470 let m0617a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p22);
2471 let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[10], x2p21);
2472 let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[4], x3p20);
2473 let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[0], x4p19);
2474 let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[6], x5p18);
2475 let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[9], x6p17);
2476 let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[3], x7p16);
2477 let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[1], x8p15);
2478 let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[7], x9p14);
2479 let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[8], x10p13);
2480 let m0617a = SseVector::fmadd(m0617a, self.twiddles_re[2], x11p12);
2481 let m0617b = SseVector::mul(self.twiddles_im[5], x1m22);
2482 let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[10], x2m21);
2483 let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[4], x3m20);
2484 let m0617b = SseVector::fmadd(m0617b, self.twiddles_im[0], x4m19);
2485 let m0617b = SseVector::fmadd(m0617b, self.twiddles_im[6], x5m18);
2486 let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[9], x6m17);
2487 let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[3], x7m16);
2488 let m0617b = SseVector::fmadd(m0617b, self.twiddles_im[1], x8m15);
2489 let m0617b = SseVector::fmadd(m0617b, self.twiddles_im[7], x9m14);
2490 let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[8], x10m13);
2491 let m0617b = SseVector::nmadd(m0617b, self.twiddles_im[2], x11m12);
2492 let [y06, y17] = SseVector::column_butterfly2([m0617a, m0617b]);
2493
2494 let m0716a = SseVector::fmadd(values[0], self.twiddles_re[6], x1p22);
2495 let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[8], x2p21);
2496 let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[1], x3p20);
2497 let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[4], x4p19);
2498 let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[10], x5p18);
2499 let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[3], x6p17);
2500 let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[2], x7p16);
2501 let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[9], x8p15);
2502 let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[5], x9p14);
2503 let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[0], x10p13);
2504 let m0716a = SseVector::fmadd(m0716a, self.twiddles_re[7], x11p12);
2505 let m0716b = SseVector::mul(self.twiddles_im[6], x1m22);
2506 let m0716b = SseVector::nmadd(m0716b, self.twiddles_im[8], x2m21);
2507 let m0716b = SseVector::nmadd(m0716b, self.twiddles_im[1], x3m20);
2508 let m0716b = SseVector::fmadd(m0716b, self.twiddles_im[4], x4m19);
2509 let m0716b = SseVector::nmadd(m0716b, self.twiddles_im[10], x5m18);
2510 let m0716b = SseVector::nmadd(m0716b, self.twiddles_im[3], x6m17);
2511 let m0716b = SseVector::fmadd(m0716b, self.twiddles_im[2], x7m16);
2512 let m0716b = SseVector::fmadd(m0716b, self.twiddles_im[9], x8m15);
2513 let m0716b = SseVector::nmadd(m0716b, self.twiddles_im[5], x9m14);
2514 let m0716b = SseVector::fmadd(m0716b, self.twiddles_im[0], x10m13);
2515 let m0716b = SseVector::fmadd(m0716b, self.twiddles_im[7], x11m12);
2516 let [y07, y16] = SseVector::column_butterfly2([m0716a, m0716b]);
2517
2518 let m0815a = SseVector::fmadd(values[0], self.twiddles_re[7], x1p22);
2519 let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[6], x2p21);
2520 let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[0], x3p20);
2521 let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[8], x4p19);
2522 let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[5], x5p18);
2523 let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[1], x6p17);
2524 let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[9], x7p16);
2525 let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[4], x8p15);
2526 let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[2], x9p14);
2527 let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[10], x10p13);
2528 let m0815a = SseVector::fmadd(m0815a, self.twiddles_re[3], x11p12);
2529 let m0815b = SseVector::mul(self.twiddles_im[7], x1m22);
2530 let m0815b = SseVector::nmadd(m0815b, self.twiddles_im[6], x2m21);
2531 let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[0], x3m20);
2532 let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[8], x4m19);
2533 let m0815b = SseVector::nmadd(m0815b, self.twiddles_im[5], x5m18);
2534 let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[1], x6m17);
2535 let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[9], x7m16);
2536 let m0815b = SseVector::nmadd(m0815b, self.twiddles_im[4], x8m15);
2537 let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[2], x9m14);
2538 let m0815b = SseVector::fmadd(m0815b, self.twiddles_im[10], x10m13);
2539 let m0815b = SseVector::nmadd(m0815b, self.twiddles_im[3], x11m12);
2540 let [y08, y15] = SseVector::column_butterfly2([m0815a, m0815b]);
2541
2542 let m0914a = SseVector::fmadd(values[0], self.twiddles_re[8], x1p22);
2543 let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[4], x2p21);
2544 let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[3], x3p20);
2545 let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[9], x4p19);
2546 let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[0], x5p18);
2547 let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[7], x6p17);
2548 let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[5], x7p16);
2549 let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[2], x8p15);
2550 let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[10], x9p14);
2551 let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[1], x10p13);
2552 let m0914a = SseVector::fmadd(m0914a, self.twiddles_re[6], x11p12);
2553 let m0914b = SseVector::mul(self.twiddles_im[8], x1m22);
2554 let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[4], x2m21);
2555 let m0914b = SseVector::fmadd(m0914b, self.twiddles_im[3], x3m20);
2556 let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[9], x4m19);
2557 let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[0], x5m18);
2558 let m0914b = SseVector::fmadd(m0914b, self.twiddles_im[7], x6m17);
2559 let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[5], x7m16);
2560 let m0914b = SseVector::fmadd(m0914b, self.twiddles_im[2], x8m15);
2561 let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[10], x9m14);
2562 let m0914b = SseVector::nmadd(m0914b, self.twiddles_im[1], x10m13);
2563 let m0914b = SseVector::fmadd(m0914b, self.twiddles_im[6], x11m12);
2564 let [y09, y14] = SseVector::column_butterfly2([m0914a, m0914b]);
2565
2566 let m1013a = SseVector::fmadd(values[0], self.twiddles_re[9], x1p22);
2567 let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[2], x2p21);
2568 let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[6], x3p20);
2569 let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[5], x4p19);
2570 let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[3], x5p18);
2571 let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[8], x6p17);
2572 let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[0], x7p16);
2573 let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[10], x8p15);
2574 let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[1], x9p14);
2575 let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[7], x10p13);
2576 let m1013a = SseVector::fmadd(m1013a, self.twiddles_re[4], x11p12);
2577 let m1013b = SseVector::mul(self.twiddles_im[9], x1m22);
2578 let m1013b = SseVector::nmadd(m1013b, self.twiddles_im[2], x2m21);
2579 let m1013b = SseVector::fmadd(m1013b, self.twiddles_im[6], x3m20);
2580 let m1013b = SseVector::nmadd(m1013b, self.twiddles_im[5], x4m19);
2581 let m1013b = SseVector::fmadd(m1013b, self.twiddles_im[3], x5m18);
2582 let m1013b = SseVector::nmadd(m1013b, self.twiddles_im[8], x6m17);
2583 let m1013b = SseVector::fmadd(m1013b, self.twiddles_im[0], x7m16);
2584 let m1013b = SseVector::fmadd(m1013b, self.twiddles_im[10], x8m15);
2585 let m1013b = SseVector::nmadd(m1013b, self.twiddles_im[1], x9m14);
2586 let m1013b = SseVector::fmadd(m1013b, self.twiddles_im[7], x10m13);
2587 let m1013b = SseVector::nmadd(m1013b, self.twiddles_im[4], x11m12);
2588 let [y10, y13] = SseVector::column_butterfly2([m1013a, m1013b]);
2589
2590 let m1112a = SseVector::fmadd(values[0], self.twiddles_re[10], x1p22);
2591 let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[0], x2p21);
2592 let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[9], x3p20);
2593 let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[1], x4p19);
2594 let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[8], x5p18);
2595 let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[2], x6p17);
2596 let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[7], x7p16);
2597 let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[3], x8p15);
2598 let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[6], x9p14);
2599 let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[4], x10p13);
2600 let m1112a = SseVector::fmadd(m1112a, self.twiddles_re[5], x11p12);
2601 let m1112b = SseVector::mul(self.twiddles_im[10], x1m22);
2602 let m1112b = SseVector::nmadd(m1112b, self.twiddles_im[0], x2m21);
2603 let m1112b = SseVector::fmadd(m1112b, self.twiddles_im[9], x3m20);
2604 let m1112b = SseVector::nmadd(m1112b, self.twiddles_im[1], x4m19);
2605 let m1112b = SseVector::fmadd(m1112b, self.twiddles_im[8], x5m18);
2606 let m1112b = SseVector::nmadd(m1112b, self.twiddles_im[2], x6m17);
2607 let m1112b = SseVector::fmadd(m1112b, self.twiddles_im[7], x7m16);
2608 let m1112b = SseVector::nmadd(m1112b, self.twiddles_im[3], x8m15);
2609 let m1112b = SseVector::fmadd(m1112b, self.twiddles_im[6], x9m14);
2610 let m1112b = SseVector::nmadd(m1112b, self.twiddles_im[4], x10m13);
2611 let m1112b = SseVector::fmadd(m1112b, self.twiddles_im[5], x11m12);
2612 let [y11, y12] = SseVector::column_butterfly2([m1112a, m1112b]);
2613
2614
2615 [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12, y13, y14, y15, y16, y17, y18, y19, y20, y21, y22]
2616 }
2617}
2618
2619struct SseF32Butterfly29<T> {
2620 direction: FftDirection,
2621 twiddles_re: [__m128; 14],
2622 twiddles_im: [__m128; 14],
2623 _phantom: std::marker::PhantomData<T>,
2624}
2625
2626boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly29, 29, |this: &SseF32Butterfly29<_>| this.direction);
2627impl<T: FftNum> SseF32Butterfly29<T> {
2628 #[target_feature(enable = "sse4.1")]
2630 unsafe fn new(direction: FftDirection) -> Self {
2631 assert_f32::<T>();
2632 let twiddles = make_twiddles(29, direction);
2633 Self {
2634 direction,
2635 twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
2636 twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
2637 _phantom: std::marker::PhantomData,
2638 }
2639 }
2640
2641 #[inline(always)]
2642 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
2643 let values = read_partial1_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28 });
2644
2645 let out = self.perform_parallel_fft_direct(values);
2646
2647 write_partial_lo_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28 } );
2648 }
2649
2650 #[inline(always)]
2651 pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
2652 let input_packed = read_complex_to_array!(buffer, { 0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56 });
2653
2654 let values = [
2655 extract_lo_hi_f32(input_packed[0], input_packed[14]),
2656 extract_hi_lo_f32(input_packed[0], input_packed[15]),
2657 extract_lo_hi_f32(input_packed[1], input_packed[15]),
2658 extract_hi_lo_f32(input_packed[1], input_packed[16]),
2659 extract_lo_hi_f32(input_packed[2], input_packed[16]),
2660 extract_hi_lo_f32(input_packed[2], input_packed[17]),
2661 extract_lo_hi_f32(input_packed[3], input_packed[17]),
2662 extract_hi_lo_f32(input_packed[3], input_packed[18]),
2663 extract_lo_hi_f32(input_packed[4], input_packed[18]),
2664 extract_hi_lo_f32(input_packed[4], input_packed[19]),
2665 extract_lo_hi_f32(input_packed[5], input_packed[19]),
2666 extract_hi_lo_f32(input_packed[5], input_packed[20]),
2667 extract_lo_hi_f32(input_packed[6], input_packed[20]),
2668 extract_hi_lo_f32(input_packed[6], input_packed[21]),
2669 extract_lo_hi_f32(input_packed[7], input_packed[21]),
2670 extract_hi_lo_f32(input_packed[7], input_packed[22]),
2671 extract_lo_hi_f32(input_packed[8], input_packed[22]),
2672 extract_hi_lo_f32(input_packed[8], input_packed[23]),
2673 extract_lo_hi_f32(input_packed[9], input_packed[23]),
2674 extract_hi_lo_f32(input_packed[9], input_packed[24]),
2675 extract_lo_hi_f32(input_packed[10], input_packed[24]),
2676 extract_hi_lo_f32(input_packed[10], input_packed[25]),
2677 extract_lo_hi_f32(input_packed[11], input_packed[25]),
2678 extract_hi_lo_f32(input_packed[11], input_packed[26]),
2679 extract_lo_hi_f32(input_packed[12], input_packed[26]),
2680 extract_hi_lo_f32(input_packed[12], input_packed[27]),
2681 extract_lo_hi_f32(input_packed[13], input_packed[27]),
2682 extract_hi_lo_f32(input_packed[13], input_packed[28]),
2683 extract_lo_hi_f32(input_packed[14], input_packed[28]),
2684 ];
2685
2686 let out = self.perform_parallel_fft_direct(values);
2687
2688 let out_packed = [
2689 extract_lo_lo_f32(out[0], out[1]),
2690 extract_lo_lo_f32(out[2], out[3]),
2691 extract_lo_lo_f32(out[4], out[5]),
2692 extract_lo_lo_f32(out[6], out[7]),
2693 extract_lo_lo_f32(out[8], out[9]),
2694 extract_lo_lo_f32(out[10], out[11]),
2695 extract_lo_lo_f32(out[12], out[13]),
2696 extract_lo_lo_f32(out[14], out[15]),
2697 extract_lo_lo_f32(out[16], out[17]),
2698 extract_lo_lo_f32(out[18], out[19]),
2699 extract_lo_lo_f32(out[20], out[21]),
2700 extract_lo_lo_f32(out[22], out[23]),
2701 extract_lo_lo_f32(out[24], out[25]),
2702 extract_lo_lo_f32(out[26], out[27]),
2703 extract_lo_hi_f32(out[28], out[0]),
2704 extract_hi_hi_f32(out[1], out[2]),
2705 extract_hi_hi_f32(out[3], out[4]),
2706 extract_hi_hi_f32(out[5], out[6]),
2707 extract_hi_hi_f32(out[7], out[8]),
2708 extract_hi_hi_f32(out[9], out[10]),
2709 extract_hi_hi_f32(out[11], out[12]),
2710 extract_hi_hi_f32(out[13], out[14]),
2711 extract_hi_hi_f32(out[15], out[16]),
2712 extract_hi_hi_f32(out[17], out[18]),
2713 extract_hi_hi_f32(out[19], out[20]),
2714 extract_hi_hi_f32(out[21], out[22]),
2715 extract_hi_hi_f32(out[23], out[24]),
2716 extract_hi_hi_f32(out[25], out[26]),
2717 extract_hi_hi_f32(out[27], out[28]),
2718 ];
2719
2720 write_complex_to_array_strided!(out_packed, buffer, 2, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28 });
2721 }
2722
2723 #[inline(always)]
2724 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 29]) -> [__m128; 29] {
2725 let rotate = SseVector::make_rotate90(FftDirection::Inverse);
2726
2727 let y00 = values[0];
2728 let [x1p28, x1m28] = SseVector::column_butterfly2([values[1], values[28]]);
2729 let x1m28 = SseVector::apply_rotate90(rotate, x1m28);
2730 let y00 = SseVector::add(y00, x1p28);
2731 let [x2p27, x2m27] = SseVector::column_butterfly2([values[2], values[27]]);
2732 let x2m27 = SseVector::apply_rotate90(rotate, x2m27);
2733 let y00 = SseVector::add(y00, x2p27);
2734 let [x3p26, x3m26] = SseVector::column_butterfly2([values[3], values[26]]);
2735 let x3m26 = SseVector::apply_rotate90(rotate, x3m26);
2736 let y00 = SseVector::add(y00, x3p26);
2737 let [x4p25, x4m25] = SseVector::column_butterfly2([values[4], values[25]]);
2738 let x4m25 = SseVector::apply_rotate90(rotate, x4m25);
2739 let y00 = SseVector::add(y00, x4p25);
2740 let [x5p24, x5m24] = SseVector::column_butterfly2([values[5], values[24]]);
2741 let x5m24 = SseVector::apply_rotate90(rotate, x5m24);
2742 let y00 = SseVector::add(y00, x5p24);
2743 let [x6p23, x6m23] = SseVector::column_butterfly2([values[6], values[23]]);
2744 let x6m23 = SseVector::apply_rotate90(rotate, x6m23);
2745 let y00 = SseVector::add(y00, x6p23);
2746 let [x7p22, x7m22] = SseVector::column_butterfly2([values[7], values[22]]);
2747 let x7m22 = SseVector::apply_rotate90(rotate, x7m22);
2748 let y00 = SseVector::add(y00, x7p22);
2749 let [x8p21, x8m21] = SseVector::column_butterfly2([values[8], values[21]]);
2750 let x8m21 = SseVector::apply_rotate90(rotate, x8m21);
2751 let y00 = SseVector::add(y00, x8p21);
2752 let [x9p20, x9m20] = SseVector::column_butterfly2([values[9], values[20]]);
2753 let x9m20 = SseVector::apply_rotate90(rotate, x9m20);
2754 let y00 = SseVector::add(y00, x9p20);
2755 let [x10p19, x10m19] = SseVector::column_butterfly2([values[10], values[19]]);
2756 let x10m19 = SseVector::apply_rotate90(rotate, x10m19);
2757 let y00 = SseVector::add(y00, x10p19);
2758 let [x11p18, x11m18] = SseVector::column_butterfly2([values[11], values[18]]);
2759 let x11m18 = SseVector::apply_rotate90(rotate, x11m18);
2760 let y00 = SseVector::add(y00, x11p18);
2761 let [x12p17, x12m17] = SseVector::column_butterfly2([values[12], values[17]]);
2762 let x12m17 = SseVector::apply_rotate90(rotate, x12m17);
2763 let y00 = SseVector::add(y00, x12p17);
2764 let [x13p16, x13m16] = SseVector::column_butterfly2([values[13], values[16]]);
2765 let x13m16 = SseVector::apply_rotate90(rotate, x13m16);
2766 let y00 = SseVector::add(y00, x13p16);
2767 let [x14p15, x14m15] = SseVector::column_butterfly2([values[14], values[15]]);
2768 let x14m15 = SseVector::apply_rotate90(rotate, x14m15);
2769 let y00 = SseVector::add(y00, x14p15);
2770
2771 let m0128a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p28);
2772 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[1], x2p27);
2773 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[2], x3p26);
2774 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[3], x4p25);
2775 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[4], x5p24);
2776 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[5], x6p23);
2777 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[6], x7p22);
2778 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[7], x8p21);
2779 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[8], x9p20);
2780 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[9], x10p19);
2781 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[10], x11p18);
2782 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[11], x12p17);
2783 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[12], x13p16);
2784 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[13], x14p15);
2785 let m0128b = SseVector::mul(self.twiddles_im[0], x1m28);
2786 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[1], x2m27);
2787 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[2], x3m26);
2788 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[3], x4m25);
2789 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[4], x5m24);
2790 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[5], x6m23);
2791 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[6], x7m22);
2792 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[7], x8m21);
2793 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[8], x9m20);
2794 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[9], x10m19);
2795 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[10], x11m18);
2796 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[11], x12m17);
2797 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[12], x13m16);
2798 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[13], x14m15);
2799 let [y01, y28] = SseVector::column_butterfly2([m0128a, m0128b]);
2800
2801 let m0227a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p28);
2802 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[3], x2p27);
2803 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[5], x3p26);
2804 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[7], x4p25);
2805 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[9], x5p24);
2806 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[11], x6p23);
2807 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[13], x7p22);
2808 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[12], x8p21);
2809 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[10], x9p20);
2810 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[8], x10p19);
2811 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[6], x11p18);
2812 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[4], x12p17);
2813 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[2], x13p16);
2814 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[0], x14p15);
2815 let m0227b = SseVector::mul(self.twiddles_im[1], x1m28);
2816 let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[3], x2m27);
2817 let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[5], x3m26);
2818 let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[7], x4m25);
2819 let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[9], x5m24);
2820 let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[11], x6m23);
2821 let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[13], x7m22);
2822 let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[12], x8m21);
2823 let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[10], x9m20);
2824 let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[8], x10m19);
2825 let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[6], x11m18);
2826 let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[4], x12m17);
2827 let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[2], x13m16);
2828 let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[0], x14m15);
2829 let [y02, y27] = SseVector::column_butterfly2([m0227a, m0227b]);
2830
2831 let m0326a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p28);
2832 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[5], x2p27);
2833 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[8], x3p26);
2834 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[11], x4p25);
2835 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[13], x5p24);
2836 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[10], x6p23);
2837 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[7], x7p22);
2838 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[4], x8p21);
2839 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[1], x9p20);
2840 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[0], x10p19);
2841 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[3], x11p18);
2842 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[6], x12p17);
2843 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[9], x13p16);
2844 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[12], x14p15);
2845 let m0326b = SseVector::mul(self.twiddles_im[2], x1m28);
2846 let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[5], x2m27);
2847 let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[8], x3m26);
2848 let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[11], x4m25);
2849 let m0326b = SseVector::nmadd(m0326b, self.twiddles_im[13], x5m24);
2850 let m0326b = SseVector::nmadd(m0326b, self.twiddles_im[10], x6m23);
2851 let m0326b = SseVector::nmadd(m0326b, self.twiddles_im[7], x7m22);
2852 let m0326b = SseVector::nmadd(m0326b, self.twiddles_im[4], x8m21);
2853 let m0326b = SseVector::nmadd(m0326b, self.twiddles_im[1], x9m20);
2854 let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[0], x10m19);
2855 let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[3], x11m18);
2856 let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[6], x12m17);
2857 let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[9], x13m16);
2858 let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[12], x14m15);
2859 let [y03, y26] = SseVector::column_butterfly2([m0326a, m0326b]);
2860
2861 let m0425a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p28);
2862 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[7], x2p27);
2863 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[11], x3p26);
2864 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[12], x4p25);
2865 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[8], x5p24);
2866 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[4], x6p23);
2867 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[0], x7p22);
2868 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[2], x8p21);
2869 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[6], x9p20);
2870 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[10], x10p19);
2871 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[13], x11p18);
2872 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[9], x12p17);
2873 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[5], x13p16);
2874 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[1], x14p15);
2875 let m0425b = SseVector::mul(self.twiddles_im[3], x1m28);
2876 let m0425b = SseVector::fmadd(m0425b, self.twiddles_im[7], x2m27);
2877 let m0425b = SseVector::fmadd(m0425b, self.twiddles_im[11], x3m26);
2878 let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[12], x4m25);
2879 let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[8], x5m24);
2880 let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[4], x6m23);
2881 let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[0], x7m22);
2882 let m0425b = SseVector::fmadd(m0425b, self.twiddles_im[2], x8m21);
2883 let m0425b = SseVector::fmadd(m0425b, self.twiddles_im[6], x9m20);
2884 let m0425b = SseVector::fmadd(m0425b, self.twiddles_im[10], x10m19);
2885 let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[13], x11m18);
2886 let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[9], x12m17);
2887 let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[5], x13m16);
2888 let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[1], x14m15);
2889 let [y04, y25] = SseVector::column_butterfly2([m0425a, m0425b]);
2890
2891 let m0524a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p28);
2892 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[9], x2p27);
2893 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[13], x3p26);
2894 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[8], x4p25);
2895 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[3], x5p24);
2896 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[0], x6p23);
2897 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[5], x7p22);
2898 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[10], x8p21);
2899 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[12], x9p20);
2900 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[7], x10p19);
2901 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[2], x11p18);
2902 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[1], x12p17);
2903 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[6], x13p16);
2904 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[11], x14p15);
2905 let m0524b = SseVector::mul(self.twiddles_im[4], x1m28);
2906 let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[9], x2m27);
2907 let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[13], x3m26);
2908 let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[8], x4m25);
2909 let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[3], x5m24);
2910 let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[0], x6m23);
2911 let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[5], x7m22);
2912 let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[10], x8m21);
2913 let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[12], x9m20);
2914 let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[7], x10m19);
2915 let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[2], x11m18);
2916 let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[1], x12m17);
2917 let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[6], x13m16);
2918 let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[11], x14m15);
2919 let [y05, y24] = SseVector::column_butterfly2([m0524a, m0524b]);
2920
2921 let m0623a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p28);
2922 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[11], x2p27);
2923 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[10], x3p26);
2924 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[4], x4p25);
2925 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[0], x5p24);
2926 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[6], x6p23);
2927 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[12], x7p22);
2928 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[9], x8p21);
2929 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[3], x9p20);
2930 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[1], x10p19);
2931 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[7], x11p18);
2932 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[13], x12p17);
2933 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[8], x13p16);
2934 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[2], x14p15);
2935 let m0623b = SseVector::mul(self.twiddles_im[5], x1m28);
2936 let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[11], x2m27);
2937 let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[10], x3m26);
2938 let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[4], x4m25);
2939 let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[0], x5m24);
2940 let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[6], x6m23);
2941 let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[12], x7m22);
2942 let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[9], x8m21);
2943 let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[3], x9m20);
2944 let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[1], x10m19);
2945 let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[7], x11m18);
2946 let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[13], x12m17);
2947 let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[8], x13m16);
2948 let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[2], x14m15);
2949 let [y06, y23] = SseVector::column_butterfly2([m0623a, m0623b]);
2950
2951 let m0722a = SseVector::fmadd(values[0], self.twiddles_re[6], x1p28);
2952 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[13], x2p27);
2953 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[7], x3p26);
2954 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[0], x4p25);
2955 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[5], x5p24);
2956 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[12], x6p23);
2957 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[8], x7p22);
2958 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[1], x8p21);
2959 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[4], x9p20);
2960 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[11], x10p19);
2961 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[9], x11p18);
2962 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[2], x12p17);
2963 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[3], x13p16);
2964 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[10], x14p15);
2965 let m0722b = SseVector::mul(self.twiddles_im[6], x1m28);
2966 let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[13], x2m27);
2967 let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[7], x3m26);
2968 let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[0], x4m25);
2969 let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[5], x5m24);
2970 let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[12], x6m23);
2971 let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[8], x7m22);
2972 let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[1], x8m21);
2973 let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[4], x9m20);
2974 let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[11], x10m19);
2975 let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[9], x11m18);
2976 let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[2], x12m17);
2977 let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[3], x13m16);
2978 let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[10], x14m15);
2979 let [y07, y22] = SseVector::column_butterfly2([m0722a, m0722b]);
2980
2981 let m0821a = SseVector::fmadd(values[0], self.twiddles_re[7], x1p28);
2982 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[12], x2p27);
2983 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[4], x3p26);
2984 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[2], x4p25);
2985 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[10], x5p24);
2986 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[9], x6p23);
2987 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[1], x7p22);
2988 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[5], x8p21);
2989 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[13], x9p20);
2990 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[6], x10p19);
2991 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[0], x11p18);
2992 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[8], x12p17);
2993 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[11], x13p16);
2994 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[3], x14p15);
2995 let m0821b = SseVector::mul(self.twiddles_im[7], x1m28);
2996 let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[12], x2m27);
2997 let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[4], x3m26);
2998 let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[2], x4m25);
2999 let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[10], x5m24);
3000 let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[9], x6m23);
3001 let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[1], x7m22);
3002 let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[5], x8m21);
3003 let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[13], x9m20);
3004 let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[6], x10m19);
3005 let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[0], x11m18);
3006 let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[8], x12m17);
3007 let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[11], x13m16);
3008 let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[3], x14m15);
3009 let [y08, y21] = SseVector::column_butterfly2([m0821a, m0821b]);
3010
3011 let m0920a = SseVector::fmadd(values[0], self.twiddles_re[8], x1p28);
3012 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[10], x2p27);
3013 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[1], x3p26);
3014 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[6], x4p25);
3015 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[12], x5p24);
3016 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[3], x6p23);
3017 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[4], x7p22);
3018 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[13], x8p21);
3019 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[5], x9p20);
3020 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[2], x10p19);
3021 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[11], x11p18);
3022 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[7], x12p17);
3023 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[0], x13p16);
3024 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[9], x14p15);
3025 let m0920b = SseVector::mul(self.twiddles_im[8], x1m28);
3026 let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[10], x2m27);
3027 let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[1], x3m26);
3028 let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[6], x4m25);
3029 let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[12], x5m24);
3030 let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[3], x6m23);
3031 let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[4], x7m22);
3032 let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[13], x8m21);
3033 let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[5], x9m20);
3034 let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[2], x10m19);
3035 let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[11], x11m18);
3036 let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[7], x12m17);
3037 let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[0], x13m16);
3038 let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[9], x14m15);
3039 let [y09, y20] = SseVector::column_butterfly2([m0920a, m0920b]);
3040
3041 let m1019a = SseVector::fmadd(values[0], self.twiddles_re[9], x1p28);
3042 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[8], x2p27);
3043 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[0], x3p26);
3044 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[10], x4p25);
3045 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[7], x5p24);
3046 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[1], x6p23);
3047 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[11], x7p22);
3048 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[6], x8p21);
3049 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[2], x9p20);
3050 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[12], x10p19);
3051 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[5], x11p18);
3052 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[3], x12p17);
3053 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[13], x13p16);
3054 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[4], x14p15);
3055 let m1019b = SseVector::mul(self.twiddles_im[9], x1m28);
3056 let m1019b = SseVector::nmadd(m1019b, self.twiddles_im[8], x2m27);
3057 let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[0], x3m26);
3058 let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[10], x4m25);
3059 let m1019b = SseVector::nmadd(m1019b, self.twiddles_im[7], x5m24);
3060 let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[1], x6m23);
3061 let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[11], x7m22);
3062 let m1019b = SseVector::nmadd(m1019b, self.twiddles_im[6], x8m21);
3063 let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[2], x9m20);
3064 let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[12], x10m19);
3065 let m1019b = SseVector::nmadd(m1019b, self.twiddles_im[5], x11m18);
3066 let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[3], x12m17);
3067 let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[13], x13m16);
3068 let m1019b = SseVector::nmadd(m1019b, self.twiddles_im[4], x14m15);
3069 let [y10, y19] = SseVector::column_butterfly2([m1019a, m1019b]);
3070
3071 let m1118a = SseVector::fmadd(values[0], self.twiddles_re[10], x1p28);
3072 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[6], x2p27);
3073 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[3], x3p26);
3074 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[13], x4p25);
3075 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[2], x5p24);
3076 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[7], x6p23);
3077 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[9], x7p22);
3078 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[0], x8p21);
3079 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[11], x9p20);
3080 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[5], x10p19);
3081 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[4], x11p18);
3082 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[12], x12p17);
3083 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[1], x13p16);
3084 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[8], x14p15);
3085 let m1118b = SseVector::mul(self.twiddles_im[10], x1m28);
3086 let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[6], x2m27);
3087 let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[3], x3m26);
3088 let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[13], x4m25);
3089 let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[2], x5m24);
3090 let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[7], x6m23);
3091 let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[9], x7m22);
3092 let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[0], x8m21);
3093 let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[11], x9m20);
3094 let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[5], x10m19);
3095 let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[4], x11m18);
3096 let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[12], x12m17);
3097 let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[1], x13m16);
3098 let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[8], x14m15);
3099 let [y11, y18] = SseVector::column_butterfly2([m1118a, m1118b]);
3100
3101 let m1217a = SseVector::fmadd(values[0], self.twiddles_re[11], x1p28);
3102 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[4], x2p27);
3103 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[6], x3p26);
3104 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[9], x4p25);
3105 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[1], x5p24);
3106 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[13], x6p23);
3107 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[2], x7p22);
3108 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[8], x8p21);
3109 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[7], x9p20);
3110 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[3], x10p19);
3111 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[12], x11p18);
3112 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[0], x12p17);
3113 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[10], x13p16);
3114 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[5], x14p15);
3115 let m1217b = SseVector::mul(self.twiddles_im[11], x1m28);
3116 let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[4], x2m27);
3117 let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[6], x3m26);
3118 let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[9], x4m25);
3119 let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[1], x5m24);
3120 let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[13], x6m23);
3121 let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[2], x7m22);
3122 let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[8], x8m21);
3123 let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[7], x9m20);
3124 let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[3], x10m19);
3125 let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[12], x11m18);
3126 let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[0], x12m17);
3127 let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[10], x13m16);
3128 let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[5], x14m15);
3129 let [y12, y17] = SseVector::column_butterfly2([m1217a, m1217b]);
3130
3131 let m1316a = SseVector::fmadd(values[0], self.twiddles_re[12], x1p28);
3132 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[2], x2p27);
3133 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[9], x3p26);
3134 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[5], x4p25);
3135 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[6], x5p24);
3136 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[8], x6p23);
3137 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[3], x7p22);
3138 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[11], x8p21);
3139 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[0], x9p20);
3140 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[13], x10p19);
3141 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[1], x11p18);
3142 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[10], x12p17);
3143 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[4], x13p16);
3144 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[7], x14p15);
3145 let m1316b = SseVector::mul(self.twiddles_im[12], x1m28);
3146 let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[2], x2m27);
3147 let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[9], x3m26);
3148 let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[5], x4m25);
3149 let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[6], x5m24);
3150 let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[8], x6m23);
3151 let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[3], x7m22);
3152 let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[11], x8m21);
3153 let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[0], x9m20);
3154 let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[13], x10m19);
3155 let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[1], x11m18);
3156 let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[10], x12m17);
3157 let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[4], x13m16);
3158 let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[7], x14m15);
3159 let [y13, y16] = SseVector::column_butterfly2([m1316a, m1316b]);
3160
3161 let m1415a = SseVector::fmadd(values[0], self.twiddles_re[13], x1p28);
3162 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[0], x2p27);
3163 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[12], x3p26);
3164 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[1], x4p25);
3165 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[11], x5p24);
3166 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[2], x6p23);
3167 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[10], x7p22);
3168 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[3], x8p21);
3169 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[9], x9p20);
3170 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[4], x10p19);
3171 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[8], x11p18);
3172 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[5], x12p17);
3173 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[7], x13p16);
3174 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[6], x14p15);
3175 let m1415b = SseVector::mul(self.twiddles_im[13], x1m28);
3176 let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[0], x2m27);
3177 let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[12], x3m26);
3178 let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[1], x4m25);
3179 let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[11], x5m24);
3180 let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[2], x6m23);
3181 let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[10], x7m22);
3182 let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[3], x8m21);
3183 let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[9], x9m20);
3184 let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[4], x10m19);
3185 let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[8], x11m18);
3186 let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[5], x12m17);
3187 let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[7], x13m16);
3188 let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[6], x14m15);
3189 let [y14, y15] = SseVector::column_butterfly2([m1415a, m1415b]);
3190
3191
3192 [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12, y13, y14, y15, y16, y17, y18, y19, y20, y21, y22, y23, y24, y25, y26, y27, y28]
3193 }
3194}
3195
3196struct SseF64Butterfly29<T> {
3197 direction: FftDirection,
3198 twiddles_re: [__m128d; 14],
3199 twiddles_im: [__m128d; 14],
3200 _phantom: std::marker::PhantomData<T>,
3201}
3202
3203boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly29, 29, |this: &SseF64Butterfly29<_>| this.direction);
3204impl<T: FftNum> SseF64Butterfly29<T> {
3205 #[target_feature(enable = "sse4.1")]
3207 unsafe fn new(direction: FftDirection) -> Self {
3208 assert_f64::<T>();
3209 let twiddles = make_twiddles(29, direction);
3210 unsafe {Self {
3211 direction,
3212 twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
3213 twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
3214 _phantom: std::marker::PhantomData,
3215 }}
3216 }
3217
3218 #[inline(always)]
3219 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
3220 let values = read_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28 });
3221
3222 let out = self.perform_fft_direct(values);
3223
3224 write_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28 });
3225 }
3226
3227 #[inline(always)]
3228 pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 29]) -> [__m128d; 29] {
3229 let rotate = SseVector::make_rotate90(FftDirection::Inverse);
3230
3231 let y00 = values[0];
3232 let [x1p28, x1m28] = SseVector::column_butterfly2([values[1], values[28]]);
3233 let x1m28 = SseVector::apply_rotate90(rotate, x1m28);
3234 let y00 = SseVector::add(y00, x1p28);
3235 let [x2p27, x2m27] = SseVector::column_butterfly2([values[2], values[27]]);
3236 let x2m27 = SseVector::apply_rotate90(rotate, x2m27);
3237 let y00 = SseVector::add(y00, x2p27);
3238 let [x3p26, x3m26] = SseVector::column_butterfly2([values[3], values[26]]);
3239 let x3m26 = SseVector::apply_rotate90(rotate, x3m26);
3240 let y00 = SseVector::add(y00, x3p26);
3241 let [x4p25, x4m25] = SseVector::column_butterfly2([values[4], values[25]]);
3242 let x4m25 = SseVector::apply_rotate90(rotate, x4m25);
3243 let y00 = SseVector::add(y00, x4p25);
3244 let [x5p24, x5m24] = SseVector::column_butterfly2([values[5], values[24]]);
3245 let x5m24 = SseVector::apply_rotate90(rotate, x5m24);
3246 let y00 = SseVector::add(y00, x5p24);
3247 let [x6p23, x6m23] = SseVector::column_butterfly2([values[6], values[23]]);
3248 let x6m23 = SseVector::apply_rotate90(rotate, x6m23);
3249 let y00 = SseVector::add(y00, x6p23);
3250 let [x7p22, x7m22] = SseVector::column_butterfly2([values[7], values[22]]);
3251 let x7m22 = SseVector::apply_rotate90(rotate, x7m22);
3252 let y00 = SseVector::add(y00, x7p22);
3253 let [x8p21, x8m21] = SseVector::column_butterfly2([values[8], values[21]]);
3254 let x8m21 = SseVector::apply_rotate90(rotate, x8m21);
3255 let y00 = SseVector::add(y00, x8p21);
3256 let [x9p20, x9m20] = SseVector::column_butterfly2([values[9], values[20]]);
3257 let x9m20 = SseVector::apply_rotate90(rotate, x9m20);
3258 let y00 = SseVector::add(y00, x9p20);
3259 let [x10p19, x10m19] = SseVector::column_butterfly2([values[10], values[19]]);
3260 let x10m19 = SseVector::apply_rotate90(rotate, x10m19);
3261 let y00 = SseVector::add(y00, x10p19);
3262 let [x11p18, x11m18] = SseVector::column_butterfly2([values[11], values[18]]);
3263 let x11m18 = SseVector::apply_rotate90(rotate, x11m18);
3264 let y00 = SseVector::add(y00, x11p18);
3265 let [x12p17, x12m17] = SseVector::column_butterfly2([values[12], values[17]]);
3266 let x12m17 = SseVector::apply_rotate90(rotate, x12m17);
3267 let y00 = SseVector::add(y00, x12p17);
3268 let [x13p16, x13m16] = SseVector::column_butterfly2([values[13], values[16]]);
3269 let x13m16 = SseVector::apply_rotate90(rotate, x13m16);
3270 let y00 = SseVector::add(y00, x13p16);
3271 let [x14p15, x14m15] = SseVector::column_butterfly2([values[14], values[15]]);
3272 let x14m15 = SseVector::apply_rotate90(rotate, x14m15);
3273 let y00 = SseVector::add(y00, x14p15);
3274
3275 let m0128a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p28);
3276 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[1], x2p27);
3277 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[2], x3p26);
3278 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[3], x4p25);
3279 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[4], x5p24);
3280 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[5], x6p23);
3281 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[6], x7p22);
3282 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[7], x8p21);
3283 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[8], x9p20);
3284 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[9], x10p19);
3285 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[10], x11p18);
3286 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[11], x12p17);
3287 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[12], x13p16);
3288 let m0128a = SseVector::fmadd(m0128a, self.twiddles_re[13], x14p15);
3289 let m0128b = SseVector::mul(self.twiddles_im[0], x1m28);
3290 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[1], x2m27);
3291 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[2], x3m26);
3292 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[3], x4m25);
3293 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[4], x5m24);
3294 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[5], x6m23);
3295 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[6], x7m22);
3296 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[7], x8m21);
3297 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[8], x9m20);
3298 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[9], x10m19);
3299 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[10], x11m18);
3300 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[11], x12m17);
3301 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[12], x13m16);
3302 let m0128b = SseVector::fmadd(m0128b, self.twiddles_im[13], x14m15);
3303 let [y01, y28] = SseVector::column_butterfly2([m0128a, m0128b]);
3304
3305 let m0227a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p28);
3306 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[3], x2p27);
3307 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[5], x3p26);
3308 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[7], x4p25);
3309 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[9], x5p24);
3310 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[11], x6p23);
3311 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[13], x7p22);
3312 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[12], x8p21);
3313 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[10], x9p20);
3314 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[8], x10p19);
3315 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[6], x11p18);
3316 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[4], x12p17);
3317 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[2], x13p16);
3318 let m0227a = SseVector::fmadd(m0227a, self.twiddles_re[0], x14p15);
3319 let m0227b = SseVector::mul(self.twiddles_im[1], x1m28);
3320 let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[3], x2m27);
3321 let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[5], x3m26);
3322 let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[7], x4m25);
3323 let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[9], x5m24);
3324 let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[11], x6m23);
3325 let m0227b = SseVector::fmadd(m0227b, self.twiddles_im[13], x7m22);
3326 let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[12], x8m21);
3327 let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[10], x9m20);
3328 let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[8], x10m19);
3329 let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[6], x11m18);
3330 let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[4], x12m17);
3331 let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[2], x13m16);
3332 let m0227b = SseVector::nmadd(m0227b, self.twiddles_im[0], x14m15);
3333 let [y02, y27] = SseVector::column_butterfly2([m0227a, m0227b]);
3334
3335 let m0326a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p28);
3336 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[5], x2p27);
3337 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[8], x3p26);
3338 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[11], x4p25);
3339 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[13], x5p24);
3340 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[10], x6p23);
3341 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[7], x7p22);
3342 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[4], x8p21);
3343 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[1], x9p20);
3344 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[0], x10p19);
3345 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[3], x11p18);
3346 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[6], x12p17);
3347 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[9], x13p16);
3348 let m0326a = SseVector::fmadd(m0326a, self.twiddles_re[12], x14p15);
3349 let m0326b = SseVector::mul(self.twiddles_im[2], x1m28);
3350 let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[5], x2m27);
3351 let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[8], x3m26);
3352 let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[11], x4m25);
3353 let m0326b = SseVector::nmadd(m0326b, self.twiddles_im[13], x5m24);
3354 let m0326b = SseVector::nmadd(m0326b, self.twiddles_im[10], x6m23);
3355 let m0326b = SseVector::nmadd(m0326b, self.twiddles_im[7], x7m22);
3356 let m0326b = SseVector::nmadd(m0326b, self.twiddles_im[4], x8m21);
3357 let m0326b = SseVector::nmadd(m0326b, self.twiddles_im[1], x9m20);
3358 let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[0], x10m19);
3359 let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[3], x11m18);
3360 let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[6], x12m17);
3361 let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[9], x13m16);
3362 let m0326b = SseVector::fmadd(m0326b, self.twiddles_im[12], x14m15);
3363 let [y03, y26] = SseVector::column_butterfly2([m0326a, m0326b]);
3364
3365 let m0425a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p28);
3366 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[7], x2p27);
3367 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[11], x3p26);
3368 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[12], x4p25);
3369 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[8], x5p24);
3370 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[4], x6p23);
3371 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[0], x7p22);
3372 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[2], x8p21);
3373 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[6], x9p20);
3374 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[10], x10p19);
3375 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[13], x11p18);
3376 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[9], x12p17);
3377 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[5], x13p16);
3378 let m0425a = SseVector::fmadd(m0425a, self.twiddles_re[1], x14p15);
3379 let m0425b = SseVector::mul(self.twiddles_im[3], x1m28);
3380 let m0425b = SseVector::fmadd(m0425b, self.twiddles_im[7], x2m27);
3381 let m0425b = SseVector::fmadd(m0425b, self.twiddles_im[11], x3m26);
3382 let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[12], x4m25);
3383 let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[8], x5m24);
3384 let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[4], x6m23);
3385 let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[0], x7m22);
3386 let m0425b = SseVector::fmadd(m0425b, self.twiddles_im[2], x8m21);
3387 let m0425b = SseVector::fmadd(m0425b, self.twiddles_im[6], x9m20);
3388 let m0425b = SseVector::fmadd(m0425b, self.twiddles_im[10], x10m19);
3389 let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[13], x11m18);
3390 let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[9], x12m17);
3391 let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[5], x13m16);
3392 let m0425b = SseVector::nmadd(m0425b, self.twiddles_im[1], x14m15);
3393 let [y04, y25] = SseVector::column_butterfly2([m0425a, m0425b]);
3394
3395 let m0524a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p28);
3396 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[9], x2p27);
3397 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[13], x3p26);
3398 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[8], x4p25);
3399 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[3], x5p24);
3400 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[0], x6p23);
3401 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[5], x7p22);
3402 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[10], x8p21);
3403 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[12], x9p20);
3404 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[7], x10p19);
3405 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[2], x11p18);
3406 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[1], x12p17);
3407 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[6], x13p16);
3408 let m0524a = SseVector::fmadd(m0524a, self.twiddles_re[11], x14p15);
3409 let m0524b = SseVector::mul(self.twiddles_im[4], x1m28);
3410 let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[9], x2m27);
3411 let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[13], x3m26);
3412 let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[8], x4m25);
3413 let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[3], x5m24);
3414 let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[0], x6m23);
3415 let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[5], x7m22);
3416 let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[10], x8m21);
3417 let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[12], x9m20);
3418 let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[7], x10m19);
3419 let m0524b = SseVector::nmadd(m0524b, self.twiddles_im[2], x11m18);
3420 let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[1], x12m17);
3421 let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[6], x13m16);
3422 let m0524b = SseVector::fmadd(m0524b, self.twiddles_im[11], x14m15);
3423 let [y05, y24] = SseVector::column_butterfly2([m0524a, m0524b]);
3424
3425 let m0623a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p28);
3426 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[11], x2p27);
3427 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[10], x3p26);
3428 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[4], x4p25);
3429 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[0], x5p24);
3430 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[6], x6p23);
3431 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[12], x7p22);
3432 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[9], x8p21);
3433 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[3], x9p20);
3434 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[1], x10p19);
3435 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[7], x11p18);
3436 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[13], x12p17);
3437 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[8], x13p16);
3438 let m0623a = SseVector::fmadd(m0623a, self.twiddles_re[2], x14p15);
3439 let m0623b = SseVector::mul(self.twiddles_im[5], x1m28);
3440 let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[11], x2m27);
3441 let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[10], x3m26);
3442 let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[4], x4m25);
3443 let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[0], x5m24);
3444 let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[6], x6m23);
3445 let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[12], x7m22);
3446 let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[9], x8m21);
3447 let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[3], x9m20);
3448 let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[1], x10m19);
3449 let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[7], x11m18);
3450 let m0623b = SseVector::fmadd(m0623b, self.twiddles_im[13], x12m17);
3451 let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[8], x13m16);
3452 let m0623b = SseVector::nmadd(m0623b, self.twiddles_im[2], x14m15);
3453 let [y06, y23] = SseVector::column_butterfly2([m0623a, m0623b]);
3454
3455 let m0722a = SseVector::fmadd(values[0], self.twiddles_re[6], x1p28);
3456 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[13], x2p27);
3457 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[7], x3p26);
3458 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[0], x4p25);
3459 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[5], x5p24);
3460 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[12], x6p23);
3461 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[8], x7p22);
3462 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[1], x8p21);
3463 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[4], x9p20);
3464 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[11], x10p19);
3465 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[9], x11p18);
3466 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[2], x12p17);
3467 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[3], x13p16);
3468 let m0722a = SseVector::fmadd(m0722a, self.twiddles_re[10], x14p15);
3469 let m0722b = SseVector::mul(self.twiddles_im[6], x1m28);
3470 let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[13], x2m27);
3471 let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[7], x3m26);
3472 let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[0], x4m25);
3473 let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[5], x5m24);
3474 let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[12], x6m23);
3475 let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[8], x7m22);
3476 let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[1], x8m21);
3477 let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[4], x9m20);
3478 let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[11], x10m19);
3479 let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[9], x11m18);
3480 let m0722b = SseVector::nmadd(m0722b, self.twiddles_im[2], x12m17);
3481 let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[3], x13m16);
3482 let m0722b = SseVector::fmadd(m0722b, self.twiddles_im[10], x14m15);
3483 let [y07, y22] = SseVector::column_butterfly2([m0722a, m0722b]);
3484
3485 let m0821a = SseVector::fmadd(values[0], self.twiddles_re[7], x1p28);
3486 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[12], x2p27);
3487 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[4], x3p26);
3488 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[2], x4p25);
3489 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[10], x5p24);
3490 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[9], x6p23);
3491 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[1], x7p22);
3492 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[5], x8p21);
3493 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[13], x9p20);
3494 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[6], x10p19);
3495 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[0], x11p18);
3496 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[8], x12p17);
3497 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[11], x13p16);
3498 let m0821a = SseVector::fmadd(m0821a, self.twiddles_re[3], x14p15);
3499 let m0821b = SseVector::mul(self.twiddles_im[7], x1m28);
3500 let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[12], x2m27);
3501 let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[4], x3m26);
3502 let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[2], x4m25);
3503 let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[10], x5m24);
3504 let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[9], x6m23);
3505 let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[1], x7m22);
3506 let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[5], x8m21);
3507 let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[13], x9m20);
3508 let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[6], x10m19);
3509 let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[0], x11m18);
3510 let m0821b = SseVector::fmadd(m0821b, self.twiddles_im[8], x12m17);
3511 let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[11], x13m16);
3512 let m0821b = SseVector::nmadd(m0821b, self.twiddles_im[3], x14m15);
3513 let [y08, y21] = SseVector::column_butterfly2([m0821a, m0821b]);
3514
3515 let m0920a = SseVector::fmadd(values[0], self.twiddles_re[8], x1p28);
3516 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[10], x2p27);
3517 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[1], x3p26);
3518 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[6], x4p25);
3519 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[12], x5p24);
3520 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[3], x6p23);
3521 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[4], x7p22);
3522 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[13], x8p21);
3523 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[5], x9p20);
3524 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[2], x10p19);
3525 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[11], x11p18);
3526 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[7], x12p17);
3527 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[0], x13p16);
3528 let m0920a = SseVector::fmadd(m0920a, self.twiddles_re[9], x14p15);
3529 let m0920b = SseVector::mul(self.twiddles_im[8], x1m28);
3530 let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[10], x2m27);
3531 let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[1], x3m26);
3532 let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[6], x4m25);
3533 let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[12], x5m24);
3534 let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[3], x6m23);
3535 let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[4], x7m22);
3536 let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[13], x8m21);
3537 let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[5], x9m20);
3538 let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[2], x10m19);
3539 let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[11], x11m18);
3540 let m0920b = SseVector::nmadd(m0920b, self.twiddles_im[7], x12m17);
3541 let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[0], x13m16);
3542 let m0920b = SseVector::fmadd(m0920b, self.twiddles_im[9], x14m15);
3543 let [y09, y20] = SseVector::column_butterfly2([m0920a, m0920b]);
3544
3545 let m1019a = SseVector::fmadd(values[0], self.twiddles_re[9], x1p28);
3546 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[8], x2p27);
3547 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[0], x3p26);
3548 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[10], x4p25);
3549 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[7], x5p24);
3550 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[1], x6p23);
3551 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[11], x7p22);
3552 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[6], x8p21);
3553 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[2], x9p20);
3554 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[12], x10p19);
3555 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[5], x11p18);
3556 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[3], x12p17);
3557 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[13], x13p16);
3558 let m1019a = SseVector::fmadd(m1019a, self.twiddles_re[4], x14p15);
3559 let m1019b = SseVector::mul(self.twiddles_im[9], x1m28);
3560 let m1019b = SseVector::nmadd(m1019b, self.twiddles_im[8], x2m27);
3561 let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[0], x3m26);
3562 let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[10], x4m25);
3563 let m1019b = SseVector::nmadd(m1019b, self.twiddles_im[7], x5m24);
3564 let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[1], x6m23);
3565 let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[11], x7m22);
3566 let m1019b = SseVector::nmadd(m1019b, self.twiddles_im[6], x8m21);
3567 let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[2], x9m20);
3568 let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[12], x10m19);
3569 let m1019b = SseVector::nmadd(m1019b, self.twiddles_im[5], x11m18);
3570 let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[3], x12m17);
3571 let m1019b = SseVector::fmadd(m1019b, self.twiddles_im[13], x13m16);
3572 let m1019b = SseVector::nmadd(m1019b, self.twiddles_im[4], x14m15);
3573 let [y10, y19] = SseVector::column_butterfly2([m1019a, m1019b]);
3574
3575 let m1118a = SseVector::fmadd(values[0], self.twiddles_re[10], x1p28);
3576 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[6], x2p27);
3577 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[3], x3p26);
3578 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[13], x4p25);
3579 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[2], x5p24);
3580 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[7], x6p23);
3581 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[9], x7p22);
3582 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[0], x8p21);
3583 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[11], x9p20);
3584 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[5], x10p19);
3585 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[4], x11p18);
3586 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[12], x12p17);
3587 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[1], x13p16);
3588 let m1118a = SseVector::fmadd(m1118a, self.twiddles_re[8], x14p15);
3589 let m1118b = SseVector::mul(self.twiddles_im[10], x1m28);
3590 let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[6], x2m27);
3591 let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[3], x3m26);
3592 let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[13], x4m25);
3593 let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[2], x5m24);
3594 let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[7], x6m23);
3595 let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[9], x7m22);
3596 let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[0], x8m21);
3597 let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[11], x9m20);
3598 let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[5], x10m19);
3599 let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[4], x11m18);
3600 let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[12], x12m17);
3601 let m1118b = SseVector::nmadd(m1118b, self.twiddles_im[1], x13m16);
3602 let m1118b = SseVector::fmadd(m1118b, self.twiddles_im[8], x14m15);
3603 let [y11, y18] = SseVector::column_butterfly2([m1118a, m1118b]);
3604
3605 let m1217a = SseVector::fmadd(values[0], self.twiddles_re[11], x1p28);
3606 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[4], x2p27);
3607 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[6], x3p26);
3608 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[9], x4p25);
3609 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[1], x5p24);
3610 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[13], x6p23);
3611 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[2], x7p22);
3612 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[8], x8p21);
3613 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[7], x9p20);
3614 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[3], x10p19);
3615 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[12], x11p18);
3616 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[0], x12p17);
3617 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[10], x13p16);
3618 let m1217a = SseVector::fmadd(m1217a, self.twiddles_re[5], x14p15);
3619 let m1217b = SseVector::mul(self.twiddles_im[11], x1m28);
3620 let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[4], x2m27);
3621 let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[6], x3m26);
3622 let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[9], x4m25);
3623 let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[1], x5m24);
3624 let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[13], x6m23);
3625 let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[2], x7m22);
3626 let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[8], x8m21);
3627 let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[7], x9m20);
3628 let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[3], x10m19);
3629 let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[12], x11m18);
3630 let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[0], x12m17);
3631 let m1217b = SseVector::fmadd(m1217b, self.twiddles_im[10], x13m16);
3632 let m1217b = SseVector::nmadd(m1217b, self.twiddles_im[5], x14m15);
3633 let [y12, y17] = SseVector::column_butterfly2([m1217a, m1217b]);
3634
3635 let m1316a = SseVector::fmadd(values[0], self.twiddles_re[12], x1p28);
3636 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[2], x2p27);
3637 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[9], x3p26);
3638 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[5], x4p25);
3639 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[6], x5p24);
3640 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[8], x6p23);
3641 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[3], x7p22);
3642 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[11], x8p21);
3643 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[0], x9p20);
3644 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[13], x10p19);
3645 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[1], x11p18);
3646 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[10], x12p17);
3647 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[4], x13p16);
3648 let m1316a = SseVector::fmadd(m1316a, self.twiddles_re[7], x14p15);
3649 let m1316b = SseVector::mul(self.twiddles_im[12], x1m28);
3650 let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[2], x2m27);
3651 let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[9], x3m26);
3652 let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[5], x4m25);
3653 let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[6], x5m24);
3654 let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[8], x6m23);
3655 let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[3], x7m22);
3656 let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[11], x8m21);
3657 let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[0], x9m20);
3658 let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[13], x10m19);
3659 let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[1], x11m18);
3660 let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[10], x12m17);
3661 let m1316b = SseVector::nmadd(m1316b, self.twiddles_im[4], x13m16);
3662 let m1316b = SseVector::fmadd(m1316b, self.twiddles_im[7], x14m15);
3663 let [y13, y16] = SseVector::column_butterfly2([m1316a, m1316b]);
3664
3665 let m1415a = SseVector::fmadd(values[0], self.twiddles_re[13], x1p28);
3666 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[0], x2p27);
3667 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[12], x3p26);
3668 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[1], x4p25);
3669 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[11], x5p24);
3670 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[2], x6p23);
3671 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[10], x7p22);
3672 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[3], x8p21);
3673 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[9], x9p20);
3674 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[4], x10p19);
3675 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[8], x11p18);
3676 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[5], x12p17);
3677 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[7], x13p16);
3678 let m1415a = SseVector::fmadd(m1415a, self.twiddles_re[6], x14p15);
3679 let m1415b = SseVector::mul(self.twiddles_im[13], x1m28);
3680 let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[0], x2m27);
3681 let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[12], x3m26);
3682 let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[1], x4m25);
3683 let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[11], x5m24);
3684 let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[2], x6m23);
3685 let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[10], x7m22);
3686 let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[3], x8m21);
3687 let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[9], x9m20);
3688 let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[4], x10m19);
3689 let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[8], x11m18);
3690 let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[5], x12m17);
3691 let m1415b = SseVector::fmadd(m1415b, self.twiddles_im[7], x13m16);
3692 let m1415b = SseVector::nmadd(m1415b, self.twiddles_im[6], x14m15);
3693 let [y14, y15] = SseVector::column_butterfly2([m1415a, m1415b]);
3694
3695
3696 [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12, y13, y14, y15, y16, y17, y18, y19, y20, y21, y22, y23, y24, y25, y26, y27, y28]
3697 }
3698}
3699
3700struct SseF32Butterfly31<T> {
3701 direction: FftDirection,
3702 twiddles_re: [__m128; 15],
3703 twiddles_im: [__m128; 15],
3704 _phantom: std::marker::PhantomData<T>,
3705}
3706
3707boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly31, 31, |this: &SseF32Butterfly31<_>| this.direction);
3708impl<T: FftNum> SseF32Butterfly31<T> {
3709 #[target_feature(enable = "sse4.1")]
3711 unsafe fn new(direction: FftDirection) -> Self {
3712 assert_f32::<T>();
3713 let twiddles = make_twiddles(31, direction);
3714 Self {
3715 direction,
3716 twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
3717 twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
3718 _phantom: std::marker::PhantomData,
3719 }
3720 }
3721
3722 #[inline(always)]
3723 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
3724 let values = read_partial1_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 });
3725
3726 let out = self.perform_parallel_fft_direct(values);
3727
3728 write_partial_lo_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 } );
3729 }
3730
3731 #[inline(always)]
3732 pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
3733 let input_packed = read_complex_to_array!(buffer, { 0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60 });
3734
3735 let values = [
3736 extract_lo_hi_f32(input_packed[0], input_packed[15]),
3737 extract_hi_lo_f32(input_packed[0], input_packed[16]),
3738 extract_lo_hi_f32(input_packed[1], input_packed[16]),
3739 extract_hi_lo_f32(input_packed[1], input_packed[17]),
3740 extract_lo_hi_f32(input_packed[2], input_packed[17]),
3741 extract_hi_lo_f32(input_packed[2], input_packed[18]),
3742 extract_lo_hi_f32(input_packed[3], input_packed[18]),
3743 extract_hi_lo_f32(input_packed[3], input_packed[19]),
3744 extract_lo_hi_f32(input_packed[4], input_packed[19]),
3745 extract_hi_lo_f32(input_packed[4], input_packed[20]),
3746 extract_lo_hi_f32(input_packed[5], input_packed[20]),
3747 extract_hi_lo_f32(input_packed[5], input_packed[21]),
3748 extract_lo_hi_f32(input_packed[6], input_packed[21]),
3749 extract_hi_lo_f32(input_packed[6], input_packed[22]),
3750 extract_lo_hi_f32(input_packed[7], input_packed[22]),
3751 extract_hi_lo_f32(input_packed[7], input_packed[23]),
3752 extract_lo_hi_f32(input_packed[8], input_packed[23]),
3753 extract_hi_lo_f32(input_packed[8], input_packed[24]),
3754 extract_lo_hi_f32(input_packed[9], input_packed[24]),
3755 extract_hi_lo_f32(input_packed[9], input_packed[25]),
3756 extract_lo_hi_f32(input_packed[10], input_packed[25]),
3757 extract_hi_lo_f32(input_packed[10], input_packed[26]),
3758 extract_lo_hi_f32(input_packed[11], input_packed[26]),
3759 extract_hi_lo_f32(input_packed[11], input_packed[27]),
3760 extract_lo_hi_f32(input_packed[12], input_packed[27]),
3761 extract_hi_lo_f32(input_packed[12], input_packed[28]),
3762 extract_lo_hi_f32(input_packed[13], input_packed[28]),
3763 extract_hi_lo_f32(input_packed[13], input_packed[29]),
3764 extract_lo_hi_f32(input_packed[14], input_packed[29]),
3765 extract_hi_lo_f32(input_packed[14], input_packed[30]),
3766 extract_lo_hi_f32(input_packed[15], input_packed[30]),
3767 ];
3768
3769 let out = self.perform_parallel_fft_direct(values);
3770
3771 let out_packed = [
3772 extract_lo_lo_f32(out[0], out[1]),
3773 extract_lo_lo_f32(out[2], out[3]),
3774 extract_lo_lo_f32(out[4], out[5]),
3775 extract_lo_lo_f32(out[6], out[7]),
3776 extract_lo_lo_f32(out[8], out[9]),
3777 extract_lo_lo_f32(out[10], out[11]),
3778 extract_lo_lo_f32(out[12], out[13]),
3779 extract_lo_lo_f32(out[14], out[15]),
3780 extract_lo_lo_f32(out[16], out[17]),
3781 extract_lo_lo_f32(out[18], out[19]),
3782 extract_lo_lo_f32(out[20], out[21]),
3783 extract_lo_lo_f32(out[22], out[23]),
3784 extract_lo_lo_f32(out[24], out[25]),
3785 extract_lo_lo_f32(out[26], out[27]),
3786 extract_lo_lo_f32(out[28], out[29]),
3787 extract_lo_hi_f32(out[30], out[0]),
3788 extract_hi_hi_f32(out[1], out[2]),
3789 extract_hi_hi_f32(out[3], out[4]),
3790 extract_hi_hi_f32(out[5], out[6]),
3791 extract_hi_hi_f32(out[7], out[8]),
3792 extract_hi_hi_f32(out[9], out[10]),
3793 extract_hi_hi_f32(out[11], out[12]),
3794 extract_hi_hi_f32(out[13], out[14]),
3795 extract_hi_hi_f32(out[15], out[16]),
3796 extract_hi_hi_f32(out[17], out[18]),
3797 extract_hi_hi_f32(out[19], out[20]),
3798 extract_hi_hi_f32(out[21], out[22]),
3799 extract_hi_hi_f32(out[23], out[24]),
3800 extract_hi_hi_f32(out[25], out[26]),
3801 extract_hi_hi_f32(out[27], out[28]),
3802 extract_hi_hi_f32(out[29], out[30]),
3803 ];
3804
3805 write_complex_to_array_strided!(out_packed, buffer, 2, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 });
3806 }
3807
3808 #[inline(always)]
3809 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 31]) -> [__m128; 31] {
3810 let rotate = SseVector::make_rotate90(FftDirection::Inverse);
3811
3812 let y00 = values[0];
3813 let [x1p30, x1m30] = SseVector::column_butterfly2([values[1], values[30]]);
3814 let x1m30 = SseVector::apply_rotate90(rotate, x1m30);
3815 let y00 = SseVector::add(y00, x1p30);
3816 let [x2p29, x2m29] = SseVector::column_butterfly2([values[2], values[29]]);
3817 let x2m29 = SseVector::apply_rotate90(rotate, x2m29);
3818 let y00 = SseVector::add(y00, x2p29);
3819 let [x3p28, x3m28] = SseVector::column_butterfly2([values[3], values[28]]);
3820 let x3m28 = SseVector::apply_rotate90(rotate, x3m28);
3821 let y00 = SseVector::add(y00, x3p28);
3822 let [x4p27, x4m27] = SseVector::column_butterfly2([values[4], values[27]]);
3823 let x4m27 = SseVector::apply_rotate90(rotate, x4m27);
3824 let y00 = SseVector::add(y00, x4p27);
3825 let [x5p26, x5m26] = SseVector::column_butterfly2([values[5], values[26]]);
3826 let x5m26 = SseVector::apply_rotate90(rotate, x5m26);
3827 let y00 = SseVector::add(y00, x5p26);
3828 let [x6p25, x6m25] = SseVector::column_butterfly2([values[6], values[25]]);
3829 let x6m25 = SseVector::apply_rotate90(rotate, x6m25);
3830 let y00 = SseVector::add(y00, x6p25);
3831 let [x7p24, x7m24] = SseVector::column_butterfly2([values[7], values[24]]);
3832 let x7m24 = SseVector::apply_rotate90(rotate, x7m24);
3833 let y00 = SseVector::add(y00, x7p24);
3834 let [x8p23, x8m23] = SseVector::column_butterfly2([values[8], values[23]]);
3835 let x8m23 = SseVector::apply_rotate90(rotate, x8m23);
3836 let y00 = SseVector::add(y00, x8p23);
3837 let [x9p22, x9m22] = SseVector::column_butterfly2([values[9], values[22]]);
3838 let x9m22 = SseVector::apply_rotate90(rotate, x9m22);
3839 let y00 = SseVector::add(y00, x9p22);
3840 let [x10p21, x10m21] = SseVector::column_butterfly2([values[10], values[21]]);
3841 let x10m21 = SseVector::apply_rotate90(rotate, x10m21);
3842 let y00 = SseVector::add(y00, x10p21);
3843 let [x11p20, x11m20] = SseVector::column_butterfly2([values[11], values[20]]);
3844 let x11m20 = SseVector::apply_rotate90(rotate, x11m20);
3845 let y00 = SseVector::add(y00, x11p20);
3846 let [x12p19, x12m19] = SseVector::column_butterfly2([values[12], values[19]]);
3847 let x12m19 = SseVector::apply_rotate90(rotate, x12m19);
3848 let y00 = SseVector::add(y00, x12p19);
3849 let [x13p18, x13m18] = SseVector::column_butterfly2([values[13], values[18]]);
3850 let x13m18 = SseVector::apply_rotate90(rotate, x13m18);
3851 let y00 = SseVector::add(y00, x13p18);
3852 let [x14p17, x14m17] = SseVector::column_butterfly2([values[14], values[17]]);
3853 let x14m17 = SseVector::apply_rotate90(rotate, x14m17);
3854 let y00 = SseVector::add(y00, x14p17);
3855 let [x15p16, x15m16] = SseVector::column_butterfly2([values[15], values[16]]);
3856 let x15m16 = SseVector::apply_rotate90(rotate, x15m16);
3857 let y00 = SseVector::add(y00, x15p16);
3858
3859 let m0130a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p30);
3860 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[1], x2p29);
3861 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[2], x3p28);
3862 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[3], x4p27);
3863 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[4], x5p26);
3864 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[5], x6p25);
3865 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[6], x7p24);
3866 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[7], x8p23);
3867 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[8], x9p22);
3868 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[9], x10p21);
3869 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[10], x11p20);
3870 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[11], x12p19);
3871 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[12], x13p18);
3872 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[13], x14p17);
3873 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[14], x15p16);
3874 let m0130b = SseVector::mul(self.twiddles_im[0], x1m30);
3875 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[1], x2m29);
3876 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[2], x3m28);
3877 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[3], x4m27);
3878 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[4], x5m26);
3879 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[5], x6m25);
3880 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[6], x7m24);
3881 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[7], x8m23);
3882 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[8], x9m22);
3883 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[9], x10m21);
3884 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[10], x11m20);
3885 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[11], x12m19);
3886 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[12], x13m18);
3887 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[13], x14m17);
3888 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[14], x15m16);
3889 let [y01, y30] = SseVector::column_butterfly2([m0130a, m0130b]);
3890
3891 let m0229a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p30);
3892 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[3], x2p29);
3893 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[5], x3p28);
3894 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[7], x4p27);
3895 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[9], x5p26);
3896 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[11], x6p25);
3897 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[13], x7p24);
3898 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[14], x8p23);
3899 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[12], x9p22);
3900 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[10], x10p21);
3901 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[8], x11p20);
3902 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[6], x12p19);
3903 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[4], x13p18);
3904 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[2], x14p17);
3905 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[0], x15p16);
3906 let m0229b = SseVector::mul(self.twiddles_im[1], x1m30);
3907 let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[3], x2m29);
3908 let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[5], x3m28);
3909 let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[7], x4m27);
3910 let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[9], x5m26);
3911 let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[11], x6m25);
3912 let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[13], x7m24);
3913 let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[14], x8m23);
3914 let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[12], x9m22);
3915 let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[10], x10m21);
3916 let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[8], x11m20);
3917 let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[6], x12m19);
3918 let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[4], x13m18);
3919 let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[2], x14m17);
3920 let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[0], x15m16);
3921 let [y02, y29] = SseVector::column_butterfly2([m0229a, m0229b]);
3922
3923 let m0328a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p30);
3924 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[5], x2p29);
3925 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[8], x3p28);
3926 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[11], x4p27);
3927 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[14], x5p26);
3928 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[12], x6p25);
3929 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[9], x7p24);
3930 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[6], x8p23);
3931 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[3], x9p22);
3932 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[0], x10p21);
3933 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[1], x11p20);
3934 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[4], x12p19);
3935 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[7], x13p18);
3936 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[10], x14p17);
3937 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[13], x15p16);
3938 let m0328b = SseVector::mul(self.twiddles_im[2], x1m30);
3939 let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[5], x2m29);
3940 let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[8], x3m28);
3941 let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[11], x4m27);
3942 let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[14], x5m26);
3943 let m0328b = SseVector::nmadd(m0328b, self.twiddles_im[12], x6m25);
3944 let m0328b = SseVector::nmadd(m0328b, self.twiddles_im[9], x7m24);
3945 let m0328b = SseVector::nmadd(m0328b, self.twiddles_im[6], x8m23);
3946 let m0328b = SseVector::nmadd(m0328b, self.twiddles_im[3], x9m22);
3947 let m0328b = SseVector::nmadd(m0328b, self.twiddles_im[0], x10m21);
3948 let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[1], x11m20);
3949 let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[4], x12m19);
3950 let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[7], x13m18);
3951 let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[10], x14m17);
3952 let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[13], x15m16);
3953 let [y03, y28] = SseVector::column_butterfly2([m0328a, m0328b]);
3954
3955 let m0427a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p30);
3956 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[7], x2p29);
3957 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[11], x3p28);
3958 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[14], x4p27);
3959 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[10], x5p26);
3960 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[6], x6p25);
3961 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[2], x7p24);
3962 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[0], x8p23);
3963 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[4], x9p22);
3964 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[8], x10p21);
3965 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[12], x11p20);
3966 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[13], x12p19);
3967 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[9], x13p18);
3968 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[5], x14p17);
3969 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[1], x15p16);
3970 let m0427b = SseVector::mul(self.twiddles_im[3], x1m30);
3971 let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[7], x2m29);
3972 let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[11], x3m28);
3973 let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[14], x4m27);
3974 let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[10], x5m26);
3975 let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[6], x6m25);
3976 let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[2], x7m24);
3977 let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[0], x8m23);
3978 let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[4], x9m22);
3979 let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[8], x10m21);
3980 let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[12], x11m20);
3981 let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[13], x12m19);
3982 let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[9], x13m18);
3983 let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[5], x14m17);
3984 let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[1], x15m16);
3985 let [y04, y27] = SseVector::column_butterfly2([m0427a, m0427b]);
3986
3987 let m0526a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p30);
3988 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[9], x2p29);
3989 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[14], x3p28);
3990 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[10], x4p27);
3991 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[5], x5p26);
3992 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[0], x6p25);
3993 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[3], x7p24);
3994 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[8], x8p23);
3995 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[13], x9p22);
3996 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[11], x10p21);
3997 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[6], x11p20);
3998 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[1], x12p19);
3999 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[2], x13p18);
4000 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[7], x14p17);
4001 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[12], x15p16);
4002 let m0526b = SseVector::mul(self.twiddles_im[4], x1m30);
4003 let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[9], x2m29);
4004 let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[14], x3m28);
4005 let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[10], x4m27);
4006 let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[5], x5m26);
4007 let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[0], x6m25);
4008 let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[3], x7m24);
4009 let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[8], x8m23);
4010 let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[13], x9m22);
4011 let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[11], x10m21);
4012 let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[6], x11m20);
4013 let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[1], x12m19);
4014 let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[2], x13m18);
4015 let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[7], x14m17);
4016 let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[12], x15m16);
4017 let [y05, y26] = SseVector::column_butterfly2([m0526a, m0526b]);
4018
4019 let m0625a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p30);
4020 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[11], x2p29);
4021 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[12], x3p28);
4022 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[6], x4p27);
4023 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[0], x5p26);
4024 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[4], x6p25);
4025 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[10], x7p24);
4026 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[13], x8p23);
4027 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[7], x9p22);
4028 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[1], x10p21);
4029 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[3], x11p20);
4030 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[9], x12p19);
4031 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[14], x13p18);
4032 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[8], x14p17);
4033 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[2], x15p16);
4034 let m0625b = SseVector::mul(self.twiddles_im[5], x1m30);
4035 let m0625b = SseVector::fmadd(m0625b, self.twiddles_im[11], x2m29);
4036 let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[12], x3m28);
4037 let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[6], x4m27);
4038 let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[0], x5m26);
4039 let m0625b = SseVector::fmadd(m0625b, self.twiddles_im[4], x6m25);
4040 let m0625b = SseVector::fmadd(m0625b, self.twiddles_im[10], x7m24);
4041 let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[13], x8m23);
4042 let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[7], x9m22);
4043 let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[1], x10m21);
4044 let m0625b = SseVector::fmadd(m0625b, self.twiddles_im[3], x11m20);
4045 let m0625b = SseVector::fmadd(m0625b, self.twiddles_im[9], x12m19);
4046 let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[14], x13m18);
4047 let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[8], x14m17);
4048 let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[2], x15m16);
4049 let [y06, y25] = SseVector::column_butterfly2([m0625a, m0625b]);
4050
4051 let m0724a = SseVector::fmadd(values[0], self.twiddles_re[6], x1p30);
4052 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[13], x2p29);
4053 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[9], x3p28);
4054 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[2], x4p27);
4055 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[3], x5p26);
4056 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[10], x6p25);
4057 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[12], x7p24);
4058 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[5], x8p23);
4059 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[0], x9p22);
4060 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[7], x10p21);
4061 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[14], x11p20);
4062 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[8], x12p19);
4063 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[1], x13p18);
4064 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[4], x14p17);
4065 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[11], x15p16);
4066 let m0724b = SseVector::mul(self.twiddles_im[6], x1m30);
4067 let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[13], x2m29);
4068 let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[9], x3m28);
4069 let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[2], x4m27);
4070 let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[3], x5m26);
4071 let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[10], x6m25);
4072 let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[12], x7m24);
4073 let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[5], x8m23);
4074 let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[0], x9m22);
4075 let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[7], x10m21);
4076 let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[14], x11m20);
4077 let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[8], x12m19);
4078 let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[1], x13m18);
4079 let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[4], x14m17);
4080 let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[11], x15m16);
4081 let [y07, y24] = SseVector::column_butterfly2([m0724a, m0724b]);
4082
4083 let m0823a = SseVector::fmadd(values[0], self.twiddles_re[7], x1p30);
4084 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[14], x2p29);
4085 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[6], x3p28);
4086 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[0], x4p27);
4087 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[8], x5p26);
4088 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[13], x6p25);
4089 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[5], x7p24);
4090 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[1], x8p23);
4091 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[9], x9p22);
4092 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[12], x10p21);
4093 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[4], x11p20);
4094 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[2], x12p19);
4095 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[10], x13p18);
4096 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[11], x14p17);
4097 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[3], x15p16);
4098 let m0823b = SseVector::mul(self.twiddles_im[7], x1m30);
4099 let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[14], x2m29);
4100 let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[6], x3m28);
4101 let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[0], x4m27);
4102 let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[8], x5m26);
4103 let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[13], x6m25);
4104 let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[5], x7m24);
4105 let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[1], x8m23);
4106 let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[9], x9m22);
4107 let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[12], x10m21);
4108 let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[4], x11m20);
4109 let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[2], x12m19);
4110 let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[10], x13m18);
4111 let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[11], x14m17);
4112 let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[3], x15m16);
4113 let [y08, y23] = SseVector::column_butterfly2([m0823a, m0823b]);
4114
4115 let m0922a = SseVector::fmadd(values[0], self.twiddles_re[8], x1p30);
4116 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[12], x2p29);
4117 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[3], x3p28);
4118 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[4], x4p27);
4119 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[13], x5p26);
4120 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[7], x6p25);
4121 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[0], x7p24);
4122 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[9], x8p23);
4123 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[11], x9p22);
4124 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[2], x10p21);
4125 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[5], x11p20);
4126 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[14], x12p19);
4127 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[6], x13p18);
4128 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[1], x14p17);
4129 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[10], x15p16);
4130 let m0922b = SseVector::mul(self.twiddles_im[8], x1m30);
4131 let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[12], x2m29);
4132 let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[3], x3m28);
4133 let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[4], x4m27);
4134 let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[13], x5m26);
4135 let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[7], x6m25);
4136 let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[0], x7m24);
4137 let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[9], x8m23);
4138 let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[11], x9m22);
4139 let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[2], x10m21);
4140 let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[5], x11m20);
4141 let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[14], x12m19);
4142 let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[6], x13m18);
4143 let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[1], x14m17);
4144 let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[10], x15m16);
4145 let [y09, y22] = SseVector::column_butterfly2([m0922a, m0922b]);
4146
4147 let m1021a = SseVector::fmadd(values[0], self.twiddles_re[9], x1p30);
4148 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[10], x2p29);
4149 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[0], x3p28);
4150 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[8], x4p27);
4151 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[11], x5p26);
4152 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[1], x6p25);
4153 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[7], x7p24);
4154 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[12], x8p23);
4155 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[2], x9p22);
4156 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[6], x10p21);
4157 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[13], x11p20);
4158 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[3], x12p19);
4159 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[5], x13p18);
4160 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[14], x14p17);
4161 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[4], x15p16);
4162 let m1021b = SseVector::mul(self.twiddles_im[9], x1m30);
4163 let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[10], x2m29);
4164 let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[0], x3m28);
4165 let m1021b = SseVector::fmadd(m1021b, self.twiddles_im[8], x4m27);
4166 let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[11], x5m26);
4167 let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[1], x6m25);
4168 let m1021b = SseVector::fmadd(m1021b, self.twiddles_im[7], x7m24);
4169 let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[12], x8m23);
4170 let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[2], x9m22);
4171 let m1021b = SseVector::fmadd(m1021b, self.twiddles_im[6], x10m21);
4172 let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[13], x11m20);
4173 let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[3], x12m19);
4174 let m1021b = SseVector::fmadd(m1021b, self.twiddles_im[5], x13m18);
4175 let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[14], x14m17);
4176 let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[4], x15m16);
4177 let [y10, y21] = SseVector::column_butterfly2([m1021a, m1021b]);
4178
4179 let m1120a = SseVector::fmadd(values[0], self.twiddles_re[10], x1p30);
4180 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[8], x2p29);
4181 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[1], x3p28);
4182 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[12], x4p27);
4183 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[6], x5p26);
4184 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[3], x6p25);
4185 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[14], x7p24);
4186 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[4], x8p23);
4187 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[5], x9p22);
4188 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[13], x10p21);
4189 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[2], x11p20);
4190 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[7], x12p19);
4191 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[11], x13p18);
4192 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[0], x14p17);
4193 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[9], x15p16);
4194 let m1120b = SseVector::mul(self.twiddles_im[10], x1m30);
4195 let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[8], x2m29);
4196 let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[1], x3m28);
4197 let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[12], x4m27);
4198 let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[6], x5m26);
4199 let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[3], x6m25);
4200 let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[14], x7m24);
4201 let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[4], x8m23);
4202 let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[5], x9m22);
4203 let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[13], x10m21);
4204 let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[2], x11m20);
4205 let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[7], x12m19);
4206 let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[11], x13m18);
4207 let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[0], x14m17);
4208 let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[9], x15m16);
4209 let [y11, y20] = SseVector::column_butterfly2([m1120a, m1120b]);
4210
4211 let m1219a = SseVector::fmadd(values[0], self.twiddles_re[11], x1p30);
4212 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[6], x2p29);
4213 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[4], x3p28);
4214 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[13], x4p27);
4215 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[1], x5p26);
4216 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[9], x6p25);
4217 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[8], x7p24);
4218 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[2], x8p23);
4219 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[14], x9p22);
4220 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[3], x10p21);
4221 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[7], x11p20);
4222 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[10], x12p19);
4223 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[0], x13p18);
4224 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[12], x14p17);
4225 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[5], x15p16);
4226 let m1219b = SseVector::mul(self.twiddles_im[11], x1m30);
4227 let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[6], x2m29);
4228 let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[4], x3m28);
4229 let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[13], x4m27);
4230 let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[1], x5m26);
4231 let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[9], x6m25);
4232 let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[8], x7m24);
4233 let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[2], x8m23);
4234 let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[14], x9m22);
4235 let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[3], x10m21);
4236 let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[7], x11m20);
4237 let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[10], x12m19);
4238 let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[0], x13m18);
4239 let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[12], x14m17);
4240 let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[5], x15m16);
4241 let [y12, y19] = SseVector::column_butterfly2([m1219a, m1219b]);
4242
4243 let m1318a = SseVector::fmadd(values[0], self.twiddles_re[12], x1p30);
4244 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[4], x2p29);
4245 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[7], x3p28);
4246 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[9], x4p27);
4247 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[2], x5p26);
4248 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[14], x6p25);
4249 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[1], x7p24);
4250 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[10], x8p23);
4251 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[6], x9p22);
4252 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[5], x10p21);
4253 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[11], x11p20);
4254 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[0], x12p19);
4255 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[13], x13p18);
4256 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[3], x14p17);
4257 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[8], x15p16);
4258 let m1318b = SseVector::mul(self.twiddles_im[12], x1m30);
4259 let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[4], x2m29);
4260 let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[7], x3m28);
4261 let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[9], x4m27);
4262 let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[2], x5m26);
4263 let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[14], x6m25);
4264 let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[1], x7m24);
4265 let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[10], x8m23);
4266 let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[6], x9m22);
4267 let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[5], x10m21);
4268 let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[11], x11m20);
4269 let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[0], x12m19);
4270 let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[13], x13m18);
4271 let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[3], x14m17);
4272 let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[8], x15m16);
4273 let [y13, y18] = SseVector::column_butterfly2([m1318a, m1318b]);
4274
4275 let m1417a = SseVector::fmadd(values[0], self.twiddles_re[13], x1p30);
4276 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[2], x2p29);
4277 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[10], x3p28);
4278 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[5], x4p27);
4279 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[7], x5p26);
4280 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[8], x6p25);
4281 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[4], x7p24);
4282 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[11], x8p23);
4283 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[1], x9p22);
4284 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[14], x10p21);
4285 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[0], x11p20);
4286 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[12], x12p19);
4287 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[3], x13p18);
4288 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[9], x14p17);
4289 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[6], x15p16);
4290 let m1417b = SseVector::mul(self.twiddles_im[13], x1m30);
4291 let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[2], x2m29);
4292 let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[10], x3m28);
4293 let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[5], x4m27);
4294 let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[7], x5m26);
4295 let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[8], x6m25);
4296 let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[4], x7m24);
4297 let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[11], x8m23);
4298 let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[1], x9m22);
4299 let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[14], x10m21);
4300 let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[0], x11m20);
4301 let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[12], x12m19);
4302 let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[3], x13m18);
4303 let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[9], x14m17);
4304 let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[6], x15m16);
4305 let [y14, y17] = SseVector::column_butterfly2([m1417a, m1417b]);
4306
4307 let m1516a = SseVector::fmadd(values[0], self.twiddles_re[14], x1p30);
4308 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[0], x2p29);
4309 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[13], x3p28);
4310 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[1], x4p27);
4311 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[12], x5p26);
4312 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[2], x6p25);
4313 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[11], x7p24);
4314 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[3], x8p23);
4315 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[10], x9p22);
4316 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[4], x10p21);
4317 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[9], x11p20);
4318 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[5], x12p19);
4319 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[8], x13p18);
4320 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[6], x14p17);
4321 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[7], x15p16);
4322 let m1516b = SseVector::mul(self.twiddles_im[14], x1m30);
4323 let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[0], x2m29);
4324 let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[13], x3m28);
4325 let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[1], x4m27);
4326 let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[12], x5m26);
4327 let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[2], x6m25);
4328 let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[11], x7m24);
4329 let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[3], x8m23);
4330 let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[10], x9m22);
4331 let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[4], x10m21);
4332 let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[9], x11m20);
4333 let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[5], x12m19);
4334 let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[8], x13m18);
4335 let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[6], x14m17);
4336 let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[7], x15m16);
4337 let [y15, y16] = SseVector::column_butterfly2([m1516a, m1516b]);
4338
4339
4340 [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12, y13, y14, y15, y16, y17, y18, y19, y20, y21, y22, y23, y24, y25, y26, y27, y28, y29, y30]
4341 }
4342}
4343
4344struct SseF64Butterfly31<T> {
4345 direction: FftDirection,
4346 twiddles_re: [__m128d; 15],
4347 twiddles_im: [__m128d; 15],
4348 _phantom: std::marker::PhantomData<T>,
4349}
4350
4351boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly31, 31, |this: &SseF64Butterfly31<_>| this.direction);
4352impl<T: FftNum> SseF64Butterfly31<T> {
4353 #[target_feature(enable = "sse4.1")]
4355 unsafe fn new(direction: FftDirection) -> Self {
4356 assert_f64::<T>();
4357 let twiddles = make_twiddles(31, direction);
4358 unsafe {Self {
4359 direction,
4360 twiddles_re: twiddles.map(|t| SseVector::broadcast_scalar(t.re)),
4361 twiddles_im: twiddles.map(|t| SseVector::broadcast_scalar(t.im)),
4362 _phantom: std::marker::PhantomData,
4363 }}
4364 }
4365
4366 #[inline(always)]
4367 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
4368 let values = read_complex_to_array!(buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 });
4369
4370 let out = self.perform_fft_direct(values);
4371
4372 write_complex_to_array!(out, buffer, { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 });
4373 }
4374
4375 #[inline(always)]
4376 pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 31]) -> [__m128d; 31] {
4377 let rotate = SseVector::make_rotate90(FftDirection::Inverse);
4378
4379 let y00 = values[0];
4380 let [x1p30, x1m30] = SseVector::column_butterfly2([values[1], values[30]]);
4381 let x1m30 = SseVector::apply_rotate90(rotate, x1m30);
4382 let y00 = SseVector::add(y00, x1p30);
4383 let [x2p29, x2m29] = SseVector::column_butterfly2([values[2], values[29]]);
4384 let x2m29 = SseVector::apply_rotate90(rotate, x2m29);
4385 let y00 = SseVector::add(y00, x2p29);
4386 let [x3p28, x3m28] = SseVector::column_butterfly2([values[3], values[28]]);
4387 let x3m28 = SseVector::apply_rotate90(rotate, x3m28);
4388 let y00 = SseVector::add(y00, x3p28);
4389 let [x4p27, x4m27] = SseVector::column_butterfly2([values[4], values[27]]);
4390 let x4m27 = SseVector::apply_rotate90(rotate, x4m27);
4391 let y00 = SseVector::add(y00, x4p27);
4392 let [x5p26, x5m26] = SseVector::column_butterfly2([values[5], values[26]]);
4393 let x5m26 = SseVector::apply_rotate90(rotate, x5m26);
4394 let y00 = SseVector::add(y00, x5p26);
4395 let [x6p25, x6m25] = SseVector::column_butterfly2([values[6], values[25]]);
4396 let x6m25 = SseVector::apply_rotate90(rotate, x6m25);
4397 let y00 = SseVector::add(y00, x6p25);
4398 let [x7p24, x7m24] = SseVector::column_butterfly2([values[7], values[24]]);
4399 let x7m24 = SseVector::apply_rotate90(rotate, x7m24);
4400 let y00 = SseVector::add(y00, x7p24);
4401 let [x8p23, x8m23] = SseVector::column_butterfly2([values[8], values[23]]);
4402 let x8m23 = SseVector::apply_rotate90(rotate, x8m23);
4403 let y00 = SseVector::add(y00, x8p23);
4404 let [x9p22, x9m22] = SseVector::column_butterfly2([values[9], values[22]]);
4405 let x9m22 = SseVector::apply_rotate90(rotate, x9m22);
4406 let y00 = SseVector::add(y00, x9p22);
4407 let [x10p21, x10m21] = SseVector::column_butterfly2([values[10], values[21]]);
4408 let x10m21 = SseVector::apply_rotate90(rotate, x10m21);
4409 let y00 = SseVector::add(y00, x10p21);
4410 let [x11p20, x11m20] = SseVector::column_butterfly2([values[11], values[20]]);
4411 let x11m20 = SseVector::apply_rotate90(rotate, x11m20);
4412 let y00 = SseVector::add(y00, x11p20);
4413 let [x12p19, x12m19] = SseVector::column_butterfly2([values[12], values[19]]);
4414 let x12m19 = SseVector::apply_rotate90(rotate, x12m19);
4415 let y00 = SseVector::add(y00, x12p19);
4416 let [x13p18, x13m18] = SseVector::column_butterfly2([values[13], values[18]]);
4417 let x13m18 = SseVector::apply_rotate90(rotate, x13m18);
4418 let y00 = SseVector::add(y00, x13p18);
4419 let [x14p17, x14m17] = SseVector::column_butterfly2([values[14], values[17]]);
4420 let x14m17 = SseVector::apply_rotate90(rotate, x14m17);
4421 let y00 = SseVector::add(y00, x14p17);
4422 let [x15p16, x15m16] = SseVector::column_butterfly2([values[15], values[16]]);
4423 let x15m16 = SseVector::apply_rotate90(rotate, x15m16);
4424 let y00 = SseVector::add(y00, x15p16);
4425
4426 let m0130a = SseVector::fmadd(values[0], self.twiddles_re[0], x1p30);
4427 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[1], x2p29);
4428 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[2], x3p28);
4429 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[3], x4p27);
4430 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[4], x5p26);
4431 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[5], x6p25);
4432 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[6], x7p24);
4433 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[7], x8p23);
4434 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[8], x9p22);
4435 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[9], x10p21);
4436 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[10], x11p20);
4437 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[11], x12p19);
4438 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[12], x13p18);
4439 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[13], x14p17);
4440 let m0130a = SseVector::fmadd(m0130a, self.twiddles_re[14], x15p16);
4441 let m0130b = SseVector::mul(self.twiddles_im[0], x1m30);
4442 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[1], x2m29);
4443 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[2], x3m28);
4444 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[3], x4m27);
4445 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[4], x5m26);
4446 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[5], x6m25);
4447 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[6], x7m24);
4448 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[7], x8m23);
4449 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[8], x9m22);
4450 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[9], x10m21);
4451 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[10], x11m20);
4452 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[11], x12m19);
4453 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[12], x13m18);
4454 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[13], x14m17);
4455 let m0130b = SseVector::fmadd(m0130b, self.twiddles_im[14], x15m16);
4456 let [y01, y30] = SseVector::column_butterfly2([m0130a, m0130b]);
4457
4458 let m0229a = SseVector::fmadd(values[0], self.twiddles_re[1], x1p30);
4459 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[3], x2p29);
4460 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[5], x3p28);
4461 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[7], x4p27);
4462 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[9], x5p26);
4463 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[11], x6p25);
4464 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[13], x7p24);
4465 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[14], x8p23);
4466 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[12], x9p22);
4467 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[10], x10p21);
4468 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[8], x11p20);
4469 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[6], x12p19);
4470 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[4], x13p18);
4471 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[2], x14p17);
4472 let m0229a = SseVector::fmadd(m0229a, self.twiddles_re[0], x15p16);
4473 let m0229b = SseVector::mul(self.twiddles_im[1], x1m30);
4474 let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[3], x2m29);
4475 let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[5], x3m28);
4476 let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[7], x4m27);
4477 let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[9], x5m26);
4478 let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[11], x6m25);
4479 let m0229b = SseVector::fmadd(m0229b, self.twiddles_im[13], x7m24);
4480 let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[14], x8m23);
4481 let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[12], x9m22);
4482 let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[10], x10m21);
4483 let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[8], x11m20);
4484 let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[6], x12m19);
4485 let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[4], x13m18);
4486 let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[2], x14m17);
4487 let m0229b = SseVector::nmadd(m0229b, self.twiddles_im[0], x15m16);
4488 let [y02, y29] = SseVector::column_butterfly2([m0229a, m0229b]);
4489
4490 let m0328a = SseVector::fmadd(values[0], self.twiddles_re[2], x1p30);
4491 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[5], x2p29);
4492 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[8], x3p28);
4493 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[11], x4p27);
4494 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[14], x5p26);
4495 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[12], x6p25);
4496 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[9], x7p24);
4497 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[6], x8p23);
4498 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[3], x9p22);
4499 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[0], x10p21);
4500 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[1], x11p20);
4501 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[4], x12p19);
4502 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[7], x13p18);
4503 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[10], x14p17);
4504 let m0328a = SseVector::fmadd(m0328a, self.twiddles_re[13], x15p16);
4505 let m0328b = SseVector::mul(self.twiddles_im[2], x1m30);
4506 let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[5], x2m29);
4507 let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[8], x3m28);
4508 let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[11], x4m27);
4509 let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[14], x5m26);
4510 let m0328b = SseVector::nmadd(m0328b, self.twiddles_im[12], x6m25);
4511 let m0328b = SseVector::nmadd(m0328b, self.twiddles_im[9], x7m24);
4512 let m0328b = SseVector::nmadd(m0328b, self.twiddles_im[6], x8m23);
4513 let m0328b = SseVector::nmadd(m0328b, self.twiddles_im[3], x9m22);
4514 let m0328b = SseVector::nmadd(m0328b, self.twiddles_im[0], x10m21);
4515 let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[1], x11m20);
4516 let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[4], x12m19);
4517 let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[7], x13m18);
4518 let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[10], x14m17);
4519 let m0328b = SseVector::fmadd(m0328b, self.twiddles_im[13], x15m16);
4520 let [y03, y28] = SseVector::column_butterfly2([m0328a, m0328b]);
4521
4522 let m0427a = SseVector::fmadd(values[0], self.twiddles_re[3], x1p30);
4523 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[7], x2p29);
4524 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[11], x3p28);
4525 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[14], x4p27);
4526 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[10], x5p26);
4527 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[6], x6p25);
4528 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[2], x7p24);
4529 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[0], x8p23);
4530 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[4], x9p22);
4531 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[8], x10p21);
4532 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[12], x11p20);
4533 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[13], x12p19);
4534 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[9], x13p18);
4535 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[5], x14p17);
4536 let m0427a = SseVector::fmadd(m0427a, self.twiddles_re[1], x15p16);
4537 let m0427b = SseVector::mul(self.twiddles_im[3], x1m30);
4538 let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[7], x2m29);
4539 let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[11], x3m28);
4540 let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[14], x4m27);
4541 let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[10], x5m26);
4542 let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[6], x6m25);
4543 let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[2], x7m24);
4544 let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[0], x8m23);
4545 let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[4], x9m22);
4546 let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[8], x10m21);
4547 let m0427b = SseVector::fmadd(m0427b, self.twiddles_im[12], x11m20);
4548 let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[13], x12m19);
4549 let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[9], x13m18);
4550 let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[5], x14m17);
4551 let m0427b = SseVector::nmadd(m0427b, self.twiddles_im[1], x15m16);
4552 let [y04, y27] = SseVector::column_butterfly2([m0427a, m0427b]);
4553
4554 let m0526a = SseVector::fmadd(values[0], self.twiddles_re[4], x1p30);
4555 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[9], x2p29);
4556 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[14], x3p28);
4557 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[10], x4p27);
4558 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[5], x5p26);
4559 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[0], x6p25);
4560 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[3], x7p24);
4561 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[8], x8p23);
4562 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[13], x9p22);
4563 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[11], x10p21);
4564 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[6], x11p20);
4565 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[1], x12p19);
4566 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[2], x13p18);
4567 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[7], x14p17);
4568 let m0526a = SseVector::fmadd(m0526a, self.twiddles_re[12], x15p16);
4569 let m0526b = SseVector::mul(self.twiddles_im[4], x1m30);
4570 let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[9], x2m29);
4571 let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[14], x3m28);
4572 let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[10], x4m27);
4573 let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[5], x5m26);
4574 let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[0], x6m25);
4575 let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[3], x7m24);
4576 let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[8], x8m23);
4577 let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[13], x9m22);
4578 let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[11], x10m21);
4579 let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[6], x11m20);
4580 let m0526b = SseVector::nmadd(m0526b, self.twiddles_im[1], x12m19);
4581 let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[2], x13m18);
4582 let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[7], x14m17);
4583 let m0526b = SseVector::fmadd(m0526b, self.twiddles_im[12], x15m16);
4584 let [y05, y26] = SseVector::column_butterfly2([m0526a, m0526b]);
4585
4586 let m0625a = SseVector::fmadd(values[0], self.twiddles_re[5], x1p30);
4587 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[11], x2p29);
4588 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[12], x3p28);
4589 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[6], x4p27);
4590 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[0], x5p26);
4591 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[4], x6p25);
4592 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[10], x7p24);
4593 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[13], x8p23);
4594 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[7], x9p22);
4595 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[1], x10p21);
4596 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[3], x11p20);
4597 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[9], x12p19);
4598 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[14], x13p18);
4599 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[8], x14p17);
4600 let m0625a = SseVector::fmadd(m0625a, self.twiddles_re[2], x15p16);
4601 let m0625b = SseVector::mul(self.twiddles_im[5], x1m30);
4602 let m0625b = SseVector::fmadd(m0625b, self.twiddles_im[11], x2m29);
4603 let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[12], x3m28);
4604 let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[6], x4m27);
4605 let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[0], x5m26);
4606 let m0625b = SseVector::fmadd(m0625b, self.twiddles_im[4], x6m25);
4607 let m0625b = SseVector::fmadd(m0625b, self.twiddles_im[10], x7m24);
4608 let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[13], x8m23);
4609 let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[7], x9m22);
4610 let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[1], x10m21);
4611 let m0625b = SseVector::fmadd(m0625b, self.twiddles_im[3], x11m20);
4612 let m0625b = SseVector::fmadd(m0625b, self.twiddles_im[9], x12m19);
4613 let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[14], x13m18);
4614 let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[8], x14m17);
4615 let m0625b = SseVector::nmadd(m0625b, self.twiddles_im[2], x15m16);
4616 let [y06, y25] = SseVector::column_butterfly2([m0625a, m0625b]);
4617
4618 let m0724a = SseVector::fmadd(values[0], self.twiddles_re[6], x1p30);
4619 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[13], x2p29);
4620 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[9], x3p28);
4621 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[2], x4p27);
4622 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[3], x5p26);
4623 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[10], x6p25);
4624 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[12], x7p24);
4625 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[5], x8p23);
4626 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[0], x9p22);
4627 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[7], x10p21);
4628 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[14], x11p20);
4629 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[8], x12p19);
4630 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[1], x13p18);
4631 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[4], x14p17);
4632 let m0724a = SseVector::fmadd(m0724a, self.twiddles_re[11], x15p16);
4633 let m0724b = SseVector::mul(self.twiddles_im[6], x1m30);
4634 let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[13], x2m29);
4635 let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[9], x3m28);
4636 let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[2], x4m27);
4637 let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[3], x5m26);
4638 let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[10], x6m25);
4639 let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[12], x7m24);
4640 let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[5], x8m23);
4641 let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[0], x9m22);
4642 let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[7], x10m21);
4643 let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[14], x11m20);
4644 let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[8], x12m19);
4645 let m0724b = SseVector::nmadd(m0724b, self.twiddles_im[1], x13m18);
4646 let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[4], x14m17);
4647 let m0724b = SseVector::fmadd(m0724b, self.twiddles_im[11], x15m16);
4648 let [y07, y24] = SseVector::column_butterfly2([m0724a, m0724b]);
4649
4650 let m0823a = SseVector::fmadd(values[0], self.twiddles_re[7], x1p30);
4651 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[14], x2p29);
4652 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[6], x3p28);
4653 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[0], x4p27);
4654 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[8], x5p26);
4655 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[13], x6p25);
4656 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[5], x7p24);
4657 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[1], x8p23);
4658 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[9], x9p22);
4659 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[12], x10p21);
4660 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[4], x11p20);
4661 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[2], x12p19);
4662 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[10], x13p18);
4663 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[11], x14p17);
4664 let m0823a = SseVector::fmadd(m0823a, self.twiddles_re[3], x15p16);
4665 let m0823b = SseVector::mul(self.twiddles_im[7], x1m30);
4666 let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[14], x2m29);
4667 let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[6], x3m28);
4668 let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[0], x4m27);
4669 let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[8], x5m26);
4670 let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[13], x6m25);
4671 let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[5], x7m24);
4672 let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[1], x8m23);
4673 let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[9], x9m22);
4674 let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[12], x10m21);
4675 let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[4], x11m20);
4676 let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[2], x12m19);
4677 let m0823b = SseVector::fmadd(m0823b, self.twiddles_im[10], x13m18);
4678 let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[11], x14m17);
4679 let m0823b = SseVector::nmadd(m0823b, self.twiddles_im[3], x15m16);
4680 let [y08, y23] = SseVector::column_butterfly2([m0823a, m0823b]);
4681
4682 let m0922a = SseVector::fmadd(values[0], self.twiddles_re[8], x1p30);
4683 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[12], x2p29);
4684 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[3], x3p28);
4685 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[4], x4p27);
4686 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[13], x5p26);
4687 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[7], x6p25);
4688 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[0], x7p24);
4689 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[9], x8p23);
4690 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[11], x9p22);
4691 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[2], x10p21);
4692 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[5], x11p20);
4693 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[14], x12p19);
4694 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[6], x13p18);
4695 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[1], x14p17);
4696 let m0922a = SseVector::fmadd(m0922a, self.twiddles_re[10], x15p16);
4697 let m0922b = SseVector::mul(self.twiddles_im[8], x1m30);
4698 let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[12], x2m29);
4699 let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[3], x3m28);
4700 let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[4], x4m27);
4701 let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[13], x5m26);
4702 let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[7], x6m25);
4703 let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[0], x7m24);
4704 let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[9], x8m23);
4705 let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[11], x9m22);
4706 let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[2], x10m21);
4707 let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[5], x11m20);
4708 let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[14], x12m19);
4709 let m0922b = SseVector::nmadd(m0922b, self.twiddles_im[6], x13m18);
4710 let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[1], x14m17);
4711 let m0922b = SseVector::fmadd(m0922b, self.twiddles_im[10], x15m16);
4712 let [y09, y22] = SseVector::column_butterfly2([m0922a, m0922b]);
4713
4714 let m1021a = SseVector::fmadd(values[0], self.twiddles_re[9], x1p30);
4715 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[10], x2p29);
4716 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[0], x3p28);
4717 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[8], x4p27);
4718 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[11], x5p26);
4719 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[1], x6p25);
4720 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[7], x7p24);
4721 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[12], x8p23);
4722 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[2], x9p22);
4723 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[6], x10p21);
4724 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[13], x11p20);
4725 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[3], x12p19);
4726 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[5], x13p18);
4727 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[14], x14p17);
4728 let m1021a = SseVector::fmadd(m1021a, self.twiddles_re[4], x15p16);
4729 let m1021b = SseVector::mul(self.twiddles_im[9], x1m30);
4730 let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[10], x2m29);
4731 let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[0], x3m28);
4732 let m1021b = SseVector::fmadd(m1021b, self.twiddles_im[8], x4m27);
4733 let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[11], x5m26);
4734 let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[1], x6m25);
4735 let m1021b = SseVector::fmadd(m1021b, self.twiddles_im[7], x7m24);
4736 let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[12], x8m23);
4737 let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[2], x9m22);
4738 let m1021b = SseVector::fmadd(m1021b, self.twiddles_im[6], x10m21);
4739 let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[13], x11m20);
4740 let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[3], x12m19);
4741 let m1021b = SseVector::fmadd(m1021b, self.twiddles_im[5], x13m18);
4742 let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[14], x14m17);
4743 let m1021b = SseVector::nmadd(m1021b, self.twiddles_im[4], x15m16);
4744 let [y10, y21] = SseVector::column_butterfly2([m1021a, m1021b]);
4745
4746 let m1120a = SseVector::fmadd(values[0], self.twiddles_re[10], x1p30);
4747 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[8], x2p29);
4748 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[1], x3p28);
4749 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[12], x4p27);
4750 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[6], x5p26);
4751 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[3], x6p25);
4752 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[14], x7p24);
4753 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[4], x8p23);
4754 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[5], x9p22);
4755 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[13], x10p21);
4756 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[2], x11p20);
4757 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[7], x12p19);
4758 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[11], x13p18);
4759 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[0], x14p17);
4760 let m1120a = SseVector::fmadd(m1120a, self.twiddles_re[9], x15p16);
4761 let m1120b = SseVector::mul(self.twiddles_im[10], x1m30);
4762 let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[8], x2m29);
4763 let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[1], x3m28);
4764 let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[12], x4m27);
4765 let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[6], x5m26);
4766 let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[3], x6m25);
4767 let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[14], x7m24);
4768 let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[4], x8m23);
4769 let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[5], x9m22);
4770 let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[13], x10m21);
4771 let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[2], x11m20);
4772 let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[7], x12m19);
4773 let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[11], x13m18);
4774 let m1120b = SseVector::nmadd(m1120b, self.twiddles_im[0], x14m17);
4775 let m1120b = SseVector::fmadd(m1120b, self.twiddles_im[9], x15m16);
4776 let [y11, y20] = SseVector::column_butterfly2([m1120a, m1120b]);
4777
4778 let m1219a = SseVector::fmadd(values[0], self.twiddles_re[11], x1p30);
4779 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[6], x2p29);
4780 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[4], x3p28);
4781 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[13], x4p27);
4782 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[1], x5p26);
4783 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[9], x6p25);
4784 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[8], x7p24);
4785 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[2], x8p23);
4786 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[14], x9p22);
4787 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[3], x10p21);
4788 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[7], x11p20);
4789 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[10], x12p19);
4790 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[0], x13p18);
4791 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[12], x14p17);
4792 let m1219a = SseVector::fmadd(m1219a, self.twiddles_re[5], x15p16);
4793 let m1219b = SseVector::mul(self.twiddles_im[11], x1m30);
4794 let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[6], x2m29);
4795 let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[4], x3m28);
4796 let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[13], x4m27);
4797 let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[1], x5m26);
4798 let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[9], x6m25);
4799 let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[8], x7m24);
4800 let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[2], x8m23);
4801 let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[14], x9m22);
4802 let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[3], x10m21);
4803 let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[7], x11m20);
4804 let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[10], x12m19);
4805 let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[0], x13m18);
4806 let m1219b = SseVector::fmadd(m1219b, self.twiddles_im[12], x14m17);
4807 let m1219b = SseVector::nmadd(m1219b, self.twiddles_im[5], x15m16);
4808 let [y12, y19] = SseVector::column_butterfly2([m1219a, m1219b]);
4809
4810 let m1318a = SseVector::fmadd(values[0], self.twiddles_re[12], x1p30);
4811 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[4], x2p29);
4812 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[7], x3p28);
4813 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[9], x4p27);
4814 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[2], x5p26);
4815 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[14], x6p25);
4816 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[1], x7p24);
4817 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[10], x8p23);
4818 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[6], x9p22);
4819 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[5], x10p21);
4820 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[11], x11p20);
4821 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[0], x12p19);
4822 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[13], x13p18);
4823 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[3], x14p17);
4824 let m1318a = SseVector::fmadd(m1318a, self.twiddles_re[8], x15p16);
4825 let m1318b = SseVector::mul(self.twiddles_im[12], x1m30);
4826 let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[4], x2m29);
4827 let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[7], x3m28);
4828 let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[9], x4m27);
4829 let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[2], x5m26);
4830 let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[14], x6m25);
4831 let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[1], x7m24);
4832 let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[10], x8m23);
4833 let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[6], x9m22);
4834 let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[5], x10m21);
4835 let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[11], x11m20);
4836 let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[0], x12m19);
4837 let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[13], x13m18);
4838 let m1318b = SseVector::nmadd(m1318b, self.twiddles_im[3], x14m17);
4839 let m1318b = SseVector::fmadd(m1318b, self.twiddles_im[8], x15m16);
4840 let [y13, y18] = SseVector::column_butterfly2([m1318a, m1318b]);
4841
4842 let m1417a = SseVector::fmadd(values[0], self.twiddles_re[13], x1p30);
4843 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[2], x2p29);
4844 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[10], x3p28);
4845 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[5], x4p27);
4846 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[7], x5p26);
4847 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[8], x6p25);
4848 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[4], x7p24);
4849 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[11], x8p23);
4850 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[1], x9p22);
4851 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[14], x10p21);
4852 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[0], x11p20);
4853 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[12], x12p19);
4854 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[3], x13p18);
4855 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[9], x14p17);
4856 let m1417a = SseVector::fmadd(m1417a, self.twiddles_re[6], x15p16);
4857 let m1417b = SseVector::mul(self.twiddles_im[13], x1m30);
4858 let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[2], x2m29);
4859 let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[10], x3m28);
4860 let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[5], x4m27);
4861 let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[7], x5m26);
4862 let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[8], x6m25);
4863 let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[4], x7m24);
4864 let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[11], x8m23);
4865 let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[1], x9m22);
4866 let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[14], x10m21);
4867 let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[0], x11m20);
4868 let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[12], x12m19);
4869 let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[3], x13m18);
4870 let m1417b = SseVector::fmadd(m1417b, self.twiddles_im[9], x14m17);
4871 let m1417b = SseVector::nmadd(m1417b, self.twiddles_im[6], x15m16);
4872 let [y14, y17] = SseVector::column_butterfly2([m1417a, m1417b]);
4873
4874 let m1516a = SseVector::fmadd(values[0], self.twiddles_re[14], x1p30);
4875 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[0], x2p29);
4876 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[13], x3p28);
4877 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[1], x4p27);
4878 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[12], x5p26);
4879 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[2], x6p25);
4880 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[11], x7p24);
4881 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[3], x8p23);
4882 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[10], x9p22);
4883 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[4], x10p21);
4884 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[9], x11p20);
4885 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[5], x12p19);
4886 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[8], x13p18);
4887 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[6], x14p17);
4888 let m1516a = SseVector::fmadd(m1516a, self.twiddles_re[7], x15p16);
4889 let m1516b = SseVector::mul(self.twiddles_im[14], x1m30);
4890 let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[0], x2m29);
4891 let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[13], x3m28);
4892 let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[1], x4m27);
4893 let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[12], x5m26);
4894 let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[2], x6m25);
4895 let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[11], x7m24);
4896 let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[3], x8m23);
4897 let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[10], x9m22);
4898 let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[4], x10m21);
4899 let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[9], x11m20);
4900 let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[5], x12m19);
4901 let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[8], x13m18);
4902 let m1516b = SseVector::nmadd(m1516b, self.twiddles_im[6], x14m17);
4903 let m1516b = SseVector::fmadd(m1516b, self.twiddles_im[7], x15m16);
4904 let [y15, y16] = SseVector::column_butterfly2([m1516a, m1516b]);
4905
4906
4907 [y00, y01, y02, y03, y04, y05, y06, y07, y08, y09, y10, y11, y12, y13, y14, y15, y16, y17, y18, y19, y20, y21, y22, y23, y24, y25, y26, y27, y28, y29, y30]
4908 }
4909}
4910
4911
4912#[cfg(test)]
4913mod unit_tests {
4914 use super::*;
4915 use crate::test_utils::check_fft_algorithm;
4916
4917 macro_rules! test_butterfly_32_func {
4918 ($test_name:ident, $struct_name:ident, $size:expr) => {
4919 #[test]
4920 fn $test_name() {
4921 assert!(std::arch::is_x86_feature_detected!("sse4.1"));
4922
4923 let fwd = unsafe { $struct_name::new(FftDirection::Forward) };
4924 check_fft_algorithm::<f32>(&fwd, $size, FftDirection::Forward);
4925
4926 let inv = unsafe { $struct_name::new(FftDirection::Inverse) };
4927 check_fft_algorithm::<f32>(&inv, $size, FftDirection::Inverse);
4928 }
4929 };
4930 }
4931 macro_rules! test_butterfly_64_func {
4932 ($test_name:ident, $struct_name:ident, $size:expr) => {
4933 #[test]
4934 fn $test_name() {
4935 assert!(std::arch::is_x86_feature_detected!("sse4.1"));
4936
4937 let fwd = unsafe { $struct_name::new(FftDirection::Forward) };
4938 check_fft_algorithm::<f64>(&fwd, $size, FftDirection::Forward);
4939
4940 let inv = unsafe { $struct_name::new(FftDirection::Inverse) };
4941 check_fft_algorithm::<f64>(&inv, $size, FftDirection::Inverse);
4942 }
4943 };
4944 }
4945 test_butterfly_32_func!(test_ssef32_butterfly7, SseF32Butterfly7, 7);
4946 test_butterfly_32_func!(test_ssef32_butterfly11, SseF32Butterfly11, 11);
4947 test_butterfly_32_func!(test_ssef32_butterfly13, SseF32Butterfly13, 13);
4948 test_butterfly_32_func!(test_ssef32_butterfly17, SseF32Butterfly17, 17);
4949 test_butterfly_32_func!(test_ssef32_butterfly19, SseF32Butterfly19, 19);
4950 test_butterfly_32_func!(test_ssef32_butterfly23, SseF32Butterfly23, 23);
4951 test_butterfly_32_func!(test_ssef32_butterfly29, SseF32Butterfly29, 29);
4952 test_butterfly_32_func!(test_ssef32_butterfly31, SseF32Butterfly31, 31);
4953 test_butterfly_64_func!(test_ssef64_butterfly7, SseF64Butterfly7, 7);
4954 test_butterfly_64_func!(test_ssef64_butterfly11, SseF64Butterfly11, 11);
4955 test_butterfly_64_func!(test_ssef64_butterfly13, SseF64Butterfly13, 13);
4956 test_butterfly_64_func!(test_ssef64_butterfly17, SseF64Butterfly17, 17);
4957 test_butterfly_64_func!(test_ssef64_butterfly19, SseF64Butterfly19, 19);
4958 test_butterfly_64_func!(test_ssef64_butterfly23, SseF64Butterfly23, 23);
4959 test_butterfly_64_func!(test_ssef64_butterfly29, SseF64Butterfly29, 29);
4960 test_butterfly_64_func!(test_ssef64_butterfly31, SseF64Butterfly31, 31);
4961}
4962