aes/
x86.rs

1pub(crate) mod ni;
2#[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
3pub(crate) mod vaes256;
4#[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))]
5pub(crate) mod vaes512;
6
7#[cfg(target_arch = "x86")]
8use core::arch::x86 as arch;
9#[cfg(target_arch = "x86_64")]
10use core::arch::x86_64 as arch;
11
12use self::arch::*;
13use crate::Block;
14#[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))]
15use cipher::consts::U64;
16use cipher::{
17    AlgorithmName, BlockCipherDecBackend, BlockCipherDecClosure, BlockCipherDecrypt,
18    BlockCipherEncBackend, BlockCipherEncClosure, BlockCipherEncrypt, BlockSizeUser, InOut, Key,
19    KeyInit, KeySizeUser, ParBlocksSizeUser,
20    consts::{U8, U16, U24, U32},
21};
22#[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
23use cipher::{Array, InOutBuf, consts::U30, typenum::Unsigned};
24#[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
25use core::cell::OnceCell;
26use core::fmt;
27
28#[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
29pub(crate) type Block30 = Array<Block, U30>;
30#[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))]
31pub(crate) type Block64 = Array<Block, U64>;
32
33pub(crate) mod features {
34    cpufeatures::new!(features_aes, "aes");
35    cpufeatures::new!(features_avx, "avx");
36    cpufeatures::new!(features_avx512f, "avx512f");
37    cpufeatures::new!(features_vaes, "vaes");
38    pub(crate) mod aes {
39        pub use super::features_aes::*;
40    }
41    #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
42    pub(crate) mod avx {
43        pub use super::features_avx::*;
44    }
45    #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))]
46    pub(crate) mod avx512f {
47        pub use super::features_avx512f::*;
48    }
49    #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
50    pub(crate) mod vaes {
51        pub use super::features_vaes::*;
52    }
53}
54
55type Simd128RoundKeys<const ROUNDS: usize> = [__m128i; ROUNDS];
56#[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
57type Simd256RoundKeys<const ROUNDS: usize> = [__m256i; ROUNDS];
58#[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))]
59type Simd512RoundKeys<const ROUNDS: usize> = [__m512i; ROUNDS];
60
61#[derive(Clone)]
62enum Backend {
63    Ni,
64    #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
65    Vaes256,
66    #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))]
67    Vaes512,
68}
69
70#[derive(Clone, Copy)]
71struct Features {
72    #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
73    avx: self::features::avx::InitToken,
74    #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))]
75    avx512f: self::features::avx512f::InitToken,
76    #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
77    vaes: self::features::vaes::InitToken,
78}
79
80impl Features {
81    fn new() -> Self {
82        Self {
83            #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
84            avx: self::features::avx::init(),
85            #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))]
86            avx512f: self::features::avx512f::init(),
87            #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
88            vaes: self::features::vaes::init(),
89        }
90    }
91
92    #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
93    fn has_vaes256(&self) -> bool {
94        #[cfg(target_arch = "x86_64")]
95        if cfg!(aes_backend = "avx256") && self.vaes.get() && self.avx.get() {
96            return true;
97        }
98        false
99    }
100
101    #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))]
102    fn has_vaes512(&self) -> bool {
103        #[cfg(target_arch = "x86_64")]
104        if cfg!(aes_backend = "avx512") && self.vaes.get() && self.avx512f.get() {
105            return true;
106        }
107        false
108    }
109
110    fn dispatch(&self) -> Backend {
111        #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))]
112        if self.has_vaes512() {
113            return self::Backend::Vaes512;
114        }
115        #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
116        if self.has_vaes256() {
117            return self::Backend::Vaes256;
118        }
119        Backend::Ni
120    }
121}
122
123macro_rules! define_aes_impl {
124    (
125        $name:tt,
126        $name_enc:ident,
127        $name_dec:ident,
128        $name_backend:ident,
129        $module:tt,
130        $key_size:ty,
131        $rounds:tt,
132        $doc:expr $(,)?
133    ) => {
134        mod $name_backend {
135            use super::*;
136
137            #[derive(Clone)]
138            pub(crate) struct Ni<'a> {
139                pub(crate) keys: &'a Simd128RoundKeys<$rounds>,
140            }
141            #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
142            impl<'a> Ni<'a> {
143                pub const fn par_blocks(&self) -> usize {
144                    <Self as ParBlocksSizeUser>::ParBlocksSize::USIZE
145                }
146            }
147            #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
148            impl<'a> From<&Vaes256<'a>> for Ni<'a> {
149                fn from(backend: &Vaes256<'a>) -> Self {
150                    Self { keys: backend.keys }
151                }
152            }
153
154            #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
155            #[derive(Clone)]
156            pub(crate) struct Vaes256<'a> {
157                #[allow(unused)] // TODO: remove once cfg flags are removed
158                pub(crate) features: Features,
159                pub(crate) keys: &'a Simd128RoundKeys<$rounds>,
160                pub(crate) simd_256_keys: OnceCell<Simd256RoundKeys<$rounds>>,
161            }
162            #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
163            impl<'a> Vaes256<'a> {
164                #[allow(unused)] // TODO: remove once cfg flags are removed
165                pub const fn par_blocks(&self) -> usize {
166                    <Self as ParBlocksSizeUser>::ParBlocksSize::USIZE
167                }
168            }
169            #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))]
170            impl<'a> From<&Vaes512<'a>> for Vaes256<'a> {
171                fn from(backend: &Vaes512<'a>) -> Self {
172                    Self {
173                        features: backend.features,
174                        keys: backend.keys,
175                        simd_256_keys: OnceCell::new(),
176                    }
177                }
178            }
179
180            #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))]
181            pub(crate) struct Vaes512<'a> {
182                pub(crate) features: Features,
183                pub(crate) keys: &'a Simd128RoundKeys<$rounds>,
184                pub(crate) simd_512_keys: OnceCell<Simd512RoundKeys<$rounds>>,
185            }
186        }
187
188        #[doc=$doc]
189        #[doc = "block cipher"]
190        #[derive(Clone)]
191        pub struct $name {
192            encrypt: $name_enc,
193            decrypt: $name_dec,
194        }
195
196        #[cfg(feature = "zeroize")]
197        impl zeroize::ZeroizeOnDrop for $name {}
198
199        impl KeySizeUser for $name {
200            type KeySize = $key_size;
201        }
202
203        impl KeyInit for $name {
204            #[inline]
205            fn new(key: &Key<Self>) -> Self {
206                let encrypt = $name_enc::new(key);
207                let decrypt = $name_dec::from(&encrypt);
208                Self { encrypt, decrypt }
209            }
210        }
211
212        impl From<$name_enc> for $name {
213            #[inline]
214            fn from(encrypt: $name_enc) -> $name {
215                let decrypt = (&encrypt).into();
216                Self { encrypt, decrypt }
217            }
218        }
219
220        impl From<&$name_enc> for $name {
221            #[inline]
222            fn from(encrypt: &$name_enc) -> $name {
223                let decrypt = encrypt.into();
224                let encrypt = encrypt.clone();
225                Self { encrypt, decrypt }
226            }
227        }
228
229        impl BlockSizeUser for $name {
230            type BlockSize = U16;
231        }
232
233        impl BlockCipherEncrypt for $name {
234            #[inline]
235            fn encrypt_with_backend(&self, f: impl BlockCipherEncClosure<BlockSize = U16>) {
236                self.encrypt.encrypt_with_backend(f)
237            }
238        }
239
240        impl BlockCipherDecrypt for $name {
241            #[inline]
242            fn decrypt_with_backend(&self, f: impl BlockCipherDecClosure<BlockSize = U16>) {
243                self.decrypt.decrypt_with_backend(f)
244            }
245        }
246
247        impl fmt::Debug for $name {
248            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
249                f.write_str(concat!(stringify!($name), " { .. }"))
250            }
251        }
252
253        impl AlgorithmName for $name {
254            fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result {
255                f.write_str(stringify!($name))
256            }
257        }
258
259        #[doc=$doc]
260        #[doc = "block cipher (encrypt-only)"]
261        #[derive(Clone)]
262        pub struct $name_enc {
263            keys: Simd128RoundKeys<$rounds>,
264            features: Features,
265        }
266
267        impl Drop for $name_enc {
268            fn drop(&mut self) {
269                #[cfg(feature = "zeroize")]
270                unsafe {
271                    zeroize::zeroize_flat_type(&mut self.keys)
272                }
273            }
274        }
275
276        #[cfg(feature = "zeroize")]
277        impl zeroize::ZeroizeOnDrop for $name_enc {}
278
279        impl KeySizeUser for $name_enc {
280            type KeySize = $key_size;
281        }
282
283        impl KeyInit for $name_enc {
284            #[inline]
285            fn new(key: &Key<Self>) -> Self {
286                // SAFETY: we enforce that this code is called only when
287                // target features required by `expand` were properly checked.
288                Self {
289                    keys: unsafe { self::ni::expand::$module::expand_key(key.as_ref()) },
290                    features: Features::new(),
291                }
292            }
293        }
294
295        impl BlockSizeUser for $name_enc {
296            type BlockSize = U16;
297        }
298
299        impl BlockCipherEncrypt for $name_enc {
300            #[inline]
301            fn encrypt_with_backend(&self, f: impl BlockCipherEncClosure<BlockSize = U16>) {
302                let features = self.features;
303                let keys = &self.keys;
304                match features.dispatch() {
305                    self::Backend::Ni => f.call(&mut $name_backend::Ni { keys }),
306                    #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
307                    self::Backend::Vaes256 => f.call(&mut $name_backend::Vaes256 {
308                        features,
309                        keys,
310                        simd_256_keys: OnceCell::new(),
311                    }),
312                    #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))]
313                    self::Backend::Vaes512 => f.call(&mut $name_backend::Vaes512 {
314                        features,
315                        keys,
316                        simd_512_keys: OnceCell::new(),
317                    }),
318                }
319            }
320        }
321
322        impl fmt::Debug for $name_enc {
323            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
324                f.write_str(concat!(stringify!($name_enc), " { .. }"))
325            }
326        }
327
328        impl AlgorithmName for $name_enc {
329            fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result {
330                f.write_str(stringify!($name_enc))
331            }
332        }
333
334        #[doc=$doc]
335        #[doc = "block cipher (decrypt-only)"]
336        #[derive(Clone)]
337        pub struct $name_dec {
338            keys: Simd128RoundKeys<$rounds>,
339            features: Features,
340        }
341
342        impl Drop for $name_dec {
343            fn drop(&mut self) {
344                #[cfg(feature = "zeroize")]
345                unsafe {
346                    zeroize::zeroize_flat_type(&mut self.keys)
347                }
348            }
349        }
350
351        #[cfg(feature = "zeroize")]
352        impl zeroize::ZeroizeOnDrop for $name_dec {}
353
354        impl KeySizeUser for $name_dec {
355            type KeySize = $key_size;
356        }
357
358        impl KeyInit for $name_dec {
359            #[inline]
360            fn new(key: &Key<Self>) -> Self {
361                $name_enc::new(key).into()
362            }
363        }
364
365        impl From<$name_enc> for $name_dec {
366            #[inline]
367            fn from(enc: $name_enc) -> $name_dec {
368                Self::from(&enc)
369            }
370        }
371
372        impl From<&$name_enc> for $name_dec {
373            #[inline]
374            fn from(enc: &$name_enc) -> $name_dec {
375                Self {
376                    keys: unsafe { self::ni::expand::inv_keys(&enc.keys) },
377                    features: enc.features.clone(),
378                }
379            }
380        }
381
382        impl BlockSizeUser for $name_dec {
383            type BlockSize = U16;
384        }
385
386        impl BlockCipherDecrypt for $name_dec {
387            #[inline]
388            fn decrypt_with_backend(&self, f: impl BlockCipherDecClosure<BlockSize = U16>) {
389                let features = self.features;
390                let keys = &self.keys;
391                match features.dispatch() {
392                    self::Backend::Ni => f.call(&mut $name_backend::Ni { keys }),
393                    #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
394                    self::Backend::Vaes256 => f.call(&mut $name_backend::Vaes256 {
395                        features,
396                        keys,
397                        simd_256_keys: OnceCell::new(),
398                    }),
399                    #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))]
400                    self::Backend::Vaes512 => f.call(&mut $name_backend::Vaes512 {
401                        features,
402                        keys,
403                        simd_512_keys: OnceCell::new(),
404                    }),
405                }
406            }
407        }
408
409        impl fmt::Debug for $name_dec {
410            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
411                f.write_str(concat!(stringify!($name_dec), " { .. }"))
412            }
413        }
414
415        impl AlgorithmName for $name_dec {
416            fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result {
417                f.write_str(stringify!($name_dec))
418            }
419        }
420
421        impl<'a> BlockSizeUser for $name_backend::Ni<'a> {
422            type BlockSize = U16;
423        }
424        #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
425        impl<'a> BlockSizeUser for $name_backend::Vaes256<'a> {
426            type BlockSize = U16;
427        }
428        #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))]
429        impl<'a> BlockSizeUser for $name_backend::Vaes512<'a> {
430            type BlockSize = U16;
431        }
432
433        impl<'a> ParBlocksSizeUser for $name_backend::Ni<'a> {
434            type ParBlocksSize = U8;
435        }
436        #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
437        impl<'a> ParBlocksSizeUser for $name_backend::Vaes256<'a> {
438            // Block size of 30 is chosen based on AVX2's 16 YMM registers.
439            //
440            // * 1 register holds 2 keys per round (loads interleaved with rounds)
441            // * 15 registers hold 2 data blocks
442            //
443            // This gives (16 <total> - 1 <round key>) * 2 <data> = 30 <data>.
444            type ParBlocksSize = U30;
445        }
446        #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))]
447        impl<'a> ParBlocksSizeUser for $name_backend::Vaes512<'a> {
448            // Block size of 64 is chosen based on AVX512's 32 ZMM registers.
449            //
450            // * 11, 13, 15 registers for keys, correspond to AES-128, AES-192, AES-256
451            // * 11, 13, 15 registers hold 4 keys each (no interleaved loading like VAES256)
452            // * 16 registers hold 4 data blocks
453            // * 1-4 registers remain unused (could use them but probably not worth it)
454            //
455            // This gives (32 <total> - 15 <AES-256 round keys> - 1 <unused>) * 4 <data> = 64 <data>.
456            type ParBlocksSize = U64;
457        }
458
459        impl<'a> BlockCipherEncBackend for $name_backend::Ni<'a> {
460            #[inline]
461            fn encrypt_block(&self, block: InOut<'_, '_, Block>) {
462                unsafe {
463                    self::ni::encdec::encrypt(self.keys, block);
464                }
465            }
466            #[inline]
467            fn encrypt_par_blocks(&self, blocks: InOut<'_, '_, cipher::ParBlocks<Self>>) {
468                unsafe {
469                    self::ni::encdec::encrypt_par(self.keys, blocks);
470                }
471            }
472        }
473        #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
474        impl<'a> BlockCipherEncBackend for $name_backend::Vaes256<'a> {
475            #[inline]
476            fn encrypt_block(&self, block: InOut<'_, '_, Block>) {
477                unsafe {
478                    self::ni::encdec::encrypt(self.keys, block);
479                }
480            }
481            #[inline]
482            fn encrypt_par_blocks(&self, blocks: InOut<'_, '_, cipher::ParBlocks<Self>>) {
483                unsafe {
484                    let simd_256_keys = self
485                        .simd_256_keys
486                        .get_or_init(|| vaes256::encdec::broadcast_keys(&self.keys));
487                    vaes256::encdec::encrypt30(simd_256_keys, blocks);
488                }
489            }
490            #[inline]
491            fn encrypt_tail_blocks(&self, blocks: InOutBuf<'_, '_, Block>) {
492                let backend = self;
493
494                let mut rem = blocks.len();
495                let (mut iptr, mut optr) = blocks.into_raw();
496
497                let backend = $name_backend::Ni::from(backend);
498                while rem >= backend.par_blocks() {
499                    let blocks = unsafe { InOut::from_raw(iptr.cast(), optr.cast()) };
500                    backend.encrypt_par_blocks(blocks);
501                    rem -= backend.par_blocks();
502                    iptr = unsafe { iptr.add(backend.par_blocks()) };
503                    optr = unsafe { optr.add(backend.par_blocks()) };
504                }
505
506                while rem > 0 {
507                    let block = unsafe { InOut::from_raw(iptr, optr) };
508                    backend.encrypt_block(block);
509                    rem -= 1;
510                    iptr = unsafe { iptr.add(1) };
511                    optr = unsafe { optr.add(1) };
512                }
513            }
514        }
515        #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))]
516        impl<'a> BlockCipherEncBackend for $name_backend::Vaes512<'a> {
517            #[inline]
518            fn encrypt_block(&self, block: InOut<'_, '_, Block>) {
519                unsafe {
520                    self::ni::encdec::encrypt(self.keys, block);
521                }
522            }
523            #[inline]
524            fn encrypt_par_blocks(&self, blocks: InOut<'_, '_, cipher::ParBlocks<Self>>) {
525                unsafe {
526                    let simd_512_keys = self
527                        .simd_512_keys
528                        .get_or_init(|| vaes512::encdec::broadcast_keys(&self.keys));
529                    vaes512::encdec::encrypt64(simd_512_keys, blocks);
530                }
531            }
532            #[inline]
533            fn encrypt_tail_blocks(&self, blocks: InOutBuf<'_, '_, Block>) {
534                let backend = self;
535
536                let mut rem = blocks.len();
537                let (mut iptr, mut optr) = blocks.into_raw();
538
539                let backend = &$name_backend::Vaes256::from(backend);
540                if backend.features.has_vaes256() {
541                    while rem >= backend.par_blocks() {
542                        let blocks = unsafe { InOut::from_raw(iptr.cast(), optr.cast()) };
543                        backend.encrypt_par_blocks(blocks);
544                        rem -= backend.par_blocks();
545                        iptr = unsafe { iptr.add(backend.par_blocks()) };
546                        optr = unsafe { optr.add(backend.par_blocks()) };
547                    }
548                }
549
550                let backend = &$name_backend::Ni::from(backend);
551                while rem >= backend.par_blocks() {
552                    let blocks = unsafe { InOut::from_raw(iptr.cast(), optr.cast()) };
553                    backend.encrypt_par_blocks(blocks);
554                    rem -= backend.par_blocks();
555                    iptr = unsafe { iptr.add(backend.par_blocks()) };
556                    optr = unsafe { optr.add(backend.par_blocks()) };
557                }
558
559                while rem > 0 {
560                    let block = unsafe { InOut::from_raw(iptr, optr) };
561                    backend.encrypt_block(block);
562                    rem -= 1;
563                    iptr = unsafe { iptr.add(1) };
564                    optr = unsafe { optr.add(1) };
565                }
566            }
567        }
568
569        impl<'a> BlockCipherDecBackend for $name_backend::Ni<'a> {
570            #[inline]
571            fn decrypt_block(&self, block: InOut<'_, '_, Block>) {
572                unsafe {
573                    self::ni::encdec::decrypt(self.keys, block);
574                }
575            }
576            #[inline]
577            fn decrypt_par_blocks(&self, blocks: InOut<'_, '_, cipher::ParBlocks<Self>>) {
578                unsafe {
579                    self::ni::encdec::decrypt_par(self.keys, blocks);
580                }
581            }
582        }
583        #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))]
584        impl<'a> BlockCipherDecBackend for $name_backend::Vaes256<'a> {
585            #[inline]
586            fn decrypt_block(&self, block: InOut<'_, '_, Block>) {
587                unsafe {
588                    self::ni::encdec::decrypt(self.keys, block);
589                }
590            }
591            #[inline]
592            fn decrypt_par_blocks(&self, blocks: InOut<'_, '_, cipher::ParBlocks<Self>>) {
593                unsafe {
594                    let simd_256_keys = self
595                        .simd_256_keys
596                        .get_or_init(|| vaes256::encdec::broadcast_keys(&self.keys));
597                    vaes256::encdec::decrypt30(simd_256_keys, blocks);
598                }
599            }
600            #[inline]
601            fn decrypt_tail_blocks(&self, blocks: InOutBuf<'_, '_, Block>) {
602                let backend = self;
603
604                let mut rem = blocks.len();
605                let (mut iptr, mut optr) = blocks.into_raw();
606
607                let backend = $name_backend::Ni::from(backend);
608                while rem >= backend.par_blocks() {
609                    let blocks = unsafe { InOut::from_raw(iptr.cast(), optr.cast()) };
610                    backend.decrypt_par_blocks(blocks);
611                    rem -= backend.par_blocks();
612                    iptr = unsafe { iptr.add(backend.par_blocks()) };
613                    optr = unsafe { optr.add(backend.par_blocks()) };
614                }
615
616                while rem > 0 {
617                    let block = unsafe { InOut::from_raw(iptr, optr) };
618                    backend.decrypt_block(block);
619                    rem -= 1;
620                    iptr = unsafe { iptr.add(1) };
621                    optr = unsafe { optr.add(1) };
622                }
623            }
624        }
625        #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))]
626        impl<'a> BlockCipherDecBackend for $name_backend::Vaes512<'a> {
627            #[inline]
628            fn decrypt_block(&self, block: InOut<'_, '_, Block>) {
629                unsafe {
630                    self::ni::encdec::decrypt(self.keys, block);
631                }
632            }
633            #[inline]
634            fn decrypt_par_blocks(&self, blocks: InOut<'_, '_, cipher::ParBlocks<Self>>) {
635                unsafe {
636                    let simd_512_keys = self
637                        .simd_512_keys
638                        .get_or_init(|| vaes512::encdec::broadcast_keys(&self.keys));
639                    vaes512::encdec::decrypt64(simd_512_keys, blocks);
640                }
641            }
642            #[inline]
643            fn decrypt_tail_blocks(&self, blocks: InOutBuf<'_, '_, Block>) {
644                let backend = self;
645
646                let mut rem = blocks.len();
647                let (mut iptr, mut optr) = blocks.into_raw();
648
649                let backend = &$name_backend::Vaes256::from(backend);
650                if backend.features.has_vaes256() {
651                    while rem >= backend.par_blocks() {
652                        let blocks = unsafe { InOut::from_raw(iptr.cast(), optr.cast()) };
653                        backend.decrypt_par_blocks(blocks);
654                        rem -= backend.par_blocks();
655                        iptr = unsafe { iptr.add(backend.par_blocks()) };
656                        optr = unsafe { optr.add(backend.par_blocks()) };
657                    }
658                }
659
660                let backend = &$name_backend::Ni::from(backend);
661                while rem >= backend.par_blocks() {
662                    let blocks = unsafe { InOut::from_raw(iptr.cast(), optr.cast()) };
663                    backend.decrypt_par_blocks(blocks);
664                    rem -= backend.par_blocks();
665                    iptr = unsafe { iptr.add(backend.par_blocks()) };
666                    optr = unsafe { optr.add(backend.par_blocks()) };
667                }
668
669                while rem > 0 {
670                    let block = unsafe { InOut::from_raw(iptr, optr) };
671                    backend.decrypt_block(block);
672                    rem -= 1;
673                    iptr = unsafe { iptr.add(1) };
674                    optr = unsafe { optr.add(1) };
675                }
676            }
677        }
678    };
679}
680
681define_aes_impl!(
682    Aes128,
683    Aes128Enc,
684    Aes128Dec,
685    aes128_backend,
686    aes128,
687    U16,
688    11,
689    "AES-128",
690);
691
692define_aes_impl!(
693    Aes192,
694    Aes192Enc,
695    Aes192Dec,
696    aes192_backend,
697    aes192,
698    U24,
699    13,
700    "AES-192",
701);
702
703define_aes_impl!(
704    Aes256,
705    Aes256Enc,
706    Aes256Dec,
707    aes256_backend,
708    aes256,
709    U32,
710    15,
711    "AES-256",
712);
aes/x86.rs

aes/
x86.rs
