aes/backends/x86_aes/
encdec.rs

1use super::{RoundKeys, utils};
2use crate::Block;
3use cipher::{
4    array::{Array, ArraySize},
5    inout::InOut,
6};
7
8#[cfg(target_arch = "x86")]
9use core::arch::x86::*;
10#[cfg(target_arch = "x86_64")]
11use core::arch::x86_64::*;
12
13#[inline]
14#[target_feature(enable = "aes")]
15pub(crate) fn encrypt<const RK: usize>(keys: &RoundKeys<RK>, mut block: InOut<'_, '_, Block>) {
16    const { assert!(matches!(RK, 11 | 13 | 15)) }
17
18    let mut b = utils::load_block(block.get_in());
19    b = _mm_xor_si128(b, keys[0]);
20    for &key in &keys[1..RK - 1] {
21        b = _mm_aesenc_si128(b, key);
22    }
23    b = _mm_aesenclast_si128(b, keys[RK - 1]);
24    utils::store_block(block.get_out(), b);
25}
26
27#[inline]
28#[target_feature(enable = "aes")]
29pub(crate) fn decrypt<const RK: usize>(keys: &RoundKeys<RK>, mut block: InOut<'_, '_, Block>) {
30    const { assert!(matches!(RK, 11 | 13 | 15)) }
31
32    let mut b = utils::load_block(block.get_in());
33    b = _mm_xor_si128(b, keys[0]);
34    for &key in &keys[1..RK - 1] {
35        b = _mm_aesdec_si128(b, key);
36    }
37    b = _mm_aesdeclast_si128(b, keys[RK - 1]);
38    utils::store_block(block.get_out(), b);
39}
40
41#[inline]
42#[target_feature(enable = "aes")]
43pub(super) fn batch_encrypt<const RK: usize, ParBlocks: ArraySize>(
44    keys: &RoundKeys<RK>,
45    mut blocks: InOut<'_, '_, Array<Block, ParBlocks>>,
46) {
47    const { assert!(matches!(RK, 11 | 13 | 15)) }
48
49    let mut b = utils::load_batch_blocks(blocks.get_in());
50
51    // Loop over keys is intentionally not used here to force inlining
52    batch_xor(&mut b, keys[0]);
53    batch_aesenc(&mut b, keys[1]);
54    batch_aesenc(&mut b, keys[2]);
55    batch_aesenc(&mut b, keys[3]);
56    batch_aesenc(&mut b, keys[4]);
57    batch_aesenc(&mut b, keys[5]);
58    batch_aesenc(&mut b, keys[6]);
59    batch_aesenc(&mut b, keys[7]);
60    batch_aesenc(&mut b, keys[8]);
61    batch_aesenc(&mut b, keys[9]);
62    if RK >= 13 {
63        batch_aesenc(&mut b, keys[10]);
64        batch_aesenc(&mut b, keys[11]);
65    }
66    if RK == 15 {
67        batch_aesenc(&mut b, keys[12]);
68        batch_aesenc(&mut b, keys[13]);
69    }
70    batch_aesenclast(&mut b, keys[RK - 1]);
71    utils::store_batch_blocks(blocks.get_out(), b);
72}
73
74#[inline]
75#[target_feature(enable = "aes")]
76pub(super) fn decrypt_par<const RK: usize, ParBlocks: ArraySize>(
77    keys: &RoundKeys<RK>,
78    mut blocks: InOut<'_, '_, Array<Block, ParBlocks>>,
79) {
80    const { assert!(matches!(RK, 11 | 13 | 15)) };
81
82    let mut b = utils::load_batch_blocks(blocks.get_in());
83
84    // Loop over keys is intentionally not used here to force inlining
85    batch_xor(&mut b, keys[0]);
86    batch_aesdec(&mut b, keys[1]);
87    batch_aesdec(&mut b, keys[2]);
88    batch_aesdec(&mut b, keys[3]);
89    batch_aesdec(&mut b, keys[4]);
90    batch_aesdec(&mut b, keys[5]);
91    batch_aesdec(&mut b, keys[6]);
92    batch_aesdec(&mut b, keys[7]);
93    batch_aesdec(&mut b, keys[8]);
94    batch_aesdec(&mut b, keys[9]);
95    if RK >= 13 {
96        batch_aesdec(&mut b, keys[10]);
97        batch_aesdec(&mut b, keys[11]);
98    }
99    if RK == 15 {
100        batch_aesdec(&mut b, keys[12]);
101        batch_aesdec(&mut b, keys[13]);
102    }
103    batch_aesdeclast(&mut b, keys[RK - 1]);
104    utils::store_batch_blocks(blocks.get_out(), b);
105}
106
107#[target_feature(enable = "sse2")]
108fn batch_xor<N: ArraySize>(blocks: &mut Array<__m128i, N>, key: __m128i) {
109    for block in blocks {
110        *block = _mm_xor_si128(*block, key);
111    }
112}
113
114#[target_feature(enable = "aes")]
115fn batch_aesenc<N: ArraySize>(blocks: &mut Array<__m128i, N>, key: __m128i) {
116    for block in blocks {
117        *block = _mm_aesenc_si128(*block, key);
118    }
119}
120
121#[target_feature(enable = "aes")]
122fn batch_aesenclast<N: ArraySize>(blocks: &mut Array<__m128i, N>, key: __m128i) {
123    for block in blocks {
124        *block = _mm_aesenclast_si128(*block, key);
125    }
126}
127
128#[target_feature(enable = "aes")]
129fn batch_aesdec<N: ArraySize>(blocks: &mut Array<__m128i, N>, key: __m128i) {
130    for block in blocks {
131        *block = _mm_aesdec_si128(*block, key);
132    }
133}
134
135#[target_feature(enable = "aes")]
136fn batch_aesdeclast<N: ArraySize>(blocks: &mut Array<__m128i, N>, key: __m128i) {
137    for block in blocks {
138        *block = _mm_aesdeclast_si128(*block, key);
139    }
140}
aes/backends/x86_aes/encdec.rs

aes/backends/x86_aes/
encdec.rs
