1#[cfg(target_arch = "x86")]
2use core::arch::x86::*;
3#[cfg(target_arch = "x86_64")]
4use core::arch::x86_64::*;
5
6pub(crate) type RoundKeys<const RK: usize> = [__m128i; RK];
7
8#[inline]
9#[target_feature(enable = "aes")]
10pub(super) fn aes128_expand_key(key: &[u8; 16]) -> RoundKeys<11> {
11 #[target_feature(enable = "aes")]
12 fn expand_round<const R: i32>(keys: &mut RoundKeys<11>, pos: usize) {
13 let mut t1 = keys[pos - 1];
14 let mut t2;
15 let mut t3;
16
17 t2 = _mm_aeskeygenassist_si128(t1, R);
18 t2 = _mm_shuffle_epi32(t2, 0xff);
19 t3 = _mm_slli_si128(t1, 0x4);
20 t1 = _mm_xor_si128(t1, t3);
21 t3 = _mm_slli_si128(t3, 0x4);
22 t1 = _mm_xor_si128(t1, t3);
23 t3 = _mm_slli_si128(t3, 0x4);
24 t1 = _mm_xor_si128(t1, t3);
25 t1 = _mm_xor_si128(t1, t2);
26
27 keys[pos] = t1;
28 }
29
30 let mut keys = [_mm_setzero_si128(); 11];
31 keys[0] = load(key);
32
33 let kr = &mut keys;
34 expand_round::<0x01>(kr, 1);
35 expand_round::<0x02>(kr, 2);
36 expand_round::<0x04>(kr, 3);
37 expand_round::<0x08>(kr, 4);
38 expand_round::<0x10>(kr, 5);
39 expand_round::<0x20>(kr, 6);
40 expand_round::<0x40>(kr, 7);
41 expand_round::<0x80>(kr, 8);
42 expand_round::<0x1B>(kr, 9);
43 expand_round::<0x36>(kr, 10);
44
45 keys
46}
47
48#[inline]
49#[target_feature(enable = "aes")]
50pub(super) fn aes192_expand_key(key: &[u8; 24]) -> RoundKeys<13> {
51 #[target_feature(enable = "aes")]
52 fn unpack_hilo(a: __m128i, b: __m128i) -> __m128i {
53 let a = _mm_shuffle_epi32(a, 0b01_00_11_10);
54 _mm_unpacklo_epi64(a, b)
55 }
56
57 #[target_feature(enable = "aes")]
58 fn expand_round<const R: i32>(mut t1: __m128i, mut t3: __m128i) -> (__m128i, __m128i) {
59 let (mut t2, mut t4);
60
61 t2 = _mm_aeskeygenassist_si128(t3, R);
62 t2 = _mm_shuffle_epi32(t2, 0x55);
63 t4 = _mm_slli_si128(t1, 0x4);
64 t1 = _mm_xor_si128(t1, t4);
65 t4 = _mm_slli_si128(t4, 0x4);
66 t1 = _mm_xor_si128(t1, t4);
67 t4 = _mm_slli_si128(t4, 0x4);
68 t1 = _mm_xor_si128(t1, t4);
69 t1 = _mm_xor_si128(t1, t2);
70 t2 = _mm_shuffle_epi32(t1, 0xff);
71 t4 = _mm_slli_si128(t3, 0x4);
72 t3 = _mm_xor_si128(t3, t4);
73 t3 = _mm_xor_si128(t3, t2);
74
75 (t1, t3)
76 }
77
78 let mut keys = [_mm_setzero_si128(); 13];
79
80 let k0 = load(&key[..16]);
81 let k1l = load(&key[16..]);
82 keys[0] = k0;
83
84 let (k1_2, k2r) = expand_round::<0x01>(k0, k1l);
85 keys[1] = _mm_unpacklo_epi64(k1l, k1_2);
86 keys[2] = unpack_hilo(k1_2, k2r);
87
88 let (k3, k4l) = expand_round::<0x02>(k1_2, k2r);
89 keys[3] = k3;
90
91 let (k4_5, k5r) = expand_round::<0x04>(k3, k4l);
92 let k4 = _mm_unpacklo_epi64(k4l, k4_5);
93 let k5 = unpack_hilo(k4_5, k5r);
94 keys[4] = k4;
95 keys[5] = k5;
96
97 let (k6, k7l) = expand_round::<0x08>(k4_5, k5r);
98 keys[6] = k6;
99
100 let (k7_8, k8r) = expand_round::<0x10>(k6, k7l);
101 keys[7] = _mm_unpacklo_epi64(k7l, k7_8);
102 keys[8] = unpack_hilo(k7_8, k8r);
103
104 let (k9, k10l) = expand_round::<0x20>(k7_8, k8r);
105 keys[9] = k9;
106
107 let (k10_11, k11r) = expand_round::<0x40>(k9, k10l);
108 keys[10] = _mm_unpacklo_epi64(k10l, k10_11);
109 keys[11] = unpack_hilo(k10_11, k11r);
110
111 let (k12, _) = expand_round::<0x80>(k10_11, k11r);
112 keys[12] = k12;
113
114 keys
115}
116
117#[inline]
118#[target_feature(enable = "aes")]
119pub(super) fn aes256_expand_key(key: &[u8; 32]) -> RoundKeys<15> {
120 #[target_feature(enable = "aes")]
121 fn expand_round<const R: i32>(keys: &mut RoundKeys<15>, pos: usize) {
122 let mut t1 = keys[pos - 2];
123 let mut t2;
124 let mut t3 = keys[pos - 1];
125 let mut t4;
126
127 t2 = _mm_aeskeygenassist_si128(t3, R);
128 t2 = _mm_shuffle_epi32(t2, 0xff);
129 t4 = _mm_slli_si128(t1, 0x4);
130 t1 = _mm_xor_si128(t1, t4);
131 t4 = _mm_slli_si128(t4, 0x4);
132 t1 = _mm_xor_si128(t1, t4);
133 t4 = _mm_slli_si128(t4, 0x4);
134 t1 = _mm_xor_si128(t1, t4);
135 t1 = _mm_xor_si128(t1, t2);
136
137 keys[pos] = t1;
138
139 t4 = _mm_aeskeygenassist_si128(t1, 0x00);
140 t2 = _mm_shuffle_epi32(t4, 0xaa);
141 t4 = _mm_slli_si128(t3, 0x4);
142 t3 = _mm_xor_si128(t3, t4);
143 t4 = _mm_slli_si128(t4, 0x4);
144 t3 = _mm_xor_si128(t3, t4);
145 t4 = _mm_slli_si128(t4, 0x4);
146 t3 = _mm_xor_si128(t3, t4);
147 t3 = _mm_xor_si128(t3, t2);
148
149 keys[pos + 1] = t3;
150 }
151
152 #[target_feature(enable = "aes")]
153 fn expand_round_last<const R: i32>(keys: &mut RoundKeys<15>, pos: usize) {
154 let mut t1 = keys[pos - 2];
155 let mut t2;
156 let t3 = keys[pos - 1];
157 let mut t4;
158
159 t2 = _mm_aeskeygenassist_si128(t3, R);
160 t2 = _mm_shuffle_epi32(t2, 0xff);
161 t4 = _mm_slli_si128(t1, 0x4);
162 t1 = _mm_xor_si128(t1, t4);
163 t4 = _mm_slli_si128(t4, 0x4);
164 t1 = _mm_xor_si128(t1, t4);
165 t4 = _mm_slli_si128(t4, 0x4);
166 t1 = _mm_xor_si128(t1, t4);
167 t1 = _mm_xor_si128(t1, t2);
168
169 keys[pos] = t1;
170 }
171
172 let mut keys = [_mm_setzero_si128(); 15];
173
174 keys[0] = load(&key[..16]);
175 keys[1] = load(&key[16..]);
176
177 let k = &mut keys;
178 expand_round::<0x01>(k, 2);
179 expand_round::<0x02>(k, 4);
180 expand_round::<0x04>(k, 6);
181 expand_round::<0x08>(k, 8);
182 expand_round::<0x10>(k, 10);
183 expand_round::<0x20>(k, 12);
184 expand_round_last::<0x40>(k, 14);
185
186 keys
187}
188
189#[inline]
190#[target_feature(enable = "aes")]
191pub(super) fn inv_expanded_keys<const N: usize>(keys: &[__m128i; N]) -> [__m128i; N] {
192 let mut inv_keys: [__m128i; N] = [_mm_setzero_si128(); N];
193 inv_keys[0] = keys[N - 1];
194 for i in 1..N - 1 {
195 inv_keys[i] = _mm_aesimc_si128(keys[N - 1 - i]);
196 }
197 inv_keys[N - 1] = keys[0];
198 inv_keys
199}
200
201#[target_feature(enable = "sse2")]
202fn load(bytes: &[u8]) -> __m128i {
203 assert!(size_of_val(bytes) <= size_of::<__m128i>());
204 let mut t = crate::Block::default();
205 t[..bytes.len()].copy_from_slice(bytes);
206 super::utils::load_block(&t)
207}