aes/backends/x86_aes/
encdec.rs

1#![allow(unsafe_op_in_unsafe_fn)]
2
3use super::RoundKeys;
4use crate::Block;
5use cipher::{
6    array::{Array, ArraySize},
7    inout::InOut,
8};
9
10#[cfg(target_arch = "x86")]
11use core::arch::x86::*;
12#[cfg(target_arch = "x86_64")]
13use core::arch::x86_64::*;
14
15#[inline]
16#[target_feature(enable = "aes")]
17pub(crate) unsafe fn encrypt<const RK: usize>(keys: &RoundKeys<RK>, block: InOut<'_, '_, Block>) {
18    const { assert!(matches!(RK, 11 | 13 | 15)) }
19
20    let (block_in, block_out) = block.into_raw();
21    let mut b = _mm_loadu_si128(block_in.cast());
22    b = _mm_xor_si128(b, keys[0]);
23    for &key in &keys[1..RK - 1] {
24        b = _mm_aesenc_si128(b, key);
25    }
26    b = _mm_aesenclast_si128(b, keys[RK - 1]);
27    _mm_storeu_si128(block_out.cast(), b);
28}
29
30#[inline]
31#[target_feature(enable = "aes")]
32pub(crate) unsafe fn decrypt<const RK: usize>(keys: &RoundKeys<RK>, block: InOut<'_, '_, Block>) {
33    const { assert!(matches!(RK, 11 | 13 | 15)) }
34
35    let (block_in, block_out) = block.into_raw();
36    let mut b = _mm_loadu_si128(block_in.cast());
37    b = _mm_xor_si128(b, keys[0]);
38    for &key in &keys[1..RK - 1] {
39        b = _mm_aesdec_si128(b, key);
40    }
41    b = _mm_aesdeclast_si128(b, keys[RK - 1]);
42    _mm_storeu_si128(block_out.cast(), b);
43}
44
45#[inline]
46#[target_feature(enable = "aes")]
47pub(super) unsafe fn encrypt_par<const RK: usize, ParBlocks: ArraySize>(
48    keys: &RoundKeys<RK>,
49    mut blocks: InOut<'_, '_, Array<Block, ParBlocks>>,
50) {
51    const { assert!(matches!(RK, 11 | 13 | 15)) }
52
53    let mut b = load(blocks.get_in());
54
55    // Loop over keys is intentionally not used here to force inlining
56    xor(&mut b, keys[0]);
57    aesenc(&mut b, keys[1]);
58    aesenc(&mut b, keys[2]);
59    aesenc(&mut b, keys[3]);
60    aesenc(&mut b, keys[4]);
61    aesenc(&mut b, keys[5]);
62    aesenc(&mut b, keys[6]);
63    aesenc(&mut b, keys[7]);
64    aesenc(&mut b, keys[8]);
65    aesenc(&mut b, keys[9]);
66    if RK >= 13 {
67        aesenc(&mut b, keys[10]);
68        aesenc(&mut b, keys[11]);
69    }
70    if RK == 15 {
71        aesenc(&mut b, keys[12]);
72        aesenc(&mut b, keys[13]);
73    }
74    aesenclast(&mut b, keys[RK - 1]);
75    store(blocks.get_out(), b);
76}
77
78#[inline]
79#[target_feature(enable = "aes")]
80pub(super) unsafe fn decrypt_par<const RK: usize, ParBlocks: ArraySize>(
81    keys: &RoundKeys<RK>,
82    mut blocks: InOut<'_, '_, Array<Block, ParBlocks>>,
83) {
84    const { assert!(matches!(RK, 11 | 13 | 15)) };
85
86    let mut b = load(blocks.get_in());
87
88    // Loop over keys is intentionally not used here to force inlining
89    xor(&mut b, keys[0]);
90    aesdec(&mut b, keys[1]);
91    aesdec(&mut b, keys[2]);
92    aesdec(&mut b, keys[3]);
93    aesdec(&mut b, keys[4]);
94    aesdec(&mut b, keys[5]);
95    aesdec(&mut b, keys[6]);
96    aesdec(&mut b, keys[7]);
97    aesdec(&mut b, keys[8]);
98    aesdec(&mut b, keys[9]);
99    if RK >= 13 {
100        aesdec(&mut b, keys[10]);
101        aesdec(&mut b, keys[11]);
102    }
103    if RK == 15 {
104        aesdec(&mut b, keys[12]);
105        aesdec(&mut b, keys[13]);
106    }
107    aesdeclast(&mut b, keys[RK - 1]);
108    store(blocks.get_out(), b);
109}
110
111#[target_feature(enable = "sse2")]
112pub(crate) unsafe fn load<N: ArraySize>(blocks: &Array<Block, N>) -> Array<__m128i, N> {
113    let p: *const __m128i = blocks.as_ptr().cast();
114    Array::from_fn(|i| unsafe { _mm_loadu_si128(p.add(i)) })
115}
116
117#[target_feature(enable = "sse2")]
118pub(crate) unsafe fn store<N: ArraySize>(dst: &mut Array<Block, N>, blocks: Array<__m128i, N>) {
119    let p: *mut __m128i = dst.as_mut_ptr().cast();
120    for (i, block) in blocks.into_iter().enumerate() {
121        unsafe { _mm_storeu_si128(p.add(i), block) }
122    }
123}
124
125#[target_feature(enable = "sse2")]
126pub(crate) unsafe fn xor<N: ArraySize>(blocks: &mut Array<__m128i, N>, key: __m128i) {
127    for block in blocks {
128        *block = _mm_xor_si128(*block, key);
129    }
130}
131
132#[target_feature(enable = "aes")]
133pub(crate) unsafe fn aesenc<N: ArraySize>(blocks: &mut Array<__m128i, N>, key: __m128i) {
134    for block in blocks {
135        *block = _mm_aesenc_si128(*block, key);
136    }
137}
138
139#[target_feature(enable = "aes")]
140pub(crate) unsafe fn aesenclast<N: ArraySize>(blocks: &mut Array<__m128i, N>, key: __m128i) {
141    for block in blocks {
142        *block = _mm_aesenclast_si128(*block, key);
143    }
144}
145
146#[target_feature(enable = "aes")]
147pub(crate) unsafe fn aesdec<N: ArraySize>(blocks: &mut Array<__m128i, N>, key: __m128i) {
148    for block in blocks {
149        *block = _mm_aesdec_si128(*block, key);
150    }
151}
152
153#[target_feature(enable = "aes")]
154pub(crate) unsafe fn aesdeclast<N: ArraySize>(blocks: &mut Array<__m128i, N>, key: __m128i) {
155    for block in blocks {
156        *block = _mm_aesdeclast_si128(*block, key);
157    }
158}
aes/backends/x86_aes/encdec.rs

aes/backends/x86_aes/
encdec.rs
