sha2/sha256/
x86_sha.rs

1//! SHA-256 `x86`/`x86_64` backend
2
3#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
4compile_error!("x86-sha backend can be used only on x86 and x86_64 target arches");
5
6#[cfg(target_arch = "x86")]
7use core::arch::x86::*;
8#[cfg(target_arch = "x86_64")]
9use core::arch::x86_64::*;
10
11#[inline]
12#[target_feature(enable = "sha")]
13fn rounds4(r: usize, abef: &mut __m128i, cdgh: &mut __m128i, rest: __m128i) {
14    use crate::consts::K32;
15    let rk = _mm_set_epi32(
16        K32[4 * r + 3] as i32,
17        K32[4 * r + 2] as i32,
18        K32[4 * r + 1] as i32,
19        K32[4 * r] as i32,
20    );
21    let t1 = _mm_add_epi32(rest, rk);
22    *cdgh = _mm_sha256rnds2_epu32(*cdgh, *abef, t1);
23    let t2 = _mm_shuffle_epi32(t1, 0x0E);
24    *abef = _mm_sha256rnds2_epu32(*abef, *cdgh, t2);
25}
26
27#[inline]
28#[target_feature(enable = "sha,ssse3")]
29fn schedule_rounds16(r: usize, abef: &mut __m128i, cdgh: &mut __m128i, w: &mut [__m128i; 4]) {
30    for i in 0..4 {
31        let w0 = w[i];
32        let w1 = w[(i + 1) % 4];
33        let w2 = w[(i + 2) % 4];
34        let w3 = w[(i + 3) % 4];
35
36        let t1 = _mm_sha256msg1_epu32(w0, w1);
37        let t2 = _mm_alignr_epi8(w3, w2, 4);
38        let t3 = _mm_add_epi32(t1, t2);
39
40        w[i] = _mm_sha256msg2_epu32(t3, w3);
41
42        rounds4(r + i, abef, cdgh, w[i]);
43    }
44}
45
46#[inline]
47#[target_feature(enable = "ssse3")]
48fn load_block(block: &[u8; 64]) -> [__m128i; 4] {
49    let block_ptr: *const __m128i = block.as_ptr().cast();
50    let mask = _mm_set_epi64x(0x0C0D_0E0F_0809_0A0B, 0x0405_0607_0001_0203);
51    core::array::from_fn(|i| {
52        let w = unsafe { _mm_loadu_si128(block_ptr.add(i)) };
53        _mm_shuffle_epi8(w, mask)
54    })
55}
56
57#[inline]
58#[target_feature(enable = "sse2")]
59fn load_state(state: &[u32; 8]) -> [__m128i; 2] {
60    let state_ptr: *const __m128i = state.as_ptr().cast();
61    core::array::from_fn(|i| unsafe { _mm_loadu_si128(state_ptr.add(i)) })
62}
63
64#[inline]
65#[target_feature(enable = "sse2")]
66fn store_state(state: &mut [u32; 8], [dcba, hgef]: [__m128i; 2]) {
67    let state_ptr: *mut __m128i = state.as_mut_ptr().cast();
68    unsafe {
69        _mm_storeu_si128(state_ptr.add(0), dcba);
70        _mm_storeu_si128(state_ptr.add(1), hgef);
71    }
72}
73
74#[target_feature(enable = "sha,sse4.1")]
75pub(super) fn compress(state: &mut [u32; 8], blocks: &[[u8; 64]]) {
76    let [dcba, hgfe] = load_state(state);
77
78    let cdab = _mm_shuffle_epi32(dcba, 0xB1);
79    let efgh = _mm_shuffle_epi32(hgfe, 0x1B);
80    let mut abef = _mm_alignr_epi8(cdab, efgh, 8);
81    let mut cdgh = _mm_blend_epi16(efgh, cdab, 0xF0);
82
83    for block in blocks {
84        let abef_save = abef;
85        let cdgh_save = cdgh;
86
87        let mut w = load_block(block);
88
89        rounds4(0, &mut abef, &mut cdgh, w[0]);
90        rounds4(1, &mut abef, &mut cdgh, w[1]);
91        rounds4(2, &mut abef, &mut cdgh, w[2]);
92        rounds4(3, &mut abef, &mut cdgh, w[3]);
93
94        schedule_rounds16(4, &mut abef, &mut cdgh, &mut w);
95        schedule_rounds16(8, &mut abef, &mut cdgh, &mut w);
96        schedule_rounds16(12, &mut abef, &mut cdgh, &mut w);
97
98        abef = _mm_add_epi32(abef, abef_save);
99        cdgh = _mm_add_epi32(cdgh, cdgh_save);
100    }
101
102    let feba = _mm_shuffle_epi32(abef, 0x1B);
103    let dchg = _mm_shuffle_epi32(cdgh, 0xB1);
104    let dcba = _mm_blend_epi16(feba, dchg, 0xF0);
105    let hgef = _mm_alignr_epi8(dchg, feba, 8);
106
107    store_state(state, [dcba, hgef]);
108}
sha2/sha256/x86_sha.rs

sha2/sha256/
x86_sha.rs
