parking_lot/
elision.rs

1// Copyright 2016 Amanieu d'Antras
2//
3// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
4// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
5// http://opensource.org/licenses/MIT>, at your option. This file may not be
6// copied, modified, or distributed except according to those terms.
7
8use std::sync::atomic::AtomicUsize;
9
10// Extension trait to add lock elision primitives to atomic types
11pub trait AtomicElisionExt {
12    type IntType;
13
14    // Perform a compare_exchange and start a transaction
15    fn elision_compare_exchange_acquire(
16        &self,
17        current: Self::IntType,
18        new: Self::IntType,
19    ) -> Result<Self::IntType, Self::IntType>;
20
21    // Perform a fetch_sub and end a transaction
22    fn elision_fetch_sub_release(&self, val: Self::IntType) -> Self::IntType;
23}
24
25// Indicates whether the target architecture supports lock elision
26#[inline]
27pub fn have_elision() -> bool {
28    cfg!(all(
29        feature = "hardware-lock-elision",
30        not(miri),
31        any(target_arch = "x86", target_arch = "x86_64"),
32    ))
33}
34
35// This implementation is never actually called because it is guarded by
36// have_elision().
37#[cfg(not(all(
38    feature = "hardware-lock-elision",
39    not(miri),
40    any(target_arch = "x86", target_arch = "x86_64")
41)))]
42impl AtomicElisionExt for AtomicUsize {
43    type IntType = usize;
44
45    #[inline]
46    fn elision_compare_exchange_acquire(&self, _: usize, _: usize) -> Result<usize, usize> {
47        unreachable!();
48    }
49
50    #[inline]
51    fn elision_fetch_sub_release(&self, _: usize) -> usize {
52        unreachable!();
53    }
54}
55
56#[cfg(all(
57    feature = "hardware-lock-elision",
58    not(miri),
59    any(target_arch = "x86", target_arch = "x86_64")
60))]
61impl AtomicElisionExt for AtomicUsize {
62    type IntType = usize;
63
64    #[inline]
65    fn elision_compare_exchange_acquire(&self, current: usize, new: usize) -> Result<usize, usize> {
66        unsafe {
67            use core::arch::asm;
68            let prev: usize;
69            #[cfg(target_pointer_width = "32")]
70            asm!(
71                "xacquire",
72                "lock",
73                "cmpxchg [{:e}], {:e}",
74                in(reg) self,
75                in(reg) new,
76                inout("eax") current => prev,
77            );
78            #[cfg(target_pointer_width = "64")]
79            asm!(
80                "xacquire",
81                "lock",
82                "cmpxchg [{}], {}",
83                in(reg) self,
84                in(reg) new,
85                inout("rax") current => prev,
86            );
87            if prev == current {
88                Ok(prev)
89            } else {
90                Err(prev)
91            }
92        }
93    }
94
95    #[inline]
96    fn elision_fetch_sub_release(&self, val: usize) -> usize {
97        unsafe {
98            use core::arch::asm;
99            let prev: usize;
100            #[cfg(target_pointer_width = "32")]
101            asm!(
102                "xrelease",
103                "lock",
104                "xadd [{:e}], {:e}",
105                in(reg) self,
106                inout(reg) val.wrapping_neg() => prev,
107            );
108            #[cfg(target_pointer_width = "64")]
109            asm!(
110                "xrelease",
111                "lock",
112                "xadd [{}], {}",
113                in(reg) self,
114                inout(reg) val.wrapping_neg() => prev,
115            );
116            prev
117        }
118    }
119}
parking_lot/elision.rs

parking_lot/
elision.rs
