#![allow(unsafe_code)]
#![allow(clippy::missing_transmute_annotations)]
#[cfg(target_arch = "x86")]
use super::reg::{ArgReg, RetReg, SyscallNumber, A0, A1, A2, A3, A4, A5, R0};
use super::vdso;
#[cfg(target_arch = "x86")]
use core::arch::global_asm;
#[cfg(feature = "process")]
#[cfg(any(
target_arch = "x86_64",
target_arch = "x86",
target_arch = "riscv64",
target_arch = "powerpc64"
))]
use core::ffi::c_void;
use core::mem::transmute;
use core::ptr::null_mut;
use core::sync::atomic::AtomicPtr;
use core::sync::atomic::Ordering::Relaxed;
#[cfg(target_pointer_width = "32")]
#[cfg(feature = "time")]
use linux_raw_sys::general::timespec as __kernel_old_timespec;
#[cfg(any(
all(
feature = "process",
any(
target_arch = "x86_64",
target_arch = "x86",
target_arch = "riscv64",
target_arch = "powerpc64"
)
),
feature = "time"
))]
use {super::c, super::conv::ret, core::mem::MaybeUninit};
#[cfg(feature = "time")]
use {
super::conv::c_int,
crate::clockid::{ClockId, DynamicClockId},
crate::io,
crate::timespec::Timespec,
linux_raw_sys::general::{__kernel_clockid_t, __kernel_timespec},
};
#[cfg(feature = "time")]
#[inline]
pub(crate) fn clock_gettime(which_clock: ClockId) -> __kernel_timespec {
unsafe {
let mut result = MaybeUninit::<__kernel_timespec>::uninit();
let callee = match transmute(CLOCK_GETTIME.load(Relaxed)) {
Some(callee) => callee,
None => init_clock_gettime(),
};
let r0 = callee(which_clock as c::c_int, result.as_mut_ptr());
assert_eq!(r0, 0);
result.assume_init()
}
}
#[cfg(feature = "time")]
#[inline]
pub(crate) fn clock_gettime_dynamic(which_clock: DynamicClockId<'_>) -> io::Result<Timespec> {
let id = match which_clock {
DynamicClockId::Known(id) => id as __kernel_clockid_t,
DynamicClockId::Dynamic(fd) => {
use crate::backend::fd::AsRawFd;
const CLOCKFD: i32 = 3;
((!fd.as_raw_fd() << 3) | CLOCKFD) as __kernel_clockid_t
}
DynamicClockId::RealtimeAlarm => c::CLOCK_REALTIME_ALARM as __kernel_clockid_t,
DynamicClockId::Tai => c::CLOCK_TAI as __kernel_clockid_t,
DynamicClockId::Boottime => c::CLOCK_BOOTTIME as __kernel_clockid_t,
DynamicClockId::BoottimeAlarm => c::CLOCK_BOOTTIME_ALARM as __kernel_clockid_t,
};
unsafe {
const EINVAL: c::c_int = -(c::EINVAL as c::c_int);
let mut timespec = MaybeUninit::<Timespec>::uninit();
let callee = match transmute(CLOCK_GETTIME.load(Relaxed)) {
Some(callee) => callee,
None => init_clock_gettime(),
};
match callee(id, timespec.as_mut_ptr()) {
0 => (),
EINVAL => return Err(io::Errno::INVAL),
_ => _rustix_clock_gettime_via_syscall(id, timespec.as_mut_ptr())?,
}
Ok(timespec.assume_init())
}
}
#[cfg(feature = "process")]
#[cfg(any(
target_arch = "x86_64",
target_arch = "x86",
target_arch = "riscv64",
target_arch = "powerpc64"
))]
#[inline]
pub(crate) fn sched_getcpu() -> usize {
unsafe {
let mut cpu = MaybeUninit::<u32>::uninit();
let callee = match transmute(GETCPU.load(Relaxed)) {
Some(callee) => callee,
None => init_getcpu(),
};
let r0 = callee(cpu.as_mut_ptr(), null_mut(), null_mut());
debug_assert_eq!(r0, 0);
cpu.assume_init() as usize
}
}
#[cfg(target_arch = "x86")]
pub(super) mod x86_via_vdso {
use super::{transmute, ArgReg, Relaxed, RetReg, SyscallNumber, A0, A1, A2, A3, A4, A5, R0};
use crate::backend::arch::asm;
#[inline]
pub(in crate::backend) unsafe fn syscall0(nr: SyscallNumber<'_>) -> RetReg<R0> {
let callee = match transmute(super::SYSCALL.load(Relaxed)) {
Some(callee) => callee,
None => super::init_syscall(),
};
asm::indirect_syscall0(callee, nr)
}
#[inline]
pub(in crate::backend) unsafe fn syscall1<'a>(
nr: SyscallNumber<'a>,
a0: ArgReg<'a, A0>,
) -> RetReg<R0> {
let callee = match transmute(super::SYSCALL.load(Relaxed)) {
Some(callee) => callee,
None => super::init_syscall(),
};
asm::indirect_syscall1(callee, nr, a0)
}
#[inline]
pub(in crate::backend) unsafe fn syscall1_noreturn<'a>(
nr: SyscallNumber<'a>,
a0: ArgReg<'a, A0>,
) -> ! {
let callee = match transmute(super::SYSCALL.load(Relaxed)) {
Some(callee) => callee,
None => super::init_syscall(),
};
asm::indirect_syscall1_noreturn(callee, nr, a0)
}
#[inline]
pub(in crate::backend) unsafe fn syscall2<'a>(
nr: SyscallNumber<'a>,
a0: ArgReg<'a, A0>,
a1: ArgReg<'a, A1>,
) -> RetReg<R0> {
let callee = match transmute(super::SYSCALL.load(Relaxed)) {
Some(callee) => callee,
None => super::init_syscall(),
};
asm::indirect_syscall2(callee, nr, a0, a1)
}
#[inline]
pub(in crate::backend) unsafe fn syscall3<'a>(
nr: SyscallNumber<'a>,
a0: ArgReg<'a, A0>,
a1: ArgReg<'a, A1>,
a2: ArgReg<'a, A2>,
) -> RetReg<R0> {
let callee = match transmute(super::SYSCALL.load(Relaxed)) {
Some(callee) => callee,
None => super::init_syscall(),
};
asm::indirect_syscall3(callee, nr, a0, a1, a2)
}
#[inline]
pub(in crate::backend) unsafe fn syscall4<'a>(
nr: SyscallNumber<'a>,
a0: ArgReg<'a, A0>,
a1: ArgReg<'a, A1>,
a2: ArgReg<'a, A2>,
a3: ArgReg<'a, A3>,
) -> RetReg<R0> {
let callee = match transmute(super::SYSCALL.load(Relaxed)) {
Some(callee) => callee,
None => super::init_syscall(),
};
asm::indirect_syscall4(callee, nr, a0, a1, a2, a3)
}
#[inline]
pub(in crate::backend) unsafe fn syscall5<'a>(
nr: SyscallNumber<'a>,
a0: ArgReg<'a, A0>,
a1: ArgReg<'a, A1>,
a2: ArgReg<'a, A2>,
a3: ArgReg<'a, A3>,
a4: ArgReg<'a, A4>,
) -> RetReg<R0> {
let callee = match transmute(super::SYSCALL.load(Relaxed)) {
Some(callee) => callee,
None => super::init_syscall(),
};
asm::indirect_syscall5(callee, nr, a0, a1, a2, a3, a4)
}
#[inline]
pub(in crate::backend) unsafe fn syscall6<'a>(
nr: SyscallNumber<'a>,
a0: ArgReg<'a, A0>,
a1: ArgReg<'a, A1>,
a2: ArgReg<'a, A2>,
a3: ArgReg<'a, A3>,
a4: ArgReg<'a, A4>,
a5: ArgReg<'a, A5>,
) -> RetReg<R0> {
let callee = match transmute(super::SYSCALL.load(Relaxed)) {
Some(callee) => callee,
None => super::init_syscall(),
};
asm::indirect_syscall6(callee, nr, a0, a1, a2, a3, a4, a5)
}
#[allow(unused_imports)]
pub(in crate::backend) use {
syscall0 as syscall0_readonly, syscall1 as syscall1_readonly,
syscall2 as syscall2_readonly, syscall3 as syscall3_readonly,
syscall4 as syscall4_readonly, syscall5 as syscall5_readonly,
syscall6 as syscall6_readonly,
};
}
#[cfg(feature = "time")]
type ClockGettimeType = unsafe extern "C" fn(c::c_int, *mut Timespec) -> c::c_int;
#[cfg(feature = "process")]
#[cfg(any(
target_arch = "x86_64",
target_arch = "x86",
target_arch = "riscv64",
target_arch = "powerpc64"
))]
type GetcpuType = unsafe extern "C" fn(*mut u32, *mut u32, *mut c_void) -> c::c_int;
#[cfg(target_arch = "x86")]
pub(super) type SyscallType = unsafe extern "C" fn();
#[cfg(feature = "time")]
#[cold]
fn init_clock_gettime() -> ClockGettimeType {
init();
unsafe { transmute(CLOCK_GETTIME.load(Relaxed)) }
}
#[cfg(feature = "process")]
#[cfg(any(
target_arch = "x86_64",
target_arch = "x86",
target_arch = "riscv64",
target_arch = "powerpc64"
))]
#[cold]
fn init_getcpu() -> GetcpuType {
init();
unsafe { transmute(GETCPU.load(Relaxed)) }
}
#[cfg(target_arch = "x86")]
#[cold]
fn init_syscall() -> SyscallType {
init();
unsafe { transmute(SYSCALL.load(Relaxed)) }
}
struct Function;
#[cfg(feature = "time")]
static CLOCK_GETTIME: AtomicPtr<Function> = AtomicPtr::new(null_mut());
#[cfg(feature = "process")]
#[cfg(any(
target_arch = "x86_64",
target_arch = "x86",
target_arch = "riscv64",
target_arch = "powerpc64"
))]
static GETCPU: AtomicPtr<Function> = AtomicPtr::new(null_mut());
#[cfg(target_arch = "x86")]
static SYSCALL: AtomicPtr<Function> = AtomicPtr::new(null_mut());
#[cfg(feature = "time")]
unsafe extern "C" fn rustix_clock_gettime_via_syscall(
clockid: c::c_int,
res: *mut Timespec,
) -> c::c_int {
match _rustix_clock_gettime_via_syscall(clockid, res) {
Ok(()) => 0,
Err(err) => err.raw_os_error().wrapping_neg(),
}
}
#[cfg(feature = "time")]
#[cfg(target_pointer_width = "32")]
unsafe fn _rustix_clock_gettime_via_syscall(
clockid: c::c_int,
res: *mut Timespec,
) -> io::Result<()> {
let r0 = syscall!(__NR_clock_gettime64, c_int(clockid), res);
match ret(r0) {
Err(io::Errno::NOSYS) => _rustix_clock_gettime_via_syscall_old(clockid, res),
otherwise => otherwise,
}
}
#[cfg(feature = "time")]
#[cfg(target_pointer_width = "32")]
unsafe fn _rustix_clock_gettime_via_syscall_old(
clockid: c::c_int,
res: *mut Timespec,
) -> io::Result<()> {
let mut old_result = MaybeUninit::<__kernel_old_timespec>::uninit();
let r0 = syscall!(__NR_clock_gettime, c_int(clockid), &mut old_result);
match ret(r0) {
Ok(()) => {
let old_result = old_result.assume_init();
*res = Timespec {
tv_sec: old_result.tv_sec.into(),
tv_nsec: old_result.tv_nsec.into(),
};
Ok(())
}
otherwise => otherwise,
}
}
#[cfg(feature = "time")]
#[cfg(target_pointer_width = "64")]
unsafe fn _rustix_clock_gettime_via_syscall(
clockid: c::c_int,
res: *mut Timespec,
) -> io::Result<()> {
ret(syscall!(__NR_clock_gettime, c_int(clockid), res))
}
#[cfg(feature = "process")]
#[cfg(any(
target_arch = "x86_64",
target_arch = "x86",
target_arch = "riscv64",
target_arch = "powerpc64"
))]
unsafe extern "C" fn rustix_getcpu_via_syscall(
cpu: *mut u32,
node: *mut u32,
unused: *mut c_void,
) -> c::c_int {
match ret(syscall!(__NR_getcpu, cpu, node, unused)) {
Ok(()) => 0,
Err(err) => err.raw_os_error().wrapping_neg(),
}
}
#[cfg(target_arch = "x86")]
extern "C" {
fn rustix_int_0x80();
}
#[cfg(target_arch = "x86")]
global_asm!(
r#"
.section .text.rustix_int_0x80,"ax",@progbits
.p2align 4
.weak rustix_int_0x80
.hidden rustix_int_0x80
.type rustix_int_0x80, @function
rustix_int_0x80:
.cfi_startproc
int 0x80
ret
.cfi_endproc
.size rustix_int_0x80, .-rustix_int_0x80
"#
);
fn minimal_init() {
#[cfg(feature = "time")]
{
CLOCK_GETTIME
.compare_exchange(
null_mut(),
rustix_clock_gettime_via_syscall as *mut Function,
Relaxed,
Relaxed,
)
.ok();
}
#[cfg(feature = "process")]
#[cfg(any(
target_arch = "x86_64",
target_arch = "x86",
target_arch = "riscv64",
target_arch = "powerpc64"
))]
{
GETCPU
.compare_exchange(
null_mut(),
rustix_getcpu_via_syscall as *mut Function,
Relaxed,
Relaxed,
)
.ok();
}
#[cfg(target_arch = "x86")]
{
SYSCALL
.compare_exchange(
null_mut(),
rustix_int_0x80 as *mut Function,
Relaxed,
Relaxed,
)
.ok();
}
}
fn init() {
minimal_init();
if let Some(vdso) = vdso::Vdso::new() {
#[cfg(feature = "time")]
{
#[cfg(target_arch = "x86_64")]
let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_clock_gettime"));
#[cfg(target_arch = "arm")]
let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_clock_gettime64"));
#[cfg(target_arch = "aarch64")]
let ptr = vdso.sym(cstr!("LINUX_2.6.39"), cstr!("__kernel_clock_gettime"));
#[cfg(target_arch = "x86")]
let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_clock_gettime64"));
#[cfg(target_arch = "riscv64")]
let ptr = vdso.sym(cstr!("LINUX_4.15"), cstr!("__vdso_clock_gettime"));
#[cfg(target_arch = "powerpc64")]
let ptr = vdso.sym(cstr!("LINUX_2.6.15"), cstr!("__kernel_clock_gettime"));
#[cfg(target_arch = "s390x")]
let ptr = vdso.sym(cstr!("LINUX_2.6.29"), cstr!("__kernel_clock_gettime"));
#[cfg(any(target_arch = "mips", target_arch = "mips32r6"))]
let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_clock_gettime64"));
#[cfg(any(target_arch = "mips64", target_arch = "mips64r6"))]
let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_clock_gettime"));
#[cfg(target_pointer_width = "64")]
let ok = true;
#[cfg(any(
target_arch = "arm",
target_arch = "mips",
target_arch = "mips32r6",
target_arch = "x86"
))]
let ok = !ptr.is_null();
if ok {
assert!(!ptr.is_null());
CLOCK_GETTIME.store(ptr.cast(), Relaxed);
}
}
#[cfg(feature = "process")]
#[cfg(any(
target_arch = "x86_64",
target_arch = "x86",
target_arch = "riscv64",
target_arch = "powerpc64"
))]
{
#[cfg(target_arch = "x86_64")]
let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_getcpu"));
#[cfg(target_arch = "x86")]
let ptr = vdso.sym(cstr!("LINUX_2.6"), cstr!("__vdso_getcpu"));
#[cfg(target_arch = "riscv64")]
let ptr = vdso.sym(cstr!("LINUX_4.15"), cstr!("__kernel_getcpu"));
#[cfg(target_arch = "powerpc64")]
let ptr = vdso.sym(cstr!("LINUX_2.6.15"), cstr!("__kernel_getcpu"));
#[cfg(any(
target_arch = "x86_64",
target_arch = "riscv64",
target_arch = "powerpc64"
))]
let ok = true;
#[cfg(target_arch = "x86")]
let ok = !ptr.is_null();
#[cfg(any(
target_arch = "aarch64",
target_arch = "arm",
target_arch = "mips",
target_arch = "mips32r6",
target_arch = "mips64",
target_arch = "mips64r6",
target_arch = "s390x",
))]
let ok = false;
if ok {
assert!(!ptr.is_null());
GETCPU.store(ptr.cast(), Relaxed);
}
}
#[cfg(target_arch = "x86")]
{
let ptr = vdso.sym(cstr!("LINUX_2.5"), cstr!("__kernel_vsyscall"));
assert!(!ptr.is_null());
SYSCALL.store(ptr.cast(), Relaxed);
}
}
}