use {
crate::{
align_up, error::AllocationError, heap::Heap, slab::Slab, unreachable_unchecked,
util::try_arc_unwrap, MemoryBounds,
},
alloc::{sync::Arc, vec::Vec},
core::{convert::TryFrom as _, mem::replace, ptr::NonNull},
gpu_alloc_types::{AllocationFlags, DeviceMapError, MemoryDevice, MemoryPropertyFlags},
};
#[derive(Debug)]
pub(crate) struct BuddyBlock<M> {
pub memory: Arc<M>,
pub ptr: Option<NonNull<u8>>,
pub offset: u64,
pub size: u64,
pub chunk: usize,
pub index: usize,
}
unsafe impl<M> Sync for BuddyBlock<M> where M: Sync {}
unsafe impl<M> Send for BuddyBlock<M> where M: Send {}
#[derive(Clone, Copy, Debug)]
enum PairState {
Exhausted,
Ready {
ready: Side,
next: usize,
prev: usize,
},
}
impl PairState {
unsafe fn replace_next(&mut self, value: usize) -> usize {
match self {
PairState::Exhausted => unreachable_unchecked(),
PairState::Ready { next, .. } => replace(next, value),
}
}
unsafe fn replace_prev(&mut self, value: usize) -> usize {
match self {
PairState::Exhausted => unreachable_unchecked(),
PairState::Ready { prev, .. } => replace(prev, value),
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Side {
Left,
Right,
}
use Side::*;
#[derive(Debug)]
struct PairEntry {
state: PairState,
chunk: usize,
offset: u64,
parent: Option<usize>,
}
struct SizeBlockEntry {
chunk: usize,
offset: u64,
index: usize,
}
#[derive(Debug)]
struct Size {
next_ready: usize,
pairs: Slab<PairEntry>,
}
#[derive(Debug)]
enum Release {
None,
Parent(usize),
Chunk(usize),
}
impl Size {
fn new() -> Self {
Size {
pairs: Slab::new(),
next_ready: 0,
}
}
unsafe fn add_pair_and_acquire_left(
&mut self,
chunk: usize,
offset: u64,
parent: Option<usize>,
) -> SizeBlockEntry {
if self.next_ready < self.pairs.len() {
unreachable_unchecked()
}
let index = self.pairs.insert(PairEntry {
state: PairState::Exhausted,
chunk,
offset,
parent,
});
let entry = self.pairs.get_unchecked_mut(index);
entry.state = PairState::Ready {
next: index,
prev: index,
ready: Right, };
self.next_ready = index;
SizeBlockEntry {
chunk,
offset,
index: index << 1,
}
}
fn acquire(&mut self, size: u64) -> Option<SizeBlockEntry> {
if self.next_ready >= self.pairs.len() {
return None;
}
let ready = self.next_ready;
let entry = unsafe { self.pairs.get_unchecked_mut(ready) };
let chunk = entry.chunk;
let offset = entry.offset;
let bit = match entry.state {
PairState::Exhausted => unsafe { unreachable_unchecked() },
PairState::Ready { ready, next, prev } => {
entry.state = PairState::Exhausted;
if prev == self.next_ready {
debug_assert_eq!(next, self.next_ready);
self.next_ready = self.pairs.len();
} else {
let prev_entry = unsafe { self.pairs.get_unchecked_mut(prev) };
let prev_next = unsafe { prev_entry.state.replace_next(next) };
debug_assert_eq!(prev_next, self.next_ready);
let next_entry = unsafe { self.pairs.get_unchecked_mut(next) };
let next_prev = unsafe { next_entry.state.replace_prev(prev) };
debug_assert_eq!(next_prev, self.next_ready);
self.next_ready = next;
}
match ready {
Left => 0,
Right => 1,
}
}
};
Some(SizeBlockEntry {
chunk,
offset: offset + bit as u64 * size,
index: (ready << 1) | bit as usize,
})
}
fn release(&mut self, index: usize) -> Release {
let side = match index & 1 {
0 => Side::Left,
1 => Side::Right,
_ => unsafe { unreachable_unchecked() },
};
let entry_index = index >> 1;
let len = self.pairs.len();
let entry = self.pairs.get_mut(entry_index);
let chunk = entry.chunk;
let offset = entry.offset;
let parent = entry.parent;
match entry.state {
PairState::Exhausted => {
if self.next_ready == len {
entry.state = PairState::Ready {
ready: side,
next: entry_index,
prev: entry_index,
};
self.next_ready = entry_index;
} else {
debug_assert!(self.next_ready < len);
let next = self.next_ready;
let next_entry = unsafe { self.pairs.get_unchecked_mut(next) };
let prev = unsafe { next_entry.state.replace_prev(entry_index) };
let prev_entry = unsafe { self.pairs.get_unchecked_mut(prev) };
let prev_next = unsafe { prev_entry.state.replace_next(entry_index) };
debug_assert_eq!(prev_next, next);
let entry = unsafe { self.pairs.get_unchecked_mut(entry_index) };
entry.state = PairState::Ready {
ready: side,
next,
prev,
};
}
Release::None
}
PairState::Ready { ready, .. } if ready == side => {
panic!("Attempt to dealloate already free block")
}
PairState::Ready { next, prev, .. } => {
unsafe {
self.pairs.remove_unchecked(entry_index);
}
if prev == entry_index {
debug_assert_eq!(next, entry_index);
self.next_ready = self.pairs.len();
} else {
let prev_entry = unsafe { self.pairs.get_unchecked_mut(prev) };
let prev_next = unsafe { prev_entry.state.replace_next(next) };
debug_assert_eq!(prev_next, entry_index);
let next_entry = unsafe { self.pairs.get_unchecked_mut(next) };
let next_prev = unsafe { next_entry.state.replace_prev(prev) };
debug_assert_eq!(next_prev, entry_index);
self.next_ready = next;
}
match parent {
Some(parent) => Release::Parent(parent),
None => {
debug_assert_eq!(offset, 0);
Release::Chunk(chunk)
}
}
}
}
}
}
#[derive(Debug)]
struct Chunk<M> {
memory: Arc<M>,
ptr: Option<NonNull<u8>>,
size: u64,
}
#[derive(Debug)]
pub(crate) struct BuddyAllocator<M> {
minimal_size: u64,
chunks: Slab<Chunk<M>>,
sizes: Vec<Size>,
memory_type: u32,
props: MemoryPropertyFlags,
atom_mask: u64,
}
unsafe impl<M> Sync for BuddyAllocator<M> where M: Sync {}
unsafe impl<M> Send for BuddyAllocator<M> where M: Send {}
impl<M> BuddyAllocator<M>
where
M: MemoryBounds + 'static,
{
pub fn new(
minimal_size: u64,
initial_dedicated_size: u64,
memory_type: u32,
props: MemoryPropertyFlags,
atom_mask: u64,
) -> Self {
assert!(
minimal_size.is_power_of_two(),
"Minimal allocation size of buddy allocator must be power of two"
);
assert!(
initial_dedicated_size.is_power_of_two(),
"Dedicated allocation size of buddy allocator must be power of two"
);
let initial_sizes = (initial_dedicated_size
.trailing_zeros()
.saturating_sub(minimal_size.trailing_zeros())) as usize;
BuddyAllocator {
minimal_size,
chunks: Slab::new(),
sizes: (0..initial_sizes).map(|_| Size::new()).collect(),
memory_type,
props,
atom_mask: atom_mask | (minimal_size - 1),
}
}
#[cfg_attr(feature = "tracing", tracing::instrument(skip(self, device)))]
pub unsafe fn alloc(
&mut self,
device: &impl MemoryDevice<M>,
size: u64,
align_mask: u64,
flags: AllocationFlags,
heap: &mut Heap,
allocations_remains: &mut u32,
) -> Result<BuddyBlock<M>, AllocationError> {
let align_mask = align_mask | self.atom_mask;
let size = align_up(size, align_mask)
.and_then(|size| size.checked_next_power_of_two())
.ok_or(AllocationError::OutOfDeviceMemory)?;
let size = size.max(self.minimal_size);
let size_index = size.trailing_zeros() - self.minimal_size.trailing_zeros();
let size_index =
usize::try_from(size_index).map_err(|_| AllocationError::OutOfDeviceMemory)?;
while self.sizes.len() <= size_index {
self.sizes.push(Size::new());
}
let host_visible = self.host_visible();
let mut candidate_size_index = size_index;
let (mut entry, entry_size_index) = loop {
let sizes_len = self.sizes.len();
let candidate_size_entry = &mut self.sizes[candidate_size_index];
let candidate_size = self.minimal_size << candidate_size_index;
if let Some(entry) = candidate_size_entry.acquire(candidate_size) {
break (entry, candidate_size_index);
}
if sizes_len == candidate_size_index + 1 {
if *allocations_remains == 0 {
return Err(AllocationError::TooManyObjects);
}
let chunk_size = self.minimal_size << (candidate_size_index + 1);
let mut memory = device.allocate_memory(chunk_size, self.memory_type, flags)?;
*allocations_remains -= 1;
heap.alloc(chunk_size);
let ptr = if host_visible {
match device.map_memory(&mut memory, 0, chunk_size) {
Ok(ptr) => Some(ptr),
Err(DeviceMapError::OutOfDeviceMemory) => {
return Err(AllocationError::OutOfDeviceMemory)
}
Err(DeviceMapError::MapFailed) | Err(DeviceMapError::OutOfHostMemory) => {
return Err(AllocationError::OutOfHostMemory)
}
}
} else {
None
};
let chunk = self.chunks.insert(Chunk {
memory: Arc::new(memory),
ptr,
size: chunk_size,
});
let entry = candidate_size_entry.add_pair_and_acquire_left(chunk, 0, None);
break (entry, candidate_size_index);
}
candidate_size_index += 1;
};
for size_index in (size_index..entry_size_index).rev() {
let size_entry = &mut self.sizes[size_index];
entry =
size_entry.add_pair_and_acquire_left(entry.chunk, entry.offset, Some(entry.index));
}
let chunk_entry = self.chunks.get_unchecked(entry.chunk);
debug_assert!(
entry
.offset
.checked_add(size)
.map_or(false, |end| end <= chunk_entry.size),
"Offset + size is not in chunk bounds"
);
Ok(BuddyBlock {
memory: chunk_entry.memory.clone(),
ptr: chunk_entry
.ptr
.map(|ptr| NonNull::new_unchecked(ptr.as_ptr().add(entry.offset as usize))),
offset: entry.offset,
size,
chunk: entry.chunk,
index: entry.index,
})
}
#[cfg_attr(feature = "tracing", tracing::instrument(skip(self, device)))]
pub unsafe fn dealloc(
&mut self,
device: &impl MemoryDevice<M>,
block: BuddyBlock<M>,
heap: &mut Heap,
allocations_remains: &mut u32,
) {
debug_assert!(block.size.is_power_of_two());
let size_index =
(block.size.trailing_zeros() - self.minimal_size.trailing_zeros()) as usize;
let mut release_index = block.index;
let mut release_size_index = size_index;
loop {
match self.sizes[release_size_index].release(release_index) {
Release::Parent(parent) => {
release_size_index += 1;
release_index = parent;
}
Release::Chunk(chunk) => {
debug_assert_eq!(chunk, block.chunk);
debug_assert_eq!(
self.chunks.get(chunk).size,
self.minimal_size << (release_size_index + 1)
);
let chunk = self.chunks.remove(chunk);
drop(block);
let memory = try_arc_unwrap(chunk.memory)
.expect("Memory shared after last block deallocated");
device.deallocate_memory(memory);
*allocations_remains += 1;
heap.dealloc(chunk.size);
return;
}
Release::None => return,
}
}
}
fn host_visible(&self) -> bool {
self.props.contains(MemoryPropertyFlags::HOST_VISIBLE)
}
}