mod entry;
pub mod raw_entry_v1;
use hashbrown::hash_table;
use crate::vec::{self, Vec};
use crate::TryReserveError;
use core::mem;
use core::ops::RangeBounds;
use crate::util::simplify_range;
use crate::{Bucket, Equivalent, HashValue};
type Indices = hash_table::HashTable<usize>;
type Entries<K, V> = Vec<Bucket<K, V>>;
pub use entry::{Entry, IndexedEntry, OccupiedEntry, VacantEntry};
#[derive(Debug)]
pub(crate) struct IndexMapCore<K, V> {
indices: Indices,
entries: Entries<K, V>,
}
struct RefMut<'a, K, V> {
indices: &'a mut Indices,
entries: &'a mut Entries<K, V>,
}
#[inline(always)]
fn get_hash<K, V>(entries: &[Bucket<K, V>]) -> impl Fn(&usize) -> u64 + '_ {
move |&i| entries[i].hash.get()
}
#[inline]
fn equivalent<'a, K, V, Q: ?Sized + Equivalent<K>>(
key: &'a Q,
entries: &'a [Bucket<K, V>],
) -> impl Fn(&usize) -> bool + 'a {
move |&i| Q::equivalent(key, &entries[i].key)
}
#[inline]
fn erase_index(table: &mut Indices, hash: HashValue, index: usize) {
if let Ok(entry) = table.find_entry(hash.get(), move |&i| i == index) {
entry.remove();
} else if cfg!(debug_assertions) {
panic!("index not found");
}
}
#[inline]
fn update_index(table: &mut Indices, hash: HashValue, old: usize, new: usize) {
let index = table
.find_mut(hash.get(), move |&i| i == old)
.expect("index not found");
*index = new;
}
fn insert_bulk_no_grow<K, V>(indices: &mut Indices, entries: &[Bucket<K, V>]) {
assert!(indices.capacity() - indices.len() >= entries.len());
for entry in entries {
indices.insert_unique(entry.hash.get(), indices.len(), |_| unreachable!());
}
}
impl<K, V> Clone for IndexMapCore<K, V>
where
K: Clone,
V: Clone,
{
fn clone(&self) -> Self {
let mut new = Self::new();
new.clone_from(self);
new
}
fn clone_from(&mut self, other: &Self) {
self.indices.clone_from(&other.indices);
if self.entries.capacity() < other.entries.len() {
let additional = other.entries.len() - self.entries.len();
self.borrow_mut().reserve_entries(additional);
}
self.entries.clone_from(&other.entries);
}
}
impl<K, V> crate::Entries for IndexMapCore<K, V> {
type Entry = Bucket<K, V>;
#[inline]
fn into_entries(self) -> Vec<Self::Entry> {
self.entries
}
#[inline]
fn as_entries(&self) -> &[Self::Entry] {
&self.entries
}
#[inline]
fn as_entries_mut(&mut self) -> &mut [Self::Entry] {
&mut self.entries
}
fn with_entries<F>(&mut self, f: F)
where
F: FnOnce(&mut [Self::Entry]),
{
f(&mut self.entries);
self.rebuild_hash_table();
}
}
impl<K, V> IndexMapCore<K, V> {
const MAX_ENTRIES_CAPACITY: usize = (isize::MAX as usize) / mem::size_of::<Bucket<K, V>>();
#[inline]
pub(crate) const fn new() -> Self {
IndexMapCore {
indices: Indices::new(),
entries: Vec::new(),
}
}
#[inline]
fn borrow_mut(&mut self) -> RefMut<'_, K, V> {
RefMut::new(&mut self.indices, &mut self.entries)
}
#[inline]
pub(crate) fn with_capacity(n: usize) -> Self {
IndexMapCore {
indices: Indices::with_capacity(n),
entries: Vec::with_capacity(n),
}
}
#[inline]
pub(crate) fn len(&self) -> usize {
self.indices.len()
}
#[inline]
pub(crate) fn capacity(&self) -> usize {
Ord::min(self.indices.capacity(), self.entries.capacity())
}
pub(crate) fn clear(&mut self) {
self.indices.clear();
self.entries.clear();
}
pub(crate) fn truncate(&mut self, len: usize) {
if len < self.len() {
self.erase_indices(len, self.entries.len());
self.entries.truncate(len);
}
}
pub(crate) fn drain<R>(&mut self, range: R) -> vec::Drain<'_, Bucket<K, V>>
where
R: RangeBounds<usize>,
{
let range = simplify_range(range, self.entries.len());
self.erase_indices(range.start, range.end);
self.entries.drain(range)
}
#[cfg(feature = "rayon")]
pub(crate) fn par_drain<R>(&mut self, range: R) -> rayon::vec::Drain<'_, Bucket<K, V>>
where
K: Send,
V: Send,
R: RangeBounds<usize>,
{
use rayon::iter::ParallelDrainRange;
let range = simplify_range(range, self.entries.len());
self.erase_indices(range.start, range.end);
self.entries.par_drain(range)
}
pub(crate) fn split_off(&mut self, at: usize) -> Self {
assert!(at <= self.entries.len());
self.erase_indices(at, self.entries.len());
let entries = self.entries.split_off(at);
let mut indices = Indices::with_capacity(entries.len());
insert_bulk_no_grow(&mut indices, &entries);
Self { indices, entries }
}
pub(crate) fn split_splice<R>(&mut self, range: R) -> (Self, vec::IntoIter<Bucket<K, V>>)
where
R: RangeBounds<usize>,
{
let range = simplify_range(range, self.len());
self.erase_indices(range.start, self.entries.len());
let entries = self.entries.split_off(range.end);
let drained = self.entries.split_off(range.start);
let mut indices = Indices::with_capacity(entries.len());
insert_bulk_no_grow(&mut indices, &entries);
(Self { indices, entries }, drained.into_iter())
}
pub(crate) fn append_unchecked(&mut self, other: &mut Self) {
self.reserve(other.len());
insert_bulk_no_grow(&mut self.indices, &other.entries);
self.entries.append(&mut other.entries);
other.indices.clear();
}
pub(crate) fn reserve(&mut self, additional: usize) {
self.indices.reserve(additional, get_hash(&self.entries));
if additional > self.entries.capacity() - self.entries.len() {
self.borrow_mut().reserve_entries(additional);
}
}
pub(crate) fn reserve_exact(&mut self, additional: usize) {
self.indices.reserve(additional, get_hash(&self.entries));
self.entries.reserve_exact(additional);
}
pub(crate) fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> {
self.indices
.try_reserve(additional, get_hash(&self.entries))
.map_err(TryReserveError::from_hashbrown)?;
if additional > self.entries.capacity() - self.entries.len() {
self.try_reserve_entries(additional)
} else {
Ok(())
}
}
fn try_reserve_entries(&mut self, additional: usize) -> Result<(), TryReserveError> {
let new_capacity = Ord::min(self.indices.capacity(), Self::MAX_ENTRIES_CAPACITY);
let try_add = new_capacity - self.entries.len();
if try_add > additional && self.entries.try_reserve_exact(try_add).is_ok() {
return Ok(());
}
self.entries
.try_reserve_exact(additional)
.map_err(TryReserveError::from_alloc)
}
pub(crate) fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> {
self.indices
.try_reserve(additional, get_hash(&self.entries))
.map_err(TryReserveError::from_hashbrown)?;
self.entries
.try_reserve_exact(additional)
.map_err(TryReserveError::from_alloc)
}
pub(crate) fn shrink_to(&mut self, min_capacity: usize) {
self.indices
.shrink_to(min_capacity, get_hash(&self.entries));
self.entries.shrink_to(min_capacity);
}
pub(crate) fn pop(&mut self) -> Option<(K, V)> {
if let Some(entry) = self.entries.pop() {
let last = self.entries.len();
erase_index(&mut self.indices, entry.hash, last);
Some((entry.key, entry.value))
} else {
None
}
}
pub(crate) fn get_index_of<Q>(&self, hash: HashValue, key: &Q) -> Option<usize>
where
Q: ?Sized + Equivalent<K>,
{
let eq = equivalent(key, &self.entries);
self.indices.find(hash.get(), eq).copied()
}
fn push_entry(&mut self, hash: HashValue, key: K, value: V) {
if self.entries.len() == self.entries.capacity() {
self.borrow_mut().reserve_entries(1);
}
self.entries.push(Bucket { hash, key, value });
}
pub(crate) fn insert_full(&mut self, hash: HashValue, key: K, value: V) -> (usize, Option<V>)
where
K: Eq,
{
let eq = equivalent(&key, &self.entries);
let hasher = get_hash(&self.entries);
match self.indices.entry(hash.get(), eq, hasher) {
hash_table::Entry::Occupied(entry) => {
let i = *entry.get();
(i, Some(mem::replace(&mut self.entries[i].value, value)))
}
hash_table::Entry::Vacant(entry) => {
let i = self.entries.len();
entry.insert(i);
self.push_entry(hash, key, value);
debug_assert_eq!(self.indices.len(), self.entries.len());
(i, None)
}
}
}
pub(crate) fn replace_full(
&mut self,
hash: HashValue,
key: K,
value: V,
) -> (usize, Option<(K, V)>)
where
K: Eq,
{
let eq = equivalent(&key, &self.entries);
let hasher = get_hash(&self.entries);
match self.indices.entry(hash.get(), eq, hasher) {
hash_table::Entry::Occupied(entry) => {
let i = *entry.get();
let entry = &mut self.entries[i];
let kv = (
mem::replace(&mut entry.key, key),
mem::replace(&mut entry.value, value),
);
(i, Some(kv))
}
hash_table::Entry::Vacant(entry) => {
let i = self.entries.len();
entry.insert(i);
self.push_entry(hash, key, value);
debug_assert_eq!(self.indices.len(), self.entries.len());
(i, None)
}
}
}
pub(crate) fn shift_remove_full<Q>(&mut self, hash: HashValue, key: &Q) -> Option<(usize, K, V)>
where
Q: ?Sized + Equivalent<K>,
{
let eq = equivalent(key, &self.entries);
match self.indices.find_entry(hash.get(), eq) {
Ok(entry) => {
let (index, _) = entry.remove();
let (key, value) = self.borrow_mut().shift_remove_finish(index);
Some((index, key, value))
}
Err(_) => None,
}
}
#[inline]
pub(crate) fn shift_remove_index(&mut self, index: usize) -> Option<(K, V)> {
self.borrow_mut().shift_remove_index(index)
}
#[inline]
pub(super) fn move_index(&mut self, from: usize, to: usize) {
self.borrow_mut().move_index(from, to);
}
#[inline]
pub(crate) fn swap_indices(&mut self, a: usize, b: usize) {
self.borrow_mut().swap_indices(a, b);
}
pub(crate) fn swap_remove_full<Q>(&mut self, hash: HashValue, key: &Q) -> Option<(usize, K, V)>
where
Q: ?Sized + Equivalent<K>,
{
let eq = equivalent(key, &self.entries);
match self.indices.find_entry(hash.get(), eq) {
Ok(entry) => {
let (index, _) = entry.remove();
let (key, value) = self.borrow_mut().swap_remove_finish(index);
Some((index, key, value))
}
Err(_) => None,
}
}
#[inline]
pub(crate) fn swap_remove_index(&mut self, index: usize) -> Option<(K, V)> {
self.borrow_mut().swap_remove_index(index)
}
fn erase_indices(&mut self, start: usize, end: usize) {
let (init, shifted_entries) = self.entries.split_at(end);
let (start_entries, erased_entries) = init.split_at(start);
let erased = erased_entries.len();
let shifted = shifted_entries.len();
let half_capacity = self.indices.capacity() / 2;
if erased == 0 {
} else if start + shifted < half_capacity && start < erased {
self.indices.clear();
insert_bulk_no_grow(&mut self.indices, start_entries);
insert_bulk_no_grow(&mut self.indices, shifted_entries);
} else if erased + shifted < half_capacity {
for (i, entry) in (start..).zip(erased_entries) {
erase_index(&mut self.indices, entry.hash, i);
}
for ((new, old), entry) in (start..).zip(end..).zip(shifted_entries) {
update_index(&mut self.indices, entry.hash, old, new);
}
} else {
let offset = end - start;
self.indices.retain(move |i| {
if *i >= end {
*i -= offset;
true
} else {
*i < start
}
});
}
debug_assert_eq!(self.indices.len(), start + shifted);
}
pub(crate) fn retain_in_order<F>(&mut self, mut keep: F)
where
F: FnMut(&mut K, &mut V) -> bool,
{
self.entries
.retain_mut(|entry| keep(&mut entry.key, &mut entry.value));
if self.entries.len() < self.indices.len() {
self.rebuild_hash_table();
}
}
fn rebuild_hash_table(&mut self) {
self.indices.clear();
insert_bulk_no_grow(&mut self.indices, &self.entries);
}
pub(crate) fn reverse(&mut self) {
self.entries.reverse();
let len = self.entries.len();
for i in &mut self.indices {
*i = len - *i - 1;
}
}
}
impl<'a, K, V> RefMut<'a, K, V> {
#[inline]
fn new(indices: &'a mut Indices, entries: &'a mut Entries<K, V>) -> Self {
Self { indices, entries }
}
fn reserve_entries(&mut self, additional: usize) {
let new_capacity = Ord::min(
self.indices.capacity(),
IndexMapCore::<K, V>::MAX_ENTRIES_CAPACITY,
);
let try_add = new_capacity - self.entries.len();
if try_add > additional && self.entries.try_reserve_exact(try_add).is_ok() {
return;
}
self.entries.reserve_exact(additional);
}
fn insert_unique(mut self, hash: HashValue, key: K, value: V) -> OccupiedEntry<'a, K, V> {
if self.entries.len() == self.entries.capacity() {
self.reserve_entries(1);
}
let i = self.indices.len();
let entry = self
.indices
.insert_unique(hash.get(), i, get_hash(self.entries));
debug_assert_eq!(i, self.entries.len());
self.entries.push(Bucket { hash, key, value });
OccupiedEntry::new(self.entries, entry)
}
fn shift_insert_unique(&mut self, index: usize, hash: HashValue, key: K, value: V) {
let end = self.indices.len();
assert!(index <= end);
self.increment_indices(index, end);
let entries = &*self.entries;
self.indices.insert_unique(hash.get(), index, move |&i| {
debug_assert_ne!(i, index);
let i = if i < index { i } else { i - 1 };
entries[i].hash.get()
});
if self.entries.len() == self.entries.capacity() {
self.reserve_entries(1);
}
self.entries.insert(index, Bucket { hash, key, value });
}
fn shift_remove_index(&mut self, index: usize) -> Option<(K, V)> {
match self.entries.get(index) {
Some(entry) => {
erase_index(self.indices, entry.hash, index);
Some(self.shift_remove_finish(index))
}
None => None,
}
}
fn shift_remove_finish(&mut self, index: usize) -> (K, V) {
self.decrement_indices(index + 1, self.entries.len());
let entry = self.entries.remove(index);
(entry.key, entry.value)
}
fn swap_remove_index(&mut self, index: usize) -> Option<(K, V)> {
match self.entries.get(index) {
Some(entry) => {
erase_index(self.indices, entry.hash, index);
Some(self.swap_remove_finish(index))
}
None => None,
}
}
fn swap_remove_finish(&mut self, index: usize) -> (K, V) {
let entry = self.entries.swap_remove(index);
if let Some(entry) = self.entries.get(index) {
let last = self.entries.len();
update_index(self.indices, entry.hash, last, index);
}
(entry.key, entry.value)
}
fn decrement_indices(&mut self, start: usize, end: usize) {
let shifted_entries = &self.entries[start..end];
if shifted_entries.len() > self.indices.capacity() / 2 {
for i in &mut *self.indices {
if start <= *i && *i < end {
*i -= 1;
}
}
} else {
for (i, entry) in (start..end).zip(shifted_entries) {
update_index(self.indices, entry.hash, i, i - 1);
}
}
}
fn increment_indices(&mut self, start: usize, end: usize) {
let shifted_entries = &self.entries[start..end];
if shifted_entries.len() > self.indices.capacity() / 2 {
for i in &mut *self.indices {
if start <= *i && *i < end {
*i += 1;
}
}
} else {
for (i, entry) in (start..end).zip(shifted_entries).rev() {
update_index(self.indices, entry.hash, i, i + 1);
}
}
}
fn move_index(&mut self, from: usize, to: usize) {
let from_hash = self.entries[from].hash;
let _ = self.entries[to]; if from != to {
update_index(self.indices, from_hash, from, usize::MAX);
if from < to {
self.decrement_indices(from + 1, to + 1);
self.entries[from..=to].rotate_left(1);
} else if to < from {
self.increment_indices(to, from);
self.entries[to..=from].rotate_right(1);
}
update_index(self.indices, from_hash, usize::MAX, to);
}
}
fn swap_indices(&mut self, a: usize, b: usize) {
if a == b && a < self.entries.len() {
return;
}
match self.indices.get_many_mut(
[self.entries[a].hash.get(), self.entries[b].hash.get()],
move |i, &x| if i == 0 { x == a } else { x == b },
) {
[Some(ref_a), Some(ref_b)] => {
mem::swap(ref_a, ref_b);
self.entries.swap(a, b);
}
_ => panic!("indices not found"),
}
}
}
#[test]
fn assert_send_sync() {
fn assert_send_sync<T: Send + Sync>() {}
assert_send_sync::<IndexMapCore<i32, i32>>();
assert_send_sync::<Entry<'_, i32, i32>>();
assert_send_sync::<IndexedEntry<'_, i32, i32>>();
assert_send_sync::<raw_entry_v1::RawEntryMut<'_, i32, i32, ()>>();
}