From 25163e22b156ba216b906d98c379886051208c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Wed, 2 Jan 2019 22:35:39 +0100 Subject: [PATCH 01/14] Add SharedWorkerLocal --- src/librustc_data_structures/sync.rs | 87 ++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/src/librustc_data_structures/sync.rs b/src/librustc_data_structures/sync.rs index 6a19f52897e5d..ace3993d4c4e0 100644 --- a/src/librustc_data_structures/sync.rs +++ b/src/librustc_data_structures/sync.rs @@ -266,6 +266,45 @@ cfg_if! { } } + #[derive(Debug, Default)] + pub struct SharedWorkerLocal(T); + + impl SharedWorkerLocal { + /// Creates a new worker local where the `initial` closure computes the + /// value this worker local should take for each thread in the thread pool. + #[inline] + pub fn new T>(mut f: F) -> SharedWorkerLocal { + SharedWorkerLocal(f(0)) + } + + #[inline] + pub fn iter(&self) -> impl Iterator { + Some(&self.0).into_iter() + } + + /// Returns the worker-local value for each thread + #[inline] + pub fn into_inner(self) -> Vec { + vec![self.0] + } + } + + impl Deref for SharedWorkerLocal { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &T { + &self.0 + } + } + + impl DerefMut for SharedWorkerLocal { + #[inline(always)] + fn deref_mut(&mut self) -> &mut T { + &mut self.0 + } + } + pub type MTRef<'a, T> = &'a mut T; #[derive(Debug, Default)] @@ -387,6 +426,54 @@ cfg_if! { } pub use rayon_core::WorkerLocal; + pub use rayon_core::Registry; + use rayon_core::current_thread_index; + + #[derive(Debug)] + pub struct SharedWorkerLocal(Vec); + + impl SharedWorkerLocal { + /// Creates a new worker local where the `initial` closure computes the + /// value this worker local should take for each thread in the thread pool. + #[inline] + pub fn new T>(mut f: F) -> SharedWorkerLocal { + SharedWorkerLocal((0..Registry::current_num_threads()).map(|i| f(i)).collect()) + } + + #[inline] + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } + + /// Returns the worker-local value for each thread + #[inline] + pub fn into_inner(self) -> Vec { + self.0 + } + } + + impl Default for SharedWorkerLocal { + #[inline] + fn default() -> Self { + SharedWorkerLocal::new(|_| Default::default()) + } + } + + impl Deref for SharedWorkerLocal { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &T { + &self.0[current_thread_index().unwrap()] + } + } + + impl DerefMut for SharedWorkerLocal { + #[inline(always)] + fn deref_mut(&mut self) -> &mut T { + &mut self.0[current_thread_index().unwrap()] + } + } pub use rayon::iter::ParallelIterator; use rayon::iter::IntoParallelIterator; From 5dcb70e4cc8bb18db23d8e82f7242a93f2c28183 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Sat, 15 Jun 2019 21:59:59 +0200 Subject: [PATCH 02/14] Make SyncDroplessArena allocations thread-local in the fast path --- src/libarena/lib.rs | 191 ++++++++++++++++++++++++++++++++------------ 1 file changed, 140 insertions(+), 51 deletions(-) diff --git a/src/libarena/lib.rs b/src/libarena/lib.rs index 854942dad3ded..4f8e2834e693c 100644 --- a/src/libarena/lib.rs +++ b/src/libarena/lib.rs @@ -21,7 +21,7 @@ extern crate alloc; use rustc_data_structures::cold_path; -use rustc_data_structures::sync::MTLock; +use rustc_data_structures::sync::{SharedWorkerLocal, WorkerLocal, Lock}; use smallvec::SmallVec; use std::cell::{Cell, RefCell}; @@ -102,6 +102,68 @@ impl TypedArenaChunk { } } +struct CurrentChunk { + /// A pointer to the next object to be allocated. + ptr: Cell<*mut T>, + + /// A pointer to the end of the allocated area. When this pointer is + /// reached, a new chunk is allocated. + end: Cell<*mut T>, +} + +impl Default for CurrentChunk { + #[inline] + fn default() -> Self { + CurrentChunk { + // We set both `ptr` and `end` to 0 so that the first call to + // alloc() will trigger a grow(). + ptr: Cell::new(0 as *mut T), + end: Cell::new(0 as *mut T), + } + } +} + +impl CurrentChunk { + #[inline] + fn align(&self, align: usize) { + let final_address = ((self.ptr.get() as usize) + align - 1) & !(align - 1); + self.ptr.set(final_address as *mut T); + assert!(self.ptr <= self.end); + } + + /// Grows the arena. + #[inline(always)] + fn grow(&self, n: usize, chunks: &mut Vec>) { + unsafe { + let (chunk, mut new_capacity); + if let Some(last_chunk) = chunks.last_mut() { + let used_bytes = self.ptr.get() as usize - last_chunk.start() as usize; + let currently_used_cap = used_bytes / mem::size_of::(); + last_chunk.entries = currently_used_cap; + if last_chunk.storage.reserve_in_place(currently_used_cap, n) { + self.end.set(last_chunk.end()); + return; + } else { + new_capacity = last_chunk.storage.capacity(); + loop { + new_capacity = new_capacity.checked_mul(2).unwrap(); + if new_capacity >= currently_used_cap + n { + break; + } + } + } + } else { + let elem_size = cmp::max(1, mem::size_of::()); + new_capacity = cmp::max(n, PAGE / elem_size); + } + chunk = TypedArenaChunk::::new(new_capacity); + self.ptr.set(chunk.start()); + self.end.set(chunk.end()); + chunks.push(chunk); + } + } +} + const PAGE: usize = 4096; impl Default for TypedArena { @@ -119,11 +181,6 @@ impl Default for TypedArena { } impl TypedArena { - pub fn in_arena(&self, ptr: *const T) -> bool { - let ptr = ptr as *const T as *mut T; - - self.chunks.borrow().iter().any(|chunk| chunk.start() <= ptr && ptr < chunk.end()) - } /// Allocates an object in the `TypedArena`, returning a reference to it. #[inline] pub fn alloc(&self, object: T) -> &mut T { @@ -339,12 +396,6 @@ impl Default for DroplessArena { } impl DroplessArena { - pub fn in_arena(&self, ptr: *const T) -> bool { - let ptr = ptr as *const u8 as *mut u8; - - self.chunks.borrow().iter().any(|chunk| chunk.start() <= ptr && ptr < chunk.end()) - } - #[inline] fn align(&self, align: usize) { let final_address = ((self.ptr.get() as usize) + align - 1) & !(align - 1); @@ -516,64 +567,102 @@ impl DroplessArena { } } -#[derive(Default)] -// FIXME(@Zoxc): this type is entirely unused in rustc -pub struct SyncTypedArena { - lock: MTLock>, -} - -impl SyncTypedArena { - #[inline(always)] - pub fn alloc(&self, object: T) -> &mut T { - // Extend the lifetime of the result since it's limited to the lock guard - unsafe { &mut *(self.lock.lock().alloc(object) as *mut T) } - } - - #[inline(always)] - pub fn alloc_slice(&self, slice: &[T]) -> &mut [T] - where - T: Copy, - { - // Extend the lifetime of the result since it's limited to the lock guard - unsafe { &mut *(self.lock.lock().alloc_slice(slice) as *mut [T]) } - } +pub struct SyncDroplessArena { + /// Pointers to the current chunk + current: WorkerLocal>, - #[inline(always)] - pub fn clear(&mut self) { - self.lock.get_mut().clear(); - } + /// A vector of arena chunks. + chunks: Lock>>>, } -#[derive(Default)] -pub struct SyncDroplessArena { - lock: MTLock, +impl Default for SyncDroplessArena { + #[inline] + fn default() -> SyncDroplessArena { + SyncDroplessArena { + current: WorkerLocal::new(|_| CurrentChunk::default()), + chunks: Default::default(), + } + } } impl SyncDroplessArena { - #[inline(always)] pub fn in_arena(&self, ptr: *const T) -> bool { - self.lock.lock().in_arena(ptr) + let ptr = ptr as *const u8 as *mut u8; + + self.chunks.lock().iter().any(|chunks| chunks.iter().any(|chunk| { + chunk.start() <= ptr && ptr < chunk.end() + })) } - #[inline(always)] + #[inline(never)] + #[cold] + fn grow(&self, needed_bytes: usize) { + self.current.grow(needed_bytes, &mut **self.chunks.lock()); + } + + #[inline] pub fn alloc_raw(&self, bytes: usize, align: usize) -> &mut [u8] { - // Extend the lifetime of the result since it's limited to the lock guard - unsafe { &mut *(self.lock.lock().alloc_raw(bytes, align) as *mut [u8]) } + unsafe { + assert!(bytes != 0); + + let current = &*self.current; + + current.align(align); + + let future_end = intrinsics::arith_offset(current.ptr.get(), bytes as isize); + if (future_end as *mut u8) >= current.end.get() { + self.grow(bytes); + } + + let ptr = current.ptr.get(); + // Set the pointer past ourselves + current.ptr.set( + intrinsics::arith_offset(current.ptr.get(), bytes as isize) as *mut u8, + ); + slice::from_raw_parts_mut(ptr, bytes) + } } - #[inline(always)] + #[inline] pub fn alloc(&self, object: T) -> &mut T { - // Extend the lifetime of the result since it's limited to the lock guard - unsafe { &mut *(self.lock.lock().alloc(object) as *mut T) } + assert!(!mem::needs_drop::()); + + let mem = self.alloc_raw( + mem::size_of::(), + mem::align_of::()) as *mut _ as *mut T; + + unsafe { + // Write into uninitialized memory. + ptr::write(mem, object); + &mut *mem + } } - #[inline(always)] + /// Allocates a slice of objects that are copied into the `SyncDroplessArena`, returning a + /// mutable reference to it. Will panic if passed a zero-sized type. + /// + /// Panics: + /// + /// - Zero-sized types + /// - Zero-length slices + #[inline] pub fn alloc_slice(&self, slice: &[T]) -> &mut [T] where T: Copy, { - // Extend the lifetime of the result since it's limited to the lock guard - unsafe { &mut *(self.lock.lock().alloc_slice(slice) as *mut [T]) } + assert!(!mem::needs_drop::()); + assert!(mem::size_of::() != 0); + assert!(!slice.is_empty()); + + let mem = self.alloc_raw( + slice.len() * mem::size_of::(), + mem::align_of::()) as *mut _ as *mut T; + + unsafe { + let arena_slice = slice::from_raw_parts_mut(mem, slice.len()); + arena_slice.copy_from_slice(slice); + arena_slice + } } } From 338b2a1d3812b7bfc4b2fe255a288a327c36bf21 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Wed, 11 Dec 2019 20:54:42 +0100 Subject: [PATCH 03/14] Split the arena into several backends. --- src/libarena/lib.rs | 601 +++++++++++++++++++++++++++++------------- src/libarena/tests.rs | 6 - 2 files changed, 424 insertions(+), 183 deletions(-) diff --git a/src/libarena/lib.rs b/src/libarena/lib.rs index 4f8e2834e693c..686cd07ac19c8 100644 --- a/src/libarena/lib.rs +++ b/src/libarena/lib.rs @@ -13,7 +13,9 @@ #![feature(core_intrinsics)] #![feature(dropck_eyepatch)] +#![feature(ptr_offset_from)] #![feature(raw_vec_internals)] +#![feature(untagged_unions)] #![cfg_attr(test, feature(test))] #![allow(deprecated)] @@ -34,75 +36,104 @@ use std::slice; use alloc::raw_vec::RawVec; -/// An arena that can hold objects of only one type. -pub struct TypedArena { - /// A pointer to the next object to be allocated. - ptr: Cell<*mut T>, +trait ChunkBackend: Sized { + type ChunkVecType: Sized; - /// A pointer to the end of the allocated area. When this pointer is - /// reached, a new chunk is allocated. - end: Cell<*mut T>, + /// Create new vec. + fn new() -> (Self, Self::ChunkVecType); - /// A vector of arena chunks. - chunks: RefCell>>, + /// Check current chunk has enough space for next allocation. + fn can_allocate(&self, len: usize, align: usize) -> bool; - /// Marker indicating that dropping the arena causes its owned - /// instances of `T` to be dropped. - _own: PhantomData, + /// Allocate a new chunk and point to it. + fn grow(&self, len: usize, align: usize, chunks: &mut Self::ChunkVecType); + + /// Allocate a slice from this chunk. Panic if space lacks. + unsafe fn alloc_raw_slice(&self, len: usize, align: usize) -> *mut T; + + /// Clear the arena. + fn clear(&self, chunks: &mut Self::ChunkVecType); } -struct TypedArenaChunk { - /// The raw storage for the arena chunk. - storage: RawVec, - /// The number of valid entries in the chunk. - entries: usize, +struct NOPCurrentChunk { + phantom: PhantomData<*mut T>, } -impl TypedArenaChunk { +impl ChunkBackend for NOPCurrentChunk { + type ChunkVecType = (); + #[inline] - unsafe fn new(capacity: usize) -> TypedArenaChunk { - TypedArenaChunk { - storage: RawVec::with_capacity(capacity), - entries: 0, - } + fn new() -> (Self, ()) { + let phantom = PhantomData; + (NOPCurrentChunk { phantom }, ()) } - /// Destroys this arena chunk. #[inline] - unsafe fn destroy(&mut self, len: usize) { - // The branch on needs_drop() is an -O1 performance optimization. - // Without the branch, dropping TypedArena takes linear time. - if mem::needs_drop::() { - let mut start = self.start(); - // Destroy all allocated objects. - for _ in 0..len { - ptr::drop_in_place(start); - start = start.offset(1); - } - } + fn can_allocate(&self, _len: usize, _align: usize) -> bool + { true } + + #[inline] + fn grow(&self, _len: usize, _align: usize, _chunks: &mut Self::ChunkVecType) + {} + + #[inline] + unsafe fn alloc_raw_slice(&self, _len: usize, align: usize) -> *mut T { + assert!(align >= mem::align_of::()); + align as *mut T } - // Returns a pointer to the first allocated object. #[inline] - fn start(&self) -> *mut T { - self.storage.ptr() + fn clear(&self, _chunk: &mut Self::ChunkVecType) + {} +} + +struct ZSTCurrentChunk { + counter: Cell, + phantom: PhantomData<*mut T>, +} + +impl ChunkBackend for ZSTCurrentChunk { + type ChunkVecType = (); + + #[inline] + fn new() -> (Self, ()) { + (ZSTCurrentChunk { + counter: Cell::new(0), + phantom: PhantomData, + }, ()) } - // Returns a pointer to the end of the allocated space. #[inline] - fn end(&self) -> *mut T { - unsafe { - if mem::size_of::() == 0 { - // A pointer as large as possible for zero-sized elements. - !0 as *mut T - } else { - self.start().add(self.storage.capacity()) - } + fn can_allocate(&self, _len: usize, _align: usize) -> bool + { true } + + #[inline] + fn grow(&self, _len: usize, _align: usize, _chunks: &mut Self::ChunkVecType) + {} + + #[inline] + unsafe fn alloc_raw_slice(&self, len: usize, align: usize) -> *mut T { + assert!(align >= mem::align_of::()); + let count = self.counter.get(); + self.counter.set(count+len); + align as *mut T + } + + #[inline] + fn clear(&self, _chunks: &mut Self::ChunkVecType) { + assert!(mem::needs_drop::()); + + let count = self.counter.get(); + for _ in 0..count { + let ptr = mem::align_of::() as *mut T; + unsafe { ptr::drop_in_place(ptr) } } + + self.counter.set(0) } } -struct CurrentChunk { +struct TypedCurrentChunk { /// A pointer to the next object to be allocated. ptr: Cell<*mut T>, @@ -111,129 +142,230 @@ struct CurrentChunk { end: Cell<*mut T>, } -impl Default for CurrentChunk { +impl ChunkBackend for TypedCurrentChunk { + type ChunkVecType = Vec>; + #[inline] - fn default() -> Self { - CurrentChunk { - // We set both `ptr` and `end` to 0 so that the first call to - // alloc() will trigger a grow(). - ptr: Cell::new(0 as *mut T), - end: Cell::new(0 as *mut T), - } + fn new() -> (Self, Self::ChunkVecType) { + (TypedCurrentChunk { + ptr: Cell::new(ptr::null_mut()), + end: Cell::new(ptr::null_mut()), + }, vec![]) } -} -impl CurrentChunk { #[inline] - fn align(&self, align: usize) { - let final_address = ((self.ptr.get() as usize) + align - 1) & !(align - 1); - self.ptr.set(final_address as *mut T); - assert!(self.ptr <= self.end); + fn can_allocate(&self, len: usize, align: usize) -> bool { + assert!(mem::size_of::() > 0); + assert!(mem::align_of::() == align); + let available_capacity = unsafe { self.end.get().offset_from(self.ptr.get()) }; + assert!(available_capacity >= 0); + let available_capacity = available_capacity as usize; + available_capacity >= len } /// Grows the arena. - #[inline(always)] - fn grow(&self, n: usize, chunks: &mut Vec>) { + #[inline(never)] + #[cold] + fn grow(&self, len: usize, align: usize, chunks: &mut Self::ChunkVecType) { + assert!(mem::size_of::() > 0); + assert!(mem::align_of::() == align); unsafe { - let (chunk, mut new_capacity); + let mut new_capacity; if let Some(last_chunk) = chunks.last_mut() { let used_bytes = self.ptr.get() as usize - last_chunk.start() as usize; let currently_used_cap = used_bytes / mem::size_of::(); last_chunk.entries = currently_used_cap; - if last_chunk.storage.reserve_in_place(currently_used_cap, n) { + if last_chunk.storage.reserve_in_place(currently_used_cap, len) { self.end.set(last_chunk.end()); return; } else { new_capacity = last_chunk.storage.capacity(); loop { new_capacity = new_capacity.checked_mul(2).unwrap(); - if new_capacity >= currently_used_cap + n { + if new_capacity >= currently_used_cap + len { break; } } } } else { let elem_size = cmp::max(1, mem::size_of::()); - new_capacity = cmp::max(n, PAGE / elem_size); + new_capacity = cmp::max(len, PAGE / elem_size); } - chunk = TypedArenaChunk::::new(new_capacity); + + let chunk = TypedArenaChunk::new(new_capacity); self.ptr.set(chunk.start()); self.end.set(chunk.end()); chunks.push(chunk); } } + + #[inline] + unsafe fn alloc_raw_slice(&self, len: usize, align: usize) -> *mut T { + assert!(mem::size_of::() > 0); + assert!(align == mem::align_of::()); + + let ptr = self.ptr.get(); + let end = ptr.add(len); + assert!(end <= self.end.get()); + + self.ptr.set(end); + ptr + } + + fn clear(&self, chunks: &mut Self::ChunkVecType) { + assert!(mem::needs_drop::()); + assert!(mem::size_of::() > 0); + + if let Some(last_chunk) = chunks.last_mut() { + // Clear the last chunk, which is partially filled. + unsafe { + let start = last_chunk.start(); + let len = self.ptr.get().offset_from(start); + assert!(len >= 0); + let slice = slice::from_raw_parts_mut(start, len as usize); + ptr::drop_in_place(slice); + self.ptr.set(start); + } + + let len = chunks.len(); + // If `T` is ZST, code below has no effect. + for mut chunk in chunks.drain(..len-1) { + unsafe { chunk.destroy(chunk.entries) } + } + } + } } -const PAGE: usize = 4096; +struct DroplessCurrentChunk { + /// A pointer to the next object to be allocated. + ptr: Cell<*mut u8>, -impl Default for TypedArena { - /// Creates a new `TypedArena`. - fn default() -> TypedArena { - TypedArena { + /// A pointer to the end of the allocated area. When this pointer is + /// reached, a new chunk is allocated. + end: Cell<*mut u8>, +} + +impl ChunkBackend for DroplessCurrentChunk { + type ChunkVecType = Vec>; + + #[inline] + fn new() -> (Self, Self::ChunkVecType) { + (DroplessCurrentChunk { // We set both `ptr` and `end` to 0 so that the first call to // alloc() will trigger a grow(). ptr: Cell::new(ptr::null_mut()), end: Cell::new(ptr::null_mut()), - chunks: RefCell::new(vec![]), - _own: PhantomData, - } + }, vec![]) } -} -impl TypedArena { - /// Allocates an object in the `TypedArena`, returning a reference to it. #[inline] - pub fn alloc(&self, object: T) -> &mut T { - if self.ptr == self.end { - self.grow(1) - } + fn can_allocate(&self, len: usize, align: usize) -> bool { + let len = len * mem::size_of::(); + let ptr = self.ptr.get(); + let ptr = unsafe { ptr.add(ptr.align_offset(align)) }; + let available_capacity = unsafe { self.end.get().offset_from(ptr) }; + assert!(available_capacity >= 0); + let available_capacity = available_capacity as usize; + available_capacity >= len + } + /// Grows the arena. + #[inline(never)] + #[cold] + fn grow(&self, len: usize, _align: usize, chunks: &mut Self::ChunkVecType) { + let len = len * mem::size_of::(); unsafe { - if mem::size_of::() == 0 { - self.ptr - .set(intrinsics::arith_offset(self.ptr.get() as *mut u8, 1) - as *mut T); - let ptr = mem::align_of::() as *mut T; - // Don't drop the object. This `write` is equivalent to `forget`. - ptr::write(ptr, object); - &mut *ptr + let mut new_capacity; + if let Some(last_chunk) = chunks.last_mut() { + let currently_used_cap = self.ptr.get() as usize - last_chunk.start() as usize; + last_chunk.entries = currently_used_cap; + if last_chunk.storage.reserve_in_place(currently_used_cap, len) { + self.end.set(last_chunk.end()); + return; + } else { + new_capacity = last_chunk.storage.capacity(); + loop { + new_capacity = new_capacity.checked_mul(2).unwrap(); + if new_capacity >= currently_used_cap + len { + break; + } + } + } } else { - let ptr = self.ptr.get(); - // Advance the pointer. - self.ptr.set(self.ptr.get().offset(1)); - // Write into uninitialized memory. - ptr::write(ptr, object); - &mut *ptr + new_capacity = cmp::max(len, PAGE); } + + let chunk = TypedArenaChunk::new(new_capacity); + self.ptr.set(chunk.start()); + self.end.set(chunk.end()); + chunks.push(chunk); } } #[inline] - fn can_allocate(&self, len: usize) -> bool { - let available_capacity_bytes = self.end.get() as usize - self.ptr.get() as usize; - let at_least_bytes = len.checked_mul(mem::size_of::()).unwrap(); - available_capacity_bytes >= at_least_bytes + unsafe fn alloc_raw_slice(&self, len: usize, align: usize) -> *mut T { + let len = len * mem::size_of::(); + let ptr = self.ptr.get(); + let ptr = ptr.add(ptr.align_offset(align)); + let end = ptr.add(len); + assert!(end <= self.end.get()); + + self.ptr.set(end); + ptr as *mut T } - /// Ensures there's enough space in the current chunk to fit `len` objects. #[inline] - fn ensure_capacity(&self, len: usize) { - if !self.can_allocate(len) { - self.grow(len); - debug_assert!(self.can_allocate(len)); + fn clear(&self, chunks: &mut Self::ChunkVecType) { + if let Some(last_chunk) = chunks.last_mut() { + // Clear the last chunk, which is partially filled. + self.ptr.set(last_chunk.start()) } } +} + +struct GenericArena> { + /// Current chunk for next allocation. + current: Chunk, + /// A vector of arena chunks. + chunks: RefCell, + + /// Marker indicating that dropping the arena causes its owned + /// instances of `T` to be dropped. + _own: PhantomData, +} + +impl> Default for GenericArena { + fn default() -> Self { + let (current, chunks) = Chunk::new(); + let chunks = RefCell::new(chunks); + GenericArena { + current, chunks, + _own: PhantomData, + } + } +} + +impl> GenericArena { #[inline] unsafe fn alloc_raw_slice(&self, len: usize) -> *mut T { - assert!(mem::size_of::() != 0); - assert!(len != 0); + let align = mem::align_of::(); + if !self.current.can_allocate(len, align) { + self.current.grow(len, align, &mut *self.chunks.borrow_mut()); + debug_assert!(self.current.can_allocate(len, align)); + } - self.ensure_capacity(len); + self.current.alloc_raw_slice(len, align) + } - let start_ptr = self.ptr.get(); - self.ptr.set(start_ptr.add(len)); - start_ptr + /// Allocates an object in the `GenericArena`, returning a reference to it. + #[inline] + pub fn alloc(&self, object: T) -> &mut T { + unsafe { + let ptr = self.alloc_raw_slice(1); + ptr::write(ptr, object); + &mut *ptr + } } /// Allocates a slice of objects that are copied into the `TypedArena`, returning a mutable @@ -274,12 +406,186 @@ impl TypedArena { } } + /// Clears the arena. Deallocates all but the longest chunk which may be reused. + pub fn clear(&mut self) { + self.current.clear(&mut *self.chunks.borrow_mut()) + } +} + +unsafe impl<#[may_dangle] T, Chunk: ChunkBackend> Drop for GenericArena { + fn drop(&mut self) { + self.clear() + // RawVec handles deallocation of `last_chunk` and `self.chunks`. + } +} + +pub union TypedArena { + nop: mem::ManuallyDrop>>, + zst: mem::ManuallyDrop>>, + dropless: mem::ManuallyDrop>, + typed: mem::ManuallyDrop>>, +} + +impl Default for TypedArena { + fn default() -> Self { + match (mem::needs_drop::(), mem::size_of::() > 0) { + (true, true) => Self { typed: mem::ManuallyDrop::new(GenericArena::default()) }, + (true, false) => Self { zst: mem::ManuallyDrop::new(GenericArena::default()) }, + (false, true) => Self { dropless: mem::ManuallyDrop::new(GenericArena::default()) }, + (false, false) => Self { nop: mem::ManuallyDrop::new(GenericArena::default()) }, + } + } +} + +macro_rules! forward_impl { + ($t:ty, &$self:ident; $call:ident($($e:expr),*)) => { + match (mem::needs_drop::<$t>(), mem::size_of::<$t>() > 0) { + (true, true) => unsafe { &*$self.typed }.$call($($e),*), + (true, false) => unsafe { &*$self.zst }.$call($($e),*), + (false, true) => unsafe { &*$self.dropless }.$call($($e),*), + (false, false) => unsafe { &*$self.nop }.$call($($e),*), + } + }; + ($t:ty, &mut $self:ident; $call:ident($($e:expr),*)) => { + match (mem::needs_drop::<$t>(), mem::size_of::<$t>() > 0) { + (true, true) => unsafe { &mut*$self.typed }.$call($($e),*), + (true, false) => unsafe { &mut*$self.zst }.$call($($e),*), + (false, true) => unsafe { &mut*$self.dropless }.$call($($e),*), + (false, false) => unsafe { &mut*$self.nop }.$call($($e),*), + } + }; +} + +impl TypedArena { + /// Allocates an object in the `GenericArena`, returning a reference to it. + #[inline] + pub fn alloc(&self, object: T) -> &mut T { + forward_impl!(T, &self; alloc(object)) + } + + /// Allocates a slice of objects that are copied into the `TypedArena`, returning a mutable + /// reference to it. Will panic if passed a zero-sized types. + /// + /// Panics: + /// + /// - Zero-sized types + /// - Zero-length slices + #[inline] + pub fn alloc_slice(&self, slice: &[T]) -> &mut [T] + where + T: Copy, + { + forward_impl!(T, &self; alloc_slice(slice)) + } + + #[inline] + pub fn alloc_from_iter>(&self, iter: I) -> &mut [T] { + forward_impl!(T, &self; alloc_from_iter(iter)) + } + + /// Clears the arena. Deallocates all but the longest chunk which may be reused. + pub fn clear(&mut self) { + forward_impl!(T, &mut self; clear()) + } +} + +unsafe impl<#[may_dangle] T> Drop for TypedArena { + fn drop(&mut self) { + match (mem::needs_drop::(), mem::size_of::() > 0) { + (true, true) => unsafe { let _ = mem::ManuallyDrop::drop(&mut self.typed); }, + (true, false) => unsafe { let _ = mem::ManuallyDrop::drop(&mut self.zst); }, + (false, true) => unsafe { let _ = mem::ManuallyDrop::drop(&mut self.dropless); }, + (false, false) => unsafe { let _ = mem::ManuallyDrop::drop(&mut self.nop); }, + } + } +} + +unsafe impl Send for TypedArena {} + +struct TypedArenaChunk { + /// The raw storage for the arena chunk. + storage: RawVec, + /// The number of valid entries in the chunk. + entries: usize, +} + +impl TypedArenaChunk { + #[inline] + unsafe fn new(capacity: usize) -> TypedArenaChunk { + TypedArenaChunk { + storage: RawVec::with_capacity(capacity), + entries: 0, + } + } + + /// Destroys this arena chunk. + #[inline] + unsafe fn destroy(&mut self, len: usize) { + // The branch on needs_drop() is an -O1 performance optimization. + // Without the branch, dropping TypedArena takes linear time. + if mem::needs_drop::() { + let mut start = self.start(); + // Destroy all allocated objects. + for _ in 0..len { + ptr::drop_in_place(start); + start = start.offset(1); + } + } + } + + // Returns a pointer to the first allocated object. + #[inline] + fn start(&self) -> *mut T { + self.storage.ptr() + } + + // Returns a pointer to the end of the allocated space. + #[inline] + fn end(&self) -> *mut T { + unsafe { + if mem::size_of::() == 0 { + // A pointer as large as possible for zero-sized elements. + !0 as *mut T + } else { + self.start().add(self.storage.capacity()) + } + } + } +} + +struct CurrentChunk { + /// A pointer to the next object to be allocated. + ptr: Cell<*mut T>, + + /// A pointer to the end of the allocated area. When this pointer is + /// reached, a new chunk is allocated. + end: Cell<*mut T>, +} + +impl Default for CurrentChunk { + #[inline] + fn default() -> Self { + CurrentChunk { + // We set both `ptr` and `end` to 0 so that the first call to + // alloc() will trigger a grow(). + ptr: Cell::new(ptr::null_mut()), + end: Cell::new(ptr::null_mut()), + } + } +} + +impl CurrentChunk { + #[inline] + fn align(&self, align: usize) { + let final_address = ((self.ptr.get() as usize) + align - 1) & !(align - 1); + self.ptr.set(final_address as *mut T); + assert!(self.ptr <= self.end); + } + /// Grows the arena. - #[inline(never)] - #[cold] - fn grow(&self, n: usize) { + #[inline(always)] + fn grow(&self, n: usize, chunks: &mut Vec>) { unsafe { - let mut chunks = self.chunks.borrow_mut(); let (chunk, mut new_capacity); if let Some(last_chunk) = chunks.last_mut() { let used_bytes = self.ptr.get() as usize - last_chunk.start() as usize; @@ -307,68 +613,9 @@ impl TypedArena { chunks.push(chunk); } } - - /// Clears the arena. Deallocates all but the longest chunk which may be reused. - pub fn clear(&mut self) { - unsafe { - // Clear the last chunk, which is partially filled. - let mut chunks_borrow = self.chunks.borrow_mut(); - if let Some(mut last_chunk) = chunks_borrow.last_mut() { - self.clear_last_chunk(&mut last_chunk); - let len = chunks_borrow.len(); - // If `T` is ZST, code below has no effect. - for mut chunk in chunks_borrow.drain(..len-1) { - chunk.destroy(chunk.entries); - } - } - } - } - - // Drops the contents of the last chunk. The last chunk is partially empty, unlike all other - // chunks. - fn clear_last_chunk(&self, last_chunk: &mut TypedArenaChunk) { - // Determine how much was filled. - let start = last_chunk.start() as usize; - // We obtain the value of the pointer to the first uninitialized element. - let end = self.ptr.get() as usize; - // We then calculate the number of elements to be dropped in the last chunk, - // which is the filled area's length. - let diff = if mem::size_of::() == 0 { - // `T` is ZST. It can't have a drop flag, so the value here doesn't matter. We get - // the number of zero-sized values in the last and only chunk, just out of caution. - // Recall that `end` was incremented for each allocated value. - end - start - } else { - (end - start) / mem::size_of::() - }; - // Pass that to the `destroy` method. - unsafe { - last_chunk.destroy(diff); - } - // Reset the chunk. - self.ptr.set(last_chunk.start()); - } -} - -unsafe impl<#[may_dangle] T> Drop for TypedArena { - fn drop(&mut self) { - unsafe { - // Determine how much was filled. - let mut chunks_borrow = self.chunks.borrow_mut(); - if let Some(mut last_chunk) = chunks_borrow.pop() { - // Drop the contents of the last chunk. - self.clear_last_chunk(&mut last_chunk); - // The last chunk will be dropped. Destroy all other chunks. - for chunk in chunks_borrow.iter_mut() { - chunk.destroy(chunk.entries); - } - } - // RawVec handles deallocation of `last_chunk` and `self.chunks`. - } - } } -unsafe impl Send for TypedArena {} +const PAGE: usize = 4096; pub struct DroplessArena { /// A pointer to the next object to be allocated. diff --git a/src/libarena/tests.rs b/src/libarena/tests.rs index fa4189409d0e8..95441af85b044 100644 --- a/src/libarena/tests.rs +++ b/src/libarena/tests.rs @@ -11,12 +11,6 @@ struct Point { z: i32, } -#[test] -pub fn test_unused() { - let arena: TypedArena = TypedArena::default(); - assert!(arena.chunks.borrow().is_empty()); -} - #[test] fn test_arena_alloc_nested() { struct Inner { From a57bc1fcbd5c1928e39cd55bda6da57a40756ddc Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Fri, 13 Dec 2019 19:41:51 +0100 Subject: [PATCH 04/14] Use refactored for DroplessArena. --- src/libarena/lib.rs | 77 ++------------------------------------------- 1 file changed, 3 insertions(+), 74 deletions(-) diff --git a/src/libarena/lib.rs b/src/libarena/lib.rs index 686cd07ac19c8..950d8a8da20e5 100644 --- a/src/libarena/lib.rs +++ b/src/libarena/lib.rs @@ -617,89 +617,18 @@ impl CurrentChunk { const PAGE: usize = 4096; +#[derive(Default)] pub struct DroplessArena { - /// A pointer to the next object to be allocated. - ptr: Cell<*mut u8>, - - /// A pointer to the end of the allocated area. When this pointer is - /// reached, a new chunk is allocated. - end: Cell<*mut u8>, - - /// A vector of arena chunks. - chunks: RefCell>>, + backend: GenericArena, } unsafe impl Send for DroplessArena {} -impl Default for DroplessArena { - #[inline] - fn default() -> DroplessArena { - DroplessArena { - ptr: Cell::new(ptr::null_mut()), - end: Cell::new(ptr::null_mut()), - chunks: Default::default(), - } - } -} - impl DroplessArena { - #[inline] - fn align(&self, align: usize) { - let final_address = ((self.ptr.get() as usize) + align - 1) & !(align - 1); - self.ptr.set(final_address as *mut u8); - assert!(self.ptr <= self.end); - } - - #[inline(never)] - #[cold] - fn grow(&self, needed_bytes: usize) { - unsafe { - let mut chunks = self.chunks.borrow_mut(); - let (chunk, mut new_capacity); - if let Some(last_chunk) = chunks.last_mut() { - let used_bytes = self.ptr.get() as usize - last_chunk.start() as usize; - if last_chunk - .storage - .reserve_in_place(used_bytes, needed_bytes) - { - self.end.set(last_chunk.end()); - return; - } else { - new_capacity = last_chunk.storage.capacity(); - loop { - new_capacity = new_capacity.checked_mul(2).unwrap(); - if new_capacity >= used_bytes + needed_bytes { - break; - } - } - } - } else { - new_capacity = cmp::max(needed_bytes, PAGE); - } - chunk = TypedArenaChunk::::new(new_capacity); - self.ptr.set(chunk.start()); - self.end.set(chunk.end()); - chunks.push(chunk); - } - } - #[inline] pub fn alloc_raw(&self, bytes: usize, align: usize) -> &mut [u8] { unsafe { - assert!(bytes != 0); - - self.align(align); - - let future_end = intrinsics::arith_offset(self.ptr.get(), bytes as isize); - if (future_end as *mut u8) >= self.end.get() { - self.grow(bytes); - } - - let ptr = self.ptr.get(); - // Set the pointer past ourselves - self.ptr.set( - intrinsics::arith_offset(self.ptr.get(), bytes as isize) as *mut u8, - ); + let ptr = self.backend.current.alloc_raw_slice(bytes, align); slice::from_raw_parts_mut(ptr, bytes) } } From 6e837482d183332f62c38ecad9498e3afc31d02c Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Fri, 13 Dec 2019 20:07:59 +0100 Subject: [PATCH 05/14] Migrate SyncDroplessArena. --- src/libarena/lib.rs | 234 ++++++++++++++++++++++++-------------------- 1 file changed, 126 insertions(+), 108 deletions(-) diff --git a/src/libarena/lib.rs b/src/libarena/lib.rs index 950d8a8da20e5..d804e5fa15b80 100644 --- a/src/libarena/lib.rs +++ b/src/libarena/lib.rs @@ -28,7 +28,6 @@ use smallvec::SmallVec; use std::cell::{Cell, RefCell}; use std::cmp; -use std::intrinsics; use std::marker::{PhantomData, Send}; use std::mem; use std::ptr; @@ -40,7 +39,10 @@ trait ChunkBackend: Sized { type ChunkVecType: Sized; /// Create new vec. - fn new() -> (Self, Self::ChunkVecType); + fn new() -> Self; + + /// Create new vec. + fn new_vec() -> Self::ChunkVecType; /// Check current chunk has enough space for next allocation. fn can_allocate(&self, len: usize, align: usize) -> bool; @@ -63,11 +65,14 @@ impl ChunkBackend for NOPCurrentChunk { type ChunkVecType = (); #[inline] - fn new() -> (Self, ()) { + fn new() -> Self { let phantom = PhantomData; - (NOPCurrentChunk { phantom }, ()) + NOPCurrentChunk { phantom } } + #[inline] + fn new_vec() {} + #[inline] fn can_allocate(&self, _len: usize, _align: usize) -> bool { true } @@ -96,13 +101,16 @@ impl ChunkBackend for ZSTCurrentChunk { type ChunkVecType = (); #[inline] - fn new() -> (Self, ()) { - (ZSTCurrentChunk { + fn new() -> Self { + ZSTCurrentChunk { counter: Cell::new(0), phantom: PhantomData, - }, ()) + } } + #[inline] + fn new_vec() {} + #[inline] fn can_allocate(&self, _len: usize, _align: usize) -> bool { true } @@ -146,11 +154,16 @@ impl ChunkBackend for TypedCurrentChunk { type ChunkVecType = Vec>; #[inline] - fn new() -> (Self, Self::ChunkVecType) { - (TypedCurrentChunk { + fn new() -> Self { + TypedCurrentChunk { ptr: Cell::new(ptr::null_mut()), end: Cell::new(ptr::null_mut()), - }, vec![]) + } + } + + #[inline] + fn new_vec() -> Self::ChunkVecType { + vec![] } #[inline] @@ -236,26 +249,35 @@ impl ChunkBackend for TypedCurrentChunk { } } -struct DroplessCurrentChunk { +struct DroplessCurrentChunk { /// A pointer to the next object to be allocated. ptr: Cell<*mut u8>, /// A pointer to the end of the allocated area. When this pointer is /// reached, a new chunk is allocated. end: Cell<*mut u8>, + + /// Ensure correct semantics. + _own: PhantomData<*mut T>, } -impl ChunkBackend for DroplessCurrentChunk { +impl ChunkBackend for DroplessCurrentChunk { type ChunkVecType = Vec>; #[inline] - fn new() -> (Self, Self::ChunkVecType) { - (DroplessCurrentChunk { + fn new() -> Self { + DroplessCurrentChunk { // We set both `ptr` and `end` to 0 so that the first call to // alloc() will trigger a grow(). ptr: Cell::new(ptr::null_mut()), end: Cell::new(ptr::null_mut()), - }, vec![]) + _own: PhantomData, + } + } + + #[inline] + fn new_vec() -> Self::ChunkVecType { + vec![] } #[inline] @@ -337,8 +359,8 @@ struct GenericArena> { impl> Default for GenericArena { fn default() -> Self { - let (current, chunks) = Chunk::new(); - let chunks = RefCell::new(chunks); + let current = Chunk::new(); + let chunks = RefCell::new(Chunk::new_vec()); GenericArena { current, chunks, _own: PhantomData, @@ -422,7 +444,7 @@ unsafe impl<#[may_dangle] T, Chunk: ChunkBackend> Drop for GenericArena { nop: mem::ManuallyDrop>>, zst: mem::ManuallyDrop>>, - dropless: mem::ManuallyDrop>, + dropless: mem::ManuallyDrop>>, typed: mem::ManuallyDrop>>, } @@ -553,73 +575,11 @@ impl TypedArenaChunk { } } -struct CurrentChunk { - /// A pointer to the next object to be allocated. - ptr: Cell<*mut T>, - - /// A pointer to the end of the allocated area. When this pointer is - /// reached, a new chunk is allocated. - end: Cell<*mut T>, -} - -impl Default for CurrentChunk { - #[inline] - fn default() -> Self { - CurrentChunk { - // We set both `ptr` and `end` to 0 so that the first call to - // alloc() will trigger a grow(). - ptr: Cell::new(ptr::null_mut()), - end: Cell::new(ptr::null_mut()), - } - } -} - -impl CurrentChunk { - #[inline] - fn align(&self, align: usize) { - let final_address = ((self.ptr.get() as usize) + align - 1) & !(align - 1); - self.ptr.set(final_address as *mut T); - assert!(self.ptr <= self.end); - } - - /// Grows the arena. - #[inline(always)] - fn grow(&self, n: usize, chunks: &mut Vec>) { - unsafe { - let (chunk, mut new_capacity); - if let Some(last_chunk) = chunks.last_mut() { - let used_bytes = self.ptr.get() as usize - last_chunk.start() as usize; - let currently_used_cap = used_bytes / mem::size_of::(); - last_chunk.entries = currently_used_cap; - if last_chunk.storage.reserve_in_place(currently_used_cap, n) { - self.end.set(last_chunk.end()); - return; - } else { - new_capacity = last_chunk.storage.capacity(); - loop { - new_capacity = new_capacity.checked_mul(2).unwrap(); - if new_capacity >= currently_used_cap + n { - break; - } - } - } - } else { - let elem_size = cmp::max(1, mem::size_of::()); - new_capacity = cmp::max(n, PAGE / elem_size); - } - chunk = TypedArenaChunk::::new(new_capacity); - self.ptr.set(chunk.start()); - self.end.set(chunk.end()); - chunks.push(chunk); - } - } -} - const PAGE: usize = 4096; #[derive(Default)] pub struct DroplessArena { - backend: GenericArena, + backend: GenericArena>, } unsafe impl Send for DroplessArena {} @@ -744,8 +704,8 @@ impl DroplessArena { } pub struct SyncDroplessArena { - /// Pointers to the current chunk - current: WorkerLocal>, + /// Current chunk for next allocation. + current: WorkerLocal>, /// A vector of arena chunks. chunks: Lock>>>, @@ -755,7 +715,7 @@ impl Default for SyncDroplessArena { #[inline] fn default() -> SyncDroplessArena { SyncDroplessArena { - current: WorkerLocal::new(|_| CurrentChunk::default()), + current: WorkerLocal::new(|_| DroplessCurrentChunk::new()), chunks: Default::default(), } } @@ -770,31 +730,10 @@ impl SyncDroplessArena { })) } - #[inline(never)] - #[cold] - fn grow(&self, needed_bytes: usize) { - self.current.grow(needed_bytes, &mut **self.chunks.lock()); - } - #[inline] pub fn alloc_raw(&self, bytes: usize, align: usize) -> &mut [u8] { unsafe { - assert!(bytes != 0); - - let current = &*self.current; - - current.align(align); - - let future_end = intrinsics::arith_offset(current.ptr.get(), bytes as isize); - if (future_end as *mut u8) >= current.end.get() { - self.grow(bytes); - } - - let ptr = current.ptr.get(); - // Set the pointer past ourselves - current.ptr.set( - intrinsics::arith_offset(current.ptr.get(), bytes as isize) as *mut u8, - ); + let ptr = self.current.alloc_raw_slice(bytes, align); slice::from_raw_parts_mut(ptr, bytes) } } @@ -814,8 +753,8 @@ impl SyncDroplessArena { } } - /// Allocates a slice of objects that are copied into the `SyncDroplessArena`, returning a - /// mutable reference to it. Will panic if passed a zero-sized type. + /// Allocates a slice of objects that are copied into the `DroplessArena`, returning a mutable + /// reference to it. Will panic if passed a zero-sized type. /// /// Panics: /// @@ -840,6 +779,85 @@ impl SyncDroplessArena { arena_slice } } + + #[inline] + unsafe fn write_from_iter>( + &self, + mut iter: I, + len: usize, + mem: *mut T, + ) -> &mut [T] { + let mut i = 0; + // Use a manual loop since LLVM manages to optimize it better for + // slice iterators + loop { + let value = iter.next(); + if i >= len || value.is_none() { + // We only return as many items as the iterator gave us, even + // though it was supposed to give us `len` + return slice::from_raw_parts_mut(mem, i); + } + ptr::write(mem.add(i), value.unwrap()); + i += 1; + } + } + + #[inline] + pub fn alloc_from_iter>(&self, iter: I) -> &mut [T] { + let iter = iter.into_iter(); + assert!(mem::size_of::() != 0); + assert!(!mem::needs_drop::()); + + let size_hint = iter.size_hint(); + + match size_hint { + (min, Some(max)) if min == max => { + // We know the exact number of elements the iterator will produce here + let len = min; + + if len == 0 { + return &mut [] + } + let size = len.checked_mul(mem::size_of::()).unwrap(); + let mem = self.alloc_raw(size, mem::align_of::()) as *mut _ as *mut T; + unsafe { + self.write_from_iter(iter, len, mem) + } + } + (_, _) => { + cold_path(move || -> &mut [T] { + let mut vec: SmallVec<[_; 8]> = iter.collect(); + if vec.is_empty() { + return &mut []; + } + // Move the content to the arena by copying it and then forgetting + // the content of the SmallVec + unsafe { + let len = vec.len(); + let start_ptr = self.alloc_raw( + len * mem::size_of::(), + mem::align_of::() + ) as *mut _ as *mut T; + vec.as_ptr().copy_to_nonoverlapping(start_ptr, len); + vec.set_len(0); + slice::from_raw_parts_mut(start_ptr, len) + } + }) + } + } + } + + /// Clears the arena. Deallocates all but the longest chunk which may be reused. + pub fn clear(&mut self) { + self.current.clear(&mut *self.chunks.borrow_mut()) + } +} + +impl Drop for SyncDroplessArena { + fn drop(&mut self) { + self.clear() + // RawVec handles deallocation of `last_chunk` and `self.chunks`. + } } #[cfg(test)] From 16ee3121f1a8ed81923fd85b8ee6c3da79e5497d Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sat, 14 Dec 2019 00:37:21 +0100 Subject: [PATCH 06/14] Grow arenas. --- src/libarena/lib.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/libarena/lib.rs b/src/libarena/lib.rs index d804e5fa15b80..5ac85e305d459 100644 --- a/src/libarena/lib.rs +++ b/src/libarena/lib.rs @@ -587,6 +587,11 @@ unsafe impl Send for DroplessArena {} impl DroplessArena { #[inline] pub fn alloc_raw(&self, bytes: usize, align: usize) -> &mut [u8] { + if !self.backend.current.can_allocate(bytes, align) { + self.backend.current.grow(bytes, align, &mut *self.backend.chunks.borrow_mut()); + debug_assert!(self.backend.current.can_allocate(bytes, align)); + } + unsafe { let ptr = self.backend.current.alloc_raw_slice(bytes, align); slice::from_raw_parts_mut(ptr, bytes) @@ -732,6 +737,11 @@ impl SyncDroplessArena { #[inline] pub fn alloc_raw(&self, bytes: usize, align: usize) -> &mut [u8] { + if !self.current.can_allocate(bytes, align) { + self.current.grow(bytes, align, &mut *self.chunks.borrow_mut()); + debug_assert!(self.current.can_allocate(bytes, align)); + } + unsafe { let ptr = self.current.alloc_raw_slice(bytes, align); slice::from_raw_parts_mut(ptr, bytes) From f3ce9edfa2f5fb13781809f67fa8aaca08879148 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sat, 14 Dec 2019 14:49:01 +0100 Subject: [PATCH 07/14] Add tests. --- src/libarena/lib.rs | 1 + src/libarena/tests.rs | 181 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 180 insertions(+), 2 deletions(-) diff --git a/src/libarena/lib.rs b/src/libarena/lib.rs index 5ac85e305d459..747191971ad50 100644 --- a/src/libarena/lib.rs +++ b/src/libarena/lib.rs @@ -13,6 +13,7 @@ #![feature(core_intrinsics)] #![feature(dropck_eyepatch)] +#![feature(iter_once_with)] #![feature(ptr_offset_from)] #![feature(raw_vec_internals)] #![feature(untagged_unions)] diff --git a/src/libarena/tests.rs b/src/libarena/tests.rs index 95441af85b044..bf3a031a415b2 100644 --- a/src/libarena/tests.rs +++ b/src/libarena/tests.rs @@ -1,7 +1,8 @@ extern crate test; use test::Bencher; -use super::TypedArena; +use super::{TypedArena, DroplessArena, SyncDroplessArena}; use std::cell::Cell; +use std::iter; #[allow(dead_code)] #[derive(Debug, Eq, PartialEq)] @@ -12,7 +13,7 @@ struct Point { } #[test] -fn test_arena_alloc_nested() { +fn test_arena_alloc_nested_typed() { struct Inner { value: u8, } @@ -54,12 +55,163 @@ fn test_arena_alloc_nested() { assert_eq!(result.inner.value, 10); } +#[test] +fn test_arena_alloc_nested_dropless() { + struct Inner { + value: u8, + } + struct Outer<'a> { + inner: &'a Inner, + } + enum EI<'e> { + I(Inner), + O(Outer<'e>), + } + + struct Wrap(DroplessArena); + + impl Wrap { + fn alloc_inner Inner>(&self, f: F) -> &Inner { + let r: &EI<'_> = self.0.alloc(EI::I(f())); + if let &EI::I(ref i) = r { + i + } else { + panic!("mismatch"); + } + } + fn alloc_outer<'a, F: Fn() -> Outer<'a>>(&'a self, f: F) -> &Outer<'a> { + let r: &EI<'_> = self.0.alloc(EI::O(f())); + if let &EI::O(ref o) = r { + o + } else { + panic!("mismatch"); + } + } + } + + let arena = Wrap(DroplessArena::default()); + + let result = arena.alloc_outer(|| Outer { + inner: arena.alloc_inner(|| Inner { value: 10 }), + }); + + assert_eq!(result.inner.value, 10); +} + +#[test] +fn test_arena_alloc_nested_sync() { + struct Inner { + value: u8, + } + struct Outer<'a> { + inner: &'a Inner, + } + enum EI<'e> { + I(Inner), + O(Outer<'e>), + } + + struct Wrap(SyncDroplessArena); + + impl Wrap { + fn alloc_inner Inner>(&self, f: F) -> &Inner { + let r: &EI<'_> = self.0.alloc(EI::I(f())); + if let &EI::I(ref i) = r { + i + } else { + panic!("mismatch"); + } + } + fn alloc_outer<'a, F: Fn() -> Outer<'a>>(&'a self, f: F) -> &Outer<'a> { + let r: &EI<'_> = self.0.alloc(EI::O(f())); + if let &EI::O(ref o) = r { + o + } else { + panic!("mismatch"); + } + } + } + + let arena = Wrap(SyncDroplessArena::default()); + + let result = arena.alloc_outer(|| Outer { + inner: arena.alloc_inner(|| Inner { value: 10 }), + }); + + assert_eq!(result.inner.value, 10); +} + +#[test] +fn test_arena_alloc_nested_iter() { + struct Inner { + value: u8, + } + struct Outer<'a> { + inner: &'a Inner, + } + enum EI<'e> { + I(Inner), + O(Outer<'e>), + } + + struct Wrap<'a>(TypedArena>); + + impl<'a> Wrap<'a> { + fn alloc_inner Inner>(&self, f: F) -> &Inner { + let r: &[EI<'_>] = self.0.alloc_from_iter(iter::once_with(|| EI::I(f()))); + if let &[EI::I(ref i)] = r { + i + } else { + panic!("mismatch"); + } + } + fn alloc_outer Outer<'a>>(&self, f: F) -> &Outer<'_> { + let r: &[EI<'_>] = self.0.alloc_from_iter(iter::once_with(|| EI::O(f()))); + if let &[EI::O(ref o)] = r { + o + } else { + panic!("mismatch"); + } + } + } + + let arena = Wrap(TypedArena::default()); + + let result = arena.alloc_outer(|| Outer { + inner: arena.alloc_inner(|| Inner { value: 10 }), + }); + + assert_eq!(result.inner.value, 10); +} + #[test] pub fn test_copy() { let arena = TypedArena::default(); for _ in 0..100000 { arena.alloc(Point { x: 1, y: 2, z: 3 }); } + + let arena = DroplessArena::default(); + for _ in 0..100000 { + arena.alloc(Point { x: 1, y: 2, z: 3 }); + } + + let arena = SyncDroplessArena::default(); + for _ in 0..100000 { + arena.alloc(Point { x: 1, y: 2, z: 3 }); + } +} + +#[test] +pub fn test_align() { + #[repr(align(32))] + struct AlignedPoint(Point); + + let arena = TypedArena::default(); + for _ in 0..100000 { + let ptr = arena.alloc(AlignedPoint(Point { x: 1, y: 2, z: 3 })); + assert_eq!((ptr as *const _ as usize) & 31, 0); + } } #[bench] @@ -159,6 +311,31 @@ fn test_typed_arena_drop_on_clear() { } } +struct DropOrder<'a> { + rank: u32, + count: &'a Cell, +} + +impl Drop for DropOrder<'_> { + fn drop(&mut self) { + assert_eq!(self.rank, self.count.get()); + self.count.set(self.count.get() + 1); + } +} + +#[test] +fn test_typed_arena_drop_order() { + let counter = Cell::new(0); + { + let arena: TypedArena> = TypedArena::default(); + for rank in 0..100 { + // Allocate something with drop glue to make sure it doesn't leak. + arena.alloc(DropOrder { rank, count: &counter }); + } + }; + assert_eq!(counter.get(), 100); +} + thread_local! { static DROP_COUNTER: Cell = Cell::new(0) } From c0faba887a72a11123097cf388502c13161eab22 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sat, 14 Dec 2019 15:24:13 +0100 Subject: [PATCH 08/14] Fortify write_from_iter. --- src/libarena/lib.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libarena/lib.rs b/src/libarena/lib.rs index 747191971ad50..b1ba496f3b4f5 100644 --- a/src/libarena/lib.rs +++ b/src/libarena/lib.rs @@ -653,11 +653,13 @@ impl DroplessArena { // slice iterators loop { let value = iter.next(); - if i >= len || value.is_none() { + if value.is_none() { // We only return as many items as the iterator gave us, even // though it was supposed to give us `len` return slice::from_raw_parts_mut(mem, i); } + // The iterator is not supposed to give us more than `len`. + assert!(i < len); ptr::write(mem.add(i), value.unwrap()); i += 1; } @@ -803,11 +805,13 @@ impl SyncDroplessArena { // slice iterators loop { let value = iter.next(); - if i >= len || value.is_none() { + if value.is_none() { // We only return as many items as the iterator gave us, even // though it was supposed to give us `len` return slice::from_raw_parts_mut(mem, i); } + // The iterator is not supposed to give us more than `len`. + assert!(i < len); ptr::write(mem.add(i), value.unwrap()); i += 1; } From 33cdafab4709a5c62f542d085039581771ee3816 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sat, 14 Dec 2019 19:00:23 +0100 Subject: [PATCH 09/14] Refactor alloc_from_iter. --- src/libarena/lib.rs | 243 ++++++++++++++++++++------------------------ 1 file changed, 109 insertions(+), 134 deletions(-) diff --git a/src/libarena/lib.rs b/src/libarena/lib.rs index b1ba496f3b4f5..058e0f51a8c9a 100644 --- a/src/libarena/lib.rs +++ b/src/libarena/lib.rs @@ -413,19 +413,11 @@ impl> GenericArena { #[inline] pub fn alloc_from_iter>(&self, iter: I) -> &mut [T] { - assert!(mem::size_of::() != 0); - let mut vec: SmallVec<[_; 8]> = iter.into_iter().collect(); - if vec.is_empty() { - return &mut []; - } - // Move the content to the arena by copying it and then forgetting - // the content of the SmallVec unsafe { - let len = vec.len(); - let start_ptr = self.alloc_raw_slice(len); - vec.as_ptr().copy_to_nonoverlapping(start_ptr, len); - vec.set_len(0); - slice::from_raw_parts_mut(start_ptr, len) + alloc_from_iter_buffer( + move |len| self.alloc_raw_slice(len), + iter.into_iter(), + ) } } @@ -641,72 +633,19 @@ impl DroplessArena { } } - #[inline] - unsafe fn write_from_iter>( - &self, - mut iter: I, - len: usize, - mem: *mut T, - ) -> &mut [T] { - let mut i = 0; - // Use a manual loop since LLVM manages to optimize it better for - // slice iterators - loop { - let value = iter.next(); - if value.is_none() { - // We only return as many items as the iterator gave us, even - // though it was supposed to give us `len` - return slice::from_raw_parts_mut(mem, i); - } - // The iterator is not supposed to give us more than `len`. - assert!(i < len); - ptr::write(mem.add(i), value.unwrap()); - i += 1; - } - } - #[inline] pub fn alloc_from_iter>(&self, iter: I) -> &mut [T] { - let iter = iter.into_iter(); assert!(mem::size_of::() != 0); assert!(!mem::needs_drop::()); - let size_hint = iter.size_hint(); - - match size_hint { - (min, Some(max)) if min == max => { - // We know the exact number of elements the iterator will produce here - let len = min; - - if len == 0 { - return &mut [] - } - let size = len.checked_mul(mem::size_of::()).unwrap(); - let mem = self.alloc_raw(size, mem::align_of::()) as *mut _ as *mut T; - unsafe { - self.write_from_iter(iter, len, mem) - } - } - (_, _) => { - cold_path(move || -> &mut [T] { - let mut vec: SmallVec<[_; 8]> = iter.collect(); - if vec.is_empty() { - return &mut []; - } - // Move the content to the arena by copying it and then forgetting - // the content of the SmallVec - unsafe { - let len = vec.len(); - let start_ptr = self.alloc_raw( - len * mem::size_of::(), - mem::align_of::() - ) as *mut _ as *mut T; - vec.as_ptr().copy_to_nonoverlapping(start_ptr, len); - vec.set_len(0); - slice::from_raw_parts_mut(start_ptr, len) - } - }) - } + unsafe { + alloc_from_iter_dropless( + move |len| self.alloc_raw( + len * mem::size_of::(), + mem::align_of::(), + ) as *mut _ as *mut T, + iter.into_iter(), + ) } } } @@ -793,72 +732,19 @@ impl SyncDroplessArena { } } - #[inline] - unsafe fn write_from_iter>( - &self, - mut iter: I, - len: usize, - mem: *mut T, - ) -> &mut [T] { - let mut i = 0; - // Use a manual loop since LLVM manages to optimize it better for - // slice iterators - loop { - let value = iter.next(); - if value.is_none() { - // We only return as many items as the iterator gave us, even - // though it was supposed to give us `len` - return slice::from_raw_parts_mut(mem, i); - } - // The iterator is not supposed to give us more than `len`. - assert!(i < len); - ptr::write(mem.add(i), value.unwrap()); - i += 1; - } - } - #[inline] pub fn alloc_from_iter>(&self, iter: I) -> &mut [T] { - let iter = iter.into_iter(); assert!(mem::size_of::() != 0); assert!(!mem::needs_drop::()); - let size_hint = iter.size_hint(); - - match size_hint { - (min, Some(max)) if min == max => { - // We know the exact number of elements the iterator will produce here - let len = min; - - if len == 0 { - return &mut [] - } - let size = len.checked_mul(mem::size_of::()).unwrap(); - let mem = self.alloc_raw(size, mem::align_of::()) as *mut _ as *mut T; - unsafe { - self.write_from_iter(iter, len, mem) - } - } - (_, _) => { - cold_path(move || -> &mut [T] { - let mut vec: SmallVec<[_; 8]> = iter.collect(); - if vec.is_empty() { - return &mut []; - } - // Move the content to the arena by copying it and then forgetting - // the content of the SmallVec - unsafe { - let len = vec.len(); - let start_ptr = self.alloc_raw( - len * mem::size_of::(), - mem::align_of::() - ) as *mut _ as *mut T; - vec.as_ptr().copy_to_nonoverlapping(start_ptr, len); - vec.set_len(0); - slice::from_raw_parts_mut(start_ptr, len) - } - }) - } + unsafe { + alloc_from_iter_dropless( + move |len| self.alloc_raw( + len * mem::size_of::(), + mem::align_of::(), + ) as *mut _ as *mut T, + iter.into_iter(), + ) } } @@ -875,5 +761,94 @@ impl Drop for SyncDroplessArena { } } +/// Helper method for slice allocation from iterators. +unsafe fn alloc_from_iter_buffer<'a, T, F, I>(alloc: F, iter: I) -> &'a mut [T] + where F: 'a + FnOnce(usize) -> *mut T, + I: Iterator, +{ + assert!(mem::size_of::() != 0); + let mut vec: SmallVec<[_; 8]> = iter.into_iter().collect(); + if vec.is_empty() { + return &mut []; + } + + // Move the content to the arena by copying it and then forgetting + // the content of the SmallVec + let len = vec.len(); + let start_ptr = alloc(len); + vec.as_ptr().copy_to_nonoverlapping(start_ptr, len); + vec.set_len(0); + slice::from_raw_parts_mut(start_ptr, len) +} + +/// Helper method for slice allocation from iterators. +/// Dropless case, optimized for known-length iterators. +unsafe fn alloc_from_iter_dropless<'a, T, F, I>(alloc: F, iter: I) -> &'a mut [T] + where F: 'a + Fn(usize) -> *mut T, + I: Iterator, +{ + assert!(!mem::needs_drop::()); + + let size_hint = iter.size_hint(); + let mut vec: SmallVec<[T; 8]> = match size_hint { + (min, Some(max)) if min == max => { + // We know the exact number of elements the iterator will produce here + let len = min; + + let mut iter = iter.peekable(); + if iter.peek().is_none() { + return &mut [] + } + + let mem = alloc(len); + + // Use a manual loop since LLVM manages to optimize it better for + // slice iterators + for i in 0..len { + let value = iter.next(); + if value.is_none() { + // We only return as many items as the iterator gave us, even + // though it was supposed to give us `len` + return slice::from_raw_parts_mut(mem, i); + } + // The iterator is not supposed to give us more than `len`. + assert!(i < len); + ptr::write(mem.add(i), value.unwrap()); + } + + if iter.peek().is_none() { + return slice::from_raw_parts_mut(mem, len) + } + + // The iterator has lied to us, and produces more elements than advertised. + // In order to handle the rest, we create a buffer, move what we already have inside, + // and extend it with the rest of the items. + cold_path(move || { + let mut vec = SmallVec::with_capacity(len + iter.size_hint().0); + + mem.copy_to_nonoverlapping(vec.as_mut_ptr(), len); + vec.set_len(len); + vec.extend(iter); + vec + }) + } + (_, _) => iter.collect() + }; + + cold_path(move || -> &mut [T] { + if vec.is_empty() { + return &mut []; + } + + // Move the content to the arena by copying it and then forgetting + // the content of the SmallVec + let len = vec.len(); + let start_ptr = alloc(len); + vec.as_ptr().copy_to_nonoverlapping(start_ptr, len); + vec.set_len(0); + slice::from_raw_parts_mut(start_ptr, len) + }) +} + #[cfg(test)] mod tests; From f4acf2469e4b5aba99d774ea7d5f21b2884d3a19 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sat, 14 Dec 2019 19:00:32 +0100 Subject: [PATCH 10/14] Comments. --- src/libarena/lib.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/libarena/lib.rs b/src/libarena/lib.rs index 058e0f51a8c9a..b333d30e5e600 100644 --- a/src/libarena/lib.rs +++ b/src/libarena/lib.rs @@ -58,6 +58,8 @@ trait ChunkBackend: Sized { fn clear(&self, chunks: &mut Self::ChunkVecType); } +/// Chunk used for zero-sized dropless types. +/// These don't require work, hence the trivial implementation. struct NOPCurrentChunk { phantom: PhantomData<*mut T>, } @@ -93,6 +95,9 @@ impl ChunkBackend for NOPCurrentChunk { {} } +/// Chunk used for zero-sized droppable types. +/// The only thing to do is counting the number of allocated items +/// in order to drop them at the end. struct ZSTCurrentChunk { counter: Cell, phantom: PhantomData<*mut T>, @@ -142,6 +147,7 @@ impl ChunkBackend for ZSTCurrentChunk { } } +/// Chunk used for droppable types. struct TypedCurrentChunk { /// A pointer to the next object to be allocated. ptr: Cell<*mut T>, @@ -250,6 +256,8 @@ impl ChunkBackend for TypedCurrentChunk { } } +/// Chunk used for types with trivial drop. +/// It is spearated from `TypedCurrentChunk` for code reuse in `DroplessArena`. struct DroplessCurrentChunk { /// A pointer to the next object to be allocated. ptr: Cell<*mut u8>, @@ -434,6 +442,8 @@ unsafe impl<#[may_dangle] T, Chunk: ChunkBackend> Drop for GenericArena { nop: mem::ManuallyDrop>>, zst: mem::ManuallyDrop>>, From 277d728d9580d1bee71535bc0930d5b57a8e5c6a Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Fri, 13 Dec 2019 21:13:52 +0100 Subject: [PATCH 11/14] Grow TypedArena backwards. --- src/libarena/lib.rs | 64 ++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/src/libarena/lib.rs b/src/libarena/lib.rs index b333d30e5e600..bb253955ef18f 100644 --- a/src/libarena/lib.rs +++ b/src/libarena/lib.rs @@ -149,12 +149,12 @@ impl ChunkBackend for ZSTCurrentChunk { /// Chunk used for droppable types. struct TypedCurrentChunk { - /// A pointer to the next object to be allocated. - ptr: Cell<*mut T>, + /// A pointer to the start of the allocatable area. + /// When this pointer is reached, a new chunk is allocated. + start: Cell<*mut T>, - /// A pointer to the end of the allocated area. When this pointer is - /// reached, a new chunk is allocated. - end: Cell<*mut T>, + /// A pointer to the end of the allocated area. + ptr: Cell<*mut T>, } impl ChunkBackend for TypedCurrentChunk { @@ -163,8 +163,8 @@ impl ChunkBackend for TypedCurrentChunk { #[inline] fn new() -> Self { TypedCurrentChunk { + start: Cell::new(ptr::null_mut()), ptr: Cell::new(ptr::null_mut()), - end: Cell::new(ptr::null_mut()), } } @@ -177,7 +177,7 @@ impl ChunkBackend for TypedCurrentChunk { fn can_allocate(&self, len: usize, align: usize) -> bool { assert!(mem::size_of::() > 0); assert!(mem::align_of::() == align); - let available_capacity = unsafe { self.end.get().offset_from(self.ptr.get()) }; + let available_capacity = unsafe { self.ptr.get().offset_from(self.start.get()) }; assert!(available_capacity >= 0); let available_capacity = available_capacity as usize; available_capacity >= len @@ -189,32 +189,30 @@ impl ChunkBackend for TypedCurrentChunk { fn grow(&self, len: usize, align: usize, chunks: &mut Self::ChunkVecType) { assert!(mem::size_of::() > 0); assert!(mem::align_of::() == align); + assert!(!self.can_allocate(len, align)); unsafe { - let mut new_capacity; + let mut new_capacity: usize; if let Some(last_chunk) = chunks.last_mut() { - let used_bytes = self.ptr.get() as usize - last_chunk.start() as usize; - let currently_used_cap = used_bytes / mem::size_of::(); + let currently_used_cap = last_chunk.end().offset_from(self.ptr.get()); + assert!(currently_used_cap >= 0); + let currently_used_cap = currently_used_cap as usize; last_chunk.entries = currently_used_cap; - if last_chunk.storage.reserve_in_place(currently_used_cap, len) { - self.end.set(last_chunk.end()); - return; - } else { - new_capacity = last_chunk.storage.capacity(); - loop { - new_capacity = new_capacity.checked_mul(2).unwrap(); - if new_capacity >= currently_used_cap + len { - break; - } + + new_capacity = last_chunk.storage.capacity(); + loop { + new_capacity = new_capacity.checked_mul(2).unwrap(); + if new_capacity >= currently_used_cap + len { + break; } } } else { - let elem_size = cmp::max(1, mem::size_of::()); + let elem_size = mem::size_of::(); new_capacity = cmp::max(len, PAGE / elem_size); } let chunk = TypedArenaChunk::new(new_capacity); - self.ptr.set(chunk.start()); - self.end.set(chunk.end()); + self.start.set(chunk.start()); + self.ptr.set(chunk.end()); chunks.push(chunk); } } @@ -225,10 +223,10 @@ impl ChunkBackend for TypedCurrentChunk { assert!(align == mem::align_of::()); let ptr = self.ptr.get(); - let end = ptr.add(len); - assert!(end <= self.end.get()); + let ptr = ptr.sub(len); + assert!(ptr >= self.start.get()); - self.ptr.set(end); + self.ptr.set(ptr); ptr } @@ -239,12 +237,14 @@ impl ChunkBackend for TypedCurrentChunk { if let Some(last_chunk) = chunks.last_mut() { // Clear the last chunk, which is partially filled. unsafe { - let start = last_chunk.start(); - let len = self.ptr.get().offset_from(start); + let end = last_chunk.end(); + let len = end.offset_from(self.ptr.get()); assert!(len >= 0); - let slice = slice::from_raw_parts_mut(start, len as usize); - ptr::drop_in_place(slice); - self.ptr.set(start); + let len = len as usize; + for i in 0..len { + ptr::drop_in_place(end.sub(i + 1)) + } + self.ptr.set(end); } let len = chunks.len(); @@ -549,7 +549,7 @@ impl TypedArenaChunk { // The branch on needs_drop() is an -O1 performance optimization. // Without the branch, dropping TypedArena takes linear time. if mem::needs_drop::() { - let mut start = self.start(); + let mut start = self.end().sub(len); // Destroy all allocated objects. for _ in 0..len { ptr::drop_in_place(start); From c623368b5f0d1cb880b8db9d48a27bc2a5fd4c93 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Fri, 13 Dec 2019 21:58:55 +0100 Subject: [PATCH 12/14] Grow DroplessArena backwards. --- src/libarena/lib.rs | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/src/libarena/lib.rs b/src/libarena/lib.rs index bb253955ef18f..c8d848dcffe94 100644 --- a/src/libarena/lib.rs +++ b/src/libarena/lib.rs @@ -260,11 +260,11 @@ impl ChunkBackend for TypedCurrentChunk { /// It is spearated from `TypedCurrentChunk` for code reuse in `DroplessArena`. struct DroplessCurrentChunk { /// A pointer to the next object to be allocated. - ptr: Cell<*mut u8>, + /// When this pointer is reached, a new chunk is allocated. + start: Cell<*mut u8>, - /// A pointer to the end of the allocated area. When this pointer is - /// reached, a new chunk is allocated. - end: Cell<*mut u8>, + /// A pointer to the end of the allocated area. + ptr: Cell<*mut u8>, /// Ensure correct semantics. _own: PhantomData<*mut T>, @@ -276,10 +276,10 @@ impl ChunkBackend for DroplessCurrentChunk { #[inline] fn new() -> Self { DroplessCurrentChunk { - // We set both `ptr` and `end` to 0 so that the first call to + // We set both `start` and `end` to 0 so that the first call to // alloc() will trigger a grow(). + start: Cell::new(ptr::null_mut()), ptr: Cell::new(ptr::null_mut()), - end: Cell::new(ptr::null_mut()), _own: PhantomData, } } @@ -290,14 +290,10 @@ impl ChunkBackend for DroplessCurrentChunk { } #[inline] - fn can_allocate(&self, len: usize, align: usize) -> bool { + fn can_allocate(&self, len: usize, _align: usize) -> bool { let len = len * mem::size_of::(); - let ptr = self.ptr.get(); - let ptr = unsafe { ptr.add(ptr.align_offset(align)) }; - let available_capacity = unsafe { self.end.get().offset_from(ptr) }; - assert!(available_capacity >= 0); - let available_capacity = available_capacity as usize; - available_capacity >= len + // start is always aligned, so removing len from ptr cannot go below it + (unsafe { self.start.get().add(len) }) <= self.ptr.get() } /// Grows the arena. @@ -308,10 +304,11 @@ impl ChunkBackend for DroplessCurrentChunk { unsafe { let mut new_capacity; if let Some(last_chunk) = chunks.last_mut() { - let currently_used_cap = self.ptr.get() as usize - last_chunk.start() as usize; + let currently_used_cap = last_chunk.end().offset_from(self.ptr.get()); + let currently_used_cap = currently_used_cap as usize; last_chunk.entries = currently_used_cap; if last_chunk.storage.reserve_in_place(currently_used_cap, len) { - self.end.set(last_chunk.end()); + self.ptr.set(last_chunk.end()); return; } else { new_capacity = last_chunk.storage.capacity(); @@ -327,21 +324,21 @@ impl ChunkBackend for DroplessCurrentChunk { } let chunk = TypedArenaChunk::new(new_capacity); - self.ptr.set(chunk.start()); - self.end.set(chunk.end()); + self.start.set(chunk.start()); + self.ptr.set(chunk.end()); chunks.push(chunk); } } #[inline] unsafe fn alloc_raw_slice(&self, len: usize, align: usize) -> *mut T { + assert!(self.can_allocate(len, align)); let len = len * mem::size_of::(); - let ptr = self.ptr.get(); - let ptr = ptr.add(ptr.align_offset(align)); - let end = ptr.add(len); - assert!(end <= self.end.get()); + let ptr = self.ptr.get().sub(len); + let ptr = ((ptr as usize) & !(align - 1)) as *mut u8; + assert!(ptr >= self.start.get()); - self.ptr.set(end); + self.ptr.set(ptr); ptr as *mut T } @@ -349,7 +346,7 @@ impl ChunkBackend for DroplessCurrentChunk { fn clear(&self, chunks: &mut Self::ChunkVecType) { if let Some(last_chunk) = chunks.last_mut() { // Clear the last chunk, which is partially filled. - self.ptr.set(last_chunk.start()) + self.ptr.set(last_chunk.end()) } } } From 16771f3524f37bc702302ab5335f03878fa239a8 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sun, 15 Dec 2019 11:31:22 +0100 Subject: [PATCH 13/14] Fix compile on mingw. --- src/librustc_data_structures/sync.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/librustc_data_structures/sync.rs b/src/librustc_data_structures/sync.rs index ace3993d4c4e0..5a69cfbcf4ce9 100644 --- a/src/librustc_data_structures/sync.rs +++ b/src/librustc_data_structures/sync.rs @@ -428,6 +428,7 @@ cfg_if! { pub use rayon_core::WorkerLocal; pub use rayon_core::Registry; use rayon_core::current_thread_index; + use rayon_core::current_num_threads; #[derive(Debug)] pub struct SharedWorkerLocal(Vec); @@ -437,7 +438,7 @@ cfg_if! { /// value this worker local should take for each thread in the thread pool. #[inline] pub fn new T>(mut f: F) -> SharedWorkerLocal { - SharedWorkerLocal((0..Registry::current_num_threads()).map(|i| f(i)).collect()) + SharedWorkerLocal((0..current_num_threads()).map(|i| f(i)).collect()) } #[inline] From bba37dfd07e7f55c89603c8690a83a03e237e706 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sun, 15 Dec 2019 14:05:25 +0100 Subject: [PATCH 14/14] Fix unsound drop example. --- src/libarena/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libarena/lib.rs b/src/libarena/lib.rs index c8d848dcffe94..3e241a4558a2d 100644 --- a/src/libarena/lib.rs +++ b/src/libarena/lib.rs @@ -446,6 +446,7 @@ pub union TypedArena { zst: mem::ManuallyDrop>>, dropless: mem::ManuallyDrop>>, typed: mem::ManuallyDrop>>, + _own: PhantomData, } impl Default for TypedArena {