From a1f237f7bfc94f52ec76bd36eaf45e55c0c358f8 Mon Sep 17 00:00:00 2001 From: Stepan Koltsov Date: Wed, 10 Apr 2024 01:29:35 +0100 Subject: [PATCH] ThinBox: move WithHeader to submodule Requested by @oli-obk. I don't have strong opinion either way. r? oli-obk --- library/alloc/src/boxed/thin.rs | 455 +++++++++++++++++--------------- 1 file changed, 235 insertions(+), 220 deletions(-) diff --git a/library/alloc/src/boxed/thin.rs b/library/alloc/src/boxed/thin.rs index 8b145b67bf186..72db3e80d1a81 100644 --- a/library/alloc/src/boxed/thin.rs +++ b/library/alloc/src/boxed/thin.rs @@ -1,20 +1,16 @@ // Based on // https://github.com/matthieu-m/rfc2580/blob/b58d1d3cba0d4b5e859d3617ea2d0943aaa31329/examples/thin.rs // by matthieu-m -use crate::alloc::{self, Layout, LayoutError}; use core::error::Error; use core::fmt::{self, Debug, Display, Formatter}; -#[cfg(not(no_global_oom_handling))] -use core::intrinsics::const_allocate; use core::marker::PhantomData; #[cfg(not(no_global_oom_handling))] use core::marker::Unsize; -use core::mem; #[cfg(not(no_global_oom_handling))] -use core::mem::SizedTypeProperties; +use core::mem; use core::ops::{Deref, DerefMut}; +use core::ptr; use core::ptr::Pointee; -use core::ptr::{self, NonNull}; /// ThinBox. /// @@ -38,7 +34,7 @@ use core::ptr::{self, NonNull}; pub struct ThinBox { // This is essentially `WithHeader<::Metadata>`, // but that would be invariant in `T`, and we want covariance. - ptr: WithOpaqueHeader, + ptr: with_header::WithOpaqueHeader, _marker: PhantomData, } @@ -68,7 +64,7 @@ impl ThinBox { #[cfg(not(no_global_oom_handling))] pub fn new(value: T) -> Self { let meta = ptr::metadata(&value); - let ptr = WithOpaqueHeader::new(meta, value); + let ptr = with_header::WithOpaqueHeader::new(meta, value); ThinBox { ptr, _marker: PhantomData } } @@ -89,7 +85,8 @@ impl ThinBox { /// [`Metadata`]: core::ptr::Pointee::Metadata pub fn try_new(value: T) -> Result { let meta = ptr::metadata(&value); - WithOpaqueHeader::try_new(meta, value).map(|ptr| ThinBox { ptr, _marker: PhantomData }) + with_header::WithOpaqueHeader::try_new(meta, value) + .map(|ptr| ThinBox { ptr, _marker: PhantomData }) } } @@ -114,11 +111,11 @@ impl ThinBox { T: Unsize, { if mem::size_of::() == 0 { - let ptr = WithOpaqueHeader::new_unsize_zst::(value); + let ptr = with_header::WithOpaqueHeader::new_unsize_zst::(value); ThinBox { ptr, _marker: PhantomData } } else { let meta = ptr::metadata(&value as &Dyn); - let ptr = WithOpaqueHeader::new(meta, value); + let ptr = with_header::WithOpaqueHeader::new(meta, value); ThinBox { ptr, _marker: PhantomData } } } @@ -183,250 +180,268 @@ impl ThinBox { self.with_header().value() } - fn with_header(&self) -> &WithHeader<::Metadata> { + fn with_header(&self) -> &with_header::WithHeader<::Metadata> { // SAFETY: both types are transparent to `NonNull` - unsafe { &*(core::ptr::addr_of!(self.ptr) as *const WithHeader<_>) } + unsafe { &*(core::ptr::addr_of!(self.ptr) as *const with_header::WithHeader<_>) } } } -/// A pointer to type-erased data, guaranteed to either be: -/// 1. `NonNull::dangling()`, in the case where both the pointee (`T`) and -/// metadata (`H`) are ZSTs. -/// 2. A pointer to a valid `T` that has a header `H` directly before the -/// pointed-to location. -#[repr(transparent)] -struct WithHeader(NonNull, PhantomData); - -/// An opaque representation of `WithHeader` to avoid the -/// projection invariance of `::Metadata`. -#[repr(transparent)] -struct WithOpaqueHeader(NonNull); - -impl WithOpaqueHeader { +mod with_header { + use crate::alloc::{self, Layout, LayoutError}; #[cfg(not(no_global_oom_handling))] - fn new(header: H, value: T) -> Self { - let ptr = WithHeader::new(header, value); - Self(ptr.0) - } - + use core::intrinsics::const_allocate; + use core::marker::PhantomData; #[cfg(not(no_global_oom_handling))] - fn new_unsize_zst(value: T) -> Self - where - Dyn: ?Sized, - T: Unsize, - { - let ptr = WithHeader::<::Metadata>::new_unsize_zst::(value); - Self(ptr.0) - } - - fn try_new(header: H, value: T) -> Result { - WithHeader::try_new(header, value).map(|ptr| Self(ptr.0)) - } -} - -impl WithHeader { + use core::marker::Unsize; + use core::mem; #[cfg(not(no_global_oom_handling))] - fn new(header: H, value: T) -> WithHeader { - let value_layout = Layout::new::(); - let Ok((layout, value_offset)) = Self::alloc_layout(value_layout) else { - // We pass an empty layout here because we do not know which layout caused the - // arithmetic overflow in `Layout::extend` and `handle_alloc_error` takes `Layout` as - // its argument rather than `Result`, also this function has been - // stable since 1.28 ._. - // - // On the other hand, look at this gorgeous turbofish! - alloc::handle_alloc_error(Layout::new::<()>()); - }; - - unsafe { - // Note: It's UB to pass a layout with a zero size to `alloc::alloc`, so - // we use `layout.dangling()` for this case, which should have a valid - // alignment for both `T` and `H`. - let ptr = if layout.size() == 0 { - // Some paranoia checking, mostly so that the ThinBox tests are - // more able to catch issues. - debug_assert!(value_offset == 0 && T::IS_ZST && H::IS_ZST); - layout.dangling() - } else { - let ptr = alloc::alloc(layout); - if ptr.is_null() { - alloc::handle_alloc_error(layout); - } - // Safety: - // - The size is at least `aligned_header_size`. - let ptr = ptr.add(value_offset) as *mut _; - - NonNull::new_unchecked(ptr) - }; + use core::mem::SizedTypeProperties; + use core::ptr; + use core::ptr::NonNull; + #[cfg(not(no_global_oom_handling))] + use core::ptr::Pointee; + + /// A pointer to type-erased data, guaranteed to either be: + /// 1. `NonNull::dangling()`, in the case where both the pointee (`T`) and + /// metadata (`H`) are ZSTs. + /// 2. A pointer to a valid `T` that has a header `H` directly before the + /// pointed-to location. + #[repr(transparent)] + pub(super) struct WithHeader(NonNull, PhantomData); + + /// An opaque representation of `WithHeader` to avoid the + /// projection invariance of `::Metadata`. + #[repr(transparent)] + pub(super) struct WithOpaqueHeader(NonNull); + + impl WithOpaqueHeader { + #[cfg(not(no_global_oom_handling))] + pub(super) fn new(header: H, value: T) -> Self { + let ptr = WithHeader::new(header, value); + Self(ptr.0) + } - let result = WithHeader(ptr, PhantomData); - ptr::write(result.header(), header); - ptr::write(result.value().cast(), value); + #[cfg(not(no_global_oom_handling))] + pub(super) fn new_unsize_zst(value: T) -> Self + where + Dyn: ?Sized, + T: Unsize, + { + let ptr = WithHeader::<::Metadata>::new_unsize_zst::(value); + Self(ptr.0) + } - result + pub(super) fn try_new(header: H, value: T) -> Result { + WithHeader::try_new(header, value).map(|ptr| Self(ptr.0)) } } - /// Non-panicking version of `new`. - /// Any error is returned as `Err(core::alloc::AllocError)`. - fn try_new(header: H, value: T) -> Result, core::alloc::AllocError> { - let value_layout = Layout::new::(); - let Ok((layout, value_offset)) = Self::alloc_layout(value_layout) else { - return Err(core::alloc::AllocError); - }; - - unsafe { - // Note: It's UB to pass a layout with a zero size to `alloc::alloc`, so - // we use `layout.dangling()` for this case, which should have a valid - // alignment for both `T` and `H`. - let ptr = if layout.size() == 0 { - // Some paranoia checking, mostly so that the ThinBox tests are - // more able to catch issues. - debug_assert!( - value_offset == 0 && mem::size_of::() == 0 && mem::size_of::() == 0 - ); - layout.dangling() - } else { - let ptr = alloc::alloc(layout); - if ptr.is_null() { - return Err(core::alloc::AllocError); - } - - // Safety: - // - The size is at least `aligned_header_size`. - let ptr = ptr.add(value_offset) as *mut _; - - NonNull::new_unchecked(ptr) + impl WithHeader { + #[cfg(not(no_global_oom_handling))] + fn new(header: H, value: T) -> WithHeader { + let value_layout = Layout::new::(); + let Ok((layout, value_offset)) = Self::alloc_layout(value_layout) else { + // We pass an empty layout here because we do not know which layout caused the + // arithmetic overflow in `Layout::extend` and `handle_alloc_error` takes `Layout` as + // its argument rather than `Result`, also this function has been + // stable since 1.28 ._. + // + // On the other hand, look at this gorgeous turbofish! + alloc::handle_alloc_error(Layout::new::<()>()); }; - let result = WithHeader(ptr, PhantomData); - ptr::write(result.header(), header); - ptr::write(result.value().cast(), value); - - Ok(result) + unsafe { + // Note: It's UB to pass a layout with a zero size to `alloc::alloc`, so + // we use `layout.dangling()` for this case, which should have a valid + // alignment for both `T` and `H`. + let ptr = if layout.size() == 0 { + // Some paranoia checking, mostly so that the ThinBox tests are + // more able to catch issues. + debug_assert!(value_offset == 0 && T::IS_ZST && H::IS_ZST); + layout.dangling() + } else { + let ptr = alloc::alloc(layout); + if ptr.is_null() { + alloc::handle_alloc_error(layout); + } + // Safety: + // - The size is at least `aligned_header_size`. + let ptr = ptr.add(value_offset) as *mut _; + + NonNull::new_unchecked(ptr) + }; + + let result = WithHeader(ptr, PhantomData); + ptr::write(result.header(), header); + ptr::write(result.value().cast(), value); + + result + } } - } - // `Dyn` is `?Sized` type like `[u32]`, and `T` is ZST type like `[u32; 0]`. - #[cfg(not(no_global_oom_handling))] - fn new_unsize_zst(value: T) -> WithHeader - where - Dyn: Pointee + ?Sized, - T: Unsize, - { - assert!(mem::size_of::() == 0); + /// Non-panicking version of `new`. + /// Any error is returned as `Err(core::alloc::AllocError)`. + fn try_new(header: H, value: T) -> Result, core::alloc::AllocError> { + let value_layout = Layout::new::(); + let Ok((layout, value_offset)) = Self::alloc_layout(value_layout) else { + return Err(core::alloc::AllocError); + }; - const fn max(a: usize, b: usize) -> usize { - if a > b { a } else { b } + unsafe { + // Note: It's UB to pass a layout with a zero size to `alloc::alloc`, so + // we use `layout.dangling()` for this case, which should have a valid + // alignment for both `T` and `H`. + let ptr = if layout.size() == 0 { + // Some paranoia checking, mostly so that the ThinBox tests are + // more able to catch issues. + debug_assert!( + value_offset == 0 && mem::size_of::() == 0 && mem::size_of::() == 0 + ); + layout.dangling() + } else { + let ptr = alloc::alloc(layout); + if ptr.is_null() { + return Err(core::alloc::AllocError); + } + + // Safety: + // - The size is at least `aligned_header_size`. + let ptr = ptr.add(value_offset) as *mut _; + + NonNull::new_unchecked(ptr) + }; + + let result = WithHeader(ptr, PhantomData); + ptr::write(result.header(), header); + ptr::write(result.value().cast(), value); + + Ok(result) + } } - // Compute a pointer to the right metadata. This will point to the beginning - // of the header, past the padding, so the assigned type makes sense. - // It also ensures that the address at the end of the header is sufficiently - // aligned for T. - let alloc: &::Metadata = const { - // FIXME: just call `WithHeader::alloc_layout` with size reset to 0. - // Currently that's blocked on `Layout::extend` not being `const fn`. + // `Dyn` is `?Sized` type like `[u32]`, and `T` is ZST type like `[u32; 0]`. + #[cfg(not(no_global_oom_handling))] + fn new_unsize_zst(value: T) -> WithHeader + where + Dyn: Pointee + ?Sized, + T: Unsize, + { + assert!(mem::size_of::() == 0); + + const fn max(a: usize, b: usize) -> usize { + if a > b { a } else { b } + } - let alloc_align = - max(mem::align_of::(), mem::align_of::<::Metadata>()); + // Compute a pointer to the right metadata. This will point to the beginning + // of the header, past the padding, so the assigned type makes sense. + // It also ensures that the address at the end of the header is sufficiently + // aligned for T. + let alloc: &::Metadata = const { + // FIXME: just call `WithHeader::alloc_layout` with size reset to 0. + // Currently that's blocked on `Layout::extend` not being `const fn`. - let alloc_size = - max(mem::align_of::(), mem::size_of::<::Metadata>()); + let alloc_align = + max(mem::align_of::(), mem::align_of::<::Metadata>()); - unsafe { - // SAFETY: align is power of two because it is the maximum of two alignments. - let alloc: *mut u8 = const_allocate(alloc_size, alloc_align); - - let metadata_offset = - alloc_size.checked_sub(mem::size_of::<::Metadata>()).unwrap(); - // SAFETY: adding offset within the allocation. - let metadata_ptr: *mut ::Metadata = - alloc.add(metadata_offset).cast(); - // SAFETY: `*metadata_ptr` is within the allocation. - metadata_ptr.write(ptr::metadata::(ptr::dangling::() as *const Dyn)); - - // SAFETY: we have just written the metadata. - &*(metadata_ptr) - } - }; - - // SAFETY: `alloc` points to `::Metadata`, so addition stays in-bounds. - let value_ptr = - unsafe { (alloc as *const ::Metadata).add(1) }.cast::().cast_mut(); - debug_assert!(value_ptr.is_aligned()); - mem::forget(value); - WithHeader(NonNull::new(value_ptr.cast()).unwrap(), PhantomData) - } + let alloc_size = + max(mem::align_of::(), mem::size_of::<::Metadata>()); - // Safety: - // - Assumes that either `value` can be dereferenced, or is the - // `NonNull::dangling()` we use when both `T` and `H` are ZSTs. - unsafe fn drop(&self, value: *mut T) { - struct DropGuard { - ptr: NonNull, - value_layout: Layout, - _marker: PhantomData, - } - - impl Drop for DropGuard { - fn drop(&mut self) { - // All ZST are allocated statically. - if self.value_layout.size() == 0 { - return; + unsafe { + // SAFETY: align is power of two because it is the maximum of two alignments. + let alloc: *mut u8 = const_allocate(alloc_size, alloc_align); + + let metadata_offset = alloc_size + .checked_sub(mem::size_of::<::Metadata>()) + .unwrap(); + // SAFETY: adding offset within the allocation. + let metadata_ptr: *mut ::Metadata = + alloc.add(metadata_offset).cast(); + // SAFETY: `*metadata_ptr` is within the allocation. + metadata_ptr.write(ptr::metadata::(ptr::dangling::() as *const Dyn)); + + // SAFETY: we have just written the metadata. + &*(metadata_ptr) } + }; - unsafe { - // SAFETY: Layout must have been computable if we're in drop - let (layout, value_offset) = - WithHeader::::alloc_layout(self.value_layout).unwrap_unchecked(); + // SAFETY: `alloc` points to `::Metadata`, so addition stays in-bounds. + let value_ptr = unsafe { (alloc as *const ::Metadata).add(1) } + .cast::() + .cast_mut(); + debug_assert!(value_ptr.is_aligned()); + mem::forget(value); + WithHeader(NonNull::new(value_ptr.cast()).unwrap(), PhantomData) + } - // Since we only allocate for non-ZSTs, the layout size cannot be zero. - debug_assert!(layout.size() != 0); - alloc::dealloc(self.ptr.as_ptr().sub(value_offset), layout); - } + // Safety: + // - Assumes that either `value` can be dereferenced, or is the + // `NonNull::dangling()` we use when both `T` and `H` are ZSTs. + pub(super) unsafe fn drop(&self, value: *mut T) { + struct DropGuard { + ptr: NonNull, + value_layout: Layout, + _marker: PhantomData, } - } - unsafe { - // `_guard` will deallocate the memory when dropped, even if `drop_in_place` unwinds. - let _guard = DropGuard { - ptr: self.0, - value_layout: Layout::for_value_raw(value), - _marker: PhantomData::, - }; + impl Drop for DropGuard { + fn drop(&mut self) { + // All ZST are allocated statically. + if self.value_layout.size() == 0 { + return; + } + + unsafe { + // SAFETY: Layout must have been computable if we're in drop + let (layout, value_offset) = + WithHeader::::alloc_layout(self.value_layout).unwrap_unchecked(); + + // Since we only allocate for non-ZSTs, the layout size cannot be zero. + debug_assert!(layout.size() != 0); + alloc::dealloc(self.ptr.as_ptr().sub(value_offset), layout); + } + } + } - // We only drop the value because the Pointee trait requires that the metadata is copy - // aka trivially droppable. - ptr::drop_in_place::(value); + unsafe { + // `_guard` will deallocate the memory when dropped, even if `drop_in_place` unwinds. + let _guard = DropGuard { + ptr: self.0, + value_layout: Layout::for_value_raw(value), + _marker: PhantomData::, + }; + + // We only drop the value because the Pointee trait requires that the metadata is copy + // aka trivially droppable. + ptr::drop_in_place::(value); + } } - } - fn header(&self) -> *mut H { - // Safety: - // - At least `size_of::()` bytes are allocated ahead of the pointer. - // - We know that H will be aligned because the middle pointer is aligned to the greater - // of the alignment of the header and the data and the header size includes the padding - // needed to align the header. Subtracting the header size from the aligned data pointer - // will always result in an aligned header pointer, it just may not point to the - // beginning of the allocation. - let hp = unsafe { self.0.as_ptr().sub(Self::header_size()) as *mut H }; - debug_assert!(hp.is_aligned()); - hp - } + pub(super) fn header(&self) -> *mut H { + // Safety: + // - At least `size_of::()` bytes are allocated ahead of the pointer. + // - We know that H will be aligned because the middle pointer is aligned to the greater + // of the alignment of the header and the data and the header size includes the padding + // needed to align the header. Subtracting the header size from the aligned data pointer + // will always result in an aligned header pointer, it just may not point to the + // beginning of the allocation. + let hp = unsafe { self.0.as_ptr().sub(Self::header_size()) as *mut H }; + debug_assert!(hp.is_aligned()); + hp + } - fn value(&self) -> *mut u8 { - self.0.as_ptr() - } + pub(super) fn value(&self) -> *mut u8 { + self.0.as_ptr() + } - const fn header_size() -> usize { - mem::size_of::() - } + const fn header_size() -> usize { + mem::size_of::() + } - fn alloc_layout(value_layout: Layout) -> Result<(Layout, usize), LayoutError> { - Layout::new::().extend(value_layout) + fn alloc_layout(value_layout: Layout) -> Result<(Layout, usize), LayoutError> { + Layout::new::().extend(value_layout) + } } -} +} // mod with_header #[unstable(feature = "thin_box", issue = "92791")] impl Error for ThinBox {