From 4a09f12ec736969a546fd4d7f4d8c3291c6acdc3 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Fri, 17 Apr 2026 08:58:51 +0200 Subject: [PATCH] transmute: fix check for whether newtypes have equal size --- compiler/rustc_middle/src/ty/layout.rs | 40 +++++++++++++++++-- tests/ui/transmute/raw-ptr-non-null.rs | 11 ----- .../ui/transmute/transmute-different-sizes.rs | 32 +++++++++++++++ .../transmute-different-sizes.stderr | 29 +++++++++++++- tests/ui/transmute/transmute-fat-pointers.rs | 37 ++++++++++++++++- .../transmute/transmute-fat-pointers.stderr | 8 ++-- 6 files changed, 135 insertions(+), 22 deletions(-) delete mode 100644 tests/ui/transmute/raw-ptr-non-null.rs diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs index 7155df08ec59d..86a43e4415ee3 100644 --- a/compiler/rustc_middle/src/ty/layout.rs +++ b/compiler/rustc_middle/src/ty/layout.rs @@ -426,13 +426,42 @@ impl<'tcx> SizeSkeleton<'tcx> { } ty::Adt(def, args) => { - // Only newtypes and enums w/ nullable pointer optimization. + // Only newtypes and enums w/ nullable pointer optimization (NPO). if def.is_union() || def.variants().is_empty() || def.variants().len() > 2 { return Err(err); } + // Only default repr types. + { + // We can ignore the seed and some particular flags that can never affect the + // layout of newtypes / NPO types, but we have to check everything else. + // If you are adding a new field to `ReprOptions`, make sure to extend the check + // below so that we bail out if it is not at its default value! + let ReprOptions { int, align, pack, flags, scalable, field_shuffle_seed: _ } = + def.repr(); + let mut ignored_flags = ReprFlags::IS_TRANSPARENT + | ReprFlags::IS_LINEAR + | ReprFlags::RANDOMIZE_LAYOUT; + if def.is_struct() { + // `repr(C)` is only okay for structs, not for enums. + // Below, the *only* thing we do for structs is propagating + // `SizeSkeleton::Pointer`. We do *not* assume that `repr(C)` preserved + // ZST-ness (which might stop being true eventually). + ignored_flags |= ReprFlags::IS_C; + } + if int.is_some() + || align.is_some() + || pack.is_some() + || flags.difference(ignored_flags) != ReprFlags::default() + || scalable.is_some() + { + return Err(err); + } + } // Get a zero-sized variant or a pointer newtype. - let zero_or_ptr_variant = |i| { + // Returns `Ok(None)` for 1-ZST types, `Ok(Some)` if (ignoring all 1-ZST fields) + // there's just a single pointer, and `Err` otherwise. + let zero_or_ptr_variant = |i| -> Result>, _> { let i = VariantIdx::from_usize(i); let fields = def.variant(i).fields.iter().map(|field| { @@ -461,7 +490,8 @@ impl<'tcx> SizeSkeleton<'tcx> { }; let v0 = zero_or_ptr_variant(0)?; - // Newtype. + // Single-variant case: Check if this is a newtype around a pointer. + // Such types are themselves pointer-sized. if def.variants().len() == 1 { if let Some(SizeSkeleton::Pointer { non_zero, tail }) = v0 { return Ok(SizeSkeleton::Pointer { non_zero, tail }); @@ -471,7 +501,9 @@ impl<'tcx> SizeSkeleton<'tcx> { } let v1 = zero_or_ptr_variant(1)?; - // Nullable pointer enum optimization. + // 2-variant case: Check if one variant is a *non-zero* pointer and the other a + // 1-ZST. Such types are eligible to for the nullable pointer enum optimization, so + // they are themselves pointer-sized. match (v0, v1) { (Some(SizeSkeleton::Pointer { non_zero: true, tail }), None) | (None, Some(SizeSkeleton::Pointer { non_zero: true, tail })) => { diff --git a/tests/ui/transmute/raw-ptr-non-null.rs b/tests/ui/transmute/raw-ptr-non-null.rs deleted file mode 100644 index af518095b3fbd..0000000000000 --- a/tests/ui/transmute/raw-ptr-non-null.rs +++ /dev/null @@ -1,11 +0,0 @@ -//! After the use of pattern types inside `NonNull`, -//! transmuting between a niche optimized enum wrapping a -//! generic `NonNull` and raw pointers stopped working. -//@ check-pass - -use std::ptr::NonNull; -pub const fn is_null<'a, T: ?Sized>(ptr: *const T) -> bool { - unsafe { matches!(core::mem::transmute::<*const T, Option>>(ptr), None) } -} - -fn main() {} diff --git a/tests/ui/transmute/transmute-different-sizes.rs b/tests/ui/transmute/transmute-different-sizes.rs index 40197a6c53f00..4f1f758677145 100644 --- a/tests/ui/transmute/transmute-different-sizes.rs +++ b/tests/ui/transmute/transmute-different-sizes.rs @@ -48,4 +48,36 @@ pub unsafe fn shouldnt_work2(from: *mut T) -> PtrAndEmptyArray { //~^ ERROR cannot transmute between types of different sizes, or dependently-sized types } +#[repr(align(16))] +struct OverAlignWrap(T); + +fn shouldnt_work3(x: OverAlignWrap<*const T>) -> *const T { + unsafe { transmute(x) } + //~^ ERROR cannot transmute between types of different sizes, or dependently-sized types +} + +// With `repr(C)`, this type has a disriminant, so it does not have the same size as `T`. +#[repr(C)] +enum NotANewtype { + SingleVariant(T), +} + +fn shouldnt_work4(x: &T) -> NotANewtype<&T> { + unsafe { transmute(x) } + //~^ ERROR cannot transmute between types of different sizes, or dependently-sized types +} + +// With `repr(C)`, this type has a disriminant; NPO does not kick in. +// Therefore this is always bigger than `T`. +#[repr(C)] +enum NotNullPointerOptimized { + Some(T), + None +} + +fn shouldnt_work5(x: &T) -> NotNullPointerOptimized<&T> { + unsafe { transmute(x) } + //~^ ERROR cannot transmute between types of different sizes, or dependently-sized types +} + fn main() {} diff --git a/tests/ui/transmute/transmute-different-sizes.stderr b/tests/ui/transmute/transmute-different-sizes.stderr index ea3a017c16e6f..fe611f7ba18d1 100644 --- a/tests/ui/transmute/transmute-different-sizes.stderr +++ b/tests/ui/transmute/transmute-different-sizes.stderr @@ -43,6 +43,33 @@ LL | std::mem::transmute(from) = note: source type: `*mut T` (pointer to `T`) = note: target type: `PtrAndEmptyArray` (size can vary because of ::Metadata) -error: aborting due to 5 previous errors +error[E0512]: cannot transmute between types of different sizes, or dependently-sized types + --> $DIR/transmute-different-sizes.rs:55:14 + | +LL | unsafe { transmute(x) } + | ^^^^^^^^^ + | + = note: source type: `OverAlignWrap<*const T>` (size can vary because of ::Metadata) + = note: target type: `*const T` (pointer to `T`) + +error[E0512]: cannot transmute between types of different sizes, or dependently-sized types + --> $DIR/transmute-different-sizes.rs:66:14 + | +LL | unsafe { transmute(x) } + | ^^^^^^^^^ + | + = note: source type: `&T` (pointer to `T`) + = note: target type: `NotANewtype<&T>` (size can vary because of ::Metadata) + +error[E0512]: cannot transmute between types of different sizes, or dependently-sized types + --> $DIR/transmute-different-sizes.rs:79:14 + | +LL | unsafe { transmute(x) } + | ^^^^^^^^^ + | + = note: source type: `&T` (pointer to `T`) + = note: target type: `NotNullPointerOptimized<&T>` (size can vary because of ::Metadata) + +error: aborting due to 8 previous errors For more information about this error, try `rustc --explain E0512`. diff --git a/tests/ui/transmute/transmute-fat-pointers.rs b/tests/ui/transmute/transmute-fat-pointers.rs index f095b80dc2da8..c03d0d34feb1b 100644 --- a/tests/ui/transmute/transmute-fat-pointers.rs +++ b/tests/ui/transmute/transmute-fat-pointers.rs @@ -5,6 +5,7 @@ #![allow(dead_code)] use std::mem::transmute; +use std::ptr::NonNull; fn a(x: &[T]) -> &U { unsafe { transmute(x) } //~ ERROR cannot transmute between types of different sizes @@ -15,11 +16,11 @@ fn b(x: &T) -> &U { } fn c(x: &T) -> &U { - unsafe { transmute(x) } + unsafe { transmute(x) } // Ok! } fn d(x: &[T]) -> &[U] { - unsafe { transmute(x) } + unsafe { transmute(x) } // Ok! } fn e(x: &T) -> &U { @@ -30,4 +31,36 @@ fn f(x: &T) -> &U { unsafe { transmute(x) } //~ ERROR cannot transmute between types of different sizes } +// We can transmute even between pointers of unknown size as long as the metadata of +// input and output type are the same. This even accounts for null pointer optimizations. +fn g1(x: &T) -> Option<&T> { + unsafe { transmute(x) } // Ok! +} + +fn g2(x: Option>) -> NonNull { + unsafe { transmute(x) } // Ok! +} + +fn g3(x: *const T) -> Option> { + unsafe { transmute(x) } // Ok! +} + +// Make sure we can see through all the layers of `Box`. +fn h(x: Box) -> &'static T { + unsafe { transmute(x) } // Ok! +} + +// Make sure we can see through newtype wrappers. +struct Wrapper1(T); + +#[repr(C)] +struct Wrapper2(T); + +fn i1(x: &T) -> Wrapper1<&T> { + unsafe { transmute(x) } // Ok! +} +fn i2(x: &T) -> Wrapper2<&T> { + unsafe { transmute(x) } // Ok! +} + fn main() { } diff --git a/tests/ui/transmute/transmute-fat-pointers.stderr b/tests/ui/transmute/transmute-fat-pointers.stderr index e8335fcbed9d0..56a4f7f33d152 100644 --- a/tests/ui/transmute/transmute-fat-pointers.stderr +++ b/tests/ui/transmute/transmute-fat-pointers.stderr @@ -1,5 +1,5 @@ error[E0512]: cannot transmute between types of different sizes, or dependently-sized types - --> $DIR/transmute-fat-pointers.rs:10:14 + --> $DIR/transmute-fat-pointers.rs:11:14 | LL | unsafe { transmute(x) } | ^^^^^^^^^ @@ -8,7 +8,7 @@ LL | unsafe { transmute(x) } = note: target type: `&U` (pointer to `U`) error[E0512]: cannot transmute between types of different sizes, or dependently-sized types - --> $DIR/transmute-fat-pointers.rs:14:14 + --> $DIR/transmute-fat-pointers.rs:15:14 | LL | unsafe { transmute(x) } | ^^^^^^^^^ @@ -17,7 +17,7 @@ LL | unsafe { transmute(x) } = note: target type: `&U` (pointer to `U`) error[E0512]: cannot transmute between types of different sizes, or dependently-sized types - --> $DIR/transmute-fat-pointers.rs:26:14 + --> $DIR/transmute-fat-pointers.rs:27:14 | LL | unsafe { transmute(x) } | ^^^^^^^^^ @@ -26,7 +26,7 @@ LL | unsafe { transmute(x) } = note: target type: `&U` (N bits) error[E0512]: cannot transmute between types of different sizes, or dependently-sized types - --> $DIR/transmute-fat-pointers.rs:30:14 + --> $DIR/transmute-fat-pointers.rs:31:14 | LL | unsafe { transmute(x) } | ^^^^^^^^^