transmute: fix check for whether newtypes have equal size

compiler/rustc_middle/src/ty/layout.rs

@@ -426,13 +426,42 @@ impl<'tcx> SizeSkeleton<'tcx> {
             }
 
             ty::Adt(def, args) => {
-                // Only newtypes and enums w/ nullable pointer optimization.
+                // Only newtypes and enums w/ nullable pointer optimization (NPO).
                 if def.is_union() || def.variants().is_empty() || def.variants().len() > 2 {
                     return Err(err);
                 }
+                // Only default repr types.
+                {
+                    // We can ignore the seed and some particular flags that can never affect the
+                    // layout of newtypes / NPO types, but we have to check everything else.
+                    // If you are adding a new field to `ReprOptions`, make sure to extend the check
+                    // below so that we bail out if it is not at its default value!
+                    let ReprOptions { int, align, pack, flags, scalable, field_shuffle_seed: _ } =
+                        def.repr();
+                    let mut ignored_flags = ReprFlags::IS_TRANSPARENT
+                        | ReprFlags::IS_LINEAR
+                        | ReprFlags::RANDOMIZE_LAYOUT;
+                    if def.is_struct() {
+                        // `repr(C)` is only okay for structs, not for enums.
+                        // Below, the *only* thing we do for structs is propagating
+                        // `SizeSkeleton::Pointer`. We do *not* assume that `repr(C)` preserved
+                        // ZST-ness (which might stop being true eventually).
+                        ignored_flags |= ReprFlags::IS_C;
+                    }
+                    if int.is_some()
+                        || align.is_some()
+                        || pack.is_some()
+                        || flags.difference(ignored_flags) != ReprFlags::default()
+                        || scalable.is_some()
+                    {
+                        return Err(err);
+                    }
+                }
 
                 // Get a zero-sized variant or a pointer newtype.
-                let zero_or_ptr_variant = |i| {
+                // Returns `Ok(None)` for 1-ZST types, `Ok(Some)` if (ignoring all 1-ZST fields)
+                // there's just a single pointer, and `Err` otherwise.
+                let zero_or_ptr_variant = |i| -> Result<Option<SizeSkeleton<'tcx>>, _> {
                     let i = VariantIdx::from_usize(i);
                     let fields =
                         def.variant(i).fields.iter().map(|field| {
@@ -461,7 +490,8 @@ impl<'tcx> SizeSkeleton<'tcx> {
                 };
 
                 let v0 = zero_or_ptr_variant(0)?;
-                // Newtype.
+                // Single-variant case: Check if this is a newtype around a pointer.
+                // Such types are themselves pointer-sized.
                 if def.variants().len() == 1 {
                     if let Some(SizeSkeleton::Pointer { non_zero, tail }) = v0 {
                         return Ok(SizeSkeleton::Pointer { non_zero, tail });
@@ -471,7 +501,9 @@ impl<'tcx> SizeSkeleton<'tcx> {
                 }
 
                 let v1 = zero_or_ptr_variant(1)?;
-                // Nullable pointer enum optimization.
+                // 2-variant case: Check if one variant is a *non-zero* pointer and the other a
+                // 1-ZST. Such types are eligible to for the nullable pointer enum optimization, so
+                // they are themselves pointer-sized.
                 match (v0, v1) {
                     (Some(SizeSkeleton::Pointer { non_zero: true, tail }), None)
                     | (None, Some(SizeSkeleton::Pointer { non_zero: true, tail })) => {

tests/ui/transmute/transmute-different-sizes.rs

@@ -48,4 +48,36 @@ pub unsafe fn shouldnt_work2<T: ?Sized>(from: *mut T) -> PtrAndEmptyArray<T> {
     //~^ ERROR cannot transmute between types of different sizes, or dependently-sized types
 }
 
+#[repr(align(16))]
+struct OverAlignWrap<T>(T);
+
+fn shouldnt_work3<T: ?Sized>(x: OverAlignWrap<*const T>) -> *const T {
+    unsafe { transmute(x) }
+    //~^ ERROR cannot transmute between types of different sizes, or dependently-sized types
+}
+
+// With `repr(C)`, this type has a disriminant, so it does not have the same size as `T`.
+#[repr(C)]
+enum NotANewtype<T> {
+    SingleVariant(T),
+}
+
+fn shouldnt_work4<T: ?Sized>(x: &T) -> NotANewtype<&T> {
+    unsafe { transmute(x) }
+    //~^ ERROR cannot transmute between types of different sizes, or dependently-sized types
+}
+
+// With `repr(C)`, this type has a disriminant; NPO does not kick in.
+// Therefore this is always bigger than `T`.
+#[repr(C)]
+enum NotNullPointerOptimized<T> {
+    Some(T),
+    None
+}
+
+fn shouldnt_work5<T: ?Sized>(x: &T) -> NotNullPointerOptimized<&T> {
+    unsafe { transmute(x) }
+    //~^ ERROR cannot transmute between types of different sizes, or dependently-sized types
+}
+
 fn main() {}

tests/ui/transmute/transmute-different-sizes.stderr

@@ -43,6 +43,33 @@ LL |     std::mem::transmute(from)
    = note: source type: `*mut T` (pointer to `T`)
    = note: target type: `PtrAndEmptyArray<T>` (size can vary because of <T as Pointee>::Metadata)
 
-error: aborting due to 5 previous errors
+error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
+  --> $DIR/transmute-different-sizes.rs:55:14
+   |
+LL |     unsafe { transmute(x) }
+   |              ^^^^^^^^^
+   |
+   = note: source type: `OverAlignWrap<*const T>` (size can vary because of <T as Pointee>::Metadata)
+   = note: target type: `*const T` (pointer to `T`)
+
+error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
+  --> $DIR/transmute-different-sizes.rs:66:14
+   |
+LL |     unsafe { transmute(x) }
+   |              ^^^^^^^^^
+   |
+   = note: source type: `&T` (pointer to `T`)
+   = note: target type: `NotANewtype<&T>` (size can vary because of <T as Pointee>::Metadata)
+
+error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
+  --> $DIR/transmute-different-sizes.rs:79:14
+   |
+LL |     unsafe { transmute(x) }
+   |              ^^^^^^^^^
+   |
+   = note: source type: `&T` (pointer to `T`)
+   = note: target type: `NotNullPointerOptimized<&T>` (size can vary because of <T as Pointee>::Metadata)
+
+error: aborting due to 8 previous errors
 
 For more information about this error, try `rustc --explain E0512`.

tests/ui/transmute/transmute-fat-pointers.rs

@@ -5,6 +5,7 @@
 #![allow(dead_code)]
 
 use std::mem::transmute;
+use std::ptr::NonNull;
 
 fn a<T, U: ?Sized>(x: &[T]) -> &U {
     unsafe { transmute(x) } //~ ERROR cannot transmute between types of different sizes
@@ -15,11 +16,11 @@ fn b<T: ?Sized, U: ?Sized>(x: &T) -> &U {
 }
 
 fn c<T, U>(x: &T) -> &U {
-    unsafe { transmute(x) }
+    unsafe { transmute(x) } // Ok!
 }
 
 fn d<T, U>(x: &[T]) -> &[U] {
-    unsafe { transmute(x) }
+    unsafe { transmute(x) } // Ok!
 }
 
 fn e<T: ?Sized, U>(x: &T) -> &U {
@@ -30,4 +31,36 @@ fn f<T, U: ?Sized>(x: &T) -> &U {
     unsafe { transmute(x) } //~ ERROR cannot transmute between types of different sizes
 }
 
+// We can transmute even between pointers of unknown size as long as the metadata of
+// input and output type are the same. This even accounts for null pointer optimizations.
+fn g1<T: ?Sized>(x: &T) -> Option<&T> {
+    unsafe { transmute(x) } // Ok!
+}
+
+fn g2<T: ?Sized>(x: Option<NonNull<T>>) -> NonNull<T> {
+    unsafe { transmute(x) } // Ok!
+}
+
+fn g3<T: ?Sized>(x: *const T) -> Option<NonNull<T>> {
+    unsafe { transmute(x) } // Ok!
+}
+
+// Make sure we can see through all the layers of `Box`.
+fn h<T: ?Sized>(x: Box<T>) -> &'static T {
+    unsafe { transmute(x) } // Ok!
+}
+
+// Make sure we can see through newtype wrappers.
+struct Wrapper1<T>(T);
+
+#[repr(C)]
+struct Wrapper2<T>(T);
+
+fn i1<T: ?Sized>(x: &T) -> Wrapper1<&T> {
+    unsafe { transmute(x) } // Ok!
+}
+fn i2<T: ?Sized>(x: &T) -> Wrapper2<&T> {
+    unsafe { transmute(x) } // Ok!
+}
+
 fn main() { }

tests/ui/transmute/transmute-fat-pointers.stderr

@@ -1,5 +1,5 @@
 error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
-  --> $DIR/transmute-fat-pointers.rs:10:14
+  --> $DIR/transmute-fat-pointers.rs:11:14
    |
 LL |     unsafe { transmute(x) }
    |              ^^^^^^^^^
@@ -8,7 +8,7 @@ LL |     unsafe { transmute(x) }
    = note: target type: `&U` (pointer to `U`)
 
 error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
-  --> $DIR/transmute-fat-pointers.rs:14:14
+  --> $DIR/transmute-fat-pointers.rs:15:14
    |
 LL |     unsafe { transmute(x) }
    |              ^^^^^^^^^
@@ -17,7 +17,7 @@ LL |     unsafe { transmute(x) }
    = note: target type: `&U` (pointer to `U`)
 
 error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
-  --> $DIR/transmute-fat-pointers.rs:26:14
+  --> $DIR/transmute-fat-pointers.rs:27:14
    |
 LL |     unsafe { transmute(x) }
    |              ^^^^^^^^^
@@ -26,7 +26,7 @@ LL |     unsafe { transmute(x) }
    = note: target type: `&U` (N bits)
 
 error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
-  --> $DIR/transmute-fat-pointers.rs:30:14
+  --> $DIR/transmute-fat-pointers.rs:31:14
    |
 LL |     unsafe { transmute(x) }
    |              ^^^^^^^^^