don't allow ZST in ScalarInt

There are several indications that we should not ZST as a ScalarInt:
- We had two ways to have ZST valtrees, either an empty `Branch` or a `Leaf` with a ZST in it.
  `ValTree::zst()` used the former, but the latter could possibly arise as well.
- Likewise, the interpreter had `Immediate::Uninit` and `Immediate::Scalar(Scalar::ZST)`.
- LLVM codegen already had to special-case ZST ScalarInt.

So instead add new ZST variants to those types that did not have other variants
which could be used for this purpose.

Author: RalfJung

compiler/rustc_codegen_llvm/src/common.rs

@@ -13,7 +13,6 @@ use rustc_codegen_ssa::traits::*;
 use rustc_middle::bug;
 use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar};
 use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
-use rustc_middle::ty::ScalarInt;
 use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer, Size};
 
 use libc::{c_char, c_uint};
@@ -223,13 +222,13 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         })
     }
 
+    fn zst_to_backend(&self, _llty: &'ll Type) -> &'ll Value {
+        self.const_undef(self.type_ix(0))
+    }
+
     fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: &'ll Type) -> &'ll Value {
         let bitsize = if layout.is_bool() { 1 } else { layout.size(self).bits() };
         match cv {
-            Scalar::Int(ScalarInt::ZST) => {
-                assert_eq!(0, layout.size(self).bytes());
-                self.const_undef(self.type_ix(0))
-            }
             Scalar::Int(int) => {
                 let data = int.assert_bits(layout.size(self));
                 let llval = self.const_uint_big(self.type_ix(bitsize), data);

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -84,6 +84,10 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
                 let llval = bx.scalar_to_backend(x, scalar, bx.immediate_backend_type(layout));
                 OperandValue::Immediate(llval)
             }
+            ConstValue::ZST => {
+                let llval = bx.zst_to_backend(bx.immediate_backend_type(layout));
+                OperandValue::Immediate(llval)
+            }
             ConstValue::Slice { data, start, end } => {
                 let Abi::ScalarPair(a_scalar, _) = layout.abi else {
                     bug!("from_const: invalid ScalarPair layout: {:#?}", layout);

compiler/rustc_codegen_ssa/src/traits/consts.rs

@@ -29,6 +29,7 @@ pub trait ConstMethods<'tcx>: BackendTypes {
     fn const_data_from_alloc(&self, alloc: ConstAllocation<'tcx>) -> Self::Value;
 
     fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: Self::Type) -> Self::Value;
+    fn zst_to_backend(&self, llty: Self::Type) -> Self::Value;
     fn from_const_alloc(
         &self,
         layout: TyAndLayout<'tcx>,

compiler/rustc_const_eval/src/const_eval/eval_queries.rs

@@ -2,7 +2,7 @@ use super::{CompileTimeEvalContext, CompileTimeInterpreter, ConstEvalErr};
 use crate::interpret::eval_nullary_intrinsic;
 use crate::interpret::{
     intern_const_alloc_recursive, Allocation, ConstAlloc, ConstValue, CtfeValidationMode, GlobalId,
-    Immediate, InternKind, InterpCx, InterpResult, MPlaceTy, MemoryKind, OpTy, RefTracking, Scalar,
+    Immediate, InternKind, InterpCx, InterpResult, MPlaceTy, MemoryKind, OpTy, RefTracking,
     ScalarMaybeUninit, StackPopCleanup,
 };
 
@@ -157,7 +157,7 @@ pub(super) fn op_to_const<'tcx>(
                     "this MPlaceTy must come from a validated constant, thus we can assume the \
                 alignment is correct",
                 );
-                ConstValue::Scalar(Scalar::ZST)
+                ConstValue::ZST
             }
         }
     };

compiler/rustc_const_eval/src/const_eval/valtrees.rs

@@ -272,7 +272,7 @@ pub fn valtree_to_const_value<'tcx>(
     match ty.kind() {
         ty::FnDef(..) => {
             assert!(valtree.unwrap_branch().is_empty());
-            ConstValue::Scalar(Scalar::ZST)
+            ConstValue::ZST
         }
         ty::Bool | ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::Char => match valtree {
             ty::ValTree::Leaf(scalar_int) => ConstValue::Scalar(Scalar::Int(scalar_int)),
@@ -344,11 +344,7 @@ fn valtree_into_mplace<'tcx>(
 
     match ty.kind() {
         ty::FnDef(_, _) => {
-            ecx.write_immediate(
-                Immediate::Scalar(ScalarMaybeUninit::Scalar(Scalar::ZST)),
-                &place.into(),
-            )
-            .unwrap();
+            ecx.write_immediate(Immediate::Uninit, &place.into()).unwrap();
         }
         ty::Bool | ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::Char => {
             let scalar_int = valtree.unwrap_leaf();

compiler/rustc_const_eval/src/interpret/operand.rs

@@ -297,8 +297,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
         let Some(alloc) = self.get_place_alloc(mplace)? else {
             return Ok(Some(ImmTy {
-                // zero-sized type
-                imm: Scalar::ZST.into(),
+                // zero-sized type can be left uninit
+                imm: Immediate::Uninit,
                 layout: mplace.layout,
             }));
         };
@@ -441,8 +441,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         // This makes several assumptions about what layouts we will encounter; we match what
         // codegen does as good as we can (see `extract_field` in `rustc_codegen_ssa/src/mir/operand.rs`).
         let field_val: Immediate<_> = match (*base, base.layout.abi) {
-            // the field contains no information
-            _ if field_layout.is_zst() => Scalar::ZST.into(),
+            // the field contains no information, can be left uninit
+            _ if field_layout.is_zst() => Immediate::Uninit,
             // the field covers the entire type
             _ if field_layout.size == base.layout.size => {
                 assert!(match (base.layout.abi, field_layout.abi) {
@@ -553,8 +553,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     ) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> {
         let layout = self.layout_of_local(frame, local, layout)?;
         let op = if layout.is_zst() {
-            // Do not read from ZST, they might not be initialized
-            Operand::Immediate(Scalar::ZST.into())
+            // Bypass `access_local` (helps in ConstProp)
+            Operand::Immediate(Immediate::Uninit)
         } else {
             *M::access_local(frame, local)?
         };
@@ -709,6 +709,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 Operand::Indirect(MemPlace::from_ptr(ptr.into()))
             }
             ConstValue::Scalar(x) => Operand::Immediate(tag_scalar(x)?.into()),
+            ConstValue::ZST => Operand::Immediate(Immediate::Uninit),
             ConstValue::Slice { data, start, end } => {
                 // We rely on mutability being set correctly in `data` to prevent writes
                 // where none should happen.

compiler/rustc_middle/src/lib.rs

@@ -59,6 +59,7 @@
 #![feature(drain_filter)]
 #![feature(intra_doc_pointers)]
 #![feature(yeet_expr)]
+#![feature(const_option)]
 #![recursion_limit = "512"]
 #![allow(rustc::potential_query_instability)]
 

compiler/rustc_middle/src/mir/interpret/value.rs

@@ -29,11 +29,14 @@ pub struct ConstAlloc<'tcx> {
 #[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable, Hash)]
 #[derive(HashStable)]
 pub enum ConstValue<'tcx> {
-    /// Used only for types with `layout::abi::Scalar` ABI and ZSTs.
+    /// Used only for types with `layout::abi::Scalar` ABI.
     ///
     /// Not using the enum `Value` to encode that this must not be `Uninit`.
     Scalar(Scalar),
 
+    /// Only used for ZSTs.
+    ZST,
+
     /// Used only for `&[u8]` and `&str`
     Slice { data: ConstAllocation<'tcx>, start: usize, end: usize },
 
@@ -55,6 +58,7 @@ impl<'a, 'tcx> Lift<'tcx> for ConstValue<'a> {
     fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<ConstValue<'tcx>> {
         Some(match self {
             ConstValue::Scalar(s) => ConstValue::Scalar(s),
+            ConstValue::ZST => ConstValue::ZST,
             ConstValue::Slice { data, start, end } => {
                 ConstValue::Slice { data: tcx.lift(data)?, start, end }
             }
@@ -69,7 +73,7 @@ impl<'tcx> ConstValue<'tcx> {
     #[inline]
     pub fn try_to_scalar(&self) -> Option<Scalar<AllocId>> {
         match *self {
-            ConstValue::ByRef { .. } | ConstValue::Slice { .. } => None,
+            ConstValue::ByRef { .. } | ConstValue::Slice { .. } | ConstValue::ZST => None,
             ConstValue::Scalar(val) => Some(val),
         }
     }
@@ -111,10 +115,6 @@ impl<'tcx> ConstValue<'tcx> {
     pub fn from_machine_usize(i: u64, cx: &impl HasDataLayout) -> Self {
         ConstValue::Scalar(Scalar::from_machine_usize(i, cx))
     }
-
-    pub fn zst() -> Self {
-        Self::Scalar(Scalar::ZST)
-    }
 }
 
 /// A `Scalar` represents an immediate, primitive value existing outside of a
@@ -194,8 +194,6 @@ impl<Tag> From<ScalarInt> for Scalar<Tag> {
 }
 
 impl<Tag> Scalar<Tag> {
-    pub const ZST: Self = Scalar::Int(ScalarInt::ZST);
-
     #[inline(always)]
     pub fn from_pointer(ptr: Pointer<Tag>, cx: &impl HasDataLayout) -> Self {
         Scalar::Ptr(ptr, u8::try_from(cx.pointer_size().bytes()).unwrap())

compiler/rustc_middle/src/mir/mod.rs

@@ -1711,7 +1711,7 @@ impl<'tcx> Operand<'tcx> {
         Operand::Constant(Box::new(Constant {
             span,
             user_ty: None,
-            literal: ConstantKind::Val(ConstValue::zst(), ty),
+            literal: ConstantKind::Val(ConstValue::ZST, ty),
         }))
     }
 
@@ -2196,7 +2196,7 @@ impl<'tcx> ConstantKind<'tcx> {
 
     #[inline]
     pub fn zero_sized(ty: Ty<'tcx>) -> Self {
-        let cv = ConstValue::Scalar(Scalar::ZST);
+        let cv = ConstValue::ZST;
         Self::Val(cv, ty)
     }
 

compiler/rustc_middle/src/mir/pretty.rs

@@ -449,6 +449,7 @@ impl<'tcx> Visitor<'tcx> for ExtraComments<'tcx> {
             }
 
             let fmt_val = |val: &ConstValue<'tcx>| match val {
+                ConstValue::ZST => format!("ZST"),
                 ConstValue::Scalar(s) => format!("Scalar({:?})", s),
                 ConstValue::Slice { .. } => format!("Slice(..)"),
                 ConstValue::ByRef { .. } => format!("ByRef(..)"),
@@ -679,6 +680,7 @@ pub fn write_allocations<'tcx>(
             ConstValue::Scalar(interpret::Scalar::Int { .. }) => {
                 Either::Left(Either::Right(std::iter::empty()))
             }
+            ConstValue::ZST => Either::Left(Either::Right(std::iter::empty())),
             ConstValue::ByRef { alloc, .. } | ConstValue::Slice { data: alloc, .. } => {
                 Either::Right(alloc_ids_from_alloc(alloc))
             }

compiler/rustc_middle/src/thir.rs

@@ -419,6 +419,10 @@ pub enum ExprKind<'tcx> {
         lit: ty::ScalarInt,
         user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
     },
+    /// A literal of a ZST type.
+    ZstLiteral {
+        user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
+    },
     /// Associated constants and named constants
     NamedConst {
         def_id: DefId,
@@ -456,7 +460,7 @@ pub enum ExprKind<'tcx> {
 
 impl<'tcx> ExprKind<'tcx> {
     pub fn zero_sized_literal(user_ty: Option<Canonical<'tcx, UserType<'tcx>>>) -> Self {
-        ExprKind::NonHirLiteral { lit: ty::ScalarInt::ZST, user_ty }
+        ExprKind::ZstLiteral { user_ty }
     }
 }
 

compiler/rustc_middle/src/thir/visit.rs

@@ -129,6 +129,7 @@ pub fn walk_expr<'a, 'tcx: 'a, V: Visitor<'a, 'tcx>>(visitor: &mut V, expr: &Exp
         Closure { closure_id: _, substs: _, upvars: _, movability: _, fake_reads: _ } => {}
         Literal { lit: _, neg: _ } => {}
         NonHirLiteral { lit: _, user_ty: _ } => {}
+        ZstLiteral { user_ty: _ } => {}
         NamedConst { def_id: _, substs: _, user_ty: _ } => {}
         ConstParam { param: _, def_id: _ } => {}
         StaticRef { alloc_id: _, ty: _, def_id: _ } => {}