From 654e3345e535913653a2bad4204c31f9ccedb3c4 Mon Sep 17 00:00:00 2001 From: Charles Lew Date: Sun, 20 Jun 2021 12:34:26 +0800 Subject: [PATCH 1/3] Change miri to use tcx allocated allocations. --- .../rustc_mir/src/interpret/eval_context.rs | 6 --- compiler/rustc_mir/src/interpret/intern.rs | 1 - compiler/rustc_mir/src/interpret/memory.rs | 4 -- .../rustc_mir/src/interpret/terminator.rs | 4 +- compiler/rustc_mir/src/interpret/traits.rs | 50 +++++++++++++------ 5 files changed, 36 insertions(+), 29 deletions(-) diff --git a/compiler/rustc_mir/src/interpret/eval_context.rs b/compiler/rustc_mir/src/interpret/eval_context.rs index 6f7519e61561..801e4b1e4785 100644 --- a/compiler/rustc_mir/src/interpret/eval_context.rs +++ b/compiler/rustc_mir/src/interpret/eval_context.rs @@ -2,7 +2,6 @@ use std::cell::Cell; use std::fmt; use std::mem; -use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_hir::{self as hir, def_id::DefId, definitions::DefPathData}; use rustc_index::vec::IndexVec; @@ -40,10 +39,6 @@ pub struct InterpCx<'mir, 'tcx, M: Machine<'mir, 'tcx>> { /// The virtual memory system. pub memory: Memory<'mir, 'tcx, M>, - - /// A cache for deduplicating vtables - pub(super) vtables: - FxHashMap<(Ty<'tcx>, Option>), Pointer>, } // The Phantomdata exists to prevent this type from being `Send`. If it were sent across a thread @@ -393,7 +388,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { tcx: tcx.at(root_span), param_env, memory: Memory::new(tcx, memory_extra), - vtables: FxHashMap::default(), } } diff --git a/compiler/rustc_mir/src/interpret/intern.rs b/compiler/rustc_mir/src/interpret/intern.rs index 23c0fe97c5f6..d5fec457fa19 100644 --- a/compiler/rustc_mir/src/interpret/intern.rs +++ b/compiler/rustc_mir/src/interpret/intern.rs @@ -107,7 +107,6 @@ fn intern_shallow<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx, const_eval: match kind { MemoryKind::Stack | MemoryKind::Machine(const_eval::MemoryKind::Heap) - | MemoryKind::Vtable | MemoryKind::CallerLocation => {} } // Set allocation mutability as appropriate. This is used by LLVM to put things into diff --git a/compiler/rustc_mir/src/interpret/memory.rs b/compiler/rustc_mir/src/interpret/memory.rs index 77de19ac674c..94506808a68c 100644 --- a/compiler/rustc_mir/src/interpret/memory.rs +++ b/compiler/rustc_mir/src/interpret/memory.rs @@ -27,8 +27,6 @@ use crate::util::pretty; pub enum MemoryKind { /// Stack memory. Error if deallocated except during a stack pop. Stack, - /// Memory backing vtables. Error if ever deallocated. - Vtable, /// Memory allocated by `caller_location` intrinsic. Error if ever deallocated. CallerLocation, /// Additional memory kinds a machine wishes to distinguish from the builtin ones. @@ -40,7 +38,6 @@ impl MayLeak for MemoryKind { fn may_leak(self) -> bool { match self { MemoryKind::Stack => false, - MemoryKind::Vtable => true, MemoryKind::CallerLocation => true, MemoryKind::Machine(k) => k.may_leak(), } @@ -51,7 +48,6 @@ impl fmt::Display for MemoryKind { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { MemoryKind::Stack => write!(f, "stack variable"), - MemoryKind::Vtable => write!(f, "vtable"), MemoryKind::CallerLocation => write!(f, "caller location"), MemoryKind::Machine(m) => write!(f, "{}", m), } diff --git a/compiler/rustc_mir/src/interpret/terminator.rs b/compiler/rustc_mir/src/interpret/terminator.rs index a5bdeb55e781..aea9933b337c 100644 --- a/compiler/rustc_mir/src/interpret/terminator.rs +++ b/compiler/rustc_mir/src/interpret/terminator.rs @@ -459,7 +459,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { }; // Find and consult vtable let vtable = receiver_place.vtable(); - let drop_fn = self.get_vtable_slot(vtable, u64::try_from(idx).unwrap())?; + let fn_val = self.get_vtable_slot(vtable, u64::try_from(idx).unwrap())?; // `*mut receiver_place.layout.ty` is almost the layout that we // want for args[0]: We have to project to field 0 because we want @@ -472,7 +472,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { OpTy::from(ImmTy::from_immediate(receiver_place.ptr.into(), this_receiver_ptr)); trace!("Patched self operand to {:#?}", args[0]); // recurse with concrete function - self.eval_fn_call(drop_fn, caller_abi, &args, ret, unwind) + self.eval_fn_call(fn_val, caller_abi, &args, ret, unwind) } } } diff --git a/compiler/rustc_mir/src/interpret/traits.rs b/compiler/rustc_mir/src/interpret/traits.rs index 072c252be2f3..539d767d62ad 100644 --- a/compiler/rustc_mir/src/interpret/traits.rs +++ b/compiler/rustc_mir/src/interpret/traits.rs @@ -1,14 +1,29 @@ use std::convert::TryFrom; -use rustc_middle::mir::interpret::{InterpResult, Pointer, PointerArithmetic, Scalar}; +use rustc_middle::mir::interpret::{ + AllocError, InterpError, InterpResult, Pointer, PointerArithmetic, Scalar, + UndefinedBehaviorInfo, UnsupportedOpInfo, +}; use rustc_middle::ty::{ self, Instance, Ty, VtblEntry, COMMON_VTABLE_ENTRIES, COMMON_VTABLE_ENTRIES_ALIGN, COMMON_VTABLE_ENTRIES_DROPINPLACE, COMMON_VTABLE_ENTRIES_SIZE, }; use rustc_target::abi::{Align, LayoutOf, Size}; +use super::alloc_range; use super::util::ensure_monomorphic_enough; -use super::{FnVal, InterpCx, Machine, MemoryKind}; +use super::{Allocation, FnVal, InterpCx, Machine}; + +fn vtable_alloc_error_to_interp_error<'tcx>(error: AllocError) -> InterpError<'tcx> { + match error { + AllocError::ReadPointerAsBytes => { + InterpError::Unsupported(UnsupportedOpInfo::ReadPointerAsBytes) + } + AllocError::InvalidUninitBytes(_info) => { + InterpError::UndefinedBehavior(UndefinedBehaviorInfo::InvalidUninitBytes(None)) + } + } +} impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// Creates a dynamic vtable for the given type and vtable origin. This is used only for @@ -60,10 +75,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // `get_vtable` in `rust_codegen_llvm/meth.rs`. // ///////////////////////////////////////////////////////////////////////////////////////// let vtable_size = ptr_size * u64::try_from(vtable_entries.len()).unwrap(); - let vtable = self.memory.allocate(vtable_size, ptr_align, MemoryKind::Vtable); - - let drop = Instance::resolve_drop_in_place(tcx, ty); - let drop = self.memory.create_fn_alloc(FnVal::Instance(drop)); + let mut vtable = Allocation::uninit(vtable_size, ptr_align); // No need to do any alignment checks on the memory accesses below, because we know the // allocation is correctly aligned as we created it above. Also we're only offsetting by @@ -72,36 +84,42 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { .iter() .map(|entry| -> InterpResult<'tcx, _> { match entry { - VtblEntry::MetadataDropInPlace => Ok(Some(drop.into())), + VtblEntry::MetadataDropInPlace => { + let instance = Instance::resolve_drop_in_place(tcx, ty); + let fn_alloc_id = tcx.create_fn_alloc(instance); + let fn_ptr = Pointer::from(fn_alloc_id); + Ok(Some(fn_ptr.into())) + } VtblEntry::MetadataSize => Ok(Some(Scalar::from_uint(size, ptr_size).into())), VtblEntry::MetadataAlign => Ok(Some(Scalar::from_uint(align, ptr_size).into())), VtblEntry::Vacant => Ok(None), VtblEntry::Method(def_id, substs) => { // Prepare the fn ptr we write into the vtable. let instance = - ty::Instance::resolve_for_vtable(tcx, self.param_env, *def_id, substs) + Instance::resolve_for_vtable(tcx, self.param_env, *def_id, substs) .ok_or_else(|| err_inval!(TooGeneric))?; - let fn_ptr = self.memory.create_fn_alloc(FnVal::Instance(instance)); + let fn_alloc_id = tcx.create_fn_alloc(instance); + let fn_ptr = Pointer::from(fn_alloc_id); Ok(Some(fn_ptr.into())) } } }) .collect::, _>>()?; - let mut vtable_alloc = - self.memory.get_mut(vtable.into(), vtable_size, ptr_align)?.expect("not a ZST"); for (idx, scalar) in scalars.into_iter().enumerate() { if let Some(scalar) = scalar { let idx: u64 = u64::try_from(idx).unwrap(); - vtable_alloc.write_ptr_sized(ptr_size * idx, scalar)?; + vtable + .write_scalar(self, alloc_range(ptr_size * idx, ptr_size), scalar) + .map_err(vtable_alloc_error_to_interp_error)?; } } - M::after_static_mem_initialized(self, vtable, vtable_size)?; + let vtable_id = tcx.create_memory_alloc(tcx.intern_const_alloc(vtable)); + let vtable_ptr = self.memory.global_base_pointer(Pointer::from(vtable_id))?; - self.memory.mark_immutable(vtable.alloc_id)?; - assert!(self.vtables.insert((ty, poly_trait_ref), vtable).is_none()); + assert!(self.vtables.insert((ty, poly_trait_ref), vtable_ptr).is_none()); - Ok(vtable) + Ok(vtable_ptr) } /// Resolves the function at the specified slot in the provided From d3ff497bec60687e3e1bdde383bd0b9dc74b5870 Mon Sep 17 00:00:00 2001 From: Charles Lew Date: Sun, 20 Jun 2021 17:43:25 +0800 Subject: [PATCH 2/3] Update other codegens to use tcx managed vtable allocations. --- .../rustc_codegen_cranelift/src/common.rs | 2 +- .../rustc_codegen_cranelift/src/constant.rs | 2 +- compiler/rustc_codegen_cranelift/src/lib.rs | 2 +- .../rustc_codegen_cranelift/src/unsize.rs | 4 +- .../rustc_codegen_cranelift/src/vtable.rs | 106 ++---------------- compiler/rustc_codegen_llvm/src/common.rs | 4 + compiler/rustc_codegen_ssa/src/meth.rs | 43 +------ .../rustc_codegen_ssa/src/traits/consts.rs | 2 + compiler/rustc_middle/src/ty/context.rs | 6 +- compiler/rustc_middle/src/ty/mod.rs | 18 +-- compiler/rustc_middle/src/ty/vtable.rs | 106 ++++++++++++++++++ compiler/rustc_mir/src/interpret/traits.rs | 97 +--------------- 12 files changed, 143 insertions(+), 249 deletions(-) create mode 100644 compiler/rustc_middle/src/ty/vtable.rs diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs index 488ff6e13495..a8a0bb52a246 100644 --- a/compiler/rustc_codegen_cranelift/src/common.rs +++ b/compiler/rustc_codegen_cranelift/src/common.rs @@ -233,7 +233,7 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> { pub(crate) module: &'m mut dyn Module, pub(crate) tcx: TyCtxt<'tcx>, pub(crate) pointer_type: Type, // Cached from module - pub(crate) vtables: FxHashMap<(Ty<'tcx>, Option>), DataId>, + pub(crate) vtables: FxHashMap<(Ty<'tcx>, Option>), Pointer>, pub(crate) constants_cx: ConstantCx, pub(crate) instance: Instance<'tcx>, diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index 3ba12c4e96d6..a87b3703949f 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -249,7 +249,7 @@ pub(crate) fn codegen_const_value<'tcx>( } } -fn pointer_for_allocation<'tcx>( +pub(crate) fn pointer_for_allocation<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, alloc: &'tcx Allocation, ) -> crate::pointer::Pointer { diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs index 6aadaf8a7cad..b817bf4aff77 100644 --- a/compiler/rustc_codegen_cranelift/src/lib.rs +++ b/compiler/rustc_codegen_cranelift/src/lib.rs @@ -98,7 +98,7 @@ mod prelude { pub(crate) use cranelift_codegen::isa::{self, CallConv}; pub(crate) use cranelift_codegen::Context; pub(crate) use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable}; - pub(crate) use cranelift_module::{self, DataContext, DataId, FuncId, Linkage, Module}; + pub(crate) use cranelift_module::{self, DataContext, FuncId, Linkage, Module}; pub(crate) use crate::abi::*; pub(crate) use crate::base::{codegen_operand, codegen_place}; diff --git a/compiler/rustc_codegen_cranelift/src/unsize.rs b/compiler/rustc_codegen_cranelift/src/unsize.rs index 042583cd5720..b9d379c6117c 100644 --- a/compiler/rustc_codegen_cranelift/src/unsize.rs +++ b/compiler/rustc_codegen_cranelift/src/unsize.rs @@ -31,9 +31,7 @@ pub(crate) fn unsized_info<'tcx>( // change to the vtable. old_info.expect("unsized_info: missing old info for trait upcast") } - (_, &ty::Dynamic(ref data, ..)) => { - crate::vtable::get_vtable(fx, fx.layout_of(source), data.principal()) - } + (_, &ty::Dynamic(ref data, ..)) => crate::vtable::get_vtable(fx, source, data.principal()), _ => bug!("unsized_info: invalid unsizing {:?} -> {:?}", source, target), } } diff --git a/compiler/rustc_codegen_cranelift/src/vtable.rs b/compiler/rustc_codegen_cranelift/src/vtable.rs index 4d1ee47b41e1..12f7092d935a 100644 --- a/compiler/rustc_codegen_cranelift/src/vtable.rs +++ b/compiler/rustc_codegen_cranelift/src/vtable.rs @@ -4,7 +4,7 @@ // FIXME dedup this logic between miri, cg_llvm and cg_clif use crate::prelude::*; -use ty::VtblEntry; +use super::constant::pointer_for_allocation; fn vtable_memflags() -> MemFlags { let mut flags = MemFlags::trusted(); // A vtable access is always aligned and will never trap. @@ -66,105 +66,19 @@ pub(crate) fn get_ptr_and_method_ref<'tcx>( pub(crate) fn get_vtable<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, - layout: TyAndLayout<'tcx>, + ty: Ty<'tcx>, trait_ref: Option>, ) -> Value { - let data_id = if let Some(data_id) = fx.vtables.get(&(layout.ty, trait_ref)) { - *data_id + let vtable_ptr = if let Some(vtable_ptr) = fx.vtables.get(&(ty, trait_ref)) { + *vtable_ptr } else { - let data_id = build_vtable(fx, layout, trait_ref); - fx.vtables.insert((layout.ty, trait_ref), data_id); - data_id - }; - - let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); - fx.bcx.ins().global_value(fx.pointer_type, local_data_id) -} - -fn build_vtable<'tcx>( - fx: &mut FunctionCx<'_, '_, 'tcx>, - layout: TyAndLayout<'tcx>, - trait_ref: Option>, -) -> DataId { - let tcx = fx.tcx; - let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes() as usize; + let vtable_alloc_id = fx.tcx.vtable_allocation(ty, trait_ref); + let vtable_allocation = fx.tcx.global_alloc(vtable_alloc_id).unwrap_memory(); + let vtable_ptr = pointer_for_allocation(fx, vtable_allocation); - let drop_in_place_fn = import_function( - tcx, - fx.module, - Instance::resolve_drop_in_place(tcx, layout.ty).polymorphize(fx.tcx), - ); - - let vtable_entries = if let Some(trait_ref) = trait_ref { - tcx.vtable_entries(trait_ref.with_self_ty(tcx, layout.ty)) - } else { - ty::COMMON_VTABLE_ENTRIES + fx.vtables.insert((ty, trait_ref), vtable_ptr); + vtable_ptr }; - let mut data_ctx = DataContext::new(); - let mut data = ::std::iter::repeat(0u8) - .take(vtable_entries.len() * usize_size) - .collect::>() - .into_boxed_slice(); - - for (idx, entry) in vtable_entries.iter().enumerate() { - match entry { - VtblEntry::MetadataSize => { - write_usize(fx.tcx, &mut data, idx, layout.size.bytes()); - } - VtblEntry::MetadataAlign => { - write_usize(fx.tcx, &mut data, idx, layout.align.abi.bytes()); - } - VtblEntry::MetadataDropInPlace | VtblEntry::Vacant | VtblEntry::Method(_, _) => {} - } - } - data_ctx.define(data); - - for (idx, entry) in vtable_entries.iter().enumerate() { - match entry { - VtblEntry::MetadataDropInPlace => { - let func_ref = fx.module.declare_func_in_data(drop_in_place_fn, &mut data_ctx); - data_ctx.write_function_addr((idx * usize_size) as u32, func_ref); - } - VtblEntry::Method(def_id, substs) => { - let func_id = import_function( - tcx, - fx.module, - Instance::resolve_for_vtable(tcx, ParamEnv::reveal_all(), *def_id, substs) - .unwrap() - .polymorphize(fx.tcx), - ); - let func_ref = fx.module.declare_func_in_data(func_id, &mut data_ctx); - data_ctx.write_function_addr((idx * usize_size) as u32, func_ref); - } - VtblEntry::MetadataSize | VtblEntry::MetadataAlign | VtblEntry::Vacant => {} - } - } - - data_ctx.set_align(fx.tcx.data_layout.pointer_align.pref.bytes()); - - let data_id = fx.module.declare_anonymous_data(false, false).unwrap(); - - fx.module.define_data(data_id, &data_ctx).unwrap(); - - data_id -} - -fn write_usize(tcx: TyCtxt<'_>, buf: &mut [u8], idx: usize, num: u64) { - let pointer_size = - tcx.layout_of(ParamEnv::reveal_all().and(tcx.types.usize)).unwrap().size.bytes() as usize; - let target = &mut buf[idx * pointer_size..(idx + 1) * pointer_size]; - - match tcx.data_layout.endian { - rustc_target::abi::Endian::Little => match pointer_size { - 4 => target.copy_from_slice(&(num as u32).to_le_bytes()), - 8 => target.copy_from_slice(&(num as u64).to_le_bytes()), - _ => todo!("pointer size {} is not yet supported", pointer_size), - }, - rustc_target::abi::Endian::Big => match pointer_size { - 4 => target.copy_from_slice(&(num as u32).to_be_bytes()), - 8 => target.copy_from_slice(&(num as u64).to_be_bytes()), - _ => todo!("pointer size {} is not yet supported", pointer_size), - }, - } + vtable_ptr.get_addr(fx) } diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 58af9d4cd04a..df5ad8ecc271 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -282,6 +282,10 @@ impl ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { } } + fn const_data_from_alloc(&self, alloc: &Allocation) -> Self::Value { + const_alloc_to_llvm(self, alloc) + } + fn from_const_alloc( &self, layout: TyAndLayout<'tcx>, diff --git a/compiler/rustc_codegen_ssa/src/meth.rs b/compiler/rustc_codegen_ssa/src/meth.rs index 4f0de7297048..63245a94c8e3 100644 --- a/compiler/rustc_codegen_ssa/src/meth.rs +++ b/compiler/rustc_codegen_ssa/src/meth.rs @@ -1,6 +1,6 @@ use crate::traits::*; -use rustc_middle::ty::{self, Instance, Ty, VtblEntry, COMMON_VTABLE_ENTRIES}; +use rustc_middle::ty::{self, Ty}; use rustc_target::abi::call::FnAbi; #[derive(Copy, Clone, Debug)] @@ -70,48 +70,13 @@ pub fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>( return val; } - // Not in the cache; build it. - let nullptr = cx.const_null(cx.type_i8p_ext(cx.data_layout().instruction_address_space)); - - let vtable_entries = if let Some(trait_ref) = trait_ref { - tcx.vtable_entries(trait_ref.with_self_ty(tcx, ty)) - } else { - COMMON_VTABLE_ENTRIES - }; - - let layout = cx.layout_of(ty); - // ///////////////////////////////////////////////////////////////////////////////////////////// - // If you touch this code, be sure to also make the corresponding changes to - // `get_vtable` in `rust_mir/interpret/traits.rs`. - // ///////////////////////////////////////////////////////////////////////////////////////////// - let components: Vec<_> = vtable_entries - .iter() - .map(|entry| match entry { - VtblEntry::MetadataDropInPlace => { - cx.get_fn_addr(Instance::resolve_drop_in_place(cx.tcx(), ty)) - } - VtblEntry::MetadataSize => cx.const_usize(layout.size.bytes()), - VtblEntry::MetadataAlign => cx.const_usize(layout.align.abi.bytes()), - VtblEntry::Vacant => nullptr, - VtblEntry::Method(def_id, substs) => cx.get_fn_addr( - ty::Instance::resolve_for_vtable( - cx.tcx(), - ty::ParamEnv::reveal_all(), - *def_id, - substs, - ) - .unwrap() - .polymorphize(cx.tcx()), - ), - }) - .collect(); - - let vtable_const = cx.const_struct(&components, false); + let vtable_alloc_id = tcx.vtable_allocation(ty, trait_ref); + let vtable_allocation = tcx.global_alloc(vtable_alloc_id).unwrap_memory(); + let vtable_const = cx.const_data_from_alloc(vtable_allocation); let align = cx.data_layout().pointer_align.abi; let vtable = cx.static_addr_of(vtable_const, align, Some("vtable")); cx.create_vtable_metadata(ty, vtable); - cx.vtables().borrow_mut().insert((ty, trait_ref), vtable); vtable } diff --git a/compiler/rustc_codegen_ssa/src/traits/consts.rs b/compiler/rustc_codegen_ssa/src/traits/consts.rs index 6b58dea794bc..20f661871232 100644 --- a/compiler/rustc_codegen_ssa/src/traits/consts.rs +++ b/compiler/rustc_codegen_ssa/src/traits/consts.rs @@ -26,6 +26,8 @@ pub trait ConstMethods<'tcx>: BackendTypes { fn const_to_opt_uint(&self, v: Self::Value) -> Option; fn const_to_opt_u128(&self, v: Self::Value, sign_ext: bool) -> Option; + fn const_data_from_alloc(&self, alloc: &Allocation) -> Self::Value; + fn scalar_to_backend(&self, cv: Scalar, layout: &abi::Scalar, llty: Self::Type) -> Self::Value; fn from_const_alloc( &self, diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 77f955e93b9d..a5a03ce0a4b6 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -11,7 +11,7 @@ use crate::middle; use crate::middle::cstore::{CrateStoreDyn, EncodedMetadata}; use crate::middle::resolve_lifetime::{self, LifetimeScopeForPath, ObjectLifetimeDefault}; use crate::middle::stability; -use crate::mir::interpret::{self, Allocation, ConstValue, Scalar}; +use crate::mir::interpret::{self, AllocId, Allocation, ConstValue, Scalar}; use crate::mir::{Body, Field, Local, Place, PlaceElem, ProjectionKind, Promoted}; use crate::thir::Thir; use crate::traits; @@ -1045,6 +1045,9 @@ pub struct GlobalCtxt<'tcx> { output_filenames: Arc, pub main_def: Option, + + pub(super) vtables_cache: + Lock, Option>), AllocId>>, } impl<'tcx> TyCtxt<'tcx> { @@ -1202,6 +1205,7 @@ impl<'tcx> TyCtxt<'tcx> { alloc_map: Lock::new(interpret::AllocMap::new()), output_filenames: Arc::new(output_filenames), main_def: resolutions.main_def, + vtables_cache: Default::default(), } } diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index a2abbec74927..859a940a6252 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -18,6 +18,7 @@ pub use adt::*; pub use assoc::*; pub use closure::*; pub use generics::*; +pub use vtable::*; use crate::hir::exports::ExportMap; use crate::ich::StableHashingContext; @@ -94,6 +95,7 @@ pub mod relate; pub mod subst; pub mod trait_def; pub mod util; +pub mod vtable; pub mod walk; mod adt; @@ -2009,19 +2011,3 @@ impl<'tcx> fmt::Debug for SymbolName<'tcx> { fmt::Display::fmt(&self.name, fmt) } } - -#[derive(Clone, Copy, Debug, PartialEq, HashStable)] -pub enum VtblEntry<'tcx> { - MetadataDropInPlace, - MetadataSize, - MetadataAlign, - Vacant, - Method(DefId, SubstsRef<'tcx>), -} - -pub const COMMON_VTABLE_ENTRIES: &[VtblEntry<'_>] = - &[VtblEntry::MetadataDropInPlace, VtblEntry::MetadataSize, VtblEntry::MetadataAlign]; - -pub const COMMON_VTABLE_ENTRIES_DROPINPLACE: usize = 0; -pub const COMMON_VTABLE_ENTRIES_SIZE: usize = 1; -pub const COMMON_VTABLE_ENTRIES_ALIGN: usize = 2; diff --git a/compiler/rustc_middle/src/ty/vtable.rs b/compiler/rustc_middle/src/ty/vtable.rs new file mode 100644 index 000000000000..3a35d8c88a47 --- /dev/null +++ b/compiler/rustc_middle/src/ty/vtable.rs @@ -0,0 +1,106 @@ +use std::convert::TryFrom; + +use crate::mir::interpret::{alloc_range, AllocId, Allocation, Pointer, Scalar}; +use crate::ty::fold::TypeFoldable; +use crate::ty::{self, DefId, SubstsRef, Ty, TyCtxt}; +use rustc_ast::Mutability; + +#[derive(Clone, Copy, Debug, PartialEq, HashStable)] +pub enum VtblEntry<'tcx> { + MetadataDropInPlace, + MetadataSize, + MetadataAlign, + Vacant, + Method(DefId, SubstsRef<'tcx>), +} + +pub const COMMON_VTABLE_ENTRIES: &[VtblEntry<'_>] = + &[VtblEntry::MetadataDropInPlace, VtblEntry::MetadataSize, VtblEntry::MetadataAlign]; + +pub const COMMON_VTABLE_ENTRIES_DROPINPLACE: usize = 0; +pub const COMMON_VTABLE_ENTRIES_SIZE: usize = 1; +pub const COMMON_VTABLE_ENTRIES_ALIGN: usize = 2; + +impl<'tcx> TyCtxt<'tcx> { + /// Retrieves an allocation that represents the contents of a vtable. + /// There's a cache within `TyCtxt` so it will be deduplicated. + pub fn vtable_allocation( + self, + ty: Ty<'tcx>, + poly_trait_ref: Option>, + ) -> AllocId { + let tcx = self; + let vtables_cache = tcx.vtables_cache.lock(); + if let Some(alloc_id) = vtables_cache.get(&(ty, poly_trait_ref)).cloned() { + return alloc_id; + } + drop(vtables_cache); + + // See https://github.com/rust-lang/rust/pull/86475#discussion_r655162674 + assert!( + !ty.needs_subst() && !poly_trait_ref.map_or(false, |trait_ref| trait_ref.needs_subst()) + ); + let param_env = ty::ParamEnv::reveal_all(); + let vtable_entries = if let Some(poly_trait_ref) = poly_trait_ref { + let trait_ref = poly_trait_ref.with_self_ty(tcx, ty); + let trait_ref = tcx.erase_regions(trait_ref); + + tcx.vtable_entries(trait_ref) + } else { + COMMON_VTABLE_ENTRIES + }; + + let layout = + tcx.layout_of(param_env.and(ty)).expect("failed to build vtable representation"); + assert!(!layout.is_unsized(), "can't create a vtable for an unsized type"); + let size = layout.size.bytes(); + let align = layout.align.abi.bytes(); + + let ptr_size = tcx.data_layout.pointer_size; + let ptr_align = tcx.data_layout.pointer_align.abi; + + let vtable_size = ptr_size * u64::try_from(vtable_entries.len()).unwrap(); + let mut vtable = Allocation::uninit(vtable_size, ptr_align); + + // No need to do any alignment checks on the memory accesses below, because we know the + // allocation is correctly aligned as we created it above. Also we're only offsetting by + // multiples of `ptr_align`, which means that it will stay aligned to `ptr_align`. + + for (idx, entry) in vtable_entries.iter().enumerate() { + let idx: u64 = u64::try_from(idx).unwrap(); + let scalar = match entry { + VtblEntry::MetadataDropInPlace => { + let instance = ty::Instance::resolve_drop_in_place(tcx, ty); + let fn_alloc_id = tcx.create_fn_alloc(instance); + let fn_ptr = Pointer::from(fn_alloc_id); + fn_ptr.into() + } + VtblEntry::MetadataSize => Scalar::from_uint(size, ptr_size).into(), + VtblEntry::MetadataAlign => Scalar::from_uint(align, ptr_size).into(), + VtblEntry::Vacant => continue, + VtblEntry::Method(def_id, substs) => { + // See https://github.com/rust-lang/rust/pull/86475#discussion_r655162674 + assert!(!substs.needs_subst()); + + // Prepare the fn ptr we write into the vtable. + let instance = + ty::Instance::resolve_for_vtable(tcx, param_env, *def_id, substs) + .expect("resolution failed during building vtable representation") + .polymorphize(tcx); + let fn_alloc_id = tcx.create_fn_alloc(instance); + let fn_ptr = Pointer::from(fn_alloc_id); + fn_ptr.into() + } + }; + vtable + .write_scalar(&tcx, alloc_range(ptr_size * idx, ptr_size), scalar) + .expect("failed to build vtable representation"); + } + + vtable.mutability = Mutability::Not; + let alloc_id = tcx.create_memory_alloc(tcx.intern_const_alloc(vtable)); + let mut vtables_cache = self.vtables_cache.lock(); + vtables_cache.insert((ty, poly_trait_ref), alloc_id); + alloc_id + } +} diff --git a/compiler/rustc_mir/src/interpret/traits.rs b/compiler/rustc_mir/src/interpret/traits.rs index 539d767d62ad..5332e615bc8e 100644 --- a/compiler/rustc_mir/src/interpret/traits.rs +++ b/compiler/rustc_mir/src/interpret/traits.rs @@ -1,29 +1,14 @@ use std::convert::TryFrom; -use rustc_middle::mir::interpret::{ - AllocError, InterpError, InterpResult, Pointer, PointerArithmetic, Scalar, - UndefinedBehaviorInfo, UnsupportedOpInfo, -}; +use rustc_middle::mir::interpret::{InterpResult, Pointer, PointerArithmetic, Scalar}; use rustc_middle::ty::{ - self, Instance, Ty, VtblEntry, COMMON_VTABLE_ENTRIES, COMMON_VTABLE_ENTRIES_ALIGN, + self, Ty, COMMON_VTABLE_ENTRIES, COMMON_VTABLE_ENTRIES_ALIGN, COMMON_VTABLE_ENTRIES_DROPINPLACE, COMMON_VTABLE_ENTRIES_SIZE, }; -use rustc_target::abi::{Align, LayoutOf, Size}; +use rustc_target::abi::{Align, Size}; -use super::alloc_range; use super::util::ensure_monomorphic_enough; -use super::{Allocation, FnVal, InterpCx, Machine}; - -fn vtable_alloc_error_to_interp_error<'tcx>(error: AllocError) -> InterpError<'tcx> { - match error { - AllocError::ReadPointerAsBytes => { - InterpError::Unsupported(UnsupportedOpInfo::ReadPointerAsBytes) - } - AllocError::InvalidUninitBytes(_info) => { - InterpError::UndefinedBehavior(UndefinedBehaviorInfo::InvalidUninitBytes(None)) - } - } -} +use super::{FnVal, InterpCx, Machine}; impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// Creates a dynamic vtable for the given type and vtable origin. This is used only for @@ -45,79 +30,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { ensure_monomorphic_enough(*self.tcx, ty)?; ensure_monomorphic_enough(*self.tcx, poly_trait_ref)?; - if let Some(&vtable) = self.vtables.get(&(ty, poly_trait_ref)) { - // This means we guarantee that there are no duplicate vtables, we will - // always use the same vtable for the same (Type, Trait) combination. - // That's not what happens in rustc, but emulating per-crate deduplication - // does not sound like it actually makes anything any better. - return Ok(vtable); - } - - let vtable_entries = if let Some(poly_trait_ref) = poly_trait_ref { - let trait_ref = poly_trait_ref.with_self_ty(*self.tcx, ty); - let trait_ref = self.tcx.erase_regions(trait_ref); - - self.tcx.vtable_entries(trait_ref) - } else { - COMMON_VTABLE_ENTRIES - }; - - let layout = self.layout_of(ty)?; - assert!(!layout.is_unsized(), "can't create a vtable for an unsized type"); - let size = layout.size.bytes(); - let align = layout.align.abi.bytes(); - - let tcx = *self.tcx; - let ptr_size = self.pointer_size(); - let ptr_align = tcx.data_layout.pointer_align.abi; - // ///////////////////////////////////////////////////////////////////////////////////////// - // If you touch this code, be sure to also make the corresponding changes to - // `get_vtable` in `rust_codegen_llvm/meth.rs`. - // ///////////////////////////////////////////////////////////////////////////////////////// - let vtable_size = ptr_size * u64::try_from(vtable_entries.len()).unwrap(); - let mut vtable = Allocation::uninit(vtable_size, ptr_align); - - // No need to do any alignment checks on the memory accesses below, because we know the - // allocation is correctly aligned as we created it above. Also we're only offsetting by - // multiples of `ptr_align`, which means that it will stay aligned to `ptr_align`. - let scalars = vtable_entries - .iter() - .map(|entry| -> InterpResult<'tcx, _> { - match entry { - VtblEntry::MetadataDropInPlace => { - let instance = Instance::resolve_drop_in_place(tcx, ty); - let fn_alloc_id = tcx.create_fn_alloc(instance); - let fn_ptr = Pointer::from(fn_alloc_id); - Ok(Some(fn_ptr.into())) - } - VtblEntry::MetadataSize => Ok(Some(Scalar::from_uint(size, ptr_size).into())), - VtblEntry::MetadataAlign => Ok(Some(Scalar::from_uint(align, ptr_size).into())), - VtblEntry::Vacant => Ok(None), - VtblEntry::Method(def_id, substs) => { - // Prepare the fn ptr we write into the vtable. - let instance = - Instance::resolve_for_vtable(tcx, self.param_env, *def_id, substs) - .ok_or_else(|| err_inval!(TooGeneric))?; - let fn_alloc_id = tcx.create_fn_alloc(instance); - let fn_ptr = Pointer::from(fn_alloc_id); - Ok(Some(fn_ptr.into())) - } - } - }) - .collect::, _>>()?; - for (idx, scalar) in scalars.into_iter().enumerate() { - if let Some(scalar) = scalar { - let idx: u64 = u64::try_from(idx).unwrap(); - vtable - .write_scalar(self, alloc_range(ptr_size * idx, ptr_size), scalar) - .map_err(vtable_alloc_error_to_interp_error)?; - } - } - - let vtable_id = tcx.create_memory_alloc(tcx.intern_const_alloc(vtable)); - let vtable_ptr = self.memory.global_base_pointer(Pointer::from(vtable_id))?; + let vtable_allocation = self.tcx.vtable_allocation(ty, poly_trait_ref); - assert!(self.vtables.insert((ty, poly_trait_ref), vtable_ptr).is_none()); + let vtable_ptr = self.memory.global_base_pointer(Pointer::from(vtable_allocation))?; Ok(vtable_ptr) } From 97772bb1f230f4981c9af6614df1ebc09b12c4f6 Mon Sep 17 00:00:00 2001 From: Charles Lew Date: Mon, 28 Jun 2021 12:39:42 +0800 Subject: [PATCH 3/3] Bless the test suite. --- src/test/ui/consts/const-eval/ub-upvars.32bit.stderr | 2 +- src/test/ui/consts/const-eval/ub-upvars.64bit.stderr | 2 +- src/test/ui/consts/issue-79690.64bit.stderr | 2 +- .../consts/miri_unleashed/mutable_references_err.32bit.stderr | 2 +- .../consts/miri_unleashed/mutable_references_err.64bit.stderr | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/ui/consts/const-eval/ub-upvars.32bit.stderr b/src/test/ui/consts/const-eval/ub-upvars.32bit.stderr index 17a96386f4b5..0818d7fbf88b 100644 --- a/src/test/ui/consts/const-eval/ub-upvars.32bit.stderr +++ b/src/test/ui/consts/const-eval/ub-upvars.32bit.stderr @@ -10,7 +10,7 @@ LL | | }; | = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior. = note: the raw bytes of the constant (size: 8, align: 4) { - ╾─alloc2──╼ ╾─alloc3──╼ │ ╾──╼╾──╼ + ╾─alloc2──╼ ╾─alloc5──╼ │ ╾──╼╾──╼ } error: aborting due to previous error diff --git a/src/test/ui/consts/const-eval/ub-upvars.64bit.stderr b/src/test/ui/consts/const-eval/ub-upvars.64bit.stderr index 11922e32ae19..59dcf7aefe41 100644 --- a/src/test/ui/consts/const-eval/ub-upvars.64bit.stderr +++ b/src/test/ui/consts/const-eval/ub-upvars.64bit.stderr @@ -10,7 +10,7 @@ LL | | }; | = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior. = note: the raw bytes of the constant (size: 16, align: 8) { - ╾───────alloc2────────╼ ╾───────alloc3────────╼ │ ╾──────╼╾──────╼ + ╾───────alloc2────────╼ ╾───────alloc5────────╼ │ ╾──────╼╾──────╼ } error: aborting due to previous error diff --git a/src/test/ui/consts/issue-79690.64bit.stderr b/src/test/ui/consts/issue-79690.64bit.stderr index 57e62d0b90f1..cab111ab69cd 100644 --- a/src/test/ui/consts/issue-79690.64bit.stderr +++ b/src/test/ui/consts/issue-79690.64bit.stderr @@ -6,7 +6,7 @@ LL | const G: Fat = unsafe { Transmute { t: FOO }.u }; | = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior. = note: the raw bytes of the constant (size: 16, align: 8) { - ╾───────alloc2────────╼ ╾───────alloc3────────╼ │ ╾──────╼╾──────╼ + ╾───────alloc2────────╼ ╾───────alloc5────────╼ │ ╾──────╼╾──────╼ } error: aborting due to previous error diff --git a/src/test/ui/consts/miri_unleashed/mutable_references_err.32bit.stderr b/src/test/ui/consts/miri_unleashed/mutable_references_err.32bit.stderr index 5374088028bf..e6db2eef955e 100644 --- a/src/test/ui/consts/miri_unleashed/mutable_references_err.32bit.stderr +++ b/src/test/ui/consts/miri_unleashed/mutable_references_err.32bit.stderr @@ -19,7 +19,7 @@ LL | const SNEAKY: &dyn Sync = &Synced { x: UnsafeCell::new(42) }; | = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior. = note: the raw bytes of the constant (size: 8, align: 4) { - ╾─alloc6──╼ ╾─alloc7──╼ │ ╾──╼╾──╼ + ╾─alloc6──╼ ╾─alloc8──╼ │ ╾──╼╾──╼ } error[E0080]: it is undefined behavior to use this value diff --git a/src/test/ui/consts/miri_unleashed/mutable_references_err.64bit.stderr b/src/test/ui/consts/miri_unleashed/mutable_references_err.64bit.stderr index 7c4842db24cc..f48978d4af6b 100644 --- a/src/test/ui/consts/miri_unleashed/mutable_references_err.64bit.stderr +++ b/src/test/ui/consts/miri_unleashed/mutable_references_err.64bit.stderr @@ -19,7 +19,7 @@ LL | const SNEAKY: &dyn Sync = &Synced { x: UnsafeCell::new(42) }; | = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior. = note: the raw bytes of the constant (size: 16, align: 8) { - ╾───────alloc6────────╼ ╾───────alloc7────────╼ │ ╾──────╼╾──────╼ + ╾───────alloc6────────╼ ╾───────alloc8────────╼ │ ╾──────╼╾──────╼ } error[E0080]: it is undefined behavior to use this value