Pass type when creating load

This makes load generation compatible with opaque pointers.

The generation of nontemporal copies still accesses the pointer
element type, as fixing this requires more movement.

compiler/rustc_codegen_llvm/src/builder.rs

@@ -410,17 +410,17 @@ fn array_alloca(&mut self, ty: &'ll Type, len: &'ll Value, align: Align) -> &'ll
         }
     }
 
-    fn load(&mut self, ptr: &'ll Value, align: Align) -> &'ll Value {
+    fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
         unsafe {
-            let load = llvm::LLVMBuildLoad(self.llbuilder, ptr, UNNAMED);
+            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
             llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
             load
         }
     }
 
-    fn volatile_load(&mut self, ptr: &'ll Value) -> &'ll Value {
+    fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value {
         unsafe {
-            let load = llvm::LLVMBuildLoad(self.llbuilder, ptr, UNNAMED);
+            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
             llvm::LLVMSetVolatile(load, llvm::True);
             load
         }
@@ -488,7 +488,7 @@ fn scalar_load_metadata<'a, 'll, 'tcx>(
                 }
             }
             let llval = const_llval.unwrap_or_else(|| {
-                let load = self.load(place.llval, place.align);
+                let load = self.load(place.layout.llvm_type(self), place.llval, place.align);
                 if let abi::Abi::Scalar(ref scalar) = place.layout.abi {
                     scalar_load_metadata(self, load, scalar);
                 }
@@ -500,7 +500,8 @@ fn scalar_load_metadata<'a, 'll, 'tcx>(
 
             let mut load = |i, scalar: &abi::Scalar, align| {
                 let llptr = self.struct_gep(place.llval, i as u64);
-                let load = self.load(llptr, align);
+                let llty = place.layout.scalar_pair_element_llvm_type(self, i, false);
+                let load = self.load(llty, llptr, align);
                 scalar_load_metadata(self, load, scalar);
                 self.to_immediate_scalar(load, scalar)
             };
@@ -817,13 +818,7 @@ fn memcpy(
         size: &'ll Value,
         flags: MemFlags,
     ) {
-        if flags.contains(MemFlags::NONTEMPORAL) {
-            // HACK(nox): This is inefficient but there is no nontemporal memcpy.
-            let val = self.load(src, src_align);
-            let ptr = self.pointercast(dst, self.type_ptr_to(self.val_ty(val)));
-            self.store_with_flags(val, ptr, dst_align, flags);
-            return;
-        }
+        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported");
         let size = self.intcast(size, self.type_isize(), false);
         let is_volatile = flags.contains(MemFlags::VOLATILE);
         let dst = self.pointercast(dst, self.type_i8p());
@@ -850,13 +845,7 @@ fn memmove(
         size: &'ll Value,
         flags: MemFlags,
     ) {
-        if flags.contains(MemFlags::NONTEMPORAL) {
-            // HACK(nox): This is inefficient but there is no nontemporal memmove.
-            let val = self.load(src, src_align);
-            let ptr = self.pointercast(dst, self.type_ptr_to(self.val_ty(val)));
-            self.store_with_flags(val, ptr, dst_align, flags);
-            return;
-        }
+        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported");
         let size = self.intcast(size, self.type_isize(), false);
         let is_volatile = flags.contains(MemFlags::VOLATILE);
         let dst = self.pointercast(dst, self.type_i8p());

compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs

@@ -20,7 +20,7 @@ pub fn insert_reference_to_gdb_debug_scripts_section_global(bx: &mut Builder<'_,
         // LLVM to keep around the reference to the global.
         let indices = [bx.const_i32(0), bx.const_i32(0)];
         let element = bx.inbounds_gep(gdb_debug_scripts_section, &indices);
-        let volative_load_instruction = bx.volatile_load(element);
+        let volative_load_instruction = bx.volatile_load(bx.type_i8(), element);
         unsafe {
             llvm::LLVMSetAlignment(volative_load_instruction, 1);
         }

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -162,11 +162,14 @@ fn codegen_intrinsic_call(
 
             sym::volatile_load | sym::unaligned_volatile_load => {
                 let tp_ty = substs.type_at(0);
-                let mut ptr = args[0].immediate();
-                if let PassMode::Cast(ty) = fn_abi.ret.mode {
-                    ptr = self.pointercast(ptr, self.type_ptr_to(ty.llvm_type(self)));
-                }
-                let load = self.volatile_load(ptr);
+                let ptr = args[0].immediate();
+                let load = if let PassMode::Cast(ty) = fn_abi.ret.mode {
+                    let llty = ty.llvm_type(self);
+                    let ptr = self.pointercast(ptr, self.type_ptr_to(llty));
+                    self.volatile_load(llty, ptr)
+                } else {
+                    self.volatile_load(self.layout_of(tp_ty).llvm_type(self), ptr)
+                };
                 let align = if name == sym::unaligned_volatile_load {
                     1
                 } else {
@@ -319,9 +322,9 @@ fn codegen_intrinsic_call(
                     let integer_ty = self.type_ix(layout.size.bits());
                     let ptr_ty = self.type_ptr_to(integer_ty);
                     let a_ptr = self.bitcast(a, ptr_ty);
-                    let a_val = self.load(a_ptr, layout.align.abi);
+                    let a_val = self.load(integer_ty, a_ptr, layout.align.abi);
                     let b_ptr = self.bitcast(b, ptr_ty);
-                    let b_val = self.load(b_ptr, layout.align.abi);
+                    let b_val = self.load(integer_ty, b_ptr, layout.align.abi);
                     self.icmp(IntPredicate::IntEQ, a_val, b_val)
                 } else {
                     let i8p_ty = self.type_i8p();
@@ -540,7 +543,7 @@ fn codegen_msvc_try(
         // Source: MicrosoftCXXABI::getAddrOfCXXCatchHandlerType in clang
         let flags = bx.const_i32(8);
         let funclet = catchpad_rust.catch_pad(cs, &[tydesc, flags, slot]);
-        let ptr = catchpad_rust.load(slot, ptr_align);
+        let ptr = catchpad_rust.load(bx.type_i8p(), slot, ptr_align);
         catchpad_rust.call(catch_func, &[data, ptr], Some(&funclet));
         catchpad_rust.catch_ret(&funclet, caught.llbb());
 

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -1385,7 +1385,12 @@ pub fn LLVMBuildArrayAlloca(
         Val: &'a Value,
         Name: *const c_char,
     ) -> &'a Value;
-    pub fn LLVMBuildLoad(B: &Builder<'a>, PointerVal: &'a Value, Name: *const c_char) -> &'a Value;
+    pub fn LLVMBuildLoad2(
+        B: &Builder<'a>,
+        Ty: &'a Type,
+        PointerVal: &'a Value,
+        Name: *const c_char,
+    ) -> &'a Value;
 
     pub fn LLVMBuildStore(B: &Builder<'a>, Val: &'a Value, Ptr: &'a Value) -> &'a Value;
 

compiler/rustc_codegen_llvm/src/va_arg.rs

@@ -32,14 +32,15 @@ fn emit_direct_ptr_va_arg(
     slot_size: Align,
     allow_higher_align: bool,
 ) -> (&'ll Value, Align) {
-    let va_list_ptr_ty = bx.cx().type_ptr_to(bx.cx.type_i8p());
+    let va_list_ty = bx.type_i8p();
+    let va_list_ptr_ty = bx.type_ptr_to(va_list_ty);
     let va_list_addr = if list.layout.llvm_type(bx.cx) != va_list_ptr_ty {
         bx.bitcast(list.immediate(), va_list_ptr_ty)
     } else {
         list.immediate()
     };
 
-    let ptr = bx.load(va_list_addr, bx.tcx().data_layout.pointer_align.abi);
+    let ptr = bx.load(va_list_ty, va_list_addr, bx.tcx().data_layout.pointer_align.abi);
 
     let (addr, addr_align) = if allow_higher_align && align > slot_size {
         (round_pointer_up_to_alignment(bx, ptr, align, bx.cx().type_i8p()), align)
@@ -82,10 +83,10 @@ fn emit_ptr_va_arg(
     let (addr, addr_align) =
         emit_direct_ptr_va_arg(bx, list, llty, size, align.abi, slot_size, allow_higher_align);
     if indirect {
-        let tmp_ret = bx.load(addr, addr_align);
-        bx.load(tmp_ret, align.abi)
+        let tmp_ret = bx.load(llty, addr, addr_align);
+        bx.load(bx.cx.layout_of(target_ty).llvm_type(bx.cx), tmp_ret, align.abi)
     } else {
-        bx.load(addr, addr_align)
+        bx.load(llty, addr, addr_align)
     }
 }
 
@@ -118,7 +119,7 @@ fn emit_aapcs_va_arg(
     };
 
     // if the offset >= 0 then the value will be on the stack
-    let mut reg_off_v = bx.load(reg_off, offset_align);
+    let mut reg_off_v = bx.load(bx.type_i32(), reg_off, offset_align);
     let use_stack = bx.icmp(IntPredicate::IntSGE, reg_off_v, zero);
     bx.cond_br(use_stack, &on_stack.llbb(), &maybe_reg.llbb());
 
@@ -139,8 +140,9 @@ fn emit_aapcs_va_arg(
     let use_stack = maybe_reg.icmp(IntPredicate::IntSGT, new_reg_off_v, zero);
     maybe_reg.cond_br(use_stack, &on_stack.llbb(), &in_reg.llbb());
 
+    let top_type = bx.type_i8p();
     let top = in_reg.struct_gep(va_list_addr, reg_top_index);
-    let top = in_reg.load(top, bx.tcx().data_layout.pointer_align.abi);
+    let top = in_reg.load(top_type, top, bx.tcx().data_layout.pointer_align.abi);
 
     // reg_value = *(@top + reg_off_v);
     let mut reg_addr = in_reg.gep(top, &[reg_off_v]);
@@ -149,8 +151,9 @@ fn emit_aapcs_va_arg(
         let offset = bx.const_i32((slot_size - layout.size.bytes()) as i32);
         reg_addr = in_reg.gep(reg_addr, &[offset]);
     }
-    let reg_addr = in_reg.bitcast(reg_addr, bx.cx.type_ptr_to(layout.llvm_type(bx)));
-    let reg_value = in_reg.load(reg_addr, layout.align.abi);
+    let reg_type = layout.llvm_type(bx);
+    let reg_addr = in_reg.bitcast(reg_addr, bx.cx.type_ptr_to(reg_type));
+    let reg_value = in_reg.load(reg_type, reg_addr, layout.align.abi);
     in_reg.br(&end.llbb());
 
     // On Stack block

compiler/rustc_codegen_ssa/src/meth.rs

@@ -20,10 +20,11 @@ pub fn get_fn<Bx: BuilderMethods<'a, 'tcx>>(
         // Load the data pointer from the object.
         debug!("get_fn({:?}, {:?})", llvtable, self);
 
-        let llvtable = bx.pointercast(llvtable, bx.type_ptr_to(bx.fn_ptr_backend_type(fn_abi)));
+        let llty = bx.fn_ptr_backend_type(fn_abi);
+        let llvtable = bx.pointercast(llvtable, bx.type_ptr_to(llty));
         let ptr_align = bx.tcx().data_layout.pointer_align.abi;
         let gep = bx.inbounds_gep(llvtable, &[bx.const_usize(self.0)]);
-        let ptr = bx.load(gep, ptr_align);
+        let ptr = bx.load(llty, gep, ptr_align);
         bx.nonnull_metadata(ptr);
         // Vtable loads are invariant.
         bx.set_invariant_load(ptr);
@@ -38,10 +39,11 @@ pub fn get_usize<Bx: BuilderMethods<'a, 'tcx>>(
         // Load the data pointer from the object.
         debug!("get_int({:?}, {:?})", llvtable, self);
 
-        let llvtable = bx.pointercast(llvtable, bx.type_ptr_to(bx.type_isize()));
+        let llty = bx.type_isize();
+        let llvtable = bx.pointercast(llvtable, bx.type_ptr_to(llty));
         let usize_align = bx.tcx().data_layout.pointer_align.abi;
         let gep = bx.inbounds_gep(llvtable, &[bx.const_usize(self.0)]);
-        let ptr = bx.load(gep, usize_align);
+        let ptr = bx.load(llty, gep, usize_align);
         // Vtable loads are invariant.
         bx.set_invariant_load(ptr);
         ptr

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -260,7 +260,7 @@ fn codegen_return_terminator(&mut self, mut bx: Bx) {
             PassMode::Direct(_) | PassMode::Pair(..) => {
                 let op = self.codegen_consume(&mut bx, mir::Place::return_place().as_ref());
                 if let Ref(llval, _, align) = op.val {
-                    bx.load(llval, align)
+                    bx.load(bx.backend_type(op.layout), llval, align)
                 } else {
                     op.immediate_or_packed_pair(&mut bx)
                 }
@@ -287,8 +287,9 @@ fn codegen_return_terminator(&mut self, mut bx: Bx) {
                         llval
                     }
                 };
-                let addr = bx.pointercast(llslot, bx.type_ptr_to(bx.cast_backend_type(&cast_ty)));
-                bx.load(addr, self.fn_abi.ret.layout.align.abi)
+                let ty = bx.cast_backend_type(&cast_ty);
+                let addr = bx.pointercast(llslot, bx.type_ptr_to(ty));
+                bx.load(ty, addr, self.fn_abi.ret.layout.align.abi)
             }
         };
         bx.ret(llval);
@@ -1086,15 +1087,16 @@ fn codegen_argument(
         if by_ref && !arg.is_indirect() {
             // Have to load the argument, maybe while casting it.
             if let PassMode::Cast(ty) = arg.mode {
-                let addr = bx.pointercast(llval, bx.type_ptr_to(bx.cast_backend_type(&ty)));
-                llval = bx.load(addr, align.min(arg.layout.align.abi));
+                let llty = bx.cast_backend_type(&ty);
+                let addr = bx.pointercast(llval, bx.type_ptr_to(llty));
+                llval = bx.load(llty, addr, align.min(arg.layout.align.abi));
             } else {
                 // We can't use `PlaceRef::load` here because the argument
                 // may have a type we don't treat as immediate, but the ABI
                 // used for this call is passing it by-value. In that case,
                 // the load would just produce `OperandValue::Ref` instead
                 // of the `OperandValue::Immediate` we need for the call.
-                llval = bx.load(llval, align);
+                llval = bx.load(bx.backend_type(arg.layout), llval, align);
                 if let abi::Abi::Scalar(ref scalar) = arg.layout.abi {
                     if scalar.is_bool() {
                         bx.range_metadata(llval, 0..2);

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -289,6 +289,15 @@ fn store_with_flags<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
         }
         match self {
             OperandValue::Ref(r, None, source_align) => {
+                if flags.contains(MemFlags::NONTEMPORAL) {
+                    // HACK(nox): This is inefficient but there is no nontemporal memcpy.
+                    // FIXME: Don't access pointer element type.
+                    let ty = bx.element_type(bx.val_ty(r));
+                    let val = bx.load(ty, r, source_align);
+                    let ptr = bx.pointercast(dest.llval, bx.type_ptr_to(ty));
+                    bx.store_with_flags(val, ptr, dest.align, flags);
+                    return;
+                }
                 base::memcpy_ty(bx, dest.llval, dest.align, r, source_align, dest.layout, flags)
             }
             OperandValue::Ref(_, Some(_), _) => {

compiler/rustc_codegen_ssa/src/mir/place.rs

@@ -407,7 +407,7 @@ pub fn project_deref<Bx: BuilderMethods<'a, 'tcx, Value = V>>(&self, bx: &mut Bx
         let layout = bx.layout_of(target_ty.ty);
 
         PlaceRef {
-            llval: bx.load(self.llval, self.align),
+            llval: bx.load(bx.backend_type(layout), self.llval, self.align),
             llextra: None,
             layout,
             align: layout.align.abi,

compiler/rustc_codegen_ssa/src/traits/builder.rs

@@ -137,8 +137,8 @@ fn to_immediate(&mut self, val: Self::Value, layout: TyAndLayout<'_>) -> Self::V
     fn dynamic_alloca(&mut self, ty: Self::Type, align: Align) -> Self::Value;
     fn array_alloca(&mut self, ty: Self::Type, len: Self::Value, align: Align) -> Self::Value;
 
-    fn load(&mut self, ptr: Self::Value, align: Align) -> Self::Value;
-    fn volatile_load(&mut self, ptr: Self::Value) -> Self::Value;
+    fn load(&mut self, ty: Self::Type, ptr: Self::Value, align: Align) -> Self::Value;
+    fn volatile_load(&mut self, ty: Self::Type, ptr: Self::Value) -> Self::Value;
     fn atomic_load(
         &mut self,
         ty: Self::Type,