From 58eddbadd224ccdd898e103a5627691fbb8d374d Mon Sep 17 00:00:00 2001 From: Rudraksh Joshi Date: Mon, 13 Apr 2026 15:04:32 +0530 Subject: [PATCH 1/5] Copy byval argument to local stackslot if alignment is insufficient --- src/abi/mod.rs | 111 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 98 insertions(+), 13 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 13f5ad5157..f919c0f2db 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -12,7 +12,7 @@ use cranelift_codegen::ir::{ }; use cranelift_codegen::isa::CallConv; use cranelift_module::ModuleError; -use rustc_abi::{CanonAbi, ExternAbi, X86Call}; +use rustc_abi::{Align, CanonAbi, ExternAbi, X86Call}; use rustc_codegen_ssa::base::is_call_from_compiler_builtins_to_upstream_monomorphization; use rustc_codegen_ssa::errors::CompilerBuiltinsCannotCall; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; @@ -243,10 +243,17 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ self::returning::codegen_return_param(fx, &ssa_analyzed, &mut block_params_iter); assert_eq!(fx.local_map.push(ret_place), RETURN_PLACE); - // None means pass_mode == NoPass + struct ArgValue<'tcx> { + value: Option>, + /// If set, the argument is a byval/byref pointer whose pointee alignment is smaller than + /// the Rust ABI alignment. In that case the argument must be copied to a sufficiently + /// aligned local stack slot before it can be treated as a place. + underaligned_pointee_align: Option, + } + enum ArgKind<'tcx> { - Normal(Option>), - Spread(Vec>>), + Normal(ArgValue<'tcx>), + Spread(Vec>), } // FIXME implement variadics in cranelift @@ -280,17 +287,34 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ let mut params = Vec::new(); for (i, _arg_ty) in tupled_arg_tys.iter().enumerate() { let arg_abi = arg_abis_iter.next().unwrap(); - let param = + let value = cvalue_for_param(fx, Some(local), Some(i), arg_abi, &mut block_params_iter); - params.push(param); + let underaligned_pointee_align = + match arg_abi.mode { + PassMode::Indirect { attrs, .. } => attrs + .pointee_align + .filter(|&pointee_align| pointee_align < arg_abi.layout.align.abi), + _ => None, + }; + params.push(ArgValue { value, underaligned_pointee_align }); } (local, ArgKind::Spread(params), arg_ty) } else { let arg_abi = arg_abis_iter.next().unwrap(); - let param = + let value = cvalue_for_param(fx, Some(local), None, arg_abi, &mut block_params_iter); - (local, ArgKind::Normal(param), arg_ty) + let underaligned_pointee_align = match arg_abi.mode { + PassMode::Indirect { attrs, .. } => attrs + .pointee_align + .filter(|&pointee_align| pointee_align < arg_abi.layout.align.abi), + _ => None, + }; + ( + local, + ArgKind::Normal(ArgValue { value, underaligned_pointee_align }), + arg_ty, + ) } }) .collect::, Ty<'tcx>)>>(); @@ -311,7 +335,9 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ for (local, arg_kind, ty) in func_params { // While this is normally an optimization to prevent an unnecessary copy when an argument is // not mutated by the current function, this is necessary to support unsized arguments. - if let ArgKind::Normal(Some(val)) = arg_kind { + if let ArgKind::Normal(ArgValue { value: Some(val), underaligned_pointee_align: None }) = + arg_kind + { if let Some((addr, meta)) = val.try_to_ptr() { // Ownership of the value at the backing storage for an argument is passed to the // callee per the ABI, so it is fine to borrow the backing storage of this argument @@ -336,15 +362,74 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ assert_eq!(fx.local_map.push(place), local); match arg_kind { - ArgKind::Normal(param) => { - if let Some(param) = param { + ArgKind::Normal(ArgValue { value: Some(param), underaligned_pointee_align }) => { + if let Some(pointee_align) = underaligned_pointee_align + && let Some(dst_ptr) = place.try_to_ptr() + && let Some((src_ptr, None)) = param.try_to_ptr() + && layout.size != Size::ZERO + { + let mut flags = MemFlags::new(); + flags.set_notrap(); + + let to_addr = dst_ptr.get_addr(fx); + let from_addr = src_ptr.get_addr(fx); + let size = layout.size.bytes(); + + // `emit_small_memory_copy` uses `u8` for alignments, just use the maximum + // alignment that fits in a `u8` if the actual alignment is larger. + let dst_align = layout.align.bytes().try_into().unwrap_or(128); + let src_align = pointee_align.bytes().try_into().unwrap_or(128); + + fx.bcx.emit_small_memory_copy( + fx.target_config, + to_addr, + from_addr, + size, + dst_align, + src_align, + true, + flags, + ); + } else { place.write_cvalue(fx, param); } } + ArgKind::Normal(ArgValue { value: None, underaligned_pointee_align: _ }) => {} ArgKind::Spread(params) => { - for (i, param) in params.into_iter().enumerate() { + for (i, ArgValue { value: param, underaligned_pointee_align }) in + params.into_iter().enumerate() + { if let Some(param) = param { - place.place_field(fx, FieldIdx::new(i)).write_cvalue(fx, param); + let field_place = place.place_field(fx, FieldIdx::new(i)); + let field_layout = field_place.layout(); + if let Some(pointee_align) = underaligned_pointee_align + && let Some(dst_ptr) = field_place.try_to_ptr() + && let Some((src_ptr, None)) = param.try_to_ptr() + && field_layout.size != Size::ZERO + { + let mut flags = MemFlags::new(); + flags.set_notrap(); + + let to_addr = dst_ptr.get_addr(fx); + let from_addr = src_ptr.get_addr(fx); + let size = field_layout.size.bytes(); + + let dst_align = field_layout.align.bytes().try_into().unwrap_or(128); + let src_align = pointee_align.bytes().try_into().unwrap_or(128); + + fx.bcx.emit_small_memory_copy( + fx.target_config, + to_addr, + from_addr, + size, + dst_align, + src_align, + true, + flags, + ); + } else { + field_place.write_cvalue(fx, param); + } } } } From c153f587cd69316058aa62a93ae9802cbe921b0f Mon Sep 17 00:00:00 2001 From: Rudraksh Joshi Date: Mon, 13 Apr 2026 15:12:37 +0530 Subject: [PATCH 2/5] correct fmt --- src/abi/mod.rs | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index f919c0f2db..e916ea0d29 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -289,13 +289,12 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ let arg_abi = arg_abis_iter.next().unwrap(); let value = cvalue_for_param(fx, Some(local), Some(i), arg_abi, &mut block_params_iter); - let underaligned_pointee_align = - match arg_abi.mode { - PassMode::Indirect { attrs, .. } => attrs - .pointee_align - .filter(|&pointee_align| pointee_align < arg_abi.layout.align.abi), - _ => None, - }; + let underaligned_pointee_align = match arg_abi.mode { + PassMode::Indirect { attrs, .. } => attrs + .pointee_align + .filter(|&pointee_align| pointee_align < arg_abi.layout.align.abi), + _ => None, + }; params.push(ArgValue { value, underaligned_pointee_align }); } @@ -310,11 +309,7 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ .filter(|&pointee_align| pointee_align < arg_abi.layout.align.abi), _ => None, }; - ( - local, - ArgKind::Normal(ArgValue { value, underaligned_pointee_align }), - arg_ty, - ) + (local, ArgKind::Normal(ArgValue { value, underaligned_pointee_align }), arg_ty) } }) .collect::, Ty<'tcx>)>>(); From 067bf06ed19043fe91f563ef9c51ddfc8d114e14 Mon Sep 17 00:00:00 2001 From: Rudraksh Joshi Date: Mon, 13 Apr 2026 16:47:46 +0530 Subject: [PATCH 3/5] Address review comments --- src/abi/mod.rs | 101 +++++++++---------------------------------- src/abi/pass_mode.rs | 29 ++++++++++--- 2 files changed, 43 insertions(+), 87 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index e916ea0d29..50b78070a1 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -31,6 +31,11 @@ use crate::base::codegen_unwind_terminate; use crate::debuginfo::EXCEPTION_HANDLER_CLEANUP; use crate::prelude::*; +pub(super) struct ArgValue<'tcx> { + pub(super) value: Option>, + pub(super) underaligned_pointee_align: Option, +} + fn clif_sig_from_fn_abi<'tcx>( tcx: TyCtxt<'tcx>, default_call_conv: CallConv, @@ -243,14 +248,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ self::returning::codegen_return_param(fx, &ssa_analyzed, &mut block_params_iter); assert_eq!(fx.local_map.push(ret_place), RETURN_PLACE); - struct ArgValue<'tcx> { - value: Option>, - /// If set, the argument is a byval/byref pointer whose pointee alignment is smaller than - /// the Rust ABI alignment. In that case the argument must be copied to a sufficiently - /// aligned local stack slot before it can be treated as a place. - underaligned_pointee_align: Option, - } - enum ArgKind<'tcx> { Normal(ArgValue<'tcx>), Spread(Vec>), @@ -287,29 +284,20 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ let mut params = Vec::new(); for (i, _arg_ty) in tupled_arg_tys.iter().enumerate() { let arg_abi = arg_abis_iter.next().unwrap(); - let value = - cvalue_for_param(fx, Some(local), Some(i), arg_abi, &mut block_params_iter); - let underaligned_pointee_align = match arg_abi.mode { - PassMode::Indirect { attrs, .. } => attrs - .pointee_align - .filter(|&pointee_align| pointee_align < arg_abi.layout.align.abi), - _ => None, - }; - params.push(ArgValue { value, underaligned_pointee_align }); + params.push(cvalue_for_param( + fx, + Some(local), + Some(i), + arg_abi, + &mut block_params_iter, + )); } (local, ArgKind::Spread(params), arg_ty) } else { let arg_abi = arg_abis_iter.next().unwrap(); - let value = - cvalue_for_param(fx, Some(local), None, arg_abi, &mut block_params_iter); - let underaligned_pointee_align = match arg_abi.mode { - PassMode::Indirect { attrs, .. } => attrs - .pointee_align - .filter(|&pointee_align| pointee_align < arg_abi.layout.align.abi), - _ => None, - }; - (local, ArgKind::Normal(ArgValue { value, underaligned_pointee_align }), arg_ty) + let arg = cvalue_for_param(fx, Some(local), None, arg_abi, &mut block_params_iter); + (local, ArgKind::Normal(arg), arg_ty) } }) .collect::, Ty<'tcx>)>>(); @@ -318,8 +306,8 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ if fx.instance.def.requires_caller_location(fx.tcx) { // Store caller location for `#[track_caller]`. let arg_abi = arg_abis_iter.next().unwrap(); - fx.caller_location = - Some(cvalue_for_param(fx, None, None, arg_abi, &mut block_params_iter).unwrap()); + let arg = cvalue_for_param(fx, None, None, arg_abi, &mut block_params_iter); + fx.caller_location = Some(arg.value.unwrap()); } assert_eq!(arg_abis_iter.next(), None, "ArgAbi left behind for {:?}", fx.fn_abi); @@ -358,33 +346,8 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ match arg_kind { ArgKind::Normal(ArgValue { value: Some(param), underaligned_pointee_align }) => { - if let Some(pointee_align) = underaligned_pointee_align - && let Some(dst_ptr) = place.try_to_ptr() - && let Some((src_ptr, None)) = param.try_to_ptr() - && layout.size != Size::ZERO - { - let mut flags = MemFlags::new(); - flags.set_notrap(); - - let to_addr = dst_ptr.get_addr(fx); - let from_addr = src_ptr.get_addr(fx); - let size = layout.size.bytes(); - - // `emit_small_memory_copy` uses `u8` for alignments, just use the maximum - // alignment that fits in a `u8` if the actual alignment is larger. - let dst_align = layout.align.bytes().try_into().unwrap_or(128); - let src_align = pointee_align.bytes().try_into().unwrap_or(128); - - fx.bcx.emit_small_memory_copy( - fx.target_config, - to_addr, - from_addr, - size, - dst_align, - src_align, - true, - flags, - ); + if underaligned_pointee_align.is_some() { + place.write_cvalue_transmute(fx, param); } else { place.write_cvalue(fx, param); } @@ -396,32 +359,8 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ { if let Some(param) = param { let field_place = place.place_field(fx, FieldIdx::new(i)); - let field_layout = field_place.layout(); - if let Some(pointee_align) = underaligned_pointee_align - && let Some(dst_ptr) = field_place.try_to_ptr() - && let Some((src_ptr, None)) = param.try_to_ptr() - && field_layout.size != Size::ZERO - { - let mut flags = MemFlags::new(); - flags.set_notrap(); - - let to_addr = dst_ptr.get_addr(fx); - let from_addr = src_ptr.get_addr(fx); - let size = field_layout.size.bytes(); - - let dst_align = field_layout.align.bytes().try_into().unwrap_or(128); - let src_align = pointee_align.bytes().try_into().unwrap_or(128); - - fx.bcx.emit_small_memory_copy( - fx.target_config, - to_addr, - from_addr, - size, - dst_align, - src_align, - true, - flags, - ); + if underaligned_pointee_align.is_some() { + field_place.write_cvalue_transmute(fx, param); } else { field_place.write_cvalue(fx, param); } diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 44b63aa95f..c04e4234b5 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -1,12 +1,13 @@ //! Argument passing use cranelift_codegen::ir::ArgumentPurpose; -use rustc_abi::{Reg, RegKind}; +use rustc_abi::{Align, Reg, RegKind}; use rustc_target::callconv::{ ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, PassMode, }; use smallvec::{SmallVec, smallvec}; +use super::ArgValue; use crate::prelude::*; use crate::value_and_place::assert_assignable; @@ -284,7 +285,7 @@ pub(super) fn cvalue_for_param<'tcx>( local_field: Option, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, block_params_iter: &mut impl Iterator, -) -> Option> { +) -> ArgValue<'tcx> { let block_params = arg_abi .get_abi_param(fx.tcx) .into_iter() @@ -305,7 +306,8 @@ pub(super) fn cvalue_for_param<'tcx>( arg_abi.layout, ); - match arg_abi.mode { + let mut underaligned_pointee_align: Option = None; + let value = match arg_abi.mode { PassMode::Ignore => None, PassMode::Direct(_) => { assert_eq!(block_params.len(), 1, "{:?}", block_params); @@ -318,9 +320,22 @@ pub(super) fn cvalue_for_param<'tcx>( PassMode::Cast { ref cast, .. } => { Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)) } - PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { + PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => { assert_eq!(block_params.len(), 1, "{:?}", block_params); - Some(CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout)) + if let Some(pointee_align) = attrs.pointee_align + && pointee_align < arg_abi.layout.align.abi + && arg_abi.layout.is_sized() + && arg_abi.layout.size != Size::ZERO + { + underaligned_pointee_align = Some(pointee_align); + // Underaligned pointer: treat as `[u8; size]` and transmute-copy into the real type. + let len = ty::Const::from_target_usize(fx.tcx, arg_abi.layout.size.bytes()); + let bytes_ty = fx.tcx.mk_ty_from_kind(ty::Array(fx.tcx.types.u8, len)); + let bytes_layout = fx.layout_of(bytes_ty); + Some(CValue::by_ref(Pointer::new(block_params[0]), bytes_layout)) + } else { + Some(CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout)) + } } PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { assert_eq!(block_params.len(), 2, "{:?}", block_params); @@ -330,5 +345,7 @@ pub(super) fn cvalue_for_param<'tcx>( arg_abi.layout, )) } - } + }; + + ArgValue { value, underaligned_pointee_align } } From d06181bd09d5decd2b97e312461bd5734a54e146 Mon Sep 17 00:00:00 2001 From: Rudraksh Joshi Date: Wed, 29 Apr 2026 15:18:34 +0530 Subject: [PATCH 4/5] fix underaligned byval pointee handling (review updates) --- src/abi/mod.rs | 20 ++++++++++++-------- src/abi/pass_mode.rs | 11 +++++------ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 50b78070a1..fcb6ecba1a 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -12,7 +12,7 @@ use cranelift_codegen::ir::{ }; use cranelift_codegen::isa::CallConv; use cranelift_module::ModuleError; -use rustc_abi::{Align, CanonAbi, ExternAbi, X86Call}; +use rustc_abi::{CanonAbi, ExternAbi, X86Call}; use rustc_codegen_ssa::base::is_call_from_compiler_builtins_to_upstream_monomorphization; use rustc_codegen_ssa::errors::CompilerBuiltinsCannotCall; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; @@ -33,7 +33,7 @@ use crate::prelude::*; pub(super) struct ArgValue<'tcx> { pub(super) value: Option>, - pub(super) underaligned_pointee_align: Option, + pub(super) is_underaligned_pointee: bool, } fn clif_sig_from_fn_abi<'tcx>( @@ -307,6 +307,10 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ // Store caller location for `#[track_caller]`. let arg_abi = arg_abis_iter.next().unwrap(); let arg = cvalue_for_param(fx, None, None, arg_abi, &mut block_params_iter); + assert!( + !arg.is_underaligned_pointee, + "caller location argument should not be underaligned", + ); fx.caller_location = Some(arg.value.unwrap()); } @@ -318,7 +322,7 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ for (local, arg_kind, ty) in func_params { // While this is normally an optimization to prevent an unnecessary copy when an argument is // not mutated by the current function, this is necessary to support unsized arguments. - if let ArgKind::Normal(ArgValue { value: Some(val), underaligned_pointee_align: None }) = + if let ArgKind::Normal(ArgValue { value: Some(val), is_underaligned_pointee: false }) = arg_kind { if let Some((addr, meta)) = val.try_to_ptr() { @@ -345,21 +349,21 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ assert_eq!(fx.local_map.push(place), local); match arg_kind { - ArgKind::Normal(ArgValue { value: Some(param), underaligned_pointee_align }) => { - if underaligned_pointee_align.is_some() { + ArgKind::Normal(ArgValue { value: Some(param), is_underaligned_pointee }) => { + if is_underaligned_pointee { place.write_cvalue_transmute(fx, param); } else { place.write_cvalue(fx, param); } } - ArgKind::Normal(ArgValue { value: None, underaligned_pointee_align: _ }) => {} + ArgKind::Normal(ArgValue { value: None, is_underaligned_pointee: _ }) => {} ArgKind::Spread(params) => { - for (i, ArgValue { value: param, underaligned_pointee_align }) in + for (i, ArgValue { value: param, is_underaligned_pointee }) in params.into_iter().enumerate() { if let Some(param) = param { let field_place = place.place_field(fx, FieldIdx::new(i)); - if underaligned_pointee_align.is_some() { + if is_underaligned_pointee { field_place.write_cvalue_transmute(fx, param); } else { field_place.write_cvalue(fx, param); diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index c04e4234b5..73986868c6 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -1,7 +1,7 @@ //! Argument passing use cranelift_codegen::ir::ArgumentPurpose; -use rustc_abi::{Align, Reg, RegKind}; +use rustc_abi::{Reg, RegKind}; use rustc_target::callconv::{ ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, PassMode, }; @@ -306,7 +306,7 @@ pub(super) fn cvalue_for_param<'tcx>( arg_abi.layout, ); - let mut underaligned_pointee_align: Option = None; + let mut is_underaligned_pointee = false; let value = match arg_abi.mode { PassMode::Ignore => None, PassMode::Direct(_) => { @@ -327,10 +327,9 @@ pub(super) fn cvalue_for_param<'tcx>( && arg_abi.layout.is_sized() && arg_abi.layout.size != Size::ZERO { - underaligned_pointee_align = Some(pointee_align); + is_underaligned_pointee = true; // Underaligned pointer: treat as `[u8; size]` and transmute-copy into the real type. - let len = ty::Const::from_target_usize(fx.tcx, arg_abi.layout.size.bytes()); - let bytes_ty = fx.tcx.mk_ty_from_kind(ty::Array(fx.tcx.types.u8, len)); + let bytes_ty = Ty::new_array(fx.tcx, fx.tcx.types.u8, arg_abi.layout.size.bytes()); let bytes_layout = fx.layout_of(bytes_ty); Some(CValue::by_ref(Pointer::new(block_params[0]), bytes_layout)) } else { @@ -347,5 +346,5 @@ pub(super) fn cvalue_for_param<'tcx>( } }; - ArgValue { value, underaligned_pointee_align } + ArgValue { value, is_underaligned_pointee } } From bfc183466062aece3f3d74deef8dad05b425e33c Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 30 Apr 2026 12:18:39 +0200 Subject: [PATCH 5/5] Couple of minor refactors --- src/abi/mod.rs | 57 +++++++++++++++++++++----------------------- src/abi/pass_mode.rs | 27 ++++++++++----------- 2 files changed, 39 insertions(+), 45 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index fcb6ecba1a..1321a96120 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -31,9 +31,9 @@ use crate::base::codegen_unwind_terminate; use crate::debuginfo::EXCEPTION_HANDLER_CLEANUP; use crate::prelude::*; -pub(super) struct ArgValue<'tcx> { - pub(super) value: Option>, - pub(super) is_underaligned_pointee: bool, +struct ArgValue<'tcx> { + value: CValue<'tcx>, + is_underaligned_pointee: bool, } fn clif_sig_from_fn_abi<'tcx>( @@ -248,9 +248,10 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ self::returning::codegen_return_param(fx, &ssa_analyzed, &mut block_params_iter); assert_eq!(fx.local_map.push(ret_place), RETURN_PLACE); + // None means pass_mode == NoPass enum ArgKind<'tcx> { - Normal(ArgValue<'tcx>), - Spread(Vec>), + Normal(Option>), + Spread(Vec>>), } // FIXME implement variadics in cranelift @@ -284,20 +285,17 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ let mut params = Vec::new(); for (i, _arg_ty) in tupled_arg_tys.iter().enumerate() { let arg_abi = arg_abis_iter.next().unwrap(); - params.push(cvalue_for_param( - fx, - Some(local), - Some(i), - arg_abi, - &mut block_params_iter, - )); + let param = + cvalue_for_param(fx, Some(local), Some(i), arg_abi, &mut block_params_iter); + params.push(param); } (local, ArgKind::Spread(params), arg_ty) } else { let arg_abi = arg_abis_iter.next().unwrap(); - let arg = cvalue_for_param(fx, Some(local), None, arg_abi, &mut block_params_iter); - (local, ArgKind::Normal(arg), arg_ty) + let param = + cvalue_for_param(fx, Some(local), None, arg_abi, &mut block_params_iter); + (local, ArgKind::Normal(param), arg_ty) } }) .collect::, Ty<'tcx>)>>(); @@ -306,12 +304,12 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ if fx.instance.def.requires_caller_location(fx.tcx) { // Store caller location for `#[track_caller]`. let arg_abi = arg_abis_iter.next().unwrap(); - let arg = cvalue_for_param(fx, None, None, arg_abi, &mut block_params_iter); + let param = cvalue_for_param(fx, None, None, arg_abi, &mut block_params_iter).unwrap(); assert!( - !arg.is_underaligned_pointee, + !param.is_underaligned_pointee, "caller location argument should not be underaligned", ); - fx.caller_location = Some(arg.value.unwrap()); + fx.caller_location = Some(param.value); } assert_eq!(arg_abis_iter.next(), None, "ArgAbi left behind for {:?}", fx.fn_abi); @@ -322,7 +320,7 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ for (local, arg_kind, ty) in func_params { // While this is normally an optimization to prevent an unnecessary copy when an argument is // not mutated by the current function, this is necessary to support unsized arguments. - if let ArgKind::Normal(ArgValue { value: Some(val), is_underaligned_pointee: false }) = + if let ArgKind::Normal(Some(ArgValue { value: val, is_underaligned_pointee: false })) = arg_kind { if let Some((addr, meta)) = val.try_to_ptr() { @@ -349,24 +347,23 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ assert_eq!(fx.local_map.push(place), local); match arg_kind { - ArgKind::Normal(ArgValue { value: Some(param), is_underaligned_pointee }) => { - if is_underaligned_pointee { - place.write_cvalue_transmute(fx, param); - } else { - place.write_cvalue(fx, param); + ArgKind::Normal(param) => { + if let Some(param) = param { + if param.is_underaligned_pointee { + place.write_cvalue_transmute(fx, param.value); + } else { + place.write_cvalue(fx, param.value); + } } } - ArgKind::Normal(ArgValue { value: None, is_underaligned_pointee: _ }) => {} ArgKind::Spread(params) => { - for (i, ArgValue { value: param, is_underaligned_pointee }) in - params.into_iter().enumerate() - { + for (i, param) in params.into_iter().enumerate() { if let Some(param) = param { let field_place = place.place_field(fx, FieldIdx::new(i)); - if is_underaligned_pointee { - field_place.write_cvalue_transmute(fx, param); + if param.is_underaligned_pointee { + field_place.write_cvalue_transmute(fx, param.value); } else { - field_place.write_cvalue(fx, param); + field_place.write_cvalue(fx, param.value); } } } diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 73986868c6..5a46e508a3 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -285,7 +285,7 @@ pub(super) fn cvalue_for_param<'tcx>( local_field: Option, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, block_params_iter: &mut impl Iterator, -) -> ArgValue<'tcx> { +) -> Option> { let block_params = arg_abi .get_abi_param(fx.tcx) .into_iter() @@ -306,19 +306,18 @@ pub(super) fn cvalue_for_param<'tcx>( arg_abi.layout, ); - let mut is_underaligned_pointee = false; let value = match arg_abi.mode { - PassMode::Ignore => None, + PassMode::Ignore => return None, PassMode::Direct(_) => { assert_eq!(block_params.len(), 1, "{:?}", block_params); - Some(CValue::by_val(block_params[0], arg_abi.layout)) + CValue::by_val(block_params[0], arg_abi.layout) } PassMode::Pair(_, _) => { assert_eq!(block_params.len(), 2, "{:?}", block_params); - Some(CValue::by_val_pair(block_params[0], block_params[1], arg_abi.layout)) + CValue::by_val_pair(block_params[0], block_params[1], arg_abi.layout) } PassMode::Cast { ref cast, .. } => { - Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)) + from_casted_value(fx, &block_params, arg_abi.layout, cast) } PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => { assert_eq!(block_params.len(), 1, "{:?}", block_params); @@ -327,24 +326,22 @@ pub(super) fn cvalue_for_param<'tcx>( && arg_abi.layout.is_sized() && arg_abi.layout.size != Size::ZERO { - is_underaligned_pointee = true; // Underaligned pointer: treat as `[u8; size]` and transmute-copy into the real type. let bytes_ty = Ty::new_array(fx.tcx, fx.tcx.types.u8, arg_abi.layout.size.bytes()); let bytes_layout = fx.layout_of(bytes_ty); - Some(CValue::by_ref(Pointer::new(block_params[0]), bytes_layout)) + return Some(ArgValue { + value: CValue::by_ref(Pointer::new(block_params[0]), bytes_layout), + is_underaligned_pointee: true, + }); } else { - Some(CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout)) + CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout) } } PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { assert_eq!(block_params.len(), 2, "{:?}", block_params); - Some(CValue::by_ref_unsized( - Pointer::new(block_params[0]), - block_params[1], - arg_abi.layout, - )) + CValue::by_ref_unsized(Pointer::new(block_params[0]), block_params[1], arg_abi.layout) } }; - ArgValue { value, is_underaligned_pointee } + Some(ArgValue { value, is_underaligned_pointee: false }) }