From 68f600d2b137a31b75c0224c60e5433f5df8564b Mon Sep 17 00:00:00 2001 From: Flakebi Date: Sun, 28 Dec 2025 02:48:15 +0100 Subject: [PATCH] Add AddrspacePtr for pointers to non-0 addrspaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A new lang item `addrspace_ptr_type` is added. It must be a `struct AddrspacePtr`. The content of this struct as written in the Rust source is ignored when computing the type layout. It is replaced by a pointer into address space `ADDRSPACE`. As the content of the struct is replaced inside the compiler, it cannot be “looked at” in the Rust source. To do something with the type (more than just holding it), rustc_intrinsics are used. Intrinsics are added to to - cast between `AddrspacePtr` and `*mut T`, this is just syntactical type conversion - cast to a different type `T` (a no-op, just type conversion) or cast to a different address space (`addrspacecast` in llvm) - convert to an integer (`ptrtoint` in llvm) - add to the offset (`ptr::[wrapping_]offset`, `getelementptr` in llvm) - and read/write from the pointer (`load/store` in llvm) This should satisfy all basic needs for operations on pointers. It is disallowed to deref an `AddrspacePtr` directly in Rust, read/write should be used instead. Rustc still uses deref internally to implement read/write. --- compiler/rustc_codegen_llvm/src/intrinsic.rs | 6 + .../rustc_codegen_ssa/src/mir/intrinsic.rs | 2 +- compiler/rustc_hir/src/lang_items.rs | 1 + .../rustc_hir_analysis/src/check/intrinsic.rs | 55 +++- compiler/rustc_hir_typeck/src/place_op.rs | 6 +- .../rustc_lint/src/types/improper_ctypes.rs | 3 + compiler/rustc_middle/src/ty/adt.rs | 11 + compiler/rustc_middle/src/ty/sty.rs | 20 ++ .../src/lower_intrinsics.rs | 6 +- compiler/rustc_mir_transform/src/shim.rs | 1 + compiler/rustc_mir_transform/src/validate.rs | 4 +- compiler/rustc_span/src/symbol.rs | 9 + .../src/traits/select/candidate_assembly.rs | 4 + .../src/traits/select/mod.rs | 2 + compiler/rustc_ty_utils/src/instance.rs | 1 + compiler/rustc_ty_utils/src/layout.rs | 15 ++ library/core/src/intrinsics/mod.rs | 133 ++++++++++ library/core/src/ptr/mod.rs | 233 ++++++++++++++++- tests/auxiliary/minicore.rs | 1 + tests/codegen-llvm/addrspace-ptr-basic.rs | 244 ++++++++++++++++++ 20 files changed, 747 insertions(+), 10 deletions(-) create mode 100644 tests/codegen-llvm/addrspace-ptr-basic.rs diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 5503f7689304b..91fa7002cf014 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -197,6 +197,12 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { &[ptr, args[1].immediate()], ) } + sym::addrspace_ptr_cast | sym::addrspace_ptr_from_ptr | sym::addrspace_ptr_to_ptr => { + self.pointercast(args[0].immediate(), result.layout.immediate_llvm_type(self.cx)) + } + sym::addrspace_ptr_to_addr => { + self.ptrtoint(args[0].immediate(), result.layout.immediate_llvm_type(self.cx)) + } sym::autodiff => { codegen_autodiff(self, tcx, instance, args, result); return Ok(()); diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index f4fae40d8828f..4501abcee5c06 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -186,7 +186,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } value } - sym::arith_offset => { + sym::arith_offset | sym::addrspace_ptr_arith_offset => { let ty = fn_args.type_at(0); let layout = bx.layout_of(ty); let ptr = args[0].immediate(); diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs index 4ac3e4e83e80a..e66e60938750c 100644 --- a/compiler/rustc_hir/src/lang_items.rs +++ b/compiler/rustc_hir/src/lang_items.rs @@ -186,6 +186,7 @@ language_item_table! { PointeeTrait, sym::pointee_trait, pointee_trait, Target::Trait, GenericRequirement::None; Metadata, sym::metadata_type, metadata_type, Target::AssocTy, GenericRequirement::None; DynMetadata, sym::dyn_metadata, dyn_metadata, Target::Struct, GenericRequirement::None; + AddrspacePtr, sym::addrspace_ptr_type, addrspace_ptr_type, Target::Struct, GenericRequirement::Exact(2); Freeze, sym::freeze, freeze_trait, Target::Trait, GenericRequirement::Exact(0); UnsafeUnpin, sym::unsafe_unpin, unsafe_unpin_trait, Target::Trait, GenericRequirement::Exact(0); diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index 4e8333f678b66..ecf2b73743d47 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -4,7 +4,7 @@ use rustc_abi::ExternAbi; use rustc_errors::DiagMessage; use rustc_hir::{self as hir, LangItem}; use rustc_middle::traits::{ObligationCause, ObligationCauseCode}; -use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_middle::ty::{self, Const, Ty, TyCtxt}; use rustc_span::def_id::LocalDefId; use rustc_span::{Span, Symbol, sym}; @@ -239,8 +239,8 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi /// Remember to add all intrinsics here, in `compiler/rustc_codegen_llvm/src/intrinsic.rs`, /// and in `library/core/src/intrinsics.rs`. -pub(crate) fn check_intrinsic_type( - tcx: TyCtxt<'_>, +pub(crate) fn check_intrinsic_type<'tcx>( + tcx: TyCtxt<'tcx>, intrinsic_id: LocalDefId, span: Span, intrinsic_name: Symbol, @@ -255,6 +255,15 @@ pub(crate) fn check_intrinsic_type( Ty::new_error_with_message(tcx, span, "expected param") } }; + let const_param = |n| { + if let &ty::GenericParamDef { name, kind: ty::GenericParamDefKind::Const { .. }, .. } = + generics.param_at(n as usize, tcx) + { + Const::new_param(tcx, ty::ParamConst::new(n, name)) + } else { + Const::new_error_with_message(tcx, span, "expected const param") + } + }; let bound_vars = tcx.mk_bound_variable_kinds(&[ ty::BoundVariableKind::Region(ty::BoundRegionKind::Anon), @@ -280,9 +289,49 @@ pub(crate) fn check_intrinsic_type( (Ty::new_ref(tcx, env_region, va_list_ty, mutbl), va_list_ty) }; + let mk_u32_const = + |i: u32| Const::new_value(tcx, ty::ValTree::from_scalar_int(tcx, i.into()), tcx.types.u32); + let mk_addrspace_ptr = |t: Ty<'tcx>, addrspace: Const<'tcx>| { + tcx.type_of(tcx.lang_items().addrspace_ptr_type().unwrap()) + .instantiate(tcx, &[t.into(), addrspace.into()]) + }; + let safety = intrinsic_operation_unsafety(tcx, intrinsic_id); let n_lts = 0; let (n_tps, n_cts, inputs, output) = match intrinsic_name { + sym::addrspace_ptr_arith_offset | sym::addrspace_ptr_offset => ( + 1, + 1, + vec![mk_addrspace_ptr(param(0), const_param(1)), tcx.types.isize], + mk_addrspace_ptr(param(0), const_param(1)), + ), + sym::addrspace_ptr_cast => ( + 2, + 2, + vec![mk_addrspace_ptr(param(0), const_param(2))], + mk_addrspace_ptr(param(1), const_param(3)), + ), + sym::addrspace_ptr_to_addr => { + (1, 1, vec![mk_addrspace_ptr(tcx.types.unit, const_param(1))], param(0)) + } + sym::addrspace_ptr_from_ptr => ( + 1, + 0, + vec![Ty::new_mut_ptr(tcx, param(0).into())], + mk_addrspace_ptr(param(0), mk_u32_const(0)), + ), + sym::addrspace_ptr_to_ptr => ( + 1, + 0, + vec![mk_addrspace_ptr(param(0), mk_u32_const(0))], + Ty::new_mut_ptr(tcx, param(0)), + ), + sym::addrspace_ptr_read_via_copy => { + (1, 1, vec![mk_addrspace_ptr(param(0), const_param(1))], param(0)) + } + sym::addrspace_ptr_write_via_move => { + (1, 1, vec![mk_addrspace_ptr(param(0), const_param(1)), param(0)], tcx.types.unit) + } sym::autodiff => (4, 0, vec![param(0), param(1), param(2)], param(3)), sym::abort => (0, 0, vec![], tcx.types.never), sym::unreachable => (0, 0, vec![], tcx.types.never), diff --git a/compiler/rustc_hir_typeck/src/place_op.rs b/compiler/rustc_hir_typeck/src/place_op.rs index a48db2cc855c0..40e52914aec37 100644 --- a/compiler/rustc_hir_typeck/src/place_op.rs +++ b/compiler/rustc_hir_typeck/src/place_op.rs @@ -23,7 +23,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { oprnd_expr: &'tcx hir::Expr<'tcx>, oprnd_ty: Ty<'tcx>, ) -> Option> { - if let Some(ty) = oprnd_ty.builtin_deref(true) { + // AddrspacePtr deref is used internally but it should not be accessed from the Rust, + // so exclude it here. + if let Some(ty) = oprnd_ty.builtin_deref(true) + && !oprnd_ty.is_addrspace_ptr() + { return Some(ty); } diff --git a/compiler/rustc_lint/src/types/improper_ctypes.rs b/compiler/rustc_lint/src/types/improper_ctypes.rs index 38094c67c34a0..cb5d2d405d000 100644 --- a/compiler/rustc_lint/src/types/improper_ctypes.rs +++ b/compiler/rustc_lint/src/types/improper_ctypes.rs @@ -468,6 +468,9 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> { if def.is_phantom_data() { return FfiPhantom(ty); } + if def.is_addrspace_ptr() { + return FfiSafe; + } match def.adt_kind() { AdtKind::Struct | AdtKind::Union => { if let Some(sym::cstring_type | sym::cstr_type) = diff --git a/compiler/rustc_middle/src/ty/adt.rs b/compiler/rustc_middle/src/ty/adt.rs index 510c546f82a4e..e1cf397288048 100644 --- a/compiler/rustc_middle/src/ty/adt.rs +++ b/compiler/rustc_middle/src/ty/adt.rs @@ -59,6 +59,8 @@ bitflags::bitflags! { const IS_PIN = 1 << 11; /// Indicates whether the type is `#[pin_project]`. const IS_PIN_PROJECT = 1 << 12; + /// Indicates whether the type is `AddrspacePtr`. + const IS_ADDRSPACE_PTR = 1 << 13; } } rustc_data_structures::external_bitflags_debug! { AdtFlags } @@ -324,6 +326,9 @@ impl AdtDefData { if tcx.is_lang_item(did, LangItem::Pin) { flags |= AdtFlags::IS_PIN; } + if tcx.is_lang_item(did, LangItem::AddrspacePtr) { + flags |= AdtFlags::IS_ADDRSPACE_PTR; + } AdtDefData { did, variants, flags, repr } } @@ -445,6 +450,12 @@ impl<'tcx> AdtDef<'tcx> { self.flags().contains(AdtFlags::IS_PIN) } + /// Returns `true` if this is `AddrspacePtr`. + #[inline] + pub fn is_addrspace_ptr(self) -> bool { + self.flags().contains(AdtFlags::IS_ADDRSPACE_PTR) + } + /// Returns `true` is this is `#[pin_v2]` for the purposes /// of structural pinning. #[inline] diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index c3834607e9236..669de60238508 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -1339,6 +1339,14 @@ impl<'tcx> Ty<'tcx> { } } + #[inline] + pub fn is_addrspace_ptr(self) -> bool { + match self.kind() { + Adt(def, _) => def.is_addrspace_ptr(), + _ => false, + } + } + /// Tests whether this is a Box definitely using the global allocator. /// /// If the allocator is still generic, the answer is `false`, but it may @@ -1366,6 +1374,13 @@ impl<'tcx> Ty<'tcx> { } } + pub fn addrspace_ptr_ty(self) -> Option> { + match self.kind() { + Adt(def, args) if def.is_addrspace_ptr() => Some(args.type_at(0)), + _ => None, + } + } + pub fn pinned_ty(self) -> Option> { match self.kind() { Adt(def, args) if def.is_pin() => Some(args.type_at(0)), @@ -1566,6 +1581,11 @@ impl<'tcx> Ty<'tcx> { pub fn builtin_deref(self, explicit: bool) -> Option> { match *self.kind() { _ if let Some(boxed) = self.boxed_ty() => Some(boxed), + _ if let Some(ty) = self.addrspace_ptr_ty() + && explicit => + { + Some(ty) + } Ref(_, ty, _) => Some(ty), RawPtr(ty, _) if explicit => Some(ty), _ => None, diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index dcee54c371080..613ea1e516593 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -145,7 +145,7 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { )); terminator.kind = TerminatorKind::Goto { target }; } - sym::read_via_copy => { + sym::read_via_copy | sym::addrspace_ptr_read_via_copy => { let Ok([arg]) = take_array(args) else { span_bug!(terminator.source_info.span, "Wrong number of arguments"); }; @@ -177,7 +177,7 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { Some(target) => TerminatorKind::Goto { target }, } } - sym::write_via_move => { + sym::write_via_move | sym::addrspace_ptr_write_via_move => { let target = target.unwrap(); let Ok([ptr, val]) = take_array(args) else { span_bug!( @@ -220,7 +220,7 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { )); terminator.kind = TerminatorKind::Goto { target }; } - sym::offset => { + sym::offset | sym::addrspace_ptr_offset => { let target = target.unwrap(); let Ok([ptr, delta]) = take_array(args) else { span_bug!( diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index 85e340c0a02ab..ac8a60aabaad7 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -543,6 +543,7 @@ fn build_clone_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) - match self_ty.kind() { ty::FnDef(..) | ty::FnPtr(..) => builder.copy_shim(), + ty::Adt(def, _) if def.is_addrspace_ptr() => builder.copy_shim(), ty::Closure(_, args) => builder.tuple_like_shim(dest, src, args.as_closure().upvar_tys()), ty::CoroutineClosure(_, args) => { builder.tuple_like_shim(dest, src, args.as_coroutine_closure().upvar_tys()) diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index 1afdb4639a0ce..521c83cdd74c3 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -1178,7 +1178,9 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { match op { Offset => { - check_kinds!(a, "Cannot offset non-pointer type {:?}", ty::RawPtr(..)); + if !a.is_addrspace_ptr() { + check_kinds!(a, "Cannot offset non-pointer type {:?}", ty::RawPtr(..)); + } if b != self.tcx.types.isize && b != self.tcx.types.usize { self.fail(location, format!("Cannot offset by non-isize type {b}")); } diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 0e30abccb62b3..04c638ab4a0a8 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -428,6 +428,15 @@ symbols! { add_assign, add_with_overflow, address, + addrspace_ptr_arith_offset, + addrspace_ptr_cast, + addrspace_ptr_from_ptr, + addrspace_ptr_offset, + addrspace_ptr_read_via_copy, + addrspace_ptr_to_addr, + addrspace_ptr_to_ptr, + addrspace_ptr_type, + addrspace_ptr_write_via_move, adt_const_params, advanced_slice_patterns, adx_target_feature, diff --git a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs index d0833f0308350..0474450e6ea49 100644 --- a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs +++ b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs @@ -1207,6 +1207,10 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { candidates.vec.push(SizedCandidate); } + ty::Adt(def, _) if def.is_addrspace_ptr() => { + candidates.vec.push(BuiltinCandidate); + } + // Fallback to whatever user-defined impls or param-env clauses exist in this case. ty::Adt(..) | ty::Alias(..) | ty::Param(..) | ty::Placeholder(..) => {} diff --git a/compiler/rustc_trait_selection/src/traits/select/mod.rs b/compiler/rustc_trait_selection/src/traits/select/mod.rs index ea903bac9d617..72b05939d443a 100644 --- a/compiler/rustc_trait_selection/src/traits/select/mod.rs +++ b/compiler/rustc_trait_selection/src/traits/select/mod.rs @@ -2256,6 +2256,8 @@ impl<'tcx> SelectionContext<'_, 'tcx> { ty::Binder::dummy(args.as_coroutine_closure().upvar_tys().to_vec()) } + ty::Adt(def, args) if def.is_addrspace_ptr() => ty::Binder::dummy(vec![]), + ty::Foreign(..) | ty::Str | ty::Slice(_) diff --git a/compiler/rustc_ty_utils/src/instance.rs b/compiler/rustc_ty_utils/src/instance.rs index 23bbd9ca6d639..82cebdcd3bf04 100644 --- a/compiler/rustc_ty_utils/src/instance.rs +++ b/compiler/rustc_ty_utils/src/instance.rs @@ -268,6 +268,7 @@ fn resolve_associated_item<'tcx>( | ty::Closure(..) | ty::CoroutineClosure(..) | ty::Tuple(..) => {} + ty::Adt(def, _) if def.is_addrspace_ptr() => {} _ => return Ok(None), }; diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs index 62f3667ad7f4f..effd6defbbbcf 100644 --- a/compiler/rustc_ty_utils/src/layout.rs +++ b/compiler/rustc_ty_utils/src/layout.rs @@ -643,6 +643,21 @@ fn layout_of_uncached<'tcx>( // ADTs. ty::Adt(def, args) => { + if def.is_addrspace_ptr() { + // Implement the AddrspacePtr lang item struct as a pointer into the + // address space from the const generic argument. + let pointee = args.type_at(0); + let addrspace = extract_const_value(cx, ty, args.const_at(1))? + .try_to_bits(tcx, ty::TypingEnv::fully_monomorphized()) + .ok_or_else(|| error(cx, LayoutError::Unknown(ty)))? + as u32; + let data_ptr = scalar_unit(Pointer(AddressSpace(addrspace))); + if !pointee.is_sized(tcx, cx.typing_env) { + panic!("AddrspacePtr pointee must be sized"); + } + return Ok(tcx.mk_layout(LayoutData::scalar(cx, data_ptr))); + } + // Cache the field layouts. let variants = def .variants() diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index 2179b451c375e..1f58207ae7261 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -56,6 +56,7 @@ use crate::ffi::va_list::{VaArgSafe, VaList}; use crate::marker::{ConstParamTy, Destruct, DiscriminantKind, PointeeSized, Tuple}; +use crate::ptr::AddrspacePtr; use crate::{mem, ptr}; mod bounds; @@ -910,6 +911,138 @@ pub const unsafe fn offset(dst: Ptr, offset: D #[rustc_intrinsic] pub const unsafe fn arith_offset(dst: *const T, offset: isize) -> *const T; +/// Casts a pointer to a pointer in a different address space. +/// +/// A pointer into one address space can sometimes be casted to a pointer into a different address space. +/// The target defines for which combination of address spaces this is possible. +/// +/// This intrinsic also allows casting the pointed-to type of the pointer. +/// +/// # Safety +/// +/// The cast must be supported by the target, casting between address spaces that do not support it +/// is undefined behavior and can fail to compile. +#[must_use = "returns a new pointer rather than modifying its argument"] +#[unstable(feature = "ptr_addrspace", issue = "none")] +#[rustc_intrinsic] +#[rustc_nounwind] +pub unsafe fn addrspace_ptr_cast< + T: 'static, + U: 'static, + const SOURCE_ADDRSPACE: u32, + const TARGET_ADDRSPACE: u32, +>( + ptr: AddrspacePtr, +) -> AddrspacePtr; + +/// Converts a raw pointer into a pointer with an explicit address space. +/// +/// If the targets raw pointer address space is the generic address space, this is a no-op, +/// just casting the type. +/// +/// # Safety +/// +/// If the targets raw pointer address space matches the generic address space, this function +/// is a no-op and is always safe to call. +/// Otherwise, if the address spaces do not match, the safety conditions of [`addrspace_ptr_cast`] apply. +#[must_use = "returns a new pointer rather than modifying its argument"] +#[unstable(feature = "ptr_addrspace", issue = "none")] +#[rustc_intrinsic] +#[rustc_nounwind] +pub unsafe fn addrspace_ptr_from_ptr( + ptr: *mut T, +) -> AddrspacePtr; + +/// Converts a pointer with an explicit address space into a raw pointer. +/// +/// If the targets raw pointer address space is the generic address space, this is a no-op, +/// just casting the type. +/// +/// # Safety +/// +/// If the targets raw pointer address space matches the generic address space, this function +/// is a no-op and is always safe to call. +/// Otherwise, if the address spaces do not match, the safety conditions of [`addrspace_ptr_cast`] apply. +#[must_use = "returns a new pointer rather than modifying its argument"] +#[unstable(feature = "ptr_addrspace", issue = "none")] +#[rustc_intrinsic] +#[rustc_nounwind] +pub unsafe fn addrspace_ptr_to_ptr( + ptr: AddrspacePtr, +) -> *mut T; + +/// Gets the "address" portion of a pointer. +/// +/// The return type `T` must be an integer of the same size as the pointer. +/// The size depends on the target and can differ between address spaces. +/// This does not expose the provenance of the pointer. +/// +/// See also [`pointer::addr`] for the raw pointer equivalent. +/// +/// # Safety +/// +/// Not all address spaces have pointers that can be converted to integers. +/// Calling this intrinsic for non-integral pointers results in undefined behavior. +/// +/// The returned type must be an integer matching the size of the pointer, otherwise calling +/// this intrinsic results in undefined behavior. +#[must_use = "returns the address and does nothing unless used"] +#[unstable(feature = "ptr_addrspace", issue = "none")] +#[rustc_intrinsic] +#[rustc_nounwind] +pub unsafe fn addrspace_ptr_to_addr(ptr: AddrspacePtr<(), ADDRSPACE>) +-> T; + +/// Adds a signed offset to a pointer. +/// +/// `count` is in units of `T`. +/// +/// See [`pointer::offset`] for details and safety concerns. +#[must_use = "returns a new pointer rather than modifying its argument"] +#[unstable(feature = "ptr_addrspace", issue = "none")] +#[rustc_intrinsic] +#[rustc_nounwind] +pub unsafe fn addrspace_ptr_offset( + ptr: AddrspacePtr, + count: isize, +) -> AddrspacePtr; + +/// Adds a signed offset to a pointer using wrapping arithmetic. +/// +/// `count` is in units of `T`. +/// +/// See [`pointer::wrapping_offset`] for details and safety concerns. +#[must_use = "returns a new pointer rather than modifying its argument"] +#[unstable(feature = "ptr_addrspace", issue = "none")] +#[rustc_intrinsic] +#[rustc_nounwind] +pub unsafe fn addrspace_ptr_arith_offset( + ptr: AddrspacePtr, + count: isize, +) -> AddrspacePtr; + +/// Reads the value from `self` without moving it. This leaves the memory in `self` unchanged. +/// +/// See [`ptr::read`] for safety concerns and examples. +#[must_use = "returns the read value and does nothing unless used"] +#[unstable(feature = "ptr_addrspace", issue = "none")] +#[rustc_intrinsic] +#[rustc_nounwind] +pub unsafe fn addrspace_ptr_read_via_copy( + ptr: AddrspacePtr, +) -> T; + +/// Overwrites a memory location with the given value without reading or dropping the old value. +/// +/// See [`ptr::write`] for safety concerns and examples. +#[unstable(feature = "ptr_addrspace", issue = "none")] +#[rustc_intrinsic] +#[rustc_nounwind] +pub unsafe fn addrspace_ptr_write_via_move( + ptr: AddrspacePtr, + value: T, +); + /// Projects to the `index`-th element of `slice_ptr`, as the same kind of pointer /// as the slice was provided -- so `&mut [T] → &mut T`, `&[T] → &T`, /// `*mut [T] → *mut T`, or `*const [T] → *const T` -- without a bounds check. diff --git a/library/core/src/ptr/mod.rs b/library/core/src/ptr/mod.rs index 29fe77390a16b..7fab9849064f0 100644 --- a/library/core/src/ptr/mod.rs +++ b/library/core/src/ptr/mod.rs @@ -403,7 +403,7 @@ use crate::cmp::Ordering; use crate::intrinsics::const_eval_select; -use crate::marker::{Destruct, FnPtr, PointeeSized}; +use crate::marker::{Destruct, FnPtr, PhantomData, PointeeSized}; use crate::mem::{self, MaybeUninit, SizedTypeProperties}; use crate::num::NonZero; use crate::{fmt, hash, intrinsics, ub_checks}; @@ -427,6 +427,237 @@ pub use unique::Unique; mod const_ptr; mod mut_ptr; +/// The list of address spaces. +/// +/// See [`AddrspacePtr`] for using pointers with an explicit associated address space. +/// +/// This module lists the address spaces for the current target. +#[unstable(feature = "ptr_addrspace", issue = "none")] +pub mod addrspace { + /// The default address space. + /// + /// On most targets, a raw pointer (`*mut/const T`) implicitly points into the generic address space. + #[unstable(feature = "ptr_addrspace", issue = "none")] + pub const GENERIC: u32 = 0; + + /// The address space corresponding to VRAM on GPUs. + /// + /// It is readable and writable from the GPU and CPU. + /// Pointers into this address space can be casted to and from the generic address space. + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[cfg(any(doc, target_arch = "amdgpu", target_arch = "nvptx64"))] + #[doc(cfg(any(target_arch = "amdgpu", target_arch = "nvptx64")))] + pub const GLOBAL: u32 = 1; + + /// The address space shared between threads within the same workgroup on GPUs. + /// + /// It is readable and writable from the GPU. + /// Pointers into this address space can be casted to and from the generic address space. + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[cfg(any(doc, target_arch = "amdgpu", target_arch = "nvptx64"))] + #[doc(cfg(any(target_arch = "amdgpu", target_arch = "nvptx64")))] + pub const WORKGROUP: u32 = 3; + + /// The address space for constant memory on GPUs. + /// + /// It is readable from the GPU but not writable. + /// Pointers into this address space can be casted to and from the generic address space. + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[cfg(any(doc, target_arch = "amdgpu", target_arch = "nvptx64"))] + #[doc(cfg(any(target_arch = "amdgpu", target_arch = "nvptx64")))] + pub const CONST: u32 = 4; +} + +/// A pointer into a target-specific address space. +/// +/// An address space is a special, target-specific memory region. +/// Each pointer has an associated address space. If no address space is explicitly mentioned, it +/// is usually the generic address space. When accessing low-level hardware capabilities or +/// intrinsics, it is sometimes necessary to explicitly specify a different address space. +/// For these cases, `AddrspacePointer` can be used. +/// +/// See also the [pointer](pointer) primitive types and the [`addrspace`] module. +#[unstable(feature = "ptr_addrspace", issue = "none")] +#[lang = "addrspace_ptr_type"] +#[allow(missing_debug_implementations)] +pub struct AddrspacePtr { + // Struct implementation is replaced by the compiler. + // This field is here for using the generic argument but cannot be set or accessed in any way. + do_not_use: PhantomData<*const T>, +} + +impl AddrspacePtr { + /// Gets the "address" portion of a pointer. + /// + /// The return type `U` must be an integer of the same size as the pointer. + /// The size depends on the target and can differ between address spaces. + /// This does not expose the provenance of the pointer. + /// + /// See also [`pointer::addr`] for the raw pointer equivalent. + /// + /// # Safety + /// + /// Not all address spaces have pointers that can be converted to integers. + /// Calling this intrinsic for non-integral pointers results in undefined behavior. + /// + /// The returned type must be an integer matching the size of the pointer, otherwise calling + /// this intrinsic results in undefined behavior. + #[must_use = "returns the address and does nothing unless used"] + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[inline(always)] + pub unsafe fn addr(self) -> U { + // SAFETY: the safety contract for `addrspace_ptr_to_addr` must be + // upheld by the caller. + unsafe { crate::intrinsics::addrspace_ptr_to_addr(self.cast::<()>()) } + } + + /// Cast to a pointer of different type. + #[must_use = "returns a new pointer rather than modifying its argument"] + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[inline(always)] + pub fn cast(self) -> AddrspacePtr { + // SAFETY: Only the pointed-to type is converted, making this a no-op just converting the type. + unsafe { crate::intrinsics::addrspace_ptr_cast(self) } + } + + /// Casts a pointer to a pointer in a different address space. + /// + /// A pointer into one address space can sometimes be casted to a pointer into a different address space. + /// The target defines for which combination of address spaces this is possible. + /// + /// # Safety + /// + /// The cast must be supported by the target, casting between address spaces that do not support it + /// is undefined behavior and can fail to compile. + #[must_use = "returns a new pointer rather than modifying its argument"] + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[inline(always)] + pub unsafe fn cast_addrspace( + self, + ) -> AddrspacePtr { + // SAFETY: the safety contract for `addrspace_ptr_cast` must be upheld by the caller. + unsafe { crate::intrinsics::addrspace_ptr_cast(self) } + } + + /// Adds a signed offset to a pointer. + /// + /// `count` is in units of `T`. + /// + /// See [`pointer::offset`] for details and safety concerns. + #[must_use = "returns a new pointer rather than modifying its argument"] + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[inline(always)] + #[track_caller] + pub unsafe fn offset(self, count: isize) -> Self { + // We cannot convert the addrspace ptr to an integer as it could be a non-integra pointer, + // so just check that the multiplication does not overflow. + #[inline] + #[rustc_allow_const_fn_unstable(const_eval_select)] + const fn runtime_offset_nowrap(count: isize, size: usize) -> bool { + // We can use const_eval_select here because this is only for UB checks. + const_eval_select!( + @capture { count: isize, size: usize } -> bool: + if const { + true + } else { + // `size` is the size of a Rust type, so we know that + // `size <= isize::MAX` and thus `as` cast here is not lossy. + count.checked_mul(size as isize).is_some() + } + ) + } + + ub_checks::assert_unsafe_precondition!( + check_language_ub, + "AddrspacePtr::offset requires the address calculation to not overflow", + ( + count: isize = count, + size: usize = size_of::(), + ) => runtime_offset_nowrap(count, size) + ); + + // SAFETY: the safety contract for `addrspace_ptr_offset` must be + // upheld by the caller. + unsafe { crate::intrinsics::addrspace_ptr_offset(self, count) } + } + + /// Adds a signed offset to a pointer using wrapping arithmetic. + /// + /// `count` is in units of `T`. + /// + /// See [`pointer::wrapping_offset`] for details and safety concerns. + #[must_use = "returns a new pointer rather than modifying its argument"] + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[inline(always)] + pub fn wrapping_offset(self, count: isize) -> Self { + // SAFETY: the `addrspace_ptr_arith_offset` intrinsic has no prerequisites to be called. + unsafe { crate::intrinsics::addrspace_ptr_arith_offset(self, count) } + } + + /// Reads the value from `self` without moving it. This leaves the memory in `self` unchanged. + /// + /// See [`core::ptr::read`] for safety concerns and examples. + #[must_use = "returns the read value and does nothing unless used"] + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[inline(always)] + pub unsafe fn read(self) -> T { + // SAFETY: the safety contract for `addrspace_ptr_read_via_copy` must be + // upheld by the caller. + unsafe { crate::intrinsics::addrspace_ptr_read_via_copy(self) } + } + + /// Overwrites a memory location with the given value without reading or dropping the old value. + /// + /// See [`core::ptr::write`] for safety concerns and examples. + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[inline(always)] + pub unsafe fn write(self, value: T) { + // SAFETY: the safety contract for `addrspace_ptr_write_via_move` must be + // upheld by the caller. + unsafe { crate::intrinsics::addrspace_ptr_write_via_move(self, value) } + } +} + +impl AddrspacePtr { + /// Converts a raw pointer into a pointer with an explicit address space. + /// + /// If the targets raw pointer address space is the generic address space, this is a no-op, + /// just casting the type. + /// + /// # Safety + /// + /// If the targets raw pointer address space matches the generic address space, this function + /// is a no-op and is always safe to call. + /// Otherwise, if the address spaces do not match, the safety conditions of [`AddrspacePtr::cast_addrspace`] apply. + #[must_use = "returns a new pointer rather than modifying its argument"] + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[inline(always)] + pub unsafe fn from_ptr(ptr: *mut T) -> Self { + // SAFETY: the safety contract for `addrspace_ptr_from_ptr` must be + // upheld by the caller. + unsafe { crate::intrinsics::addrspace_ptr_from_ptr(ptr) } + } + + /// Converts a pointer with an explicit address space into a raw pointer. + /// + /// If the targets raw pointer address space is the generic address space, this is a no-op, + /// just casting the type. + /// + /// # Safety + /// + /// If the targets raw pointer address space matches the generic address space, this function + /// is a no-op and is always safe to call. + /// Otherwise, if the address spaces do not match, the safety conditions of [`AddrspacePtr::cast_addrspace`] apply. + #[must_use = "returns a new pointer rather than modifying its argument"] + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[inline(always)] + pub unsafe fn as_ptr(self) -> *mut T { + // SAFETY: the safety contract for `addrspace_ptr_to_ptr` must be + // upheld by the caller. + unsafe { crate::intrinsics::addrspace_ptr_to_ptr(self) } + } +} + // Some functions are defined here because they accidentally got made // available in this module on stable. See . // (`transmute` also falls into this category, but it cannot be wrapped due to the diff --git a/tests/auxiliary/minicore.rs b/tests/auxiliary/minicore.rs index 288a5b50dd5ef..bfc4db6b6cf1c 100644 --- a/tests/auxiliary/minicore.rs +++ b/tests/auxiliary/minicore.rs @@ -293,3 +293,4 @@ pub enum SimdAlign { } impl ConstParamTy_ for SimdAlign {} +impl ConstParamTy_ for u32 {} diff --git a/tests/codegen-llvm/addrspace-ptr-basic.rs b/tests/codegen-llvm/addrspace-ptr-basic.rs new file mode 100644 index 0000000000000..a80053b0c909d --- /dev/null +++ b/tests/codegen-llvm/addrspace-ptr-basic.rs @@ -0,0 +1,244 @@ +//! Checks the basic usage pointers into the non-generic address space. + +//@ add-minicore +//@ compile-flags: --target=amdgcn-amd-amdhsa -Ctarget-cpu=gfx900 +//@ needs-llvm-components: amdgpu + +#![crate_type = "lib"] +#![no_core] +#![feature(abi_unadjusted, intrinsics, lang_items, link_llvm_intrinsics, no_core, rustc_attrs)] + +extern crate minicore; +use minicore::*; + +#[rustc_intrinsic] +const unsafe fn size_of_val(x: *const T) -> usize; + +mod addrspace { + pub const GENERIC: u32 = 0; + pub const WORKGROUP: u32 = 3; + pub const CONST: u32 = 4; +} + +#[rustc_intrinsic] +#[rustc_nounwind] +unsafe fn addrspace_ptr_cast< + T: 'static, + U: 'static, + const SOURCE_ADDRSPACE: u32, + const TARGET_ADDRSPACE: u32, +>( + ptr: AddrspacePtr, +) -> AddrspacePtr; + +#[rustc_intrinsic] +#[rustc_nounwind] +unsafe fn addrspace_ptr_from_ptr( + ptr: *mut T, +) -> AddrspacePtr; + +#[rustc_intrinsic] +#[rustc_nounwind] +unsafe fn addrspace_ptr_to_ptr(ptr: AddrspacePtr) -> *mut T; + +#[rustc_intrinsic] +#[rustc_nounwind] +unsafe fn addrspace_ptr_to_addr(ptr: AddrspacePtr<(), ADDRSPACE>) -> T; + +#[rustc_intrinsic] +#[rustc_nounwind] +unsafe fn addrspace_ptr_offset( + ptr: AddrspacePtr, + count: isize, +) -> AddrspacePtr; + +#[rustc_intrinsic] +#[rustc_nounwind] +unsafe fn addrspace_ptr_arith_offset( + ptr: AddrspacePtr, + count: isize, +) -> AddrspacePtr; + +#[rustc_intrinsic] +#[rustc_nounwind] +unsafe fn addrspace_ptr_read_via_copy( + ptr: AddrspacePtr, +) -> T; + +#[rustc_intrinsic] +#[rustc_nounwind] +unsafe fn addrspace_ptr_write_via_move( + ptr: AddrspacePtr, + value: T, +); + +#[repr(C)] +struct DispatchPacket { + header: u16, +} + +extern "unadjusted" { + #[link_name = "llvm.amdgcn.dispatch.ptr"] + fn dispatch_ptr() -> AddrspacePtr; +} + +#[lang = "addrspace_ptr_type"] +pub struct AddrspacePtr { + // Struct implementation is replaced by compiler. + // This field is here for using the generic arguments but cannot be set or used in any way. + do_not_use: PhantomData<*const T>, +} + +impl AddrspacePtr { + unsafe fn addr(self) -> U { + unsafe { addrspace_ptr_to_addr(self.cast::<()>()) } + } + + unsafe fn cast(self) -> AddrspacePtr { + unsafe { addrspace_ptr_cast(self) } + } + + unsafe fn cast_addrspace( + self, + ) -> AddrspacePtr { + unsafe { addrspace_ptr_cast(self) } + } + + unsafe fn offset(self, count: isize) -> Self { + unsafe { addrspace_ptr_offset(self, count) } + } + + unsafe fn wrapping_offset(self, count: isize) -> Self { + unsafe { addrspace_ptr_arith_offset(self, count) } + } + + unsafe fn read(self) -> T { + unsafe { addrspace_ptr_read_via_copy(self) } + } + + unsafe fn write(self, value: T) { + unsafe { addrspace_ptr_write_via_move(self, value) } + } +} + +impl AddrspacePtr { + unsafe fn from_ptr(ptr: *mut T) -> Self { + unsafe { addrspace_ptr_from_ptr(ptr) } + } + + unsafe fn as_ptr(self) -> *mut T { + unsafe { addrspace_ptr_to_ptr(self) } + } +} + +#[unsafe(no_mangle)] +fn addr(ptr: AddrspacePtr) -> u32 { + // CHECK-LABEL: @addr + // CHECK: %[[val:[^ ]+]] = ptrtoint ptr addrspace(3) %ptr to i32 + // CHECK: ret i32 %[[val]] + unsafe { ptr.addr() } +} + +#[unsafe(no_mangle)] +fn cast( + ptr: AddrspacePtr, +) -> AddrspacePtr { + // CHECK-LABEL: @cast + // CHECK: ret ptr addrspace(3) %ptr + unsafe { ptr.cast() } +} + +#[unsafe(no_mangle)] +fn cast_addrspace( + ptr: AddrspacePtr, +) -> AddrspacePtr { + // CHECK-LABEL: @cast_addrspace + // CHECK: %[[val:[^ ]+]] = addrspacecast ptr addrspace(3) %ptr to ptr + // CHECK: ret ptr %[[val]] + unsafe { ptr.cast_addrspace() } +} + +#[unsafe(no_mangle)] +fn offset( + ptr: AddrspacePtr, +) -> AddrspacePtr { + // CHECK-LABEL: @offset + // CHECK: %[[val:[^ ]+]] = getelementptr inbounds i8, ptr addrspace(3) %ptr, i32 -20 + // CHECK: ret ptr addrspace(3) %[[val]] + unsafe { ptr.offset(-20) } +} + +#[unsafe(no_mangle)] +fn wrapping_offset( + ptr: AddrspacePtr, +) -> AddrspacePtr { + // CHECK-LABEL: @wrapping_offset + // CHECK: %[[val:[^ ]+]] = getelementptr i8, ptr addrspace(3) %ptr, i32 -15 + // CHECK: ret ptr addrspace(3) %[[val]] + unsafe { ptr.wrapping_offset(-15) } +} + +#[unsafe(no_mangle)] +fn read(ptr: AddrspacePtr) -> u32 { + // CHECK-LABEL: @read + // CHECK: %[[val:[^ ]+]] = load i32, ptr addrspace(3) %ptr, align 4 + // CHECK: ret i32 %[[val]] + unsafe { ptr.read() } +} + +#[unsafe(no_mangle)] +fn write(ptr: AddrspacePtr, val: u32) { + // CHECK-LABEL: @write + // CHECK: store i32 %val, ptr addrspace(3) %ptr, align 4 + // CHECK: ret void + unsafe { ptr.write(val) } +} + +#[unsafe(no_mangle)] +fn fun(ptr: *mut u8) -> (AddrspacePtr, usize) { + // CHECK-LABEL: @fun + // CHECK: %[[v0:[^ ]+]] = addrspacecast ptr %ptr to ptr addrspace(3) + // CHECK: %[[v1:[^ ]+]] = insertvalue { ptr addrspace(3), i64 } poison, ptr addrspace(3) %[[v0]], 0 + // CHECK: %[[res:[^ ]+]] = insertvalue { ptr addrspace(3), i64 } %[[v1]], i64 4, 1 + // CHECK: ret { ptr addrspace(3), i64 } %[[res]] + + unsafe { + let p: AddrspacePtr = + AddrspacePtr::from_ptr(ptr).cast_addrspace(); + let size = size_of_val(&p); + (p, size) + } +} + +#[unsafe(no_mangle)] +fn get_raw_dispatch_ptr() -> AddrspacePtr { + // CHECK-LABEL: @get_raw_dispatch_ptr + // CHECK: %[[val:[^ ]+]] = tail call noundef ptr addrspace(4) @llvm.amdgcn.dispatch.ptr() + // CHECK: ret ptr addrspace(4) %[[val]] + unsafe { dispatch_ptr() } +} + +#[unsafe(no_mangle)] +fn get_dispatch_ptr() -> &'static DispatchPacket { + // CHECK-LABEL: @get_dispatch_ptr + // CHECK: %[[val:[^ ]+]] = tail call noundef ptr addrspace(4) @llvm.amdgcn.dispatch.ptr() + // CHECK: %[[res:[^ ]+]] = addrspacecast ptr addrspace(4) %[[val]] to ptr + // CHECK: ret ptr %[[res]] + unsafe { &*dispatch_ptr().cast_addrspace::<{ addrspace::GENERIC }>().as_ptr() } +} + +#[unsafe(no_mangle)] +fn read_ptr( + ptr: AddrspacePtr, +) -> AddrspacePtr { + // Read a pointer to a specific addrspace from a pointer + // CHECK-LABEL: @read + // CHECK: %[[val:[^ ]+]] = load ptr addrspace(3), ptr addrspace(4) %ptr, align 4 + // CHECK: ret ptr addrspace(3) %[[val]] + unsafe { ptr.cast::>().read() } +} + +const EXPECTED: usize = 4; +const ACTUAL: usize = mem::size_of::>(); +// Validate that the size is 4 byte, which is different from the generic pointer size +const _: [(); EXPECTED] = [(); ACTUAL];