diff --git a/.gitignore b/.gitignore index 1bbd3a99580..13bd0d0ffde 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,5 @@ llvm build_system/target config.toml build -rustlantis \ No newline at end of file +rustlantis +stuff/ diff --git a/Cargo.lock b/Cargo.lock index 44aeab75c29..c174628d0d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -56,18 +56,18 @@ dependencies = [ [[package]] name = "gccjit" -version = "6.0.0" +version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb358d2563af5e32af92620915e6b05839ae60645343473735619441f45eb04" +checksum = "6d85b5754389edaad832ba320709a25086b3081a8c6c0fab2322965e5fb512b3" dependencies = [ "gccjit_sys", ] [[package]] name = "gccjit_sys" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2389fb01673e9cc63684d996a58079edccc5de89008274f3be59f1b16ac1f017" +checksum = "e081669728b490723537f9def7eb674b7c9acd8de0b92ad4f4abf5f5cc75ea4b" dependencies = [ "libc", ] @@ -145,9 +145,9 @@ dependencies = [ [[package]] name = "object" -version = "0.37.1" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03fd943161069e1768b4b3d050890ba48730e590f57e56d4aa04e7e090e61b4a" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" dependencies = [ "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index 1aff8ed115e..02be6d56c23 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,9 +18,9 @@ master = ["gccjit/master"] default = ["master"] [dependencies] -object = { version = "0.37.0", default-features = false, features = ["std", "read"] } +object = { version = "0.39.0", default-features = false, features = ["std", "read"] } tempfile = "3.20" -gccjit = { version = "6.0.0", features = ["dlopen"] } +gccjit = { version = "6.1.0", features = ["dlopen"] } #gccjit = { git = "https://github.com/rust-lang/gccjit.rs", branch = "error-dlopen", features = ["dlopen"] } # Local copy. diff --git a/build_system/src/test.rs b/build_system/src/test.rs index 3ea62b95798..299cae6fb13 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -1263,13 +1263,12 @@ fn run_ui_tests(env: &Env, args: &TestArg) -> Result<(), String> { let rust_path = setup_rustc(&mut env, args)?; let extra = - if args.is_using_gcc_master_branch() { "" } else { " -Csymbol-mangling-version=v0" }; + if args.is_using_gcc_master_branch() { "" } else { "-Csymbol-mangling-version=v0" }; let rustc_args = format!( - "{test_flags} -Zcodegen-backend={backend} --sysroot {sysroot}{extra}", + "{test_flags} -Zcodegen-backend={backend} {extra}", test_flags = env.get("TEST_FLAGS").unwrap_or(&String::new()), backend = args.config_info.cg_backend_path, - sysroot = args.config_info.sysroot_path, extra = extra, ); @@ -1286,6 +1285,8 @@ fn run_ui_tests(env: &Env, args: &TestArg) -> Result<(), String> { &"build.compiletest-allow-stage0=true", &"--compiletest-rustc-args", &rustc_args, + &"--sysroot", + &args.config_info.sysroot_path, &"--bypass-ignore-backends", &"--force-rerun", ]; diff --git a/libgccjit.version b/libgccjit.version index 47539d889df..62417a80f82 100644 --- a/libgccjit.version +++ b/libgccjit.version @@ -1 +1 @@ -201ca90ac810d1c6509c252cc9c87d3ace0661d7 +badf78d09d16e66f4ca07971c51aa6a227558d4f diff --git a/rust-toolchain b/rust-toolchain index d777360fd42..ba9ce799ba3 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2026-08-04" +channel = "nightly-2026-08-21" components = ["rust-src", "rustc-dev", "llvm-tools-preview"] diff --git a/src/attributes.rs b/src/attributes.rs index 95d12480efa..9ff6c19f6f1 100644 --- a/src/attributes.rs +++ b/src/attributes.rs @@ -15,6 +15,8 @@ use rustc_target::callconv::FnAbi; #[cfg(feature = "master")] use rustc_target::spec::Arch; +#[cfg(feature = "master")] +use crate::base; use crate::context::CodegenCx; use crate::gcc_util::to_gcc_features; @@ -116,6 +118,16 @@ pub fn from_fn_attrs<'gcc, 'tcx>( } else { codegen_fn_attrs.inline }; + // GCC drops `weak` from a function that is also `inline`, leaving the symbol strong, and + // the linkage is what has to survive. `inline(never)` does not conflict. + let inline = match inline { + InlineAttr::Always | InlineAttr::Hint | InlineAttr::Force { .. } + if codegen_fn_attrs.linkage.is_some_and(base::linkage_needs_weak_attribute) => + { + InlineAttr::None + } + inline => inline, + }; if let Some(attr) = inline_attr(cx, inline, instance) { if let FnAttribute::AlwaysInline = attr { func.add_attribute(FnAttribute::Inline); diff --git a/src/base.rs b/src/base.rs index 9c06c7090c8..3346ff85074 100644 --- a/src/base.rs +++ b/src/base.rs @@ -1,6 +1,8 @@ use std::sync::Arc; use std::time::Instant; +#[cfg(feature = "master")] +use gccjit::VarAttribute; use gccjit::{CType, FunctionType, GlobalKind}; use rustc_codegen_ssa::ModuleCodegen; use rustc_codegen_ssa::base::maybe_create_entry_wrapper; @@ -39,32 +41,72 @@ pub fn symbol_visibility_to_gcc(visibility: SymbolVisibility) -> gccjit::Visibil } } +/// The kind of a global *definition* with an explicit `#[linkage]`. +/// +/// The flavours that another object file is allowed to override also need +/// `global_linkage_attribute` from the caller: `GlobalKind` alone cannot express weakness. pub fn global_linkage_to_gcc(linkage: Linkage) -> GlobalKind { match linkage { - Linkage::External => GlobalKind::Imported, - Linkage::AvailableExternally => GlobalKind::Imported, - Linkage::LinkOnceAny => unimplemented!(), - Linkage::LinkOnceODR => unimplemented!(), - Linkage::WeakAny => unimplemented!(), - Linkage::WeakODR => unimplemented!(), - Linkage::Internal => GlobalKind::Internal, - Linkage::ExternalWeak => GlobalKind::Imported, // FIXME(antoyo): should be weak linkage. - Linkage::Common => unimplemented!(), + Linkage::External => GlobalKind::Exported, + // libgccjit cannot emit a definition that the linker discards in favour of the one in + // another object file, so emit a private copy of it instead. + Linkage::AvailableExternally | Linkage::Internal => GlobalKind::Internal, + // libgccjit exposes no comdat, so `weak` stands in for the linkonce flavours. + Linkage::LinkOnceAny + | Linkage::LinkOnceODR + | Linkage::WeakAny + | Linkage::WeakODR + | Linkage::ExternalWeak + | Linkage::Common => GlobalKind::Exported, + } +} + +/// The attribute a global *definition* needs on top of its [`GlobalKind`] to get this linkage. +#[cfg(feature = "master")] +pub fn global_linkage_attribute<'gcc>(linkage: Linkage) -> Option> { + match linkage { + Linkage::Common => Some(VarAttribute::Common), + _ if linkage_needs_weak_attribute(linkage) => Some(VarAttribute::Weak), + _ => None, } } +/// The type of a function *definition* with an explicit `#[linkage]`. +/// +/// The flavours that another object file is allowed to override also need +/// `linkage_needs_weak_attribute` from the caller: `FunctionType` alone cannot express weakness. pub fn linkage_to_gcc(linkage: Linkage) -> FunctionType { match linkage { Linkage::External => FunctionType::Exported, - // FIXME(antoyo): set the attribute externally_visible. - Linkage::AvailableExternally => FunctionType::Extern, - Linkage::LinkOnceAny => unimplemented!(), - Linkage::LinkOnceODR => unimplemented!(), - Linkage::WeakAny => FunctionType::Exported, // FIXME(antoyo): should be similar to linkonce. - Linkage::WeakODR => unimplemented!(), - Linkage::Internal => FunctionType::Internal, - Linkage::ExternalWeak => unimplemented!(), - Linkage::Common => unimplemented!(), + // libgccjit cannot emit a definition that the linker discards in favour of the one in + // another object file, so emit a private copy of it instead. + Linkage::AvailableExternally | Linkage::Internal => FunctionType::Internal, + // libgccjit exposes no comdat, so `weak` stands in for every overridable flavour. + Linkage::LinkOnceAny + | Linkage::LinkOnceODR + | Linkage::WeakAny + | Linkage::WeakODR + | Linkage::ExternalWeak + | Linkage::Common => FunctionType::Exported, + } +} + +/// Whether a definition with this linkage must carry the `weak` attribute, so that a strong +/// definition in another object file wins over it instead of clashing with it. +/// +/// `common` is in here for functions only: GCC honours that attribute on a variable, but drops it +/// on a function, so a common function falls back to weak. Globals go through +/// `global_linkage_attribute` instead. +#[cfg(feature = "master")] +pub fn linkage_needs_weak_attribute(linkage: Linkage) -> bool { + match linkage { + Linkage::LinkOnceAny + | Linkage::LinkOnceODR + | Linkage::WeakAny + | Linkage::WeakODR + | Linkage::ExternalWeak + | Linkage::Common => true, + Linkage::External | Linkage::AvailableExternally | Linkage::Internal => false, } } diff --git a/src/builder.rs b/src/builder.rs index 550f09615ef..dd7fc3ddc4e 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -66,6 +66,33 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { self.value_counter.get() } + /// Tell GCC that `pointer` is `align`-aligned, so that the bulk memory builtins can widen their + /// accesses: a pointer cast to an aligned type would be dropped as a useless conversion. + fn assume_aligned(&mut self, pointer: RValue<'gcc>, align: Align) -> RValue<'gcc> { + if align.bytes() <= 1 { + return pointer; + } + let assume_aligned = self.context.get_builtin_function("__builtin_assume_aligned"); + let alignment = self.context.new_rvalue_from_long(self.type_size_t(), align.bytes() as i64); + let pointer_type = pointer.get_type(); + let const_void_ptr_type = self.context.new_type::<()>().make_const().make_pointer(); + let pointer = self.context.new_cast(self.location, pointer, const_void_ptr_type); + let aligned = self.context.new_call(self.location, assume_aligned, &[pointer, alignment]); + self.context.new_cast(self.location, aligned, pointer_type) + } + + /// GCC ignores a volatile qualifier on the pointers given to `memcpy`/`memmove`/`memset` and + /// happily deletes the call, so a barrier is what keeps the operation observable. The pointers + /// are fed to it because a clobber alone does not reach memory GCC believes never escapes. + fn volatile_barrier(&mut self, pointers: &[RValue<'gcc>]) { + let barrier = self.block.add_extended_asm(self.location, ""); + for pointer in pointers { + barrier.add_input_operand(None, "r", *pointer); + } + barrier.add_clobber("memory"); + barrier.set_volatile_flag(true); + } + fn atomic_extremum( &mut self, operation: ExtremumOperation, @@ -82,8 +109,13 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { AtomicOrdering::AcqRel | AtomicOrdering::Release => AtomicOrdering::Acquire, _ => order, }; - let previous_value = - self.atomic_load(dst.get_type(), dst, load_ordering, Size::from_bytes(size)); + let previous_value = self.atomic_load( + dst.get_type(), + dst, + load_ordering, + /* volatile */ false, + Size::from_bytes(size), + ); let previous_var = func.new_local(self.location, previous_value.get_type(), "previous_value"); let return_value = self.new_temp(func, self.location, previous_value.get_type()); @@ -719,8 +751,8 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { if return_type == void_type { self.block.end_with_void_return(self.location) } else { - let abort = self.context.get_builtin_function("abort"); - self.block.add_eval(self.location, self.context.new_call(self.location, abort, &[])); + let trap = self.context.get_builtin_function("__builtin_trap"); + self.block.add_eval(self.location, self.context.new_call(self.location, trap, &[])); let return_value = self.new_temp(self.current_func(), self.location, return_type); self.block.end_with_return(self.location, return_value) } @@ -1060,6 +1092,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { _ty: Type<'gcc>, ptr: RValue<'gcc>, order: AtomicOrdering, + _volatile: bool, // FIXME we are always making the load volatile size: Size, ) -> RValue<'gcc> { // FIXME(antoyo): use ty. @@ -1229,6 +1262,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { value: RValue<'gcc>, ptr: RValue<'gcc>, order: AtomicOrdering, + _volatile: bool, // FIXME we are always making the store volatile size: Size, ) { // FIXME(antoyo): handle alignment. @@ -1448,47 +1482,53 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { fn memcpy( &mut self, dst: RValue<'gcc>, - _dst_align: Align, + dst_align: Align, src: RValue<'gcc>, - _src_align: Align, + src_align: Align, size: RValue<'gcc>, flags: MemFlags, _tt: Option, // Autodiff TypeTrees are LLVM-only, ignored in GCC backend ) { assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported"); let size = self.intcast(size, self.type_size_t(), false); - let _is_volatile = flags.contains(MemFlags::VOLATILE); let dst = self.pointercast(dst, self.type_i8p()); + let dst = self.assume_aligned(dst, dst_align); let src = self.pointercast(src, self.type_ptr_to(self.type_void())); + let src = self.assume_aligned(src, src_align); let memcpy = self.context.get_builtin_function("memcpy"); - // FIXME(antoyo): handle aligns and is_volatile. self.block.add_eval( self.location, self.context.new_call(self.location, memcpy, &[dst, src, size]), ); + if flags.contains(MemFlags::VOLATILE) { + self.volatile_barrier(&[dst, src]); + } } fn memmove( &mut self, dst: RValue<'gcc>, - _dst_align: Align, + dst_align: Align, src: RValue<'gcc>, - _src_align: Align, + src_align: Align, size: RValue<'gcc>, flags: MemFlags, ) { assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported"); let size = self.intcast(size, self.type_size_t(), false); - let _is_volatile = flags.contains(MemFlags::VOLATILE); let dst = self.pointercast(dst, self.type_i8p()); + let dst = self.assume_aligned(dst, dst_align); let src = self.pointercast(src, self.type_ptr_to(self.type_void())); + let src = self.assume_aligned(src, src_align); let memmove = self.context.get_builtin_function("memmove"); - // FIXME(antoyo): handle is_volatile. self.block.add_eval( self.location, self.context.new_call(self.location, memmove, &[dst, src, size]), ); + if flags.contains(MemFlags::VOLATILE) { + self.volatile_barrier(&[dst, src]); + } } fn memset( @@ -1496,20 +1536,22 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { ptr: RValue<'gcc>, fill_byte: RValue<'gcc>, size: RValue<'gcc>, - _align: Align, + align: Align, flags: MemFlags, ) { assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memset not supported"); - let _is_volatile = flags.contains(MemFlags::VOLATILE); let ptr = self.pointercast(ptr, self.type_i8p()); + let ptr = self.assume_aligned(ptr, align); let memset = self.context.get_builtin_function("memset"); - // FIXME(antoyo): handle align and is_volatile. let fill_byte = self.context.new_cast(self.location, fill_byte, self.i32_type); let size = self.intcast(size, self.type_size_t(), false); self.block.add_eval( self.location, self.context.new_call(self.location, memset, &[ptr, fill_byte, size]), ); + if flags.contains(MemFlags::VOLATILE) { + self.volatile_barrier(&[ptr]); + } } fn vscale(&mut self, _: Self::Type) -> Self::Value { diff --git a/src/common.rs b/src/common.rs index a503c1b3451..21d92c6cc29 100644 --- a/src/common.rs +++ b/src/common.rs @@ -126,68 +126,86 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { } } +/// The element type and element count of the array used to represent a run of `len` constant bytes. +/// +/// Larger integers are used where possible: this reduces the number of rvalues, which is a +/// significant memory saving on constant-heavy crates. +fn byte_run_shape<'gcc>(cx: &CodegenCx<'gcc, '_>, len: usize) -> (Type<'gcc>, u64) { + match len % 8 { + 0 => (cx.context.new_type::(), len as u64 / 8), + 4 => (cx.context.new_type::(), len as u64 / 4), + _ => (cx.context.new_type::(), len as u64), + } +} + +/// The type [`bytes_in_context`] gives a run of `len` constant bytes. +/// +/// Exposed separately so that the type of a constant allocation can be computed before any of its +/// rvalues exist; see [`crate::consts::const_alloc_type`]. +/// +/// The result is cached because `gcc_jit_context_new_array_type` mints a fresh type every call. +/// Two equal-but-distinct array types would key [`CodegenCx::type_struct`] differently and so +/// produce two distinct anonymous structs, and libgccjit compares struct types by identity. +pub fn bytes_type_in_context<'gcc>(cx: &CodegenCx<'gcc, '_>, len: usize) -> Type<'gcc> { + let (element_type, count) = byte_run_shape(cx, len); + if let Some(&typ) = cx.byte_array_types.borrow().get(&(element_type, count)) { + return typ; + } + let typ = new_array_type(cx.context, None, element_type, count); + cx.byte_array_types.borrow_mut().insert((element_type, count), typ); + typ +} + +// FIXME(FractalFir): Consider using `global_set_initializer` instead. Before this is done, we need to confirm that +// `global_set_initializer` is more memory efficient than the current solution. +// `global_set_initializer` calls `global_set_initializer_rvalue` under the hood - does it generate an array of rvalues, +// or is it using a more efficient representation? pub fn bytes_in_context<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, bytes: &[u8]) -> RValue<'gcc> { - // Instead of always using an array of bytes, use an array of larger integers of target endianness - // if possible. This reduces the amount of `rvalues` we use, which reduces memory usage significantly. - // - // FIXME(FractalFir): Consider using `global_set_initializer` instead. Before this is done, we need to confirm that - // `global_set_initializer` is more memory efficient than the current solution. - // `global_set_initializer` calls `global_set_initializer_rvalue` under the hood - does it generate an array of rvalues, - // or is it using a more efficient representation? - match bytes.len() % 8 { + let typ = bytes_type_in_context(cx, bytes.len()); + let (element_type, _) = byte_run_shape(cx, bytes.len()); + let context = &cx.context; + // Since we are representing arbitrary byte runs as integers, we need to follow the target + // endianness. + let endian = cx.sess().target.options.endian; + let elements: Vec<_> = match bytes.len() % 8 { 0 => { - let context = &cx.context; - let byte_type = context.new_type::(); - let typ = new_array_type(context, None, byte_type, bytes.len() as u64 / 8); let (arrays, remainder) = bytes.as_chunks::<8>(); debug_assert!(remainder.is_empty()); - let elements: Vec<_> = arrays + arrays .iter() .map(|&arr| { context.new_rvalue_from_long( - byte_type, - // Since we are representing arbitrary byte runs as integers, we need to follow the target - // endianness. - match cx.sess().target.options.endian { + element_type, + match endian { rustc_abi::Endian::Little => u64::from_le_bytes(arr) as i64, rustc_abi::Endian::Big => u64::from_be_bytes(arr) as i64, }, ) }) - .collect(); - context.new_array_constructor(None, typ, &elements) + .collect() } 4 => { - let context = &cx.context; - let byte_type = context.new_type::(); - let typ = new_array_type(context, None, byte_type, bytes.len() as u64 / 4); let (arrays, remainder) = bytes.as_chunks::<4>(); debug_assert!(remainder.is_empty()); - let elements: Vec<_> = arrays + arrays .iter() .map(|&arr| { context.new_rvalue_from_int( - byte_type, - match cx.sess().target.options.endian { + element_type, + match endian { rustc_abi::Endian::Little => u32::from_le_bytes(arr) as i32, rustc_abi::Endian::Big => u32::from_be_bytes(arr) as i32, }, ) }) - .collect(); - context.new_array_constructor(None, typ, &elements) - } - _ => { - let context = cx.context; - let byte_type = context.new_type::(); - let typ = new_array_type(context, None, byte_type, bytes.len() as u64); - let elements: Vec<_> = bytes - .iter() - .map(|&byte| context.new_rvalue_from_int(byte_type, byte as i32)) - .collect(); - context.new_array_constructor(None, typ, &elements) + .collect() } - } + _ => bytes + .iter() + .map(|&byte| context.new_rvalue_from_int(element_type, byte as i32)) + .collect(), + }; + context.new_array_constructor(None, typ, &elements) } pub fn type_is_pointer(typ: Type<'_>) -> bool { diff --git a/src/consts.rs b/src/consts.rs index 5ebdf91fe20..06e945a6a45 100644 --- a/src/consts.rs +++ b/src/consts.rs @@ -1,6 +1,8 @@ +use std::ops::Range; + #[cfg(feature = "master")] -use gccjit::{FnAttribute, ToRValue, VarAttribute, Visibility}; -use gccjit::{Function, GlobalKind, LValue, RValue, Type}; +use gccjit::{FnAttribute, VarAttribute, Visibility}; +use gccjit::{Function, GlobalKind, LValue, RValue, ToRValue, Type}; use rustc_abi::{self as abi, Align, HasDataLayout, Primitive, Size, WrappingRange}; use rustc_codegen_ssa::traits::{ BaseTypeCodegenMethods, ConstCodegenMethods, StaticCodegenMethods, @@ -11,15 +13,18 @@ use rustc_hir::def_id::LOCAL_CRATE; use rustc_log::tracing::trace; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::mir::interpret::{ - self, ConstAllocation, ErrorHandled, Scalar as InterpScalar, read_target_uint, + self, Allocation, ConstAllocation, CtfeProvenance, ErrorHandled, Scalar as InterpScalar, + read_target_uint, }; +use rustc_middle::mono::MonoItem; use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, Instance}; use rustc_middle::{bug, span_bug}; use rustc_span::def_id::DefId; -use crate::base; +use crate::common::bytes_type_in_context; use crate::context::CodegenCx; +use crate::type_::struct_attributes; use crate::type_of::LayoutGccExt; pub(crate) fn const_alloc_to_gcc<'gcc, 'tcx>( @@ -99,15 +104,21 @@ impl<'gcc, 'tcx> StaticCodegenMethods for CodegenCx<'gcc, 'tcx> { let is_thread_local = attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL); let global = self.get_static_inner(def_id, val_llty); - #[cfg(feature = "master")] - if global.to_rvalue().get_type() != val_llty { - global.to_rvalue().set_type(val_llty); - } + debug_assert_eq!( + global.to_rvalue().get_type(), + val_llty, + "`predefine_static` declared this global with a type its initializer does not have" + ); // NOTE: Alignment from attributes has already been applied to the allocation. set_global_alignment(self, global, alloc.align); - global.global_set_initializer_rvalue(value); + // A common symbol is storage the linker allocates and zero-fills, so giving the definition + // an initializer — even an all-zero one — takes it back out of `.comm`. A non-zero one is + // kept: the symbol is then an ordinary definition, which is what GCC does with it too. + if attrs.linkage != Some(Linkage::Common) || !is_zero_initializer(alloc) { + global.global_set_initializer_rvalue(value); + } // As an optimization, all shared statics which do not have interior // mutability are placed into read-only memory. @@ -260,15 +271,14 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { return global; } - // FIXME: Once we stop removing globals in `codegen_static`, we can uncomment this code. - // let defined_in_current_codegen_unit = - // self.codegen_unit.items().contains_key(&MonoItem::Static(def_id)); - // assert!( - // !defined_in_current_codegen_unit, - // "consts::get_static() should always hit the cache for \ - // statics defined in the same CGU, but did not for `{:?}`", - // def_id - // ); + let defined_in_current_codegen_unit = + self.codegen_unit.items().contains_key(&MonoItem::Static(def_id)); + assert!( + !defined_in_current_codegen_unit, + "consts::get_static() should always hit the cache for \ + statics defined in the same CGU, but did not for `{:?}`", + def_id + ); let sym = self.tcx.symbol_name(instance).name; let fn_attrs = self.tcx.codegen_fn_attrs(def_id); @@ -332,75 +342,133 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { global } } -/// Converts a given const alloc to a gcc Rvalue, without any caching or deduplication. -/// YOU SHOULD NOT call this function directly - that may break the semantics of Rust. -/// Use `const_data_from_alloc` instead. -pub(crate) fn const_alloc_to_gcc_uncached<'gcc>( - cx: &CodegenCx<'gcc, '_>, - alloc: ConstAllocation<'_>, -) -> RValue<'gcc> { - let alloc = alloc.inner(); - let mut llvals = Vec::with_capacity(alloc.provenance().ptrs().len() + 1); - let dl = cx.data_layout(); - let pointer_size = dl.pointer_size().bytes() as usize; +/// One field of the packed struct that a constant allocation is lowered to. +enum AllocField { + /// A run of bytes carrying no provenance. + Bytes { range: Range }, + /// A pointer with provenance, occupying one target pointer worth of bytes. + Pointer { offset: usize, prov: CtfeProvenance }, +} + +/// The field-by-field shape of `alloc`. +/// +/// [`const_alloc_to_gcc_uncached`] and [`const_alloc_type`] have to agree exactly on this, down to +/// the empty trailing run an allocation ending on a pointer produces, so both derive the shape here +/// instead of each walking the allocation on its own. +fn alloc_fields(cx: &CodegenCx<'_, '_>, alloc: &interpret::Allocation) -> Vec { + let pointer_size = cx.data_layout().pointer_size().bytes() as usize; + let mut fields = Vec::with_capacity(alloc.provenance().ptrs().len() + 1); let mut next_offset = 0; for &(offset, prov) in alloc.provenance().ptrs().iter() { - let alloc_id = prov.alloc_id(); let offset = offset.bytes(); assert_eq!(offset as usize as u64, offset); let offset = offset as usize; if offset > next_offset { - // This `inspect` is okay since we have checked that it is not within a pointer with provenance, it - // is within the bounds of the allocation, and it doesn't affect interpreter execution - // (we inspect the result after interpreter execution). Any undef byte is replaced with - // some arbitrary byte value. - // - // FIXME: relay undef bytes to codegen as undef const bytes - let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(next_offset..offset); - llvals.push(cx.const_bytes(bytes)); + fields.push(AllocField::Bytes { range: next_offset..offset }); } - let ptr_offset = read_target_uint( - dl.endian, - // This `inspect` is okay since it is within the bounds of the allocation, it doesn't - // affect interpreter execution (we inspect the result after interpreter execution), - // and we properly interpret the provenance as a relocation pointer offset. - alloc.inspect_with_uninit_and_ptr_outside_interpreter(offset..(offset + pointer_size)), - ) - .expect("const_alloc_to_gcc_uncached: could not read relocation pointer") - as u64; - - let address_space = cx.tcx.global_alloc(alloc_id).address_space(cx); - - llvals.push(cx.scalar_to_backend( - InterpScalar::from_pointer( - interpret::Pointer::new(prov, Size::from_bytes(ptr_offset)), - &cx.tcx, - ), - abi::Scalar::Initialized { - value: Primitive::Pointer(address_space), - valid_range: WrappingRange::full(dl.pointer_size()), - }, - cx.type_i8p_ext(address_space), - )); + fields.push(AllocField::Pointer { offset, prov }); next_offset = offset + pointer_size; } if alloc.len() >= next_offset { - let range = next_offset..alloc.len(); - // This `inspect` is okay since we have check that it is after all provenance, it is - // within the bounds of the allocation, and it doesn't affect interpreter execution (we - // inspect the result after interpreter execution). Any undef byte is replaced with some - // arbitrary byte value. - // - // FIXME: relay undef bytes to codegen as undef const bytes - let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range); - llvals.push(cx.const_bytes(bytes)); + fields.push(AllocField::Bytes { range: next_offset..alloc.len() }); } + fields +} + +/// The type [`const_alloc_to_gcc`] gives `alloc`, computed without building any rvalue. +/// +/// This lets `predefine_static` declare a static's global with the type its initializer will have, +/// so that the two never disagree. It must not reach for the rvalue of anything it points at: +/// during the predefine pass the pointee may not be declared yet, and `alloc_to_backend` would +/// declare it with the wrong type behind our back. +pub(crate) fn const_alloc_type<'gcc>( + cx: &CodegenCx<'gcc, '_>, + alloc: ConstAllocation<'_>, +) -> Type<'gcc> { + let fields: Vec<_> = alloc_fields(cx, alloc.inner()) + .into_iter() + .map(|field| match field { + AllocField::Bytes { range } => bytes_type_in_context(cx, range.len()), + AllocField::Pointer { prov, .. } => { + let address_space = cx.tcx.global_alloc(prov.alloc_id()).address_space(cx); + cx.type_i8p_ext(address_space) + } + }) + .collect(); + cx.type_struct(&fields, &struct_attributes(true, None)) +} + +/// Converts a given const alloc to a gcc Rvalue, without any caching or deduplication. +/// YOU SHOULD NOT call this function directly - that may break the semantics of Rust. +/// Use `const_data_from_alloc` instead. +pub(crate) fn const_alloc_to_gcc_uncached<'gcc>( + cx: &CodegenCx<'gcc, '_>, + alloc: ConstAllocation<'_>, +) -> RValue<'gcc> { + let alloc = alloc.inner(); + let dl = cx.data_layout(); + let pointer_size = dl.pointer_size(); + + let llvals: Vec<_> = alloc_fields(cx, alloc) + .into_iter() + .map(|field| match field { + AllocField::Bytes { range } => { + // This `inspect` is okay since we have checked that it is not within a pointer with + // provenance, it is within the bounds of the allocation, and it doesn't affect + // interpreter execution (we inspect the result after interpreter execution). Any + // undef byte is replaced with some arbitrary byte value. + // + // FIXME: relay undef bytes to codegen as undef const bytes + cx.const_bytes(alloc.inspect_with_uninit_and_ptr_outside_interpreter(range)) + } + AllocField::Pointer { offset, prov } => { + let ptr_offset = read_target_uint( + dl.endian, + // This `inspect` is okay since it is within the bounds of the allocation, it + // doesn't affect interpreter execution (we inspect the result after interpreter + // execution), and we properly interpret the provenance as a relocation pointer + // offset. + alloc.inspect_with_uninit_and_ptr_outside_interpreter( + offset..(offset + pointer_size.bytes() as usize), + ), + ) + .expect("const_alloc_to_gcc_uncached: could not read relocation pointer") + as u64; + + let address_space = cx.tcx.global_alloc(prov.alloc_id()).address_space(cx); + + cx.scalar_to_backend( + InterpScalar::from_pointer( + interpret::Pointer::new(prov, Size::from_bytes(ptr_offset)), + &cx.tcx, + ), + abi::Scalar::Initialized { + value: Primitive::Pointer(address_space), + valid_range: WrappingRange::full(pointer_size), + }, + cx.type_i8p_ext(address_space), + ) + } + }) + .collect(); + // FIXME(bjorn3) avoid wrapping in a struct when there is only a single element. cx.const_struct(&llvals, true) } +/// Whether this allocation is all zeroes, and so needs no initializer to be spelled out. +fn is_zero_initializer(alloc: &Allocation) -> bool { + alloc.provenance().ptrs().is_empty() + // This `inspect` is okay: it is within the bounds of the allocation, there is no provenance + // to misread, and it does not affect interpreter execution. + && alloc + .inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.size().bytes_usize()) + .iter() + .all(|&byte| byte == 0) +} + fn codegen_static_initializer<'gcc, 'tcx>( cx: &CodegenCx<'gcc, 'tcx>, def_id: DefId, @@ -417,10 +485,10 @@ fn check_and_apply_linkage<'gcc, 'tcx>( ) -> LValue<'gcc> { let is_tls = attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL); if let Some(linkage) = attrs.import_linkage { - // Declare a symbol `foo` with the desired linkage. - let global1 = - cx.declare_global_with_linkage(sym, cx.type_i8(), base::global_linkage_to_gcc(linkage)); + // Whatever the flavour, an import is an undefined reference to a symbol defined elsewhere. + let global1 = cx.declare_global_with_linkage(sym, cx.type_i8(), GlobalKind::Imported); + // Only `extern_weak` lets the symbol stay unresolved, in which case it reads as null. if linkage == Linkage::ExternalWeak { #[cfg(feature = "master")] global1.add_attribute(VarAttribute::Weak); @@ -434,8 +502,13 @@ fn check_and_apply_linkage<'gcc, 'tcx>( // zero. let real_name = format!("_rust_extern_with_linkage_{:016x}_{sym}", cx.tcx.stable_crate_id(LOCAL_CRATE)); - let global2 = cx.define_global(&real_name, gcc_type, is_tls, attrs.link_section); - // FIXME(antoyo): set linkage. + let global2 = cx.define_global( + &real_name, + gcc_type, + GlobalKind::Internal, + is_tls, + attrs.link_section, + ); let value = cx.const_ptrcast(global1.get_address(None), gcc_type); global2.global_set_initializer_rvalue(value); global2 diff --git a/src/context.rs b/src/context.rs index ebbdbb72516..1453ea012f2 100644 --- a/src/context.rs +++ b/src/context.rs @@ -97,6 +97,12 @@ pub struct CodegenCx<'gcc, 'tcx> { /// Cache of the anonymous struct types. pub struct_types: RefCell, Type<'gcc>>>, + /// Cache of the array types used for runs of constant bytes, keyed by element type and count. + /// + /// libgccjit mints a fresh type on every `new_array_type`, and struct types are keyed on their + /// field types, so without this two equal byte runs would yield two distinct anonymous structs. + pub byte_array_types: RefCell, u64), Type<'gcc>>>, + /// Cache instances of monomorphic and polymorphic items pub instances: RefCell, LValue<'gcc>>>, /// Cache function instances of monomorphic and polymorphic items @@ -242,12 +248,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { let isize_type = usize_type; let bool_type = context.new_type::(); - let mut functions = FxHashMap::default(); - let builtins = ["abort"]; - - for builtin in builtins.iter() { - functions.insert(builtin.to_string(), context.get_builtin_function(builtin)); - } + let functions = FxHashMap::default(); let mut cx = Self { int128_align: tcx @@ -314,6 +315,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { types: Default::default(), tcx, struct_types: Default::default(), + byte_array_types: Default::default(), local_gen_sym_counter: Cell::new(0), global_gen_sym_counter: Cell::new(0), eh_personality: Cell::new(None), diff --git a/src/declare.rs b/src/declare.rs index 9bf57fbf75b..32bb7c3aa34 100644 --- a/src/declare.rs +++ b/src/declare.rs @@ -14,6 +14,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { &self, name: &str, ty: Type<'gcc>, + global_kind: GlobalKind, is_tls: bool, link_section: Option, ) -> LValue<'gcc> { @@ -31,7 +32,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { } global } else { - self.declare_global(name, ty, GlobalKind::Exported, is_tls, link_section) + self.declare_global(name, ty, global_kind, is_tls, link_section) } } @@ -141,10 +142,11 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { &self, name: &str, ty: Type<'gcc>, + global_kind: GlobalKind, is_tls: bool, link_section: Option, ) -> LValue<'gcc> { - self.get_or_insert_global(name, ty, is_tls, link_section) + self.get_or_insert_global(name, ty, global_kind, is_tls, link_section) } pub fn get_declared_value(&self, name: &str) -> Option> { diff --git a/src/int.rs b/src/int.rs index 9633539a16b..4e4b9116661 100644 --- a/src/int.rs +++ b/src/int.rs @@ -21,12 +21,12 @@ use crate::context::CodegenCx; impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { pub fn gcc_urem(&self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> { // 128-bit unsigned %: __umodti3 - self.multiplicative_operation(BinaryOp::Modulo, "mod", false, a, b) + self.division_operation(BinaryOp::Modulo, "mod", false, a, b) } pub fn gcc_srem(&self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> { // 128-bit signed %: __modti3 - self.multiplicative_operation(BinaryOp::Modulo, "mod", true, a, b) + self.division_operation(BinaryOp::Modulo, "mod", true, a, b) } pub fn gcc_not(&self, a: RValue<'gcc>) -> RValue<'gcc> { @@ -215,6 +215,27 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { self.additive_operation(BinaryOp::Minus, a, b) } + fn division_operation( + &self, + operation: BinaryOp, + operation_name: &str, + signed: bool, + mut a: RValue<'gcc>, + mut b: RValue<'gcc>, + ) -> RValue<'gcc> { + let a_type = a.get_type(); + if self.is_native_int_type(a_type) && self.is_native_int_type(b.get_type()) { + let typ = if signed { a_type.to_signed(self.cx) } else { a_type.to_unsigned(self.cx) }; + if !typ.is_compatible_with(a_type) { + a = self.context.new_cast(self.location, a, typ); + } + if !typ.is_compatible_with(b.get_type()) { + b = self.context.new_cast(self.location, b, typ); + } + } + self.multiplicative_operation(operation, operation_name, signed, a, b) + } + fn multiplicative_operation( &self, operation: BinaryOp, @@ -261,15 +282,13 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { } pub fn gcc_sdiv(&self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> { - // FIXME(antoyo): check if the types are signed? // 128-bit, signed: __divti3 - // FIXME(antoyo): convert the arguments to signed? - self.multiplicative_operation(BinaryOp::Divide, "div", true, a, b) + self.division_operation(BinaryOp::Divide, "div", true, a, b) } pub fn gcc_udiv(&self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> { // 128-bit, unsigned: __udivti3 - self.multiplicative_operation(BinaryOp::Divide, "div", false, a, b) + self.division_operation(BinaryOp::Divide, "div", false, a, b) } pub fn gcc_checked_binop( diff --git a/src/intrinsic/mod.rs b/src/intrinsic/mod.rs index 9704fd7614e..2bbbe5faf0e 100644 --- a/src/intrinsic/mod.rs +++ b/src/intrinsic/mod.rs @@ -103,7 +103,6 @@ fn get_simple_intrinsic<'gcc, 'tcx>( sym::round_ties_even_f64 => "rint", sym::roundf32 => "roundf", sym::roundf64 => "round", - sym::abort => "abort", _ => return None, }; Some(cx.context.get_builtin_function(gcc_name)) @@ -641,9 +640,8 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc } fn abort(&mut self) { - let func = self.context.get_builtin_function("abort"); - let func: RValue<'gcc> = unsafe { std::mem::transmute(func) }; - self.call(self.type_void(), None, None, func, &[], None, None); + let func = self.context.get_builtin_function("__builtin_trap"); + self.block.add_eval(self.location, self.context.new_call(self.location, func, &[])); } fn assume(&mut self, value: Self::Value) { diff --git a/src/lib.rs b/src/lib.rs index 7e22d6db50c..7abb2fa0d8a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -85,7 +85,7 @@ use rustc_codegen_ssa::back::write::{ CodegenContext, FatLtoInput, ModuleConfig, SharedEmitter, TargetMachineFactoryFn, ThinLtoInput, }; use rustc_codegen_ssa::base::codegen_crate; -use rustc_codegen_ssa::target_features::cfg_target_feature; +use rustc_codegen_ssa::target_features::internal_target_features; use rustc_codegen_ssa::traits::{CodegenBackend, ExtraBackendMethods, WriteBackendMethods}; use rustc_codegen_ssa::{CompiledModule, CompiledModules, CrateInfo, ModuleCodegen, TargetConfig}; use rustc_data_structures::profiling::SelfProfilerRef; @@ -94,8 +94,8 @@ use rustc_errors::{DiagCtxt, DiagCtxtHandle}; use rustc_middle::dep_graph::{WorkProduct, WorkProductMap}; use rustc_middle::ty::TyCtxt; use rustc_middle::util::Providers; -use rustc_session::Session; use rustc_session::config::{OptLevel, OutputFilenames}; +use rustc_session::{IncrCompSession, Session}; use rustc_span::{Symbol, sym}; use rustc_target::spec::{RelocModel, TargetTuple}; use tempfile::TempDir; @@ -306,13 +306,14 @@ impl CodegenBackend for GccCodegenBackend { &self, ongoing_codegen: Box, sess: &Session, + incr_comp_session: Option<&IncrCompSession>, _outputs: &OutputFilenames, crate_info: &CrateInfo, ) -> (CompiledModules, WorkProductMap) { ongoing_codegen .downcast::>() .expect("Expected GccCodegenBackend's OngoingCodegen, found Box") - .join(sess, crate_info) + .join(sess, incr_comp_session, crate_info) } fn target_config(&self, sess: &Session) -> TargetConfig { @@ -519,7 +520,7 @@ fn to_gcc_opt_level(optlevel: Option) -> OptimizationLevel { /// Returns the features that should be set in `cfg(target_feature)`. fn target_config(sess: &Session, target_info: &LockedTargetInfo) -> TargetConfig { - let (unstable_target_features, target_features) = cfg_target_feature( + let internal_target_features = internal_target_features( sess, |feature| to_gcc_features(sess, feature), |feature| { @@ -543,8 +544,7 @@ fn target_config(sess: &Session, target_info: &LockedTargetInfo) -> TargetConfig let has_reliable_f128 = target_info.supports_target_dependent_type(CType::Float128); TargetConfig { - target_features, - unstable_target_features, + internal_target_features, // There are no known bugs with GCC support for f16 or f128 has_reliable_f16, has_reliable_f16_math: has_reliable_f16, diff --git a/src/mono_item.rs b/src/mono_item.rs index 7513978b122..57411c85477 100644 --- a/src/mono_item.rs +++ b/src/mono_item.rs @@ -1,6 +1,6 @@ use gccjit::Function; #[cfg(feature = "master")] -use gccjit::{FnAttribute, LValue, ToRValue, VarAttribute}; +use gccjit::{FnAttribute, GlobalKind, ToRValue, Type, VarAttribute}; use rustc_codegen_ssa::traits::PreDefineCodegenMethods; use rustc_hir::attrs::Linkage; use rustc_hir::def::DefKind; @@ -11,6 +11,7 @@ use rustc_middle::mono::Visibility; use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, Instance, TypeVisitableExt}; +use crate::consts::const_alloc_type; use crate::context::CodegenCx; use crate::type_of::LayoutGccExt; use crate::{attributes, base}; @@ -20,33 +21,58 @@ impl<'gcc, 'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> { fn predefine_static( &mut self, def_id: DefId, - _linkage: Linkage, + linkage: Linkage, visibility: Visibility, global_name: &str, ) { let attrs = self.tcx.codegen_fn_attrs(def_id); let instance = Instance::mono(self.tcx, def_id); - let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() }; - // Nested statics do not have a type, so pick a dummy type and let `codegen_static` figure out - // the gcc type from the actual evaluated initializer. - let ty = - if nested { self.tcx.types.unit } else { instance.ty(self.tcx, self.typing_env()) }; - let gcc_type = self.layout_of(ty).gcc_type(self); + // Declare the global with the type its initializer will have, so that `codegen_static` + // never has to retype it afterwards. The initializer is lowered as a packed struct of byte + // runs and relocations, which almost never matches the layout type. + let gcc_type = match self.tcx.eval_static_initializer(def_id) { + Ok(alloc) => const_alloc_type(self, alloc), + // The initializer failed to evaluate; `codegen_static` bails out on it too, so this + // type is never used to hold one. + Err(_) => { + let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() }; + // Nested statics do not have a type, so pick a dummy one. + let ty = if nested { + self.tcx.types.unit + } else { + instance.ty(self.tcx, self.typing_env()) + }; + self.layout_of(ty).gcc_type(self) + } + }; let is_tls = attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL); + let global_kind = base::global_linkage_to_gcc(linkage); + let global = + self.define_global(global_name, gcc_type, global_kind, is_tls, attrs.link_section); + #[cfg(feature = "master")] + { + // Visibility is meaningless on an internal global: GCC ignores the attribute and + // warns about it. + if !matches!(global_kind, GlobalKind::Internal) { + // If we're compiling the compiler-builtins crate, e.g., the equivalent of + // compiler-rt, then we want to implicitly compile everything with hidden + // visibility as we're going to link this object all over the place but + // don't want the symbols to get exported. + let visibility = if self.tcx.is_compiler_builtins(LOCAL_CRATE) { + gccjit::Visibility::Hidden + } else { + base::visibility_to_gcc(visibility) + }; + global.add_attribute(VarAttribute::Visibility(visibility)); + } + if let Some(attribute) = base::global_linkage_attribute(linkage) { + global.add_attribute(attribute); + } + } - let create_global = |this: &CodegenCx<'gcc, 'tcx>, name: &str, visibility: Visibility| { - let global = this.define_global(name, gcc_type, is_tls, attrs.link_section); - #[cfg(feature = "master")] - global.add_attribute(VarAttribute::Visibility(base::visibility_to_gcc(visibility))); - // FIXME(antoyo): set linkage. - global - }; - let global = create_global(self, global_name, visibility); - - let attrs = self.tcx.codegen_instance_attrs(instance.def); #[cfg(feature = "master")] - self.add_static_aliases(&attrs.foreign_item_symbol_aliases, global_name, &create_global); + self.add_static_aliases(gcc_type, global_name, attrs, &attrs.foreign_item_symbol_aliases); self.instances.borrow_mut().insert(instance, global); } @@ -75,20 +101,31 @@ impl<'gcc, 'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> { impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { #[cfg(feature = "master")] - fn add_static_aliases( + fn add_static_aliases( &self, - aliases: &[(DefId, Linkage, Visibility)], + gcc_type: Type<'gcc>, aliased: &str, - create_global: &F, - ) where - F: Fn(&CodegenCx<'gcc, 'tcx>, &str, Visibility) -> LValue<'gcc>, - { - for &(alias, _linkage, visibility) in aliases { + attrs: &CodegenFnAttrs, + aliases: &[(DefId, Linkage, Visibility)], + ) { + let is_tls = attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL); + + for &(alias, linkage, visibility) in aliases { let instance = Instance::mono(self.tcx, alias); let symbol_name = self.tcx.symbol_name(instance); - let alias = create_global(self, symbol_name.name, visibility); + let alias = self.declare_global( + symbol_name.name, + gcc_type, + GlobalKind::Imported, + is_tls, + attrs.link_section, + ); + alias.add_attribute(VarAttribute::Visibility(base::visibility_to_gcc(visibility))); alias.add_attribute(VarAttribute::Alias(aliased)); + if linkage == Linkage::WeakAny { + alias.add_attribute(VarAttribute::Weak); + } // Add the alias name to the set of cached items, so there is no duplicate // instance added to it during the normal `external static` codegen @@ -153,6 +190,11 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { attributes::from_fn_attrs(self, fn_decl, instance, Some(fn_abi)); + #[cfg(feature = "master")] + if base::linkage_needs_weak_attribute(linkage) { + fn_decl.add_attribute(FnAttribute::Weak); + } + // If we're compiling the compiler-builtins crate, e.g., the equivalent of // compiler-rt, then we want to implicitly compile everything with hidden // visibility as we're going to link this object all over the place but diff --git a/tests/asm/bulk_memory_alignment.rs b/tests/asm/bulk_memory_alignment.rs new file mode 100644 index 00000000000..61154674471 --- /dev/null +++ b/tests/asm/bulk_memory_alignment.rs @@ -0,0 +1,45 @@ +//@ assembly-output: emit-asm +//@ only-x86_64-unknown-linux-gnu +//@ compile-flags: -Copt-level=3 + +#![crate_type = "lib"] + +// The alignment reaches GCC's `memcpy`/`memset` expansion only through +// `__builtin_assume_aligned`; a pointer cast to an aligned type is stripped as a useless +// conversion. An over-aligned type therefore has to expand to aligned moves and a packed one +// to unaligned moves. The alignment is 64 so that the contrast holds whatever vector width +// the host picks. + +#[repr(align(64))] +pub struct Aligned([u8; 64]); + +#[repr(C, packed)] +pub struct Packed([u8; 64]); + +// CHECK-LABEL: "copy_aligned": +// CHECK: {{(v)?mov(dqa|aps)}} +#[no_mangle] +pub unsafe fn copy_aligned(destination: *mut Aligned, source: *const Aligned) { + core::ptr::copy_nonoverlapping(source, destination, 1); +} + +// CHECK-LABEL: "copy_packed": +// CHECK: {{(v)?mov(dqu|ups)}} +#[no_mangle] +pub unsafe fn copy_packed(destination: *mut Packed, source: *const Packed) { + core::ptr::copy_nonoverlapping(source, destination, 1); +} + +// CHECK-LABEL: "set_aligned": +// CHECK: {{(v)?mov(dqa|aps)}} +#[no_mangle] +pub unsafe fn set_aligned(destination: *mut Aligned) { + core::ptr::write_bytes(destination, 0, 1); +} + +// CHECK-LABEL: "set_packed": +// CHECK: {{(v)?mov(dqu|ups)}} +#[no_mangle] +pub unsafe fn set_packed(destination: *mut Packed) { + core::ptr::write_bytes(destination, 0, 1); +} diff --git a/tests/asm/volatile_bulk_memory.rs b/tests/asm/volatile_bulk_memory.rs new file mode 100644 index 00000000000..6233b8ac74c --- /dev/null +++ b/tests/asm/volatile_bulk_memory.rs @@ -0,0 +1,37 @@ +//@ assembly-output: emit-asm +//@ only-x86_64-unknown-linux-gnu +//@ compile-flags: -Copt-level=3 + +#![feature(core_intrinsics)] +#![crate_type = "lib"] + +use std::intrinsics::{ + volatile_copy_memory, volatile_copy_nonoverlapping_memory, volatile_set_memory, +}; + +// The buffers below are never read back, so the writes only survive because they are volatile. +// The functions are ordered alphabetically because that is the order they are emitted in. + +// CHECK-LABEL: "volatile_copy": +// CHECK: mov +#[no_mangle] +pub unsafe fn volatile_copy(source: *const u8) { + let mut buffer = [1u8; 64]; + volatile_copy_memory(buffer.as_mut_ptr(), source, 64); +} + +// CHECK-LABEL: "volatile_copy_nonoverlapping": +// CHECK: mov +#[no_mangle] +pub unsafe fn volatile_copy_nonoverlapping(source: *const u8) { + let mut buffer = [1u8; 64]; + volatile_copy_nonoverlapping_memory(buffer.as_mut_ptr(), source, 64); +} + +// CHECK-LABEL: "volatile_set": +// CHECK: mov +#[no_mangle] +pub unsafe fn volatile_set() { + let mut buffer = [1u8; 64]; + volatile_set_memory(buffer.as_mut_ptr(), 0, 64); +} diff --git a/tests/c/import_linkage.c b/tests/c/import_linkage.c new file mode 100644 index 00000000000..f2beb9603d0 --- /dev/null +++ b/tests/c/import_linkage.c @@ -0,0 +1,17 @@ +/* The symbols that `tests/run/import_linkage.rs` imports with an explicit `#[linkage]`. + * + * Such an import is a pointer whose value is the address of the symbol, so what the Rust side + * reads back is `&value_*`, not the pointer stored in it. The distinct values make a mix-up + * visible. */ + +#include + +int32_t external_value = 1; +int32_t available_externally_value = 2; +int32_t linkonce_value = 3; +int32_t linkonce_odr_value = 4; +int32_t weak_value = 5; +int32_t weak_odr_value = 6; +int32_t common_value = 7; +int32_t extern_weak_value = 8; +int32_t internal_value = 9; diff --git a/tests/c/static_linkage.c b/tests/c/static_linkage.c new file mode 100644 index 00000000000..787e61f9cf1 --- /dev/null +++ b/tests/c/static_linkage.c @@ -0,0 +1,37 @@ +/* Strong definitions of the statics that `tests/run/static_linkage.rs` also defines, but weakly. + * The linker has to keep these and drop the Rust ones; a backend that emits the Rust definitions + * as ordinary global symbols fails the link with a duplicate definition instead. + * + * `internal_static` is the opposite case: the Rust side keeps its own, and the two definitions + * coexist because the Rust one is local. */ + +#include + +int32_t weak_static = 1; +int32_t weak_odr_static = 2; +int32_t linkonce_static = 3; +int32_t linkonce_odr_static = 4; +int32_t common_static = 5; +int32_t internal_static = 200; + +/* `available_externally` promises the real definition lives elsewhere: a backend may read this one + * or emit an equivalent copy of the Rust initializer, so the two have to hold the same value. */ +int32_t available_externally_static = 7; + +/* Called from Rust, so that the reads also happen in a translation unit GCC compiled. */ +int32_t c_read_all(void) +{ + if (weak_static != 1) + return 11; + if (weak_odr_static != 2) + return 12; + if (linkonce_static != 3) + return 13; + if (linkonce_odr_static != 4) + return 14; + if (common_static != 5) + return 15; + if (internal_static != 200) + return 16; + return 0; +} diff --git a/tests/c/weak_function_linkage.c b/tests/c/weak_function_linkage.c new file mode 100644 index 00000000000..25dcdedbcd9 --- /dev/null +++ b/tests/c/weak_function_linkage.c @@ -0,0 +1,54 @@ +/* Strong definitions of the functions that `tests/run/weak_function_linkage.rs` also defines, but + * weakly. The linker has to keep these and drop the Rust ones. + * + * A backend that emits the Rust definitions as ordinary global symbols does not merely pick the + * wrong one: the link fails outright with a duplicate definition. */ + +#include + +int32_t weak_function(void) +{ + return 1; +} + +int32_t weak_odr_function(void) +{ + return 2; +} + +int32_t linkonce_function(void) +{ + return 3; +} + +int32_t linkonce_odr_function(void) +{ + return 4; +} + +int32_t weak_inline_function(void) +{ + return 8; +} + +/* `available_externally` promises the real definition lives elsewhere: a backend may call this one + * or emit an equivalent copy of the Rust body, so the two have to return the same value. */ +int32_t available_externally_function(void) +{ + return 7; +} + +/* Called from Rust, so that the calls also go through a caller that GCC compiled: a cg_gcc caller + * could inline the weak body it can see instead of calling the symbol. */ +int32_t c_call_all(void) +{ + if (weak_function() != 1) + return 11; + if (weak_odr_function() != 2) + return 12; + if (linkonce_function() != 3) + return 13; + if (linkonce_odr_function() != 4) + return 14; + return 0; +} diff --git a/tests/compile/asm_noreturn_call.rs b/tests/compile/asm_noreturn_call.rs new file mode 100644 index 00000000000..c9238697d80 --- /dev/null +++ b/tests/compile/asm_noreturn_call.rs @@ -0,0 +1,15 @@ +// Compiler: + +// Regression test for https://github.com/rust-lang/rustc_codegen_gcc/issues/827 + +#![crate_type = "lib"] + +#[cfg(target_arch = "x86_64")] +pub type NoReturn = extern "sysv64" fn(&'static u8) -> !; + +#[cfg(target_arch = "x86_64")] +pub fn call_no_return(function: *const NoReturn) -> ! { + unsafe { + std::arch::asm!("call {}", in(reg) function, options(noreturn)); + } +} diff --git a/tests/failing-ui-tests.txt b/tests/failing-ui-tests.txt index 2cf6925c0b5..ce614fecba2 100644 --- a/tests/failing-ui-tests.txt +++ b/tests/failing-ui-tests.txt @@ -14,17 +14,8 @@ tests/ui/attributes/fn-align-dyn.rs tests/ui/linkage-attr/raw-dylib/elf/glibc-x86_64.rs tests/ui/statics/const_generics.rs tests/ui/thir-print/offset_of.rs -tests/ui/eii/default/call_impl.rs tests/ui/asm/x86_64/global_asm_escape.rs tests/ui/lto/all-crates.rs -tests/ui/eii/static/cross_crate_decl.rs -tests/ui/eii/static/cross_crate_def.rs -tests/ui/eii/static/same_address.rs -tests/ui/eii/static/simple.rs -tests/ui/eii/static/default.rs -tests/ui/eii/static/default_cross_crate.rs -tests/ui/eii/static/default_explicit.rs -tests/ui/eii/static/default_cross_crate_explicit.rs tests/ui/explicit-tail-calls/tailcc-no-signature-restriction.rs tests/ui/abi/rust-tail-cc.rs tests/ui/abi/rust-preserve-none-cc.rs diff --git a/tests/run/custom_abort.rs b/tests/run/custom_abort.rs new file mode 100644 index 00000000000..eafa4321a43 --- /dev/null +++ b/tests/run/custom_abort.rs @@ -0,0 +1,27 @@ +// Compiler: +// +// Run-time: +// status: 42 + +// Check that a program can define its own `abort`. + +#![feature(no_core)] +#![no_std] +#![no_core] +#![no_main] + +extern crate mini_core; +use mini_core::*; + +#[no_mangle] +extern "C" fn abort() { + unsafe { + libc::exit(42); + } +} + +#[no_mangle] +extern "C" fn main(_argc: i32, _argv: *const *const u8) -> i32 { + abort(); + 0 +} diff --git a/tests/run/import_linkage.rs b/tests/run/import_linkage.rs new file mode 100644 index 00000000000..bf5cb9e5327 --- /dev/null +++ b/tests/run/import_linkage.rs @@ -0,0 +1,84 @@ +// Compiler: +// +// Run-time: +// status: 0 + +// Checks the `#[linkage]` flavours an `extern` static can be imported with, against the symbols +// `tests/c/import_linkage.c` defines. +// +// The value of such an import is the address of the symbol rather than its contents, which is why +// the types are pointers: an `extern_weak` import of a symbol nobody defines reads as null instead +// of failing the link. + +#![feature(linkage, no_core)] +#![no_std] +#![no_core] +#![no_main] + +extern crate mini_core; +use mini_core::*; + +extern "C" { + #[linkage = "external"] + static external_value: *const i32; + #[linkage = "available_externally"] + static available_externally_value: *const i32; + #[linkage = "linkonce"] + static linkonce_value: *const i32; + #[linkage = "linkonce_odr"] + static linkonce_odr_value: *const i32; + #[linkage = "weak"] + static weak_value: *const i32; + #[linkage = "weak_odr"] + static weak_odr_value: *const i32; + #[linkage = "common"] + static common_value: *const i32; + #[linkage = "extern_weak"] + static extern_weak_value: *const i32; + // An import is an undefined reference whatever the flavour says. Upstream bug: rustc lowers + // this one to an internal declaration, which LLVM's verifier rejects ("Global is external, but + // doesn't have external or weak linkage!") and which crashes cg_llvm at -O3. + #[linkage = "internal"] + static internal_value: *const i32; + + // Nothing defines this one, so it stays null instead of breaking the link. + #[linkage = "extern_weak"] + static undefined_value: *const i32; +} + +#[no_mangle] +extern "C" fn main(_argc: i32, _argv: *const *const u8) -> i32 { + unsafe { + if *external_value != 1 { + return 1; + } + if *available_externally_value != 2 { + return 2; + } + if *linkonce_value != 3 { + return 3; + } + if *linkonce_odr_value != 4 { + return 4; + } + if *weak_value != 5 { + return 5; + } + if *weak_odr_value != 6 { + return 6; + } + if *common_value != 7 { + return 7; + } + if *extern_weak_value != 8 { + return 8; + } + if *internal_value != 9 { + return 9; + } + if undefined_value as usize != 0 { + return 10; + } + } + 0 +} diff --git a/tests/run/ptr_to_int_div.rs b/tests/run/ptr_to_int_div.rs new file mode 100644 index 00000000000..afc563d6c97 --- /dev/null +++ b/tests/run/ptr_to_int_div.rs @@ -0,0 +1,19 @@ +// Compiler: +// +// Run-time: +// status: 0 + +use std::hint::black_box; +use std::mem::transmute; + +fn main() { + let pointer = black_box(usize::MAX) as *const (); + + let unsigned = unsafe { transmute::<*const (), usize>(pointer) }; + assert_eq!(unsigned / black_box(2), usize::MAX / 2); + assert_eq!(unsigned % black_box(2), usize::MAX % 2); + + let signed = unsafe { transmute::<*const (), isize>(pointer) }; + assert_eq!(signed / black_box(2), -1isize / 2); + assert_eq!(signed % black_box(2), -1isize % 2); +} diff --git a/tests/run/static_alloc_shapes.rs b/tests/run/static_alloc_shapes.rs new file mode 100644 index 00000000000..39a4d07bdf6 --- /dev/null +++ b/tests/run/static_alloc_shapes.rs @@ -0,0 +1,50 @@ +// Compiler: +// +// Run-time: +// status: 0 +// stdout: 8 +// 12 +// 5 +// 7 +// 7 +// 9 + +#![feature(no_core)] +#![no_std] +#![no_core] +#![no_main] + +extern crate mini_core; +use mini_core::*; + +// One byte run of each length class that maps to a distinct array element type. +static mut BYTES8: [u8; 8] = [1, 2, 3, 4, 5, 6, 7, 8]; +static mut BYTES12: [u8; 12] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; +static mut BYTES5: [u8; 5] = [1, 2, 3, 4, 5]; + +static mut VALUE: isize = 7; +static mut OTHER: isize = 9; + +// An allocation that is exactly one relocation, so it ends on a pointer with no trailing bytes. +static mut PTR: &isize = unsafe { &VALUE }; + +struct TwoRefs { + first: &'static isize, + second: &'static isize, +} + +// Two adjacent relocations, with no byte run between them. +static mut TWO_REFS: TwoRefs = TwoRefs { first: unsafe { &VALUE }, second: unsafe { &OTHER } }; + +#[no_mangle] +extern "C" fn main(_argc: isize, _argv: *const *const u8) -> i32 { + unsafe { + libc::printf(b"%ld\n\0" as *const u8 as *const i8, BYTES8[7] as isize); + libc::printf(b"%ld\n\0" as *const u8 as *const i8, BYTES12[11] as isize); + libc::printf(b"%ld\n\0" as *const u8 as *const i8, BYTES5[4] as isize); + libc::printf(b"%ld\n\0" as *const u8 as *const i8, *PTR); + libc::printf(b"%ld\n\0" as *const u8 as *const i8, *TWO_REFS.first); + libc::printf(b"%ld\n\0" as *const u8 as *const i8, *TWO_REFS.second); + } + 0 +} diff --git a/tests/run/static_linkage.rs b/tests/run/static_linkage.rs new file mode 100644 index 00000000000..7b911c064d7 --- /dev/null +++ b/tests/run/static_linkage.rs @@ -0,0 +1,79 @@ +// Compiler: +// +// Run-time: +// status: 0 + +// Checks that `#[linkage]` on a static that this crate defines reaches the symbol, against +// `tests/c/static_linkage.c`, which defines the overridable ones strongly. +// +// If `predefine_static` were to ignore its `linkage` argument outright, every static would come out as +// an ordinary global symbol: the overridable ones would clash with the C definitions at link time, and +// `internal` would export a symbol it should have kept private. + +#![feature(linkage, no_core)] +#![no_std] +#![no_core] +#![no_main] + +extern crate mini_core; +use mini_core::*; + +#[linkage = "weak"] +#[no_mangle] +pub static weak_static: i32 = 0; + +#[linkage = "weak_odr"] +#[no_mangle] +pub static weak_odr_static: i32 = 0; + +#[linkage = "linkonce"] +#[no_mangle] +pub static linkonce_static: i32 = 0; + +#[linkage = "linkonce_odr"] +#[no_mangle] +pub static linkonce_odr_static: i32 = 0; + +// `common` is only valid on a mutable global: LLVM rejects a constant one. +#[linkage = "common"] +#[no_mangle] +pub static mut common_static: i32 = 0; + +// Private to this crate, so the C definition of the same name is a different object. +#[linkage = "internal"] +#[no_mangle] +pub static internal_static: i32 = 100; + +// Not overridden by the C side: the definition here is the one that survives. +#[linkage = "weak"] +#[no_mangle] +pub static only_weak_static: i32 = 6; + +// The real definition is the one in the C file; a backend may read it or emit an equivalent copy of +// this initializer, so both spell the same value. +#[linkage = "available_externally"] +#[no_mangle] +pub static available_externally_static: i32 = 7; + +extern "C" { + fn c_read_all() -> i32; +} + +#[no_mangle] +extern "C" fn main(_argc: i32, _argv: *const *const u8) -> i32 { + let result = unsafe { c_read_all() }; + if result != 0 { + return result; + } + + if internal_static != 100 { + return 1; + } + if only_weak_static != 6 { + return 2; + } + if available_externally_static != 7 { + return 3; + } + 0 +} diff --git a/tests/run/weak_function_linkage.rs b/tests/run/weak_function_linkage.rs new file mode 100644 index 00000000000..677f0135340 --- /dev/null +++ b/tests/run/weak_function_linkage.rs @@ -0,0 +1,106 @@ +// Compiler: +// +// Run-time: +// status: 0 + +// Checks that the `#[linkage]` flavours another object file is allowed to override are emitted as +// weak symbols, by linking against `tests/c/weak_function_linkage.c`, which defines the same +// symbols strongly. + +#![feature(linkage, no_core)] +#![no_std] +#![no_core] +#![no_main] + +extern crate mini_core; +use mini_core::*; + +#[linkage = "weak"] +#[no_mangle] +extern "C" fn weak_function() -> i32 { + 0 +} + +// `_odr` promises every definition of the symbol is equivalent, which lets a backend call this body +// instead of the one in the C file. They spell the same value for that reason. +#[linkage = "weak_odr"] +#[no_mangle] +extern "C" fn weak_odr_function() -> i32 { + 2 +} + +#[linkage = "linkonce"] +#[no_mangle] +extern "C" fn linkonce_function() -> i32 { + 0 +} + +#[linkage = "linkonce_odr"] +#[no_mangle] +extern "C" fn linkonce_odr_function() -> i32 { + 4 +} + +// `#[linkage = "common"]` is absent on purpose: a common symbol is `SHN_COMMON`, which the object +// format only allows for objects, so no backend can give a function that linkage. + +// Not overridden by the C side: the definition here is the one that runs. +#[linkage = "weak"] +#[no_mangle] +extern "C" fn only_weak_function() -> i32 { + 6 +} + +// The real definition is the one in the C file; a backend may call it or emit an equivalent copy of +// this body, so both spell the same value. +#[linkage = "available_externally"] +#[no_mangle] +extern "C" fn available_externally_function() -> i32 { + 7 +} + +// GCC drops `weak` from a function that is also `inline`: a backend that keeps the hint emits this +// as an ordinary global symbol and clashes with the C definition. rustc lints the hint as ignored +// on a function with an explicit `#[linkage]`, hence the `allow`. +#[linkage = "weak"] +#[inline] +#[no_mangle] +#[allow(unused_attributes)] +extern "C" fn weak_inline_function() -> i32 { + 0 +} + +extern "C" { + fn c_call_all() -> i32; +} + +#[no_mangle] +extern "C" fn main(_argc: i32, _argv: *const *const u8) -> i32 { + let result = unsafe { c_call_all() }; + if result != 0 { + return result; + } + + if weak_function() != 1 { + return 1; + } + if weak_odr_function() != 2 { + return 2; + } + if linkonce_function() != 3 { + return 3; + } + if linkonce_odr_function() != 4 { + return 4; + } + if only_weak_function() != 6 { + return 6; + } + if available_externally_function() != 7 { + return 7; + } + if weak_inline_function() != 8 { + return 8; + } + 0 +}