runtime: mark all global variables explicitly for GC

CHANGELOG.md

@@ -84,6 +84,7 @@
   - gc: correct the old size calculation in block realloc
   - gc: correct the leaking allocator bounds and overflow checks
   - gc: move objHeader to the end of the block header
+  - cgo: keep malloc allocations alive until free and stop the program for invalid or repeated free calls
   - fix the leaking GC build with the cores scheduler
   - rp2040: fix -gc=leaking and -gc=none
   - rp2: handle the RP2350 shared FIFO IRQ for GC (#5482)

builder/bdwgc.go

@@ -30,10 +30,11 @@ var BoehmGC = Library{
 			// Use a minimal environment.
 			"-DNO_MSGBOX_ON_ERROR", // don't call MessageBoxA on Windows
 			"-DDONT_USE_ATEXIT",
-			"-DNO_GETENV",          // smaller binary, more predictable configuration
-			"-DNO_CLOCK",           // don't use system clock
-			"-DNO_DEBUGGING",       // reduce code size
-			"-DGC_NO_FINALIZATION", // finalization is not used at the moment
+			"-DNO_GETENV",               // smaller binary, more predictable configuration
+			"-DNO_CLOCK",                // don't use system clock
+			"-DNO_DEBUGGING",            // reduce code size
+			"-DGC_NO_FINALIZATION",      // finalization is not used at the moment
+			"-DGC_ATOMIC_UNCOLLECTABLE", // pointer-free storage retained until GC_free
 
 			// Special flag to work around the lack of __data_start in ld.lld.
 			// TODO: try to fix this in LLVM/lld directly so we don't have to

builder/build.go

@@ -1293,6 +1293,9 @@ func makeGlobalsModule(ctx llvm.Context, globals map[string]map[string]string, m
 			global := llvm.AddGlobal(mod, stringType, globalName)
 			global.SetInitializer(initializer)
 			global.SetAlignment(targetData.PrefTypeAlignment(stringType))
+			// Keep external linkage for module resolution. Hidden visibility permits internalization.
+			// See https://llvm.org/docs/LangRef.html#visibility-styles.
+			global.SetVisibility(llvm.HiddenVisibility)
 		}
 	}
 

builder/testdata/binary-size.txt

@@ -1,4 +1,4 @@
 target      package               code rodata data  bss
-hifive1b    examples/echo         4321    323    0 2268
-microbit    examples/serial       2842    382    8 2264
-wioterminal examples/pininterrupt 8039   1665  132 7496
+hifive1b    examples/echo         4533    323    0 2268
+microbit    examples/serial       3002    382    8 2264
+wioterminal examples/pininterrupt 8331   1717  148 7496

compileopts/config.go

@@ -24,7 +24,7 @@ import (
 // library path in advance in several places).
 var libVersions = map[string]int{
 	"musl":         3,
-	"bdwgc":        2,
+	"bdwgc":        3,
 	"picolibc":     2,
 	"wasmbuiltins": 1,
 }

compiler/channel.go

@@ -14,16 +14,18 @@ import (
 )
 
 func (b *builder) createMakeChan(expr *ssa.MakeChan) llvm.Value {
-	elementSize := b.targetData.TypeAllocSize(b.getLLVMType(expr.Type().Underlying().(*types.Chan).Elem()))
+	elementType := b.getLLVMType(expr.Type().Underlying().(*types.Chan).Elem())
+	elementSize := b.targetData.TypeAllocSize(elementType)
 	elementSizeValue := llvm.ConstInt(b.uintptrType, elementSize, false)
+	elementLayout := b.createObjectLayout(elementType, expr.Pos())
 	bufSize := b.getValue(expr.Size, getPos(expr))
 	b.createChanBoundsCheck(elementSize, bufSize, expr.Size.Type().Underlying().(*types.Basic), expr.Pos())
 	if bufSize.Type().IntTypeWidth() < b.uintptrType.IntTypeWidth() {
 		bufSize = b.CreateZExt(bufSize, b.uintptrType, "")
 	} else if bufSize.Type().IntTypeWidth() > b.uintptrType.IntTypeWidth() {
 		bufSize = b.CreateTrunc(bufSize, b.uintptrType, "")
 	}
-	return b.createRuntimeCall("chanMake", []llvm.Value{elementSizeValue, bufSize}, "")
+	return b.createRuntimeCall("chanMake", []llvm.Value{elementSizeValue, bufSize, elementLayout}, "")
 }
 
 // createChanSend emits a pseudo chan send operation. It is lowered to the

compiler/interface.go

@@ -284,13 +284,15 @@ func (c *compilerContext) getTypeCode(typ types.Type) llvm.Value {
 				types.NewVar(token.NoPos, nil, "elementType", types.Typ[types.UnsafePointer]),
 				types.NewVar(token.NoPos, nil, "length", types.Typ[types.Uintptr]),
 				types.NewVar(token.NoPos, nil, "sliceOf", types.Typ[types.UnsafePointer]),
+				types.NewVar(token.NoPos, nil, "layout", types.Typ[types.UnsafePointer]),
 			)
 		case *types.Map:
 			typeFieldTypes = append(typeFieldTypes,
 				types.NewVar(token.NoPos, nil, "numMethods", types.Typ[types.Uint16]),
 				types.NewVar(token.NoPos, nil, "ptrTo", types.Typ[types.UnsafePointer]),
 				types.NewVar(token.NoPos, nil, "elementType", types.Typ[types.UnsafePointer]),
 				types.NewVar(token.NoPos, nil, "keyType", types.Typ[types.UnsafePointer]),
+				types.NewVar(token.NoPos, nil, "hashmapTypeInfo", types.Typ[types.UnsafePointer]),
 			)
 		case *types.Struct:
 			typeFieldTypes = append(typeFieldTypes,
@@ -299,6 +301,7 @@ func (c *compilerContext) getTypeCode(typ types.Type) llvm.Value {
 				types.NewVar(token.NoPos, nil, "pkgpath", types.Typ[types.UnsafePointer]),
 				types.NewVar(token.NoPos, nil, "size", types.Typ[types.Uint32]),
 				types.NewVar(token.NoPos, nil, "numFields", types.Typ[types.Uint16]),
+				types.NewVar(token.NoPos, nil, "layout", types.Typ[types.UnsafePointer]),
 				types.NewVar(token.NoPos, nil, "fields", types.NewArray(c.getRuntimeType("structField"), int64(typ.NumFields()))),
 			)
 			if len(methods) > 0 {
@@ -418,13 +421,15 @@ func (c *compilerContext) getTypeCode(typ types.Type) llvm.Value {
 				c.getTypeCode(typ.Elem()),                              // elementType
 				llvm.ConstInt(c.uintptrType, uint64(typ.Len()), false), // length
 				c.getTypeCode(types.NewSlice(typ.Elem())),              // slicePtr
+				c.createObjectLayout(c.getLLVMType(typ), token.NoPos),  // layout
 			}
 		case *types.Map:
 			typeFields = []llvm.Value{
 				llvm.ConstInt(c.ctx.Int16Type(), 0, false), // numMethods
 				c.getTypeCode(types.NewPointer(typ)),       // ptrTo
 				c.getTypeCode(typ.Elem()),                  // elem
 				c.getTypeCode(typ.Key()),                   // key
+				c.getHashmapTypeInfo(typ, token.NoPos),     // hashmapTypeInfo
 			}
 		case *types.Struct:
 			var pkgpath string
@@ -450,6 +455,7 @@ func (c *compilerContext) getTypeCode(typ types.Type) llvm.Value {
 				pkgPathPtr,
 				llvm.ConstInt(c.ctx.Int32Type(), uint64(size), false),            // size
 				llvm.ConstInt(c.ctx.Int16Type(), uint64(typ.NumFields()), false), // numFields
+				c.createObjectLayout(llvmStructType, token.NoPos),                // layout
 			}
 			structFieldType := c.getLLVMRuntimeType("structField")
 
@@ -510,7 +516,7 @@ func (c *compilerContext) getTypeCode(typ types.Type) llvm.Value {
 			typeFields = []llvm.Value{c.getTypeCode(types.NewPointer(typ))}
 			// TODO: params, return values, etc
 		}
-		// Prepend metadata byte.
+		// Prepend the common RawType field.
 		typeFields = append([]llvm.Value{
 			llvm.ConstInt(c.ctx.Int8Type(), uint64(metabyte), false),
 		}, typeFields...)

compiler/map.go

@@ -13,6 +13,12 @@ import (
 
 const hashArrayUnrollLimit = 4
 
+const (
+	hashmapBucketSlots  = 8
+	hashmapMaxKeySize   = 128
+	hashmapMaxValueSize = 128
+)
+
 // createMakeMap creates a new map object (runtime.hashmap) by allocating and
 // initializing an appropriately sized object.
 func (b *builder) createMakeMap(expr *ssa.MakeMap) (llvm.Value, error) {
@@ -25,6 +31,8 @@ func (b *builder) createMakeMap(expr *ssa.MakeMap) (llvm.Value, error) {
 	valueSize := b.targetData.TypeAllocSize(llvmValueType)
 	llvmKeySize := llvm.ConstInt(b.uintptrType, keySize, false)
 	llvmValueSize := llvm.ConstInt(b.uintptrType, valueSize, false)
+	mapLayout := b.getHashmapTypeInfo(mapType, expr.Pos())
+
 	sizeHint := llvm.ConstInt(b.uintptrType, 8, false)
 	if expr.Reserve != nil {
 		sizeHint = b.getValue(expr.Reserve, getPos(expr))
@@ -54,11 +62,72 @@ func (b *builder) createMakeMap(expr *ssa.MakeMap) (llvm.Value, error) {
 
 	hashmap := b.createRuntimeCall("hashmapMakeGeneric", []llvm.Value{
 		llvmKeySize, llvmValueSize, sizeHint,
+		mapLayout,
 		hashFn, equalFn,
 	}, "")
 	return hashmap, nil
 }
 
+func (c *compilerContext) getHashmapTypeInfo(mapType *types.Map, pos token.Pos) llvm.Value {
+	llvmKeyType := c.getLLVMType(mapType.Key().Underlying())
+	llvmValueType := c.getLLVMType(mapType.Elem().Underlying())
+	keySize := c.targetData.TypeAllocSize(llvmKeyType)
+	valueSize := c.targetData.TypeAllocSize(llvmValueType)
+	keyLayout := c.createObjectLayout(llvmKeyType, pos)
+	valueLayout := c.createObjectLayout(llvmValueType, pos)
+
+	llvmKeySlotType := llvmKeyType
+	if keySize > hashmapMaxKeySize {
+		llvmKeySlotType = c.dataPtrType
+	}
+	llvmValueSlotType := llvmValueType
+	if valueSize > hashmapMaxValueSize {
+		llvmValueSlotType = c.dataPtrType
+	}
+
+	// Keep this in sync with runtime.hashmapBucket and
+	// runtime.hashmapBucketHeaderSize.
+	pointerSize := c.targetData.TypeAllocSize(c.dataPtrType)
+	headerSize := (uint64(8) + pointerSize + 7) &^ 7
+	headerPadding := headerSize - uint64(8) - pointerSize
+	bucketFields := []llvm.Type{
+		llvm.ArrayType(c.ctx.Int8Type(), hashmapBucketSlots),
+		c.dataPtrType,
+	}
+	if headerPadding != 0 {
+		bucketFields = append(bucketFields, llvm.ArrayType(c.ctx.Int8Type(), int(headerPadding)))
+	}
+	bucketFields = append(bucketFields,
+		llvm.ArrayType(llvmKeySlotType, hashmapBucketSlots),
+		llvm.ArrayType(llvmValueSlotType, hashmapBucketSlots),
+	)
+	bucketType := c.ctx.StructType(bucketFields, true)
+	bucketSize := headerSize +
+		c.targetData.TypeAllocSize(llvmKeySlotType)*hashmapBucketSlots +
+		c.targetData.TypeAllocSize(llvmValueSlotType)*hashmapBucketSlots
+	if c.targetData.TypeAllocSize(bucketType) != bucketSize {
+		panic("compiler hashmap bucket layout does not match runtime")
+	}
+	bucketLayout := c.createObjectLayout(bucketType, pos)
+	mapLayoutName := "runtime.hashmapType:" +
+		hashmapCanonicalTypeName(mapType.Key()) + ":" +
+		hashmapCanonicalTypeName(mapType.Elem())
+	mapLayout := c.mod.NamedGlobal(mapLayoutName)
+	if mapLayout.IsNil() {
+		initializer := c.ctx.ConstStruct([]llvm.Value{
+			keyLayout,
+			valueLayout,
+			bucketLayout,
+		}, false)
+		mapLayout = llvm.AddGlobal(c.mod, initializer.Type(), mapLayoutName)
+		mapLayout.SetInitializer(initializer)
+		mapLayout.SetGlobalConstant(true)
+		mapLayout.SetUnnamedAddr(true)
+		mapLayout.SetLinkage(llvm.LinkOnceODRLinkage)
+	}
+	return mapLayout
+}
+
 // getRuntimeFunctionValue returns a TinyGo function value (with nil context)
 // for the named runtime function.
 func (b *builder) getRuntimeFunctionValue(name string, sig *types.Signature) llvm.Value {

compiler/testdata/go1.21.ll

@@ -166,13 +166,13 @@ entry:
 }
 
 ; Function Attrs: nounwind
-define hidden void @main.clearMap(ptr dereferenceable_or_null(48) %m, ptr %context) unnamed_addr #1 {
+define hidden void @main.clearMap(ptr dereferenceable_or_null(52) %m, ptr %context) unnamed_addr #1 {
 entry:
   call void @runtime.hashmapClear(ptr %m, ptr undef) #4
   ret void
 }
 
-declare void @runtime.hashmapClear(ptr dereferenceable_or_null(48), ptr) #0
+declare void @runtime.hashmapClear(ptr dereferenceable_or_null(52), ptr) #0
 
 attributes #0 = { "target-features"="+bulk-memory,+bulk-memory-opt,+call-indirect-overlong,+mutable-globals,+nontrapping-fptoint,+sign-ext,-multivalue,-reference-types" }
 attributes #1 = { nounwind "target-features"="+bulk-memory,+bulk-memory-opt,+call-indirect-overlong,+mutable-globals,+nontrapping-fptoint,+sign-ext,-multivalue,-reference-types" }

compiler/testdata/go1.27.ll

@@ -14,7 +14,7 @@ target triple = "wasm32-unknown-wasi"
 @"main$string" = internal unnamed_addr constant [18 x i8] c"main.genericMethod", align 1
 @"main$string.1" = internal unnamed_addr constant [7 x i8] c"Regular", align 1
 @"pointer:named:main.genericMethod$methodset" = linkonce_odr unnamed_addr constant { i32, [1 x ptr], { ptr } } { i32 1, [1 x ptr] [ptr @"reflect/methods.Regular:func:{basic:int}{basic:int}"], { ptr } { ptr @"(*main.genericMethod).Regular" } }
-@"reflect/types.type:struct:{}" = linkonce_odr constant { i8, i16, ptr, ptr, i32, i16, [0 x %runtime.structField] } { i8 90, i16 0, ptr @"reflect/types.type:pointer:struct:{}", ptr @"reflect/types.type.pkgpath.empty", i32 0, i16 0, [0 x %runtime.structField] zeroinitializer }, align 4
+@"reflect/types.type:struct:{}" = linkonce_odr constant { i8, i16, ptr, ptr, i32, i16, ptr, [0 x %runtime.structField] } { i8 90, i16 0, ptr @"reflect/types.type:pointer:struct:{}", ptr @"reflect/types.type.pkgpath.empty", i32 0, i16 0, ptr inttoptr (i32 3 to ptr), [0 x %runtime.structField] zeroinitializer }, align 4
 @"reflect/types.type.pkgpath.empty" = linkonce_odr unnamed_addr constant [1 x i8] zeroinitializer, align 1
 @"reflect/types.type:pointer:struct:{}" = linkonce_odr constant { i8, i16, ptr } { i8 -43, i16 0, ptr @"reflect/types.type:struct:{}" }, align 4
 @"named:main.genericMethod$methodset" = linkonce_odr unnamed_addr constant { i32, [1 x ptr], { ptr } } { i32 1, [1 x ptr] [ptr @"reflect/methods.Regular:func:{basic:int}{basic:int}"], { ptr } { ptr @"(main.genericMethod).Regular$invoke" } }

compiler/testdata/large.ll

@@ -9,6 +9,7 @@ target triple = "wasm32-unknown-wasi"
 
 @"runtime/gc.layout:258-000000000000000000000000000000000000000000000000000000000000000002" = linkonce_odr unnamed_addr constant { i32, [33 x i8] } { i32 258, [33 x i8] c"\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\02" }
 @"reflect/types.typeid:named:main.largeValue" = external constant i8
+@"runtime.hashmapType:[1025]byte:[1025]byte" = linkonce_odr unnamed_addr constant { ptr, ptr, ptr } { ptr inttoptr (i32 3 to ptr), ptr inttoptr (i32 3 to ptr), ptr inttoptr (i32 67108137 to ptr) }
 @llvm.used = appending global [15 x ptr] [ptr @"(main.largeReceiver).makeLargeValue", ptr @"(main.largeReceiver).readLargeValue", ptr @main.makeLargeValue, ptr @main.makeZeroLargeValue, ptr @main.readLargeValue, ptr @main.deferLargeValue, ptr @main.goLargeValue, ptr @main.makeLargeResults, ptr @main.makeTwoLargeResults, ptr @main.makeMixedLargeResults, ptr @main.chooseLargeValue, ptr @main.makePointerLargeValue, ptr @main.useLargeMap, ptr @main.useLargeChannel, ptr @main.selectLargeChannel]
 @"main$string" = internal unnamed_addr constant [31 x i8] c"blocking select matched no case", align 1
 @"main$pack" = internal unnamed_addr constant { %runtime._string } { %runtime._string { ptr @"main$string", i32 31 } }
@@ -350,7 +351,7 @@ declare void @llvm.memset.p0.i32(ptr nocapture writeonly, i8, i32, i1 immarg) #7
 define hidden i8 @main.useLargeMap(ptr readonly dereferenceable_or_null(1025) %key, ptr readonly dereferenceable_or_null(1025) %value, ptr %context) unnamed_addr #1 {
 entry:
   %stackalloc = alloca i8, align 1
-  %0 = call ptr @runtime.hashmapMakeGeneric(i32 1025, i32 1025, i32 1, ptr null, ptr nonnull @runtime.hash32, ptr null, ptr nonnull @runtime.memequal, ptr undef) #9
+  %0 = call ptr @runtime.hashmapMakeGeneric(i32 1025, i32 1025, i32 1, ptr nonnull @"runtime.hashmapType:[1025]byte:[1025]byte", ptr null, ptr nonnull @runtime.hash32, ptr null, ptr nonnull @runtime.memequal, ptr undef) #9
   call void @runtime.trackPointer(ptr %0, ptr nonnull %stackalloc, ptr undef) #9
   call void @runtime.hashmapBinarySet(ptr %0, ptr %key, ptr %value, ptr undef) #9
   %result = call align 1 dereferenceable(1025) ptr @runtime.alloc(i32 1025, ptr nonnull inttoptr (i32 3 to ptr), ptr undef) #9
@@ -381,11 +382,11 @@ declare i32 @runtime.hash32(ptr, i32, i32, ptr) #0
 
 declare i1 @runtime.memequal(ptr, ptr, i32, ptr) #0
 
-declare ptr @runtime.hashmapMakeGeneric(i32, i32, i32, ptr, ptr, ptr, ptr, ptr) #0
+declare ptr @runtime.hashmapMakeGeneric(i32, i32, i32, ptr, ptr, ptr, ptr, ptr, ptr) #0
 
-declare void @runtime.hashmapBinarySet(ptr dereferenceable_or_null(48), ptr, ptr, ptr) #0
+declare void @runtime.hashmapBinarySet(ptr dereferenceable_or_null(52), ptr, ptr, ptr) #0
 
-declare i1 @runtime.hashmapBinaryGet(ptr dereferenceable_or_null(48), ptr, ptr, i32, ptr) #0
+declare i1 @runtime.hashmapBinaryGet(ptr dereferenceable_or_null(52), ptr, ptr, i32, ptr) #0
 
 ; Function Attrs: nounwind
 define hidden i8 @main.useLargeChannel(ptr dereferenceable_or_null(36) %ch, ptr readonly dereferenceable_or_null(1025) %value, ptr %context) unnamed_addr #1 {