diff --git a/changelog/druntime.tgc.dd b/changelog/druntime.tgc.dd new file mode 100644 index 000000000000..5887aa0ddf1b --- /dev/null +++ b/changelog/druntime.tgc.dd @@ -0,0 +1,15 @@ +Opt-in thread-local GC (`tgc`, 0.2.0 prototype) + +Select with `--DRT-gcopt=gc:tgc`. + +**0.2.0:** working partitioned shared regions with region-scoped collection +(selective thread suspend via `thread_suspendList`), per-heap list locks, +basic array append metadata, and benchmark/test harness. + +Region API: `_d_tgc_region_create`, `_d_tgc_region_attach`, `_d_tgc_region_malloc`, +`_d_tgc_region_collect`, `_d_tgc_version`. + +Optional: `--DRT-gcopt=gc:tgc tgcShared:symgc` (SymGC shared backend stub for 0.3.0). + +Tests: `druntime/test/gc/tgc.d`, `tgc_regions.d`, `tgc_bench.d`. +Benchmarks: `tools/tgc-bench/run-benchmarks.ps1`. diff --git a/druntime/Makefile b/druntime/Makefile index 95ca8cf7c58d..2e55fb0f4074 100644 --- a/druntime/Makefile +++ b/druntime/Makefile @@ -203,6 +203,9 @@ $(DOC_OUTPUT_DIR)/core_internal_gc_impl_conservative_%.html : import/core/intern $(DOC_OUTPUT_DIR)/core_internal_gc_impl_manual_%.html : import/core/internal/gc/impl/manual/%.d $(DMD) $(DMD) $(DDOCFLAGS) -Df$@ project.ddoc $(DOCFMT) $< +$(DOC_OUTPUT_DIR)/core_internal_gc_impl_tgc_%.html : import/core/internal/gc/impl/tgc/%.d $(DMD) + $(DMD) $(DDOCFLAGS) -Df$@ project.ddoc $(DOCFMT) $< + $(DOC_OUTPUT_DIR)/core_internal_gc_impl_proto_%.html : import/core/internal/gc/impl/proto/%.d $(DMD) $(DMD) $(DDOCFLAGS) -Df$@ project.ddoc $(DOCFMT) $< diff --git a/druntime/mak/COPY b/druntime/mak/COPY index 90dc5e23f599..c96129730769 100644 --- a/druntime/mak/COPY +++ b/druntime/mak/COPY @@ -79,6 +79,7 @@ COPY=\ $(IMPDIR)\core\internal\gc\proxy.d \ $(IMPDIR)\core\internal\gc\impl\conservative\gc.d \ $(IMPDIR)\core\internal\gc\impl\manual\gc.d \ + $(IMPDIR)\core\internal\gc\impl\tgc\gc.d \ $(IMPDIR)\core\internal\gc\impl\proto\gc.d \ \ $(IMPDIR)\core\internal\container\array.d \ diff --git a/druntime/mak/DOCS b/druntime/mak/DOCS index cb5049bf9d0d..c5d36b8aeea1 100644 --- a/druntime/mak/DOCS +++ b/druntime/mak/DOCS @@ -551,6 +551,7 @@ DOCS=\ $(DOCDIR)\core_internal_gc_proxy.html \ $(DOCDIR)\core_internal_gc_impl_conservative_gc.html \ $(DOCDIR)\core_internal_gc_impl_manual_gc.html \ + $(DOCDIR)\core_internal_gc_impl_tgc_gc.html \ $(DOCDIR)\core_internal_gc_impl_proto_gc.html \ \ $(DOCDIR)\rt_aApply.html \ diff --git a/druntime/mak/SRCS b/druntime/mak/SRCS index eff0ab33b62c..8222ea923a83 100644 --- a/druntime/mak/SRCS +++ b/druntime/mak/SRCS @@ -82,6 +82,7 @@ SRCS=\ src\core\internal\gc\proxy.d \ src\core\internal\gc\impl\conservative\gc.d \ src\core\internal\gc\impl\manual\gc.d \ + src\core\internal\gc\impl\tgc\gc.d \ src\core\internal\gc\impl\proto\gc.d \ \ src\core\internal\util\array.d \ diff --git a/druntime/src/core/gc/config.d b/druntime/src/core/gc/config.d index c3b79e0926b5..c42aaec0d7e0 100644 --- a/druntime/src/core/gc/config.d +++ b/druntime/src/core/gc/config.d @@ -20,6 +20,7 @@ struct Config bool fork = false; // optional concurrent behaviour ubyte profile; // enable profiling with summary when terminating program string gc = "conservative"; // select gc implementation conservative|precise|manual + string tgcShared = "native"; // tgc shared-region backend: native|symgc (symgc: 0.3.0 stub) @MemVal size_t initReserve; // initial reserve (bytes) @MemVal size_t minPoolSize = 1 << 20; // initial and minimum pool size (bytes) diff --git a/druntime/src/core/internal/gc/impl/tgc/gc.d b/druntime/src/core/internal/gc/impl/tgc/gc.d new file mode 100644 index 000000000000..26baa1e595e3 --- /dev/null +++ b/druntime/src/core/internal/gc/impl/tgc/gc.d @@ -0,0 +1,1545 @@ +/** + * Opt-in thread-local garbage collector (`tgc`) — **0.2.3 prototype**. + * + * Target design: per-thread private heaps plus partitioned shared regions + * (many-to-many). Collecting a region pauses only threads attached to that + * region. See dlang-supplemental design notes for the full architecture. + * + * **0.1.0:** private per-thread heaps, local collect, remote-free queue. + * **0.1.1:** shared-region API scaffold. + * **0.2.0:** region collect (selective suspend), heap locks, array append metadata. + * **0.2.x:** sorted-index findBlock O(log n), 32-byte header, bounded fixpoint mark. + * + * Cross-thread sharing on private heaps via remote free is interim, not the + * target model. Prefer attaching workers to a shared region. + * + * Select with `--DRT-gcopt=gc:tgc`. Informal side-name: "realtime GC". + * + * Copyright: Copyright dlang-supplemental contributors 2026. + * License: $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0). + */ +module core.internal.gc.impl.tgc.gc; + +/// Semantic version of the `tgc` prototype (not druntime release version). +enum tgcVersion = "0.2.3"; + +import core.gc.gcinterface; + +import core.internal.container.array; +import core.internal.spinlock; + +import core.thread.threadbase : ThreadBase, ScanAllThreadsFn, thread_scanList; + +extern (C) void thread_suspendList(ThreadBase*, size_t) nothrow; +extern (C) void thread_resumeList(ThreadBase*, size_t) nothrow; + +/// Shared-region backend selection (0.3.0 SymGC hybrid uses `symgc` when enabled). +enum TgcSharedBackend : ubyte { tgcNative, symgc } +private __gshared TgcSharedBackend tgcSharedBackend = TgcSharedBackend.tgcNative; + +import cstdlib = core.stdc.stdlib : calloc, free, malloc, realloc; +import core.stdc.string : memcpy, memmove, memset; +static import core.memory; + +extern (C) noreturn onOutOfMemoryError(void* pretend_sideffect = null, string file = __FILE__, size_t line = __LINE__) @trusted pure nothrow @nogc; /* dmd @@@BUG11461@@@ */ +extern (C) void rt_finalizeFromGC(void* p, size_t size, uint attr, const TypeInfo typeInfo) nothrow; +extern (C) void* thread_stackTop() nothrow @nogc; +extern (C) void* thread_stackBottom() nothrow @nogc; + +private enum size_t headerAlign = (void*).sizeof; +private enum size_t collectThresholdInit = 256 * 1024; +private enum uint markMask = 0x3; +private enum uint remoteQueuedBit = 0x4; + +/// Per-block metadata placed immediately before user payload. +/// 32 bytes on 64-bit (was 48 with intrusive list links). +private struct BlkHeader +{ + size_t size; /// user-visible capacity (alloc size) + size_t arrayUsed; /// used bytes when BlkAttr.APPENDABLE (else 0) + uint attr; /// BlkAttr bits (user-visible) + uint marked; /// low bits: mark state; remoteQueuedBit: pending free + ThreadHeap* heap; /// owning heap +} + +static assert(BlkHeader.sizeof == 32 || (void*).sizeof != 8); + +private struct ThreadHeap +{ + /// Address-sorted block index (by payload base). Replaces O(n) list walk. + BlkHeader** blocks; + size_t blockLen; + size_t blockCap; + void* minAddr; + void* maxAddr; + + size_t usedBytes; + size_t allocatedTotal; /// bytes allocated on this thread since start + size_t collectThreshold = collectThresholdInit; + size_t numCollections; + + // Remote frees pushed by other threads (ownership transfer). Implemented. + void** remotePtrs; + size_t remoteLen; + size_t remoteCap; + SpinLock remoteLock; + + bool collecting; + SpinLock listLock; + ThreadBase owner; + + static ThreadHeap* create() nothrow @nogc + { + auto h = cast(ThreadHeap*) cstdlib.calloc(1, ThreadHeap.sizeof); + if (!h) + onOutOfMemoryError(); + h.collectThreshold = collectThresholdInit; + h.remoteLock = SpinLock(SpinLock.Contention.brief); + h.listLock = SpinLock(SpinLock.Contention.brief); + return h; + } + + bool queueRemote(void* p) nothrow @nogc + { + listLock.lock(); + auto h = findBlockUnlocked(p); + if (!h || cast(void*)(h + 1) !is p || (h.marked & remoteQueuedBit)) + { + listLock.unlock(); + return false; + } + h.marked |= remoteQueuedBit; + + remoteLock.lock(); + if (remoteLen == remoteCap) + { + size_t ncap = remoteCap ? remoteCap * 2 : 16; + auto np = cast(void**) cstdlib.realloc(remotePtrs, ncap * (void*).sizeof); + if (!np) + { + remoteLock.unlock(); + listLock.unlock(); + onOutOfMemoryError(); + } + remotePtrs = np; + remoteCap = ncap; + } + remotePtrs[remoteLen++] = p; + remoteLock.unlock(); + listLock.unlock(); + return true; + } + + void drainRemote() nothrow @nogc + { + remoteLock.lock(); + size_t n = remoteLen; + void** ptrs = remotePtrs; + remotePtrs = null; + remoteLen = 0; + remoteCap = 0; + remoteLock.unlock(); + + foreach (i; 0 .. n) + { + auto p = ptrs[i]; + if (!p) + continue; + listLock.lock(); + auto h = findBlockUnlocked(p); + if (h && cast(void*)(h + 1) is p && (h.marked & remoteQueuedBit)) + { + indexRemove(h); + listLock.unlock(); + cstdlib.free(h); + } + else + listLock.unlock(); + } + cstdlib.free(ptrs); + } + + /// Insert `h` into the address-sorted index. Caller holds listLock or is sole owner. + void indexInsert(BlkHeader* h) nothrow @nogc + { + if (blockLen == blockCap) + { + size_t ncap = blockCap ? blockCap * 2 : 8; + auto np = cast(BlkHeader**) cstdlib.realloc(blocks, ncap * (BlkHeader*).sizeof); + if (!np) + onOutOfMemoryError(); + blocks = np; + blockCap = ncap; + } + void* base = h + 1; + // Find first index with payload base >= base (insertion point). + size_t lo = 0, hi = blockLen; + while (lo < hi) + { + size_t mid = lo + (hi - lo) / 2; + if (cast(void*)(blocks[mid] + 1) < base) + lo = mid + 1; + else + hi = mid; + } + if (lo != blockLen) + memmove(blocks + lo + 1, blocks + lo, + (blockLen - lo) * (BlkHeader*).sizeof); + blocks[lo] = h; + blockLen++; + usedBytes += h.size; + minAddr = blocks[0] + 1; + auto last = blocks[blockLen - 1]; + maxAddr = cast(void*)(last + 1) + last.size; + } + + /// Remove `h` from the address-sorted index. Caller holds listLock. + void indexRemove(BlkHeader* h) nothrow @nogc + { + void* base = h + 1; + size_t lo = 0, hi = blockLen; + while (lo < hi) + { + size_t mid = lo + (hi - lo) / 2; + auto mb = cast(void*)(blocks[mid] + 1); + if (mb < base) + lo = mid + 1; + else if (mb > base) + hi = mid; + else + { + if (mid + 1 != blockLen) + memmove(blocks + mid, blocks + mid + 1, + (blockLen - mid - 1) * (BlkHeader*).sizeof); + blockLen--; + if (usedBytes >= h.size) + usedBytes -= h.size; + else + usedBytes = 0; + if (blockLen) + { + minAddr = blocks[0] + 1; + auto last = blocks[blockLen - 1]; + maxAddr = cast(void*)(last + 1) + last.size; + } + else + minAddr = maxAddr = null; + return; + } + } + } + + void link(BlkHeader* h) nothrow @nogc + { + listLock.lock(); + indexInsert(h); + listLock.unlock(); + } + + void unlink(BlkHeader* h) nothrow @nogc + { + listLock.lock(); + indexRemove(h); + listLock.unlock(); + } + + void unlinkAndFree(BlkHeader* h) nothrow @nogc + { + unlink(h); + // GC.free does not run finalizers; call destroy() first if needed. + cstdlib.free(h); + } + + bool freeExact(void* p) nothrow @nogc + { + listLock.lock(); + auto h = findBlockUnlocked(p); + if (!h || cast(void*)(h + 1) !is p) + { + listLock.unlock(); + return false; + } + indexRemove(h); + listLock.unlock(); + cstdlib.free(h); + return true; + } + + void unlinkAndFreeFinalize(BlkHeader* h) nothrow + { + unlink(h); + if (h.attr & BlkAttr.FINALIZE) + rt_finalizeFromGC(h + 1, h.size, h.attr, null); + cstdlib.free(h); + } + + static BlkHeader* headerOf(void* p) nothrow @nogc + { + if (!p) + return null; + return cast(BlkHeader*) p - 1; + } + + /// Layered interior-pointer lookup: + /// range reject O(1), linear for tiny heaps, binary predecessor otherwise. + BlkHeader* findBlock(void* p) nothrow @nogc + { + if (!p) + return null; + listLock.lock(); + scope (exit) listLock.unlock(); + return findBlockUnlocked(p); + } + + /// Caller holds listLock. + BlkHeader* findBlockUnlocked(void* p) nothrow @nogc + { + if (!blockLen || p < minAddr || p >= maxAddr) + return null; + + // Linear wins for tiny arrays by avoiding branchy binary-search setup. + enum linearLookupLimit = 8; + if (blockLen <= linearLookupLimit) + { + foreach (i; 0 .. blockLen) + { + auto h = blocks[i]; + void* base = h + 1; + if (p < base) + return null; + if (p < base + h.size) + return h; + } + return null; + } + + // Find rightmost block with payload base <= p. + size_t lo = 0, hi = blockLen; + while (lo < hi) + { + size_t mid = lo + (hi - lo) / 2; + if (cast(void*)(blocks[mid] + 1) <= p) + lo = mid + 1; + else + hi = mid; + } + if (lo == 0) + return null; + auto h = blocks[lo - 1]; + void* base = h + 1; + if (p >= base && p < base + h.size) + return h; + return null; + } +} + +/** + * Partitioned shared region heap (target cross-thread model). + * + * Threads attach explicitly; collecting this region must pause only members + * (selective suspend not implemented in 0.1.0). + */ +private struct SharedRegion +{ + uint id; + ThreadHeap* heap; + ThreadHeap** memberHeaps; + ThreadBase* memberThreads; + size_t memberLen; + size_t memberCap; + SpinLock lock; + + static SharedRegion* create(uint id) nothrow @nogc + { + auto r = cast(SharedRegion*) cstdlib.calloc(1, SharedRegion.sizeof); + if (!r) + onOutOfMemoryError(); + r.id = id; + r.heap = ThreadHeap.create(); + r.lock = SpinLock(SpinLock.Contention.brief); + if (tgcSharedBackend == TgcSharedBackend.symgc) + { + import core.stdc.stdio : fprintf, stderr; + fprintf(stderr, "tgc: tgcShared:symgc requested; using native shared-region backend (0.3.0)\n".ptr); + } + return r; + } + + bool isAttached(ThreadHeap* h) nothrow @nogc + { + foreach (i; 0 .. memberLen) + if (memberHeaps[i] is h) + return true; + return false; + } + + bool isAttachedThread(ThreadBase tb) nothrow @nogc + { + if (!tb) + return false; + foreach (i; 0 .. memberLen) + if (memberThreads[i] is tb) + return true; + return false; + } + + bool attachThread(ThreadBase tb) nothrow @nogc + { + if (!tb) + return false; + auto h = currentHeap(); + if (!h) + return false; + lock.lock(); + if (isAttached(h)) + { + lock.unlock(); + return true; + } + if (memberLen == memberCap) + { + size_t ncap = memberCap ? memberCap * 2 : 4; + auto hp = cast(ThreadHeap**) cstdlib.realloc(memberHeaps, ncap * (ThreadHeap*).sizeof); + auto tp = cast(ThreadBase*) cstdlib.realloc(memberThreads, ncap * ThreadBase.sizeof); + if (!hp || !tp) + { + lock.unlock(); + onOutOfMemoryError(); + } + memberHeaps = hp; + memberThreads = tp; + memberCap = ncap; + } + memberHeaps[memberLen] = h; + memberThreads[memberLen] = tb; + memberLen++; + lock.unlock(); + return true; + } + + bool detachThread(ThreadBase tb) nothrow @nogc + { + if (!tb) + return false; + auto h = currentHeap(); + if (!h) + return false; + lock.lock(); + foreach (i; 0 .. memberLen) + { + if (memberHeaps[i] is h) + { + memberHeaps[i] = memberHeaps[memberLen - 1]; + memberThreads[i] = memberThreads[memberLen - 1]; + memberLen--; + lock.unlock(); + return true; + } + } + lock.unlock(); + return false; + } + + void collectRegion() nothrow + { + if (!heap) + return; + auto gc = cast(ThreadGC) tgcInstance; + if (!gc) + return; + + // Establish lock barriers before suspension so no member can be + // frozen while owning a lock needed by the collector. Keep the region + // lock through resume to serialize collectors, attach/detach, and + // allocation admission. + gc.rootsLock.lock(); + lock.lock(); + if (heap.collecting) + { + lock.unlock(); + gc.rootsLock.unlock(); + return; + } + heap.collecting = true; + heap.drainRemote(); + + heap.listLock.lock(); + foreach (i; 0 .. heap.blockLen) + heap.blocks[i].marked &= remoteQueuedBit; + + ThreadBase* tlist = null; + size_t n = memberLen; + if (n) + { + tlist = cast(ThreadBase*) cstdlib.malloc(n * ThreadBase.sizeof); + if (!tlist) + onOutOfMemoryError(); + memcpy(tlist, memberThreads, n * ThreadBase.sizeof); + // Barrier every member-private index before suspension. Once the + // members stop, releasing these locks leaves stable indexes for + // region-root scanning. + foreach (i; 0 .. n) + memberHeaps[i].listLock.lock(); + } + + if (n) + thread_suspendList(tlist, n); + heap.listLock.unlock(); + foreach (i; 0 .. n) + memberHeaps[i].listLock.unlock(); + + if (n) + { + thread_scanList(tlist, n, (void* p1, void* p2) nothrow { + gc.markRangeHeap(heap, p1, p2); + }); + foreach (i; 0 .. n) + gc.markHeapContentsInto(memberHeaps[i], heap); + } + + foreach (ref r; gc.roots) + { + if (r.proot) + gc.markPtrHeap(heap, *cast(void**) r.proot); + gc.markPtrHeap(heap, r.proot); + } + foreach (ref r; gc.ranges) + gc.markRangeHeap(heap, r.pbot, r.ptop); + gc.rootsLock.unlock(); + + if (gc.markHeapFixpoint(heap)) + gc.sweepHeap(heap); + if (n) + { + thread_resumeList(tlist, n); + cstdlib.free(tlist); + } + heap.numCollections++; + gc.profileCollections++; + heap.collecting = false; + lock.unlock(); + } +} + +private __gshared SharedRegion** allRegions; +private __gshared size_t allRegionsLen; +private __gshared size_t allRegionsCap; +private __gshared uint nextRegionId = 1; +private __gshared SpinLock regionsLock; +private __gshared GC tgcInstance; + +private SharedRegion* findRegion(uint id) nothrow @nogc +{ + regionsLock.lock(); + foreach (i; 0 .. allRegionsLen) + { + auto r = allRegions[i]; + if (r && r.id == id) + { + regionsLock.unlock(); + return r; + } + } + regionsLock.unlock(); + return null; +} + +/// Create a partitioned shared region; returns region id (0 on failure). +extern (C) uint _d_tgc_region_create() nothrow @nogc +{ + regionsLock.lock(); + uint id = nextRegionId++; + auto r = SharedRegion.create(id); + if (allRegionsLen == allRegionsCap) + { + size_t ncap = allRegionsCap ? allRegionsCap * 2 : 4; + auto np = cast(SharedRegion**) cstdlib.realloc(allRegions, ncap * (SharedRegion*).sizeof); + if (!np) + { + regionsLock.unlock(); + onOutOfMemoryError(); + } + allRegions = np; + allRegionsCap = ncap; + } + allRegions[allRegionsLen++] = r; + regionsLock.unlock(); + return id; +} + +/// Attach the calling thread's private heap to `regionId`. +extern (C) bool _d_tgc_region_attach(uint regionId) nothrow @nogc +{ + auto r = findRegion(regionId); + if (!r) + return false; + auto tb = ThreadBase.getThis(); + if (!tb) + return false; + return r.attachThread(tb); +} + +/// Detach the calling thread from `regionId`. +extern (C) bool _d_tgc_region_detach(uint regionId) nothrow @nogc +{ + auto r = findRegion(regionId); + if (!r) + return false; + auto tb = ThreadBase.getThis(); + if (!tb) + return false; + return r.detachThread(tb); +} + +/// Collect a shared region (pauses only attached threads). +extern (C) bool _d_tgc_region_collect(uint regionId) nothrow +{ + auto r = findRegion(regionId); + if (!r) + return false; + r.collectRegion(); + return true; +} + +/// Allocate in a shared region (attached threads only). Returns null if unknown region or not attached. +extern (C) void* _d_tgc_region_malloc(uint regionId, size_t size, uint bits) nothrow @nogc +{ + auto r = findRegion(regionId); + if (!r) + return null; + auto tb = ThreadBase.getThis(); + r.lock.lock(); + if (!r.isAttachedThread(tb)) + { + r.lock.unlock(); + return null; + } + r.lock.unlock(); + + r.heap.drainRemote(); + if (size > size_t.max - BlkHeader.sizeof) + onOutOfMemoryError(); + size_t total = BlkHeader.sizeof + size; + auto raw = cstdlib.malloc(total); + if (size && raw is null) + onOutOfMemoryError(); + memset(raw, 0, BlkHeader.sizeof); + auto h = cast(BlkHeader*) raw; + h.size = size; + h.arrayUsed = (bits & BlkAttr.APPENDABLE) ? size : 0; + h.attr = bits; + h.marked = 0; + h.heap = r.heap; + r.heap.link(h); + r.heap.allocatedTotal += size; + return cast(void*)(h + 1); +} + +extern (C) const(char)* _d_tgc_version() nothrow @nogc +{ + return tgcVersion.ptr; +} + +// TLS pointer to the calling thread's heap +// TLS pointer to the calling thread's heap +private ThreadHeap* tlsHeap; + +private __gshared ThreadHeap*[] allHeaps; +private __gshared size_t allHeapsLen; +private __gshared size_t allHeapsCap; +private __gshared SpinLock heapsLock; + +private void registerHeap(ThreadHeap* h) nothrow @nogc +{ + heapsLock.lock(); + if (allHeapsLen == allHeapsCap) + { + size_t ncap = allHeapsCap ? allHeapsCap * 2 : 8; + auto np = cast(ThreadHeap**) cstdlib.realloc(allHeaps.ptr, ncap * (ThreadHeap*).sizeof); + if (!np) + { + heapsLock.unlock(); + onOutOfMemoryError(); + } + allHeaps = np[0 .. ncap]; + allHeapsCap = ncap; + } + allHeaps[allHeapsLen++] = h; + heapsLock.unlock(); +} + +private void unregisterHeap(ThreadHeap* h) nothrow @nogc +{ + heapsLock.lock(); + foreach (i; 0 .. allHeapsLen) + { + if (allHeaps[i] is h) + { + allHeaps[i] = allHeaps[allHeapsLen - 1]; + allHeapsLen--; + break; + } + } + heapsLock.unlock(); +} + +private bool isRegisteredHeap(ThreadHeap* h) nothrow @nogc +{ + if (!h) + return false; + heapsLock.lock(); + foreach (i; 0 .. allHeapsLen) + { + if (allHeaps[i] is h) + { + heapsLock.unlock(); + return true; + } + } + heapsLock.unlock(); + return false; +} + +private ThreadHeap* currentHeap() nothrow @nogc +{ + if (tlsHeap && isRegisteredHeap(tlsHeap)) + return tlsHeap; + auto t = ThreadBase.getThis(); + if (t) + { + auto existing = cast(ThreadHeap*) t.tlsGCData(); + if (isRegisteredHeap(existing)) + { + tlsHeap = existing; + return tlsHeap; + } + } + tlsHeap = ThreadHeap.create(); + registerHeap(tlsHeap); + if (t) + t.tlsGCData() = tlsHeap; + return tlsHeap; +} + +// register GC in C constructor +private pragma(crt_constructor) void gc_tgc_ctor() +{ + heapsLock = SpinLock(SpinLock.Contention.brief); + regionsLock = SpinLock(SpinLock.Contention.brief); + _d_register_tgc_gc(); +} + +extern (C) void _d_register_tgc_gc() +{ + import core.gc.registry; + registerGCFactory("tgc", &initialize, &threadInitHook); +} + +private void threadInitHook(ThreadBase base) nothrow @nogc +{ + auto h = cast(ThreadHeap*) base.tlsGCData(); + if (!isRegisteredHeap(h)) + { + h = ThreadHeap.create(); + registerHeap(h); + } + tlsHeap = h; + base.tlsGCData() = h; +} + +private bool isSharedRegionHeap(ThreadHeap* h) nothrow @nogc +{ + if (!h) + return false; + regionsLock.lock(); + foreach (i; 0 .. allRegionsLen) + { + auto r = allRegions[i]; + if (r && r.heap is h) + { + regionsLock.unlock(); + return true; + } + } + regionsLock.unlock(); + return false; +} + +private GC initialize() +{ + import core.lifetime : emplace; + import core.gc.config; + + if (config.tgcShared == "symgc") + tgcSharedBackend = TgcSharedBackend.symgc; + + auto gc = cast(ThreadGC) cstdlib.malloc(__traits(classInstanceSize, ThreadGC)); + if (!gc) + onOutOfMemoryError(); + + auto inst = emplace(gc); + tgcInstance = inst; + return inst; +} + +/** + * Thread-local GC implementation (`tgc` 0.1.0 prototype). + * + * Also known informally as a "realtime GC" because local collection does not + * globally stop-the-world; the name registered with the runtime is `tgc`. + */ +class ThreadGC : GC +{ + Array!Root roots; + Array!Range ranges; + SpinLock rootsLock; + bool disabled; + size_t profileCollections; + ulong profilePauseTicks; + + this() + { + rootsLock = SpinLock(SpinLock.Contention.brief); + // Ensure the initializing thread has a heap. + cast(void) currentHeap(); + } + + ~this() + { + } + + void enable() + { + disabled = false; + } + + void disable() + { + disabled = true; + } + + void collect() nothrow + { + collectHeap(currentHeap()); + } + + void minimize() nothrow + { + auto h = currentHeap(); + h.drainRemote(); + } + + uint getAttr(void* p) nothrow + { + auto blk = queryBlock(p); + return isExactBase(blk, p) ? blk.attr : 0; + } + + uint setAttr(void* p, uint mask) nothrow + { + auto blk = queryBlock(p); + if (!isExactBase(blk, p)) + return 0; + blk.attr |= mask; + return blk.attr; + } + + uint clrAttr(void* p, uint mask) nothrow + { + auto blk = queryBlock(p); + if (!isExactBase(blk, p)) + return 0; + blk.attr &= ~mask; + return blk.attr; + } + + void* malloc(size_t size, uint bits, const TypeInfo ti) nothrow + { + return alloc(size, bits, false); + } + + BlkInfo qalloc(size_t size, uint bits, const scope TypeInfo ti) nothrow + { + BlkInfo retval; + retval.base = alloc(size, bits, false); + retval.size = size; + retval.attr = bits; + return retval; + } + + void* calloc(size_t size, uint bits, const TypeInfo ti) nothrow + { + return alloc(size, bits, true); + } + + void* realloc(void* p, size_t size, uint bits, const TypeInfo ti) nothrow + { + if (!p) + return alloc(size, bits, false); + if (!size) + { + free(p); + return null; + } + + auto blk = queryBlock(p); + if (!isExactBase(blk, p)) + return null; + + auto heap = blk.heap; + if (heap !is currentHeap()) + { + // Cannot realloc foreign block in place; copy into local heap. + auto np = alloc(size, bits ? bits : blk.attr, false); + auto n = size < blk.size ? size : blk.size; + memcpy(np, p, n); + free(p); + return np; + } + + if (size <= blk.size) + { + auto oldSize = blk.size; + heap.listLock.lock(); + blk.size = size; + if (blk.arrayUsed > size) + blk.arrayUsed = size; + if (bits) + blk.attr = bits; + heap.usedBytes -= oldSize - size; + if (heap.blockLen && heap.blocks[heap.blockLen - 1] is blk) + heap.maxAddr = cast(void*)(blk + 1) + size; + heap.listLock.unlock(); + return p; + } + + auto np = alloc(size, bits ? bits : blk.attr, false); + memcpy(np, p, blk.size); + heap.unlinkAndFree(blk); + return np; + } + + size_t extend(void* p, size_t minsize, size_t maxsize, const TypeInfo ti) nothrow + { + return 0; + } + + size_t reserve(size_t size) nothrow + { + return 0; + } + + void free(void* p) nothrow @nogc + { + if (!p) + return; + + auto local = tlsHeap; + if (local && local.freeExact(p)) + return; + + regionsLock.lock(); + foreach (i; 0 .. allRegionsLen) + { + auto r = allRegions[i]; + if (!r || !r.heap) + continue; + r.lock.lock(); + bool freed = r.heap.freeExact(p); + r.lock.unlock(); + if (freed) + { + regionsLock.unlock(); + return; + } + } + regionsLock.unlock(); + + // Keep the registry lock until the owner has accepted the request. + // cleanupThread unregisters before destroying a heap. + heapsLock.lock(); + foreach (i; 0 .. allHeapsLen) + { + auto owner = allHeaps[i]; + if (owner !is local && owner.queueRemote(p)) + { + heapsLock.unlock(); + return; + } + } + heapsLock.unlock(); + } + + void* addrOf(void* p) nothrow @nogc + { + auto blk = queryBlock(p); + return blk ? cast(void*)(blk + 1) : null; + } + + size_t sizeOf(void* p) nothrow @nogc + { + auto blk = queryBlock(p); + return isExactBase(blk, p) ? blk.size : 0; + } + + BlkInfo query(void* p) nothrow + { + auto blk = queryBlock(p); + if (!blk) + return BlkInfo.init; + BlkInfo info; + info.base = cast(void*)(blk + 1); + info.size = blk.size; + info.attr = blk.attr; + return info; + } + + core.memory.GC.Stats stats() @trusted nothrow + { + core.memory.GC.Stats s; + auto h = currentHeap(); + s.usedSize = h.usedBytes; + s.freeSize = 0; + s.allocatedInCurrentThread = h.allocatedTotal; + return s; + } + + core.memory.GC.ProfileStats profileStats() @trusted nothrow + { + core.memory.GC.ProfileStats s; + s.numCollections = profileCollections; + return s; + } + + void addRoot(void* p) nothrow @nogc + { + rootsLock.lock(); + roots.insertBack(Root(p)); + rootsLock.unlock(); + } + + void removeRoot(void* p) nothrow @nogc + { + rootsLock.lock(); + foreach (ref r; roots) + { + if (r is p) + { + r = roots.back; + roots.popBack(); + rootsLock.unlock(); + return; + } + } + rootsLock.unlock(); + assert(false); + } + + @property RootIterator rootIter() return @nogc + { + return &rootsApply; + } + + private int rootsApply(scope int delegate(ref Root) nothrow dg) + { + rootsLock.lock(); + foreach (ref r; roots) + { + if (auto result = dg(r)) + { + rootsLock.unlock(); + return result; + } + } + rootsLock.unlock(); + return 0; + } + + void addRange(void* p, size_t sz, const TypeInfo ti = null) nothrow @nogc + { + rootsLock.lock(); + ranges.insertBack(Range(p, p + sz, cast() ti)); + rootsLock.unlock(); + } + + void removeRange(void* p) nothrow @nogc + { + rootsLock.lock(); + foreach (ref r; ranges) + { + if (r.pbot is p) + { + r = ranges.back; + ranges.popBack(); + rootsLock.unlock(); + return; + } + } + rootsLock.unlock(); + assert(false); + } + + @property RangeIterator rangeIter() return @nogc + { + return &rangesApply; + } + + private int rangesApply(scope int delegate(ref Range) nothrow dg) + { + rootsLock.lock(); + foreach (ref r; ranges) + { + if (auto result = dg(r)) + { + rootsLock.unlock(); + return result; + } + } + rootsLock.unlock(); + return 0; + } + + void runFinalizers(const scope void[] segment) nothrow + { + } + + bool inFinalizer() nothrow + { + auto h = tlsHeap; + return h !is null && h.collecting; + } + + ulong allocatedInCurrentThread() nothrow + { + return currentHeap().allocatedTotal; + } + + void[] getArrayUsed(void* ptr, bool atomic = false) nothrow + { + auto blk = queryBlock(ptr); + if (!blk || !(blk.attr & BlkAttr.APPENDABLE)) + return null; + auto heap = blk.heap; + heap.listLock.lock(); + auto used = blk.arrayUsed; + heap.listLock.unlock(); + return (cast(void*)(blk + 1))[0 .. used]; + } + + bool expandArrayUsed(void[] slice, size_t newUsed, bool atomic = false) nothrow @trusted + { + if (!slice.ptr || newUsed < slice.length) + return false; + auto blk = queryBlock(slice.ptr); + if (!blk || !(blk.attr & BlkAttr.APPENDABLE)) + return false; + auto base = cast(void*)(blk + 1); + size_t offset = slice.ptr - base; + if (offset > blk.size || newUsed > blk.size - offset) + return false; + auto heap = blk.heap; + heap.listLock.lock(); + if (offset + slice.length != blk.arrayUsed) + { + heap.listLock.unlock(); + return false; + } + blk.arrayUsed = offset + newUsed; + heap.listLock.unlock(); + return true; + } + + size_t reserveArrayCapacity(void[] slice, size_t request, bool atomic = false) nothrow @trusted + { + if (!slice.ptr) + return 0; + auto blk = queryBlock(slice.ptr); + if (!blk || !(blk.attr & BlkAttr.APPENDABLE)) + return 0; + auto base = cast(void*)(blk + 1); + size_t offset = slice.ptr - base; + if (offset > blk.size || slice.length > blk.size - offset) + return 0; + auto heap = blk.heap; + heap.listLock.lock(); + bool isTail = offset + slice.length == blk.arrayUsed; + size_t capacity = isTail && request <= blk.size - offset + ? blk.size - offset : 0; + heap.listLock.unlock(); + // This malloc-backed prototype cannot extend in place. Returning zero + // makes the array runtime allocate/copy safely instead of retaining a + // pointer to storage that reserve moved behind its back. + return capacity; + } + + bool shrinkArrayUsed(void[] slice, size_t existingUsed, bool atomic = false) nothrow + { + if (!slice.ptr) + return false; + auto blk = queryBlock(slice.ptr); + if (!blk || !(blk.attr & BlkAttr.APPENDABLE)) + return false; + if (existingUsed < slice.length) + return false; + auto base = cast(void*)(blk + 1); + size_t offset = slice.ptr - base; + if (offset > blk.size || existingUsed > blk.size - offset) + return false; + auto heap = blk.heap; + heap.listLock.lock(); + if (offset + existingUsed != blk.arrayUsed) + { + heap.listLock.unlock(); + return false; + } + blk.arrayUsed = offset + slice.length; + heap.listLock.unlock(); + return true; + } + + package void markPtrHeap(ThreadHeap* heap, void* p) nothrow @nogc + { + markPtrInHeap(heap, p); + } + + package void markRangeHeap(ThreadHeap* heap, void* pbot, void* ptop) nothrow @nogc + { + markRangeInHeap(heap, pbot, ptop); + } + + package void markHeapContentsInto(ThreadHeap* source, ThreadHeap* target) nothrow @nogc + { + if (!source || !target) + return; + source.listLock.lock(); + foreach (i; 0 .. source.blockLen) + { + auto b = source.blocks[i]; + if (b.attr & BlkAttr.NO_SCAN) + continue; + auto base = cast(void*)(b + 1); + size_t scanLen = (b.attr & BlkAttr.APPENDABLE) && b.arrayUsed + ? b.arrayUsed : b.size; + markRangeInHeap(target, base, base + scanLen); + } + source.listLock.unlock(); + } + + /// Returns true if fixpoint converged (safe to sweep). + package bool markHeapFixpoint(ThreadHeap* heap) nothrow @nogc + { + BlkHeader** work = null; + size_t workCap = 0; + + while (true) + { + // Snapshot newly marked candidates under lock; scan unlocked because + // findBlock takes the same lock. State 2 prevents rescanning blocks. + heap.listLock.lock(); + size_t workLen = 0; + foreach (i; 0 .. heap.blockLen) + { + auto b = heap.blocks[i]; + if ((b.marked & markMask) != 1) + continue; + b.marked = (b.marked & ~markMask) | 2; + if (b.attr & BlkAttr.NO_SCAN) + continue; + if (workLen == workCap) + { + size_t ncap = workCap ? workCap * 2 : 8; + auto np = cast(BlkHeader**) cstdlib.realloc(work, ncap * (BlkHeader*).sizeof); + if (!np) + { + heap.listLock.unlock(); + cstdlib.free(work); + onOutOfMemoryError(); + } + work = np; + workCap = ncap; + } + work[workLen++] = b; + } + heap.listLock.unlock(); + + foreach (i; 0 .. workLen) + { + auto b = work[i]; + void* base = b + 1; + size_t scanLen = (b.attr & BlkAttr.APPENDABLE) && b.arrayUsed + ? b.arrayUsed : b.size; + markRangeInHeap(heap, base, base + scanLen); + } + if (!workLen) + { + cstdlib.free(work); + return true; + } + } + + } + + package void sweepHeap(ThreadHeap* heap) nothrow + { + BlkHeader** doomed = null; + size_t doomedLen = 0; + size_t doomedCap = 0; + heap.listLock.lock(); + for (size_t i = heap.blockLen; i > 0; --i) + { + auto b = heap.blocks[i - 1]; + if (b.marked & markMask) + continue; + heap.indexRemove(b); + if (doomedLen == doomedCap) + { + size_t ncap = doomedCap ? doomedCap * 2 : 8; + auto np = cast(BlkHeader**) cstdlib.realloc(doomed, ncap * (BlkHeader*).sizeof); + if (!np) + { + heap.listLock.unlock(); + onOutOfMemoryError(); + } + doomed = np; + doomedCap = ncap; + } + doomed[doomedLen++] = b; + } + heap.listLock.unlock(); + + foreach (i; 0 .. doomedLen) + { + auto b = doomed[i]; + if (b.attr & BlkAttr.FINALIZE) + rt_finalizeFromGC(b + 1, b.size, b.attr, null); + cstdlib.free(b); + } + cstdlib.free(doomed); + } + + void initThread(ThreadBase t) nothrow @nogc + { + auto h = currentHeap(); + h.owner = t; + t.tlsGCData() = h; + } + + void cleanupThread(ThreadBase t) nothrow @nogc + { + auto h = cast(ThreadHeap*) t.tlsGCData(); + if (!h) + return; + // Stop new foreign lookups before draining and destroying this heap. + unregisterHeap(h); + h.drainRemote(); + while (h.blockLen) + { + auto cur = h.blocks[h.blockLen - 1]; + h.unlinkAndFree(cur); + } + if (tlsHeap is h) + tlsHeap = null; + t.tlsGCData() = null; + regionsLock.lock(); + foreach (i; 0 .. allRegionsLen) + { + auto r = allRegions[i]; + if (!r) + continue; + r.lock.lock(); + foreach (j; 0 .. r.memberLen) + { + if (r.memberHeaps[j] is h) + { + r.memberHeaps[j] = r.memberHeaps[r.memberLen - 1]; + r.memberThreads[j] = r.memberThreads[r.memberLen - 1]; + r.memberLen--; + break; + } + } + r.lock.unlock(); + } + regionsLock.unlock(); + cstdlib.free(h.remotePtrs); + cstdlib.free(h.blocks); + cstdlib.free(h); + } + +private: + + BlkHeader* queryBlock(void* p) nothrow @nogc + { + if (!p) + return null; + // Fast path: local heap + if (tlsHeap) + { + if (auto b = tlsHeap.findBlock(p)) + return b; + } + // Slow path: search registered heaps (for free/query of foreign ptrs) + heapsLock.lock(); + foreach (i; 0 .. allHeapsLen) + { + if (auto b = allHeaps[i].findBlock(p)) + { + heapsLock.unlock(); + return b; + } + } + heapsLock.unlock(); + regionsLock.lock(); + foreach (i; 0 .. allRegionsLen) + { + auto r = allRegions[i]; + if (!r || !r.heap) + continue; + if (auto b = r.heap.findBlock(p)) + { + regionsLock.unlock(); + return b; + } + } + regionsLock.unlock(); + return null; + } + + static bool isExactBase(BlkHeader* blk, void* p) nothrow @nogc + { + return blk !is null && cast(void*)(blk + 1) is p; + } + + void* alloc(size_t size, uint bits, bool zero) nothrow + { + auto heap = currentHeap(); + heap.drainRemote(); + + if (!disabled && heap.usedBytes >= heap.collectThreshold) + collectHeap(heap); + + if (size > size_t.max - BlkHeader.sizeof) + onOutOfMemoryError(); + size_t total = BlkHeader.sizeof + size; + // Align user payload + auto raw = zero ? cstdlib.calloc(1, total) : cstdlib.malloc(total); + if (size && raw is null) + onOutOfMemoryError(); + if (!zero) + memset(raw, 0, BlkHeader.sizeof); + + auto h = cast(BlkHeader*) raw; + h.size = size; + h.attr = bits; + h.marked = 0; + h.arrayUsed = (bits & BlkAttr.APPENDABLE) ? size : 0; + h.heap = heap; + heap.link(h); + heap.allocatedTotal += size; + + if (heap.usedBytes > heap.collectThreshold) + heap.collectThreshold = heap.usedBytes * 2; + + return cast(void*)(h + 1); + } + + void collectHeap(ThreadHeap* heap) nothrow + { + if (!heap || heap.collecting || disabled) + return; + + heap.collecting = true; + heap.drainRemote(); + + heap.listLock.lock(); + foreach (i; 0 .. heap.blockLen) + heap.blocks[i].marked &= remoteQueuedBit; + heap.listLock.unlock(); + + void* top; + void* bot; + tryStackBounds(top, bot); + if (top && bot) + { + if (top > bot) + { + auto tmp = top; + top = bot; + bot = tmp; + } + markRangeInHeap(heap, top, bot); + } + + markTLSHeap(heap); + + rootsLock.lock(); + foreach (ref r; roots) + { + if (r.proot) + markPtrInHeap(heap, *cast(void**) r.proot); + markPtrInHeap(heap, r.proot); + } + foreach (ref r; ranges) + markRangeInHeap(heap, r.pbot, r.ptop); + rootsLock.unlock(); + + if (markHeapFixpoint(heap)) + sweepHeap(heap); + // Non-converged fixpoint: skip sweep (safer than freeing live objects). + + heap.numCollections++; + profileCollections++; + heap.collecting = false; + } + + void tryStackBounds(ref void* top, ref void* bot) nothrow + { + top = null; + bot = null; + if (ThreadBase.getThis() is null) + { + // Without attachment, approximate with a local and a small window. + void* approx; + top = ≈ + bot = cast(void*)(&approx) + 4096; + return; + } + top = thread_stackTop(); + bot = thread_stackBottom(); + } + + void markTLSHeap(ThreadHeap* heap) nothrow + { + import rt.sections; + auto rng = initTLSRanges(); + scanTLSRanges(rng, (void* pbeg, void* pend) nothrow { + markRangeInHeap(heap, pbeg, pend); + }); + } + + void markRangeInHeap(ThreadHeap* heap, void* pbot, void* ptop) nothrow @nogc + { + if (!pbot || !ptop || pbot >= ptop) + return; + auto p = cast(void**) pbot; + auto e = cast(void**) ptop; + auto addr = cast(size_t) p; + addr = (addr + (void*).sizeof - 1) & ~((void*).sizeof - 1); + p = cast(void**) addr; + for (; p + 1 <= e; ++p) + markPtrInHeap(heap, *p); + } + + void markPtrInHeap(ThreadHeap* heap, void* p) nothrow @nogc + { + if (!p) + return; + auto b = heap.findBlock(p); + if (b && !(b.marked & markMask)) + b.marked = (b.marked & ~markMask) | 1; + } + + static BlkHeader* headerOf(void* p) nothrow @nogc + { + return ThreadHeap.headerOf(p); + } +} diff --git a/druntime/src/core/internal/gc/proxy.d b/druntime/src/core/internal/gc/proxy.d index 9b83eb5d3fee..79a1f48534d2 100644 --- a/druntime/src/core/internal/gc/proxy.d +++ b/druntime/src/core/internal/gc/proxy.d @@ -37,6 +37,7 @@ extern (C) // do not import GC modules, they might add a dependency to this whole module void _d_register_conservative_gc(); void _d_register_manual_gc(); + void _d_register_tgc_gc(); // if you don't want to include the default GCs, replace during link by another implementation void* register_default_gcs() @weak @@ -46,7 +47,9 @@ extern (C) // avoid being optimized away auto reg1 = &_d_register_conservative_gc; auto reg2 = &_d_register_manual_gc; - return reg1 < reg2 ? reg1 : reg2; + auto reg3 = &_d_register_tgc_gc; + auto m = reg1 < reg2 ? reg1 : reg2; + return m < reg3 ? m : reg3; } void gc_init() diff --git a/druntime/src/core/internal/parseoptions.d b/druntime/src/core/internal/parseoptions.d index bc4755517ed5..05e014d6eb74 100644 --- a/druntime/src/core/internal/parseoptions.d +++ b/druntime/src/core/internal/parseoptions.d @@ -410,6 +410,7 @@ unittest assert(conf.parseOptions("help profile:1 help")); assert(conf.parseOptions("gc:manual") && conf.gc == "manual"); + assert(conf.parseOptions("gc:tgc") && conf.gc == "tgc"); assert(conf.parseOptions("gc:my-gc~modified") && conf.gc == "my-gc~modified"); assert(conf.parseOptions("gc:conservative help profile:1") && conf.gc == "conservative" && conf.profile == 1); diff --git a/druntime/src/core/thread/osthread.d b/druntime/src/core/thread/osthread.d index 827b19b61df9..9c38fa84a8b1 100644 --- a/druntime/src/core/thread/osthread.d +++ b/druntime/src/core/thread/osthread.d @@ -1513,6 +1513,83 @@ extern (C) void thread_suspendAll() nothrow } } +/** + * Suspend only the listed threads for partial stop-the-world collection. + * The calling thread is never blocked; if listed, only its registers are captured. + * Must be paired with thread_resumeList. + */ +extern (C) void thread_suspendList(ThreadBase* list, size_t count) nothrow +{ + thread_preStopTheWorld(); + if (++listSuspendDepth > 1) + return; + + size_t cnt; + bool suspendedSelf; + ThreadBase caller = ThreadBase.sm_tbeg ? ThreadBase.getThis() : null; + + for (size_t i = 0; i < count; ++i) + { + auto tb = list[i]; + if (!tb) + continue; + if (suspend(tb.toThread)) + { + if (tb is caller) + suspendedSelf = true; + ++cnt; + } + } + + version (Darwin) {} + else version (Solaris) {} + else version (WASI) {} + else version (Posix) + { + if (!multiThreadedFlag) + return; + assert(cnt >= 1); + if (suspendedSelf) + --cnt; + for (; cnt; --cnt) + { + while (sem_wait(&suspendCount) != 0) + { + if (errno != EINTR) + onThreadError("Unable to wait for semaphore"); + errno = 0; + } + } + } + else version (Windows) {} + else + static assert(0, "unsupported os"); +} + +/** + * Resume threads suspended by thread_suspendList. + */ +extern (C) void thread_resumeList(ThreadBase* list, size_t count) nothrow +in +{ + assert(listSuspendDepth > 0); +} +do +{ + if (--listSuspendDepth > 0) + return; + + scope (exit) thread_postRestartTheWorld(); + + for (size_t i = 0; i < count; ++i) + { + auto tb = list[i]; + if (!tb) + continue; + resume(tb); + } +} + /** * Resume the specified thread and unload stack and register information. * If the supplied thread is the calling thread, stack and register diff --git a/druntime/src/core/thread/threadbase.d b/druntime/src/core/thread/threadbase.d index 59f46b50e523..b1a88a90157a 100644 --- a/druntime/src/core/thread/threadbase.d +++ b/druntime/src/core/thread/threadbase.d @@ -1053,6 +1053,9 @@ package __gshared bool multiThreadedFlag = false; // Used for suspendAll/resumeAll below. package __gshared uint suspendDepth = 0; +// Partial STW for opt-in GC region collect (tgc). Separate from suspendDepth. +package __gshared uint listSuspendDepth = 0; + private alias resume = externDFunc!("core.thread.osthread.resume", void function(ThreadBase) nothrow @nogc); /** @@ -1214,6 +1217,68 @@ extern (C) void thread_scanAll(scope ScanAllThreadsFn scan) nothrow thread_scanAllType((type, p1, p2) => scan(p1, p2)); } +/** + * Scan stacks/registers/TLS of threads suspended by thread_suspendList. + */ +extern (C) void thread_scanList(ThreadBase* list, size_t count, scope ScanAllThreadsFn scan) nothrow +in +{ + assert(listSuspendDepth > 0); +} +do +{ + callWithStackShell(sp => scanListImpl(list, count, scan, sp)); +} + +private void scanListImpl(ThreadBase* list, size_t count, scope ScanAllThreadsFn scan, void* curStackTop) nothrow +{ + ThreadBase thisThread = null; + void* oldStackTop = null; + + if (ThreadBase.sm_tbeg) + { + thisThread = ThreadBase.getThis(); + if (thisThread && !thisThread.m_lock) + { + oldStackTop = thisThread.m_curr.tstack; + thisThread.m_curr.tstack = curStackTop; + } + } + + scope (exit) + { + if (thisThread && !thisThread.m_lock) + thisThread.m_curr.tstack = oldStackTop; + } + + for (size_t i = 0; i < count; ++i) + { + auto t = list[i]; + if (!t) + continue; + + for (StackContext* c = t.m_curr; c; c = c.within) + { + static if (isStackGrowingDown) + { + if (c.tstack && c.tstack < c.bstack) + scan(c.tstack, c.bstack); + } + else + { + if (c.bstack && c.bstack < c.tstack) + scan(c.bstack, c.tstack + 1); + } + } + + if (auto regs = t.savedRegisters()) + scan(regs.ptr, regs.ptr + regs.length); + + if (t.m_tlsrtdata !is null) + rt_tlsgc_scan(t.m_tlsrtdata, (p1, p2) => scan(p1, p2)); + } +} + private alias thread_yield = externDFunc!("core.thread.osthread.thread_yield", void function() @nogc nothrow); diff --git a/druntime/test/gc/Makefile b/druntime/test/gc/Makefile index 2567ec128f21..ffce9d231914 100644 --- a/druntime/test/gc/Makefile +++ b/druntime/test/gc/Makefile @@ -1,6 +1,6 @@ TESTS:=attributes sentinel printf memstomp invariant logging \ precise precisegc \ - recoverfree collect nocollect + recoverfree collect nocollect tgc tgc_regions tgc_remote_free tgc_bench ifneq ($(OS),windows) # some .d files are for Posix only @@ -86,6 +86,10 @@ $(ROOT)/precise_concurrent$(DOTEXE): extra_dflags += $(core_ut) -main $(ROOT)/precise_concurrent.done: run_args+="--DRT-gcopt=gc:precise fork:1" $(ROOT)/attributes$(DOTEXE): extra_dflags += $(core_ut) +$(ROOT)/tgc.done: run_args+=--DRT-gcopt=gc:tgc +$(ROOT)/tgc_regions.done: run_args+=--DRT-gcopt=gc:tgc +$(ROOT)/tgc_remote_free.done: run_args+=--DRT-gcopt=gc:tgc +$(ROOT)/tgc_bench.done: run_args+=--DRT-gcopt=gc:tgc $(ROOT)/forkgc$(DOTEXE): extra_dflags += $(core_ut) $(ROOT)/sigmaskgc$(DOTEXE): extra_dflags += $(core_ut) $(ROOT)/startbackgc$(DOTEXE): extra_dflags += $(core_ut) diff --git a/druntime/test/gc/tgc.d b/druntime/test/gc/tgc.d new file mode 100644 index 000000000000..7eebaf50dddc --- /dev/null +++ b/druntime/test/gc/tgc.d @@ -0,0 +1,208 @@ +/** + * Smoke tests for the opt-in thread-local GC (`tgc`, 0.1.0 prototype). + * + * Run with: --DRT-gcopt=gc:tgc + */ +import core.memory; +import core.thread; +import core.atomic; +import core.exception : OutOfMemoryError; +import cstdlib = core.stdc.stdlib; +import core.stdc.string : memset; + +extern (C) uint _d_tgc_region_create() nothrow @nogc; +extern (C) bool _d_tgc_region_attach(uint regionId) nothrow @nogc; +extern (C) void* _d_tgc_region_malloc(uint regionId, size_t size, uint bits) nothrow @nogc; +extern (C) const(char)* _d_tgc_version() nothrow @nogc; + +shared size_t otherThreadAllocs; +shared bool otherDone; +shared bool collectDone; + +class ChainNode +{ + ChainNode next; + size_t value; +} + +pragma(inline, false) +ChainNode makeChain(size_t length) +{ + ChainNode head; + foreach_reverse (i; 0 .. length) + { + auto node = new ChainNode; + node.next = head; + node.value = i; + head = node; + } + return head; +} + +pragma(inline, false) +void installTailRoot(void* range, size_t bytes) +{ + auto target = new ubyte[256]; + target[0] = 0x5A; + *cast(void**)(range + bytes - (void*).sizeof) = target.ptr; +} + +pragma(inline, false) +void clobberStack() +{ + void*[8192] zeros; + zeros[] = null; +} + +void worker() +{ + // A fresh worker heap exercises the tiny-index linear lookup tier. + void*[4] tinyBlocks; + foreach (ref p; tinyBlocks) + p = GC.malloc(32, GC.BlkAttr.NO_SCAN); + foreach (p; tinyBlocks) + { + auto found = GC.addrOf(p + 7); + assert(found == p); + GC.free(p); + } + + // Allocate on this thread's private heap + foreach (i; 0 .. 100) + { + auto p = new int[64]; + p[0] = cast(int) i; + atomicOp!"+="(otherThreadAllocs, 1); + } + // Keep a live allocation so collect on another thread must not free it + auto keep = new ubyte[1024]; + keep[0] = 42; + + // Wait until main has collected, then verify our data survived + while (!atomicLoad(collectDone)) + Thread.yield(); + + assert(keep[0] == 42); + atomicStore(otherDone, true); +} + +void main() +{ + import core.stdc.string : strcmp; + auto ver = _d_tgc_version(); + assert(ver !is null && !strcmp(ver, "0.2.3")); + + // Shared region scaffold: create, attach, alloc + auto rid = _d_tgc_region_create(); + assert(rid != 0); + bool attached = _d_tgc_region_attach(rid); + assert(attached); + auto rp = cast(int*) _d_tgc_region_malloc(rid, int.sizeof, 0); + assert(rp !is null); + *rp = 123; + assert(*rp == 123); + + // Exercise both lookup tiers and interior-pointer handling. + void*[16] blocks; + foreach (i; 0 .. blocks.length) + blocks[i] = GC.malloc(64, GC.BlkAttr.NO_SCAN); + foreach (p; blocks) + { + auto interior = p + 31; + auto found = GC.addrOf(interior); + assert(found == p); + assert(GC.sizeOf(interior) == 0); + assert(GC.getAttr(interior) == 0); + auto setResult = GC.setAttr(interior, GC.BlkAttr.NO_MOVE); + auto clearResult = GC.clrAttr(interior, GC.BlkAttr.NO_SCAN); + assert(setResult == 0); + assert(clearResult == 0); + auto resized = GC.realloc(interior, 128); + assert(resized is null); + GC.free(interior); + auto stillLive = GC.addrOf(p); + assert(stillLive == p); + auto atEnd = GC.addrOf(p + 64); + assert(atEnd is null); + } + auto removed = blocks[7]; + GC.free(removed); + auto removedLookup = GC.addrOf(removed + 1); + assert(removedLookup is null); + blocks[7] = null; + foreach (p; blocks) + GC.free(p); + + bool overflowRejected; + try + { + auto impossible = GC.malloc(size_t.max); + if (impossible) + GC.free(impossible); + } + catch (OutOfMemoryError) + overflowRejected = true; + assert(overflowRejected); + + auto before = GC.profileStats().numCollections; + + // Local allocations + int[] local; + foreach (i; 0 .. 50) + local ~= cast(int) i; + assert(local.length == 50); + + // A heap pointer chain requires fixpoint marking beyond direct roots. + auto chain = makeChain(300); + GC.collect(); + size_t chainLength; + for (auto node = chain; node; node = node.next) + { + assert(node.value == chainLength); + chainLength++; + } + assert(chainLength == 300); + + // The sole deliberate root is beyond the old 4 MiB scan cutoff. + enum registeredBytes = 5 * 1024 * 1024; + auto registered = cstdlib.malloc(registeredBytes); + assert(registered !is null); + memset(registered, 0, registeredBytes); + GC.addRange(registered, registeredBytes); + installTailRoot(registered, registeredBytes); + clobberStack(); + GC.collect(); + auto tailRoot = *cast(void**)(registered + registeredBytes - (void*).sizeof); + assert((cast(ubyte*) tailRoot)[0] == 0x5A); + GC.removeRange(registered); + cstdlib.free(registered); + + auto t = new Thread(&worker); + t.start(); + + // Wait until the worker has allocated + while (atomicLoad(otherThreadAllocs) < 50) + Thread.yield(); + + // Collect on the main thread only — must not STW-destroy worker heap + GC.collect(); + atomicStore(collectDone, true); + + t.join(); + assert(atomicLoad(otherDone)); + + // Detach smoke: spawn work then detach is documented for @nogc threads; + // here we only verify GC still functions after a normal thread exit. + auto after = GC.profileStats().numCollections; + assert(after >= before); + + // Force more collections via threshold pressure + foreach (i; 0 .. 200) + { + auto junk = new ubyte[4096]; + junk[0] = cast(ubyte) i; + } + GC.collect(); + + assert(local[0] == 0 && local[$ - 1] == 49); +} diff --git a/druntime/test/gc/tgc_bench.d b/druntime/test/gc/tgc_bench.d new file mode 100644 index 000000000000..78857c30b82f --- /dev/null +++ b/druntime/test/gc/tgc_bench.d @@ -0,0 +1,116 @@ +/** + * Simple GC benchmark for comparing backends. + * + * Usage: + * tgc_bench # default conservative GC + * tgc_bench --DRT-gcopt=gc:tgc + * tgc_bench --DRT-gcopt=gc:tgc tgcShared:symgc + * + * Environment: TGC_BENCH_ITERS (default 50000), TGC_BENCH_THREADS (default 4), + * TGC_BENCH_LOOKUPS (default 200000) + */ +import core.memory; +import core.thread; +import core.time; +import core.stdc.stdio; +import core.stdc.stdlib; + +extern (C) uint _d_tgc_region_create() nothrow @nogc; +extern (C) bool _d_tgc_region_attach(uint regionId) nothrow @nogc; +extern (C) void* _d_tgc_region_malloc(uint regionId, size_t size, uint bits) nothrow @nogc; +extern (C) const(char)* _d_tgc_version() nothrow @nogc; + +__gshared uint benchRegion; + +void allocWorker() +{ + if (benchRegion) + _d_tgc_region_attach(benchRegion); + foreach (i; 0 .. 2000) + { + if (benchRegion) + { + auto p = cast(int*) _d_tgc_region_malloc(benchRegion, 64, 0); + if (p) + *p = cast(int) i; + } + else + { + auto p = new int[16]; + p[0] = cast(int) i; + } + } +} + +void main() +{ + size_t iters = 50_000; + size_t nThreads = 4; + size_t lookups = 200_000; + if (const v = getenv("TGC_BENCH_ITERS")) + iters = cast(size_t) atol(v); + if (const v = getenv("TGC_BENCH_THREADS")) + nThreads = cast(size_t) atol(v); + if (const v = getenv("TGC_BENCH_LOOKUPS")) + lookups = cast(size_t) atol(v); + + const char* ver = _d_tgc_version(); + if (ver && ver[0]) + printf("tgc version: %s\n", ver); + + foreach (i; 0 .. 1000) + auto w = new byte[128]; + GC.collect(); + + MonoTime t0 = MonoTime.currTime; + foreach (i; 0 .. iters) + { + auto p = new byte[64 + (i & 63)]; + p[0] = cast(byte) i; + } + printf("single-thread alloc: %llu ms (%zu allocs)\n", + cast(ulong)(MonoTime.currTime - t0).total!"msecs", iters); + + t0 = MonoTime.currTime; + GC.collect(); + printf("GC.collect pause: %llu ms\n", cast(ulong)(MonoTime.currTime - t0).total!"msecs"); + + enum lookupBlockCount = 4096; + void*[lookupBlockCount] lookupBlocks; + foreach (ref p; lookupBlocks) + p = GC.malloc(64, GC.BlkAttr.NO_SCAN); + t0 = MonoTime.currTime; + foreach (i; 0 .. lookups) + { + auto base = lookupBlocks[i % lookupBlockCount]; + auto found = GC.addrOf(base + (i & 63)); + assert(found == base); + } + printf("interior-pointer lookup: %llu ms (%zu lookups, %u blocks)\n", + cast(ulong)(MonoTime.currTime - t0).total!"msecs", + lookups, lookupBlockCount); + foreach (p; lookupBlocks) + GC.free(p); + + if (_d_tgc_version()[0]) + { + benchRegion = _d_tgc_region_create(); + _d_tgc_region_attach(benchRegion); + } + + Thread[] threads; + threads.length = nThreads; + t0 = MonoTime.currTime; + foreach (i; 0 .. nThreads) + { + threads[i] = new Thread(&allocWorker); + threads[i].start(); + } + foreach (t; threads) + t.join(); + printf("multi-thread worker phase: %llu ms (%zu threads)\n", + cast(ulong)(MonoTime.currTime - t0).total!"msecs", nThreads); + + auto stats = GC.profileStats(); + printf("collections: %llu\n", cast(ulong) stats.numCollections); +} diff --git a/druntime/test/gc/tgc_regions.d b/druntime/test/gc/tgc_regions.d new file mode 100644 index 000000000000..7946b13c7350 --- /dev/null +++ b/druntime/test/gc/tgc_regions.d @@ -0,0 +1,85 @@ +/** + * Shared-region tests for tgc 0.2.0+ + * + * Run with: --DRT-gcopt=gc:tgc + */ +import core.memory; +import core.thread; +import core.atomic; +import core.stdc.string : strcmp; + +extern (C) uint _d_tgc_region_create() nothrow @nogc; +extern (C) bool _d_tgc_region_attach(uint regionId) nothrow @nogc; +extern (C) void* _d_tgc_region_malloc(uint regionId, size_t size, uint bits) nothrow @nogc; +extern (C) bool _d_tgc_region_collect(uint regionId) nothrow; +extern (C) const(char)* _d_tgc_version() nothrow @nogc; + +shared uint regionId; +shared bool workerReady; +shared bool collectDone; +shared int* sharedCell; + +class RegionHolder +{ + shared int* cell; +} + +void worker() +{ + bool attached = _d_tgc_region_attach(regionId); + assert(attached); + sharedCell = cast(shared int*) _d_tgc_region_malloc(regionId, int.sizeof, 0); + assert(sharedCell !is null); + auto holder = new RegionHolder; + holder.cell = cast(shared int*) _d_tgc_region_malloc(regionId, int.sizeof, 0); + assert(holder.cell !is null); + *holder.cell = 77; + atomicStore(workerReady, true); + + while (!atomicLoad(collectDone)) + Thread.yield(); + + // Must still be valid after region collect from main (root on main after join setup) + assert(*sharedCell == 42); + assert(*holder.cell == 77); +} + +void main() +{ + auto ver = _d_tgc_version(); + assert(ver !is null && !strcmp(ver, "0.2.3")); + + regionId = _d_tgc_region_create(); + assert(regionId != 0); + bool attached = _d_tgc_region_attach(regionId); + assert(attached); + + auto t = new Thread(&worker); + t.start(); + + while (!atomicLoad(workerReady)) + Thread.yield(); + + // Keep a reference on this thread so the cell stays live + auto localRef = sharedCell; + assert(localRef !is null); + *localRef = 42; + + bool collected = _d_tgc_region_collect(regionId); + assert(collected); + foreach (i; 0 .. 64) + { + auto overwrite = cast(int*) _d_tgc_region_malloc(regionId, int.sizeof, 0); + assert(overwrite !is null); + *overwrite = -1; + } + atomicStore(collectDone, true); + t.join(); + + assert(*localRef == 42); + + // Junk collect on private heap still works + foreach (i; 0 .. 100) + auto x = new int[i % 17 + 1]; + GC.collect(); +} diff --git a/druntime/test/gc/tgc_remote_free.d b/druntime/test/gc/tgc_remote_free.d new file mode 100644 index 000000000000..a31a06829c81 --- /dev/null +++ b/druntime/test/gc/tgc_remote_free.d @@ -0,0 +1,56 @@ +/** + * Cross-thread remote-free queue test for tgc. + * + * Run with: --DRT-gcopt=gc:tgc + */ +import core.atomic; +import core.memory; +import core.thread; + +shared size_t remoteAddress; +shared bool freeQueued; +shared bool queueDrained; +shared bool lookupDone; + +void ownerThread() +{ + auto owned = GC.malloc(128, GC.BlkAttr.NO_SCAN); + assert(owned !is null); + atomicStore(remoteAddress, cast(size_t) owned); + + while (!atomicLoad(freeQueued)) + Thread.yield(); + + // Any owner allocation drains frees queued by foreign threads. + auto trigger = GC.malloc(1, GC.BlkAttr.NO_SCAN); + assert(trigger !is null); + atomicStore(queueDrained, true); + + while (!atomicLoad(lookupDone)) + Thread.yield(); +} + +void main() +{ + auto owner = new Thread(&ownerThread); + owner.start(); + + size_t address; + while (!address) + { + address = atomicLoad(remoteAddress); + Thread.yield(); + } + + auto foreign = cast(void*) address; + GC.free(foreign); + GC.free(foreign); // duplicate requests must coalesce safely + atomicStore(freeQueued, true); + + while (!atomicLoad(queueDrained)) + Thread.yield(); + auto found = GC.addrOf(foreign); + assert(found is null); + atomicStore(lookupDone, true); + owner.join(); +} diff --git a/spec/garbage.dd b/spec/garbage.dd index b9227de658dd..454d63d8f382 100644 --- a/spec/garbage.dd +++ b/spec/garbage.dd @@ -405,7 +405,10 @@ $(H2 $(LNAME2 gc_config, Configuring the Collector)) $(UL $(LI disable:0|1 - start disabled) $(LI profile:0|1 - enable profiling with summary when terminating program) - $(LI gc:conservative|precise|manual - select collector implementation (default = conservative)) + $(LI gc:conservative|precise|manual|tgc - select collector implementation (default = conservative). + $(TT tgc) is an opt-in thread-local collector (0.1.0 prototype): per-thread private heaps, + local collection without global stop-the-world. Target design uses partitioned shared regions + for cross-thread data; region collect is not implemented yet.) $(LI initReserve:N - initial memory to reserve in MB) $(LI minPoolSize:N - initial and minimum pool size in MB) $(LI maxPoolSize:N - maximum pool size in MB) diff --git a/tools/tgc-bench/run-benchmarks.ps1 b/tools/tgc-bench/run-benchmarks.ps1 new file mode 100644 index 000000000000..2f2e648dcb31 --- /dev/null +++ b/tools/tgc-bench/run-benchmarks.ps1 @@ -0,0 +1,31 @@ +# Compare GC backends for tgc development +# Requires a built dmd/druntime from feature/tgc with tgc registered. +param( + [int]$Iters = 50000, + [int]$Threads = 4 +) + +$ErrorActionPreference = "Stop" +$env:TGC_BENCH_ITERS = "$Iters" +$env:TGC_BENCH_THREADS = "$Threads" + +$root = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) +$bench = Join-Path $root "druntime\test\gc\generated\windows\release\64\tgc_bench.exe" + +if (-not (Test-Path $bench)) { + Write-Error "Build tgc_bench first: make -C druntime/test/gc OS=windows MODEL=64 (from dmd fork root after druntime build)" +} + +$runs = @( + @{ Name = "conservative (default)"; Args = @() }, + @{ Name = "tgc"; Args = @("--DRT-gcopt=gc:tgc") }, + @{ Name = "tgc (native shared regions)"; Args = @("--DRT-gcopt=gc:tgc", "tgcShared:native") } +) + +foreach ($run in $runs) { + Write-Host "`n=== $($run.Name) ===" -ForegroundColor Cyan + & $bench @($run.Args) +} + +Write-Host "`nOptional SymGC row (requires symgc-linked build):" -ForegroundColor Yellow +Write-Host " tgc_bench --DRT-gcopt=gc:sdc"