diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 84cbb56a9192156..71a0a4c6a1c3eb8 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -118,7 +118,20 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) { /* Bit 1 is set when the object is in generation which is GCed currently. */ #define _PyGC_PREV_MASK_COLLECTING ((uintptr_t)2) -#define _PyGC_PREV_SHIFT 2 +/* Use an epoch bit only where the existing header alignment permits it. */ +#if !defined(Py_GIL_DISABLED) && SIZEOF_VOID_P == 8 +# ifdef __cplusplus +# define _PyGC_PREV_MASK_EPOCH \ + ((uintptr_t)(alignof(PyGC_Head) >= 8 ? 4 : 0)) +# else +# define _PyGC_PREV_MASK_EPOCH \ + ((uintptr_t)(_Alignof(PyGC_Head) >= 8 ? 4 : 0)) +# endif +#else +# define _PyGC_PREV_MASK_EPOCH ((uintptr_t)0) +#endif + +#define _PyGC_PREV_SHIFT (2 + (_PyGC_PREV_MASK_EPOCH != 0)) #define _PyGC_PREV_MASK (((uintptr_t) -1) << _PyGC_PREV_SHIFT) /* set for debugging information */ @@ -151,7 +164,7 @@ static inline void _PyGCHead_SET_NEXT(PyGC_Head *gc, PyGC_Head *next) { gc->_gc_next = (uintptr_t)next; } -// Lowest two bits of _gc_prev is used for _PyGC_PREV_MASK_* flags. +// Low bits of _gc_prev are used for _PyGC_PREV_MASK_* flags. static inline PyGC_Head* _PyGCHead_PREV(PyGC_Head *gc) { uintptr_t prev = (gc->_gc_prev & _PyGC_PREV_MASK); return (PyGC_Head*)prev; @@ -228,6 +241,10 @@ static inline void _PyObject_GC_TRACK( PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); _PyGCHead_SET_NEXT(last, gc); _PyGCHead_SET_PREV(gc, last); + if (_PyGC_PREV_MASK_EPOCH) { + gc->_gc_prev = (gc->_gc_prev & ~_PyGC_PREV_MASK_EPOCH) + | gcstate->mark_epoch; + } _PyGCHead_SET_NEXT(gc, generation0); generation0->_gc_prev = (uintptr_t)gc; gcstate->heap_size++; diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 9cc25eda44f9de1..c0ab263a244f4f1 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -163,8 +163,8 @@ typedef struct { _Py_ALIGNED_DEF(_PyObject_MIN_ALIGNMENT, uintptr_t) _gc_next; // Tagged pointer to previous object in the list. - // Lowest two bits are used for flags documented later. - // Those bits are made available by the struct's minimum alignment. + // Low bits are used for flags documented in pycore_gc.h. + // The available bits depend on the header's existing alignment. uintptr_t _gc_prev; } PyGC_Head; @@ -244,6 +244,10 @@ struct _gc_runtime_state { #endif /* true if we are currently running the collector */ int collecting; +#ifndef Py_GIL_DISABLED + /* Epoch bit stamped on tracked objects; toggled by eligible full GCs. */ + unsigned char mark_epoch; +#endif // The frame that started the current collection. It might be NULL even when // collecting (if no Python frame is running): _PyInterpreterFrame *frame; diff --git a/Include/object.h b/Include/object.h index 549b91ebecf4f22..cc5211e6d89e4ac 100644 --- a/Include/object.h +++ b/Include/object.h @@ -112,7 +112,7 @@ whose size is determined when the object is allocated. /* PyObjects are given a minimum alignment so that the least significant bits * of an object pointer become available for other purposes. - * This must be an integer literal with the value (1 << _PyGC_PREV_SHIFT), number of bytes. + * This must be an integer literal specifying the minimum number of bytes. */ #define _PyObject_MIN_ALIGNMENT 4 diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md index 0ef45ff8e02bc5f..de2d649e715777c 100644 --- a/InternalDocs/garbage_collector.md +++ b/InternalDocs/garbage_collector.md @@ -661,10 +661,10 @@ of `PyGC_Head` discussed in the `Memory layout and object structure`_ section: - The `_gc_prev` field is normally used as the "previous" pointer to maintain the doubly linked list but its lowest two bits are used to keep the flags `PREV_MASK_COLLECTING` and `_PyGC_PREV_MASK_FINALIZED`. Between collections, - the only flag that can be present is `_PyGC_PREV_MASK_FINALIZED` that indicates + the `_PyGC_PREV_MASK_FINALIZED` flag indicates if an object has been already finalized. During collections `_gc_prev` is temporarily used for storing a copy of the reference count (`gc_ref`), in - addition to two flags, and the GC linked list becomes a singly linked list until + addition to these flags, and the GC linked list becomes a singly linked list until `_gc_prev` is restored. - The `_gc_next` field is used as the "next" pointer to maintain the doubly linked @@ -676,6 +676,12 @@ of `PyGC_Head` discussed in the `Memory layout and object structure`_ section: currently in. Instead, when that's needed, ad hoc tricks (like the `NEXT_MASK_UNREACHABLE` flag) are employed. +On supported 64-bit GIL builds, `_gc_prev` also holds a mark-epoch bit. +Eligible full collections first trace interpreter dictionaries and active frames, +moving known-live objects to a separate list using the existing links. Only the +remaining objects need trial reference counting. This shortcut is disabled for +frozen heaps, secondary interpreters and after interpreter finalization. + Optimization: delayed untracking containers =========================================== diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-11-00-45-00.gh-issue-159137.gcRoots.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-11-00-45-00.gh-issue-159137.gcRoots.rst new file mode 100644 index 000000000000000..1e748a01e8b5774 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-11-00-45-00.gh-issue-159137.gcRoots.rst @@ -0,0 +1,2 @@ +Restore known-root marking in the GIL garbage collector to reduce trial +reference counting during full collections. diff --git a/Python/gc.c b/Python/gc.c index 5ce94819ae3f608..c02f51af110c63e 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -22,6 +22,11 @@ typedef struct _gc_runtime_state GCState; +/* Root marking requires unique interpreter ownership. Keep this state outside + * _PyRuntime so secondary interpreters and runtime reinitialization disable it. + */ +static int gc_ownership_state; // 0: never initialized, 1: unique, 2: unknown + #ifdef Py_DEBUG # define GC_DEBUG #endif @@ -82,10 +87,10 @@ gc_set_refs(PyGC_Head *g, Py_ssize_t refs) } static inline void -gc_reset_refs(PyGC_Head *g, Py_ssize_t refs) +gc_reset_refs(PyGC_Head *g, Py_ssize_t refs, uintptr_t epoch) { g->_gc_prev = (g->_gc_prev & _PyGC_PREV_MASK_FINALIZED) - | PREV_MASK_COLLECTING + | epoch | PREV_MASK_COLLECTING | ((uintptr_t)(refs) << _PyGC_PREV_SHIFT); } @@ -113,6 +118,10 @@ get_gc_state(void) void _PyGC_InitState(GCState *gcstate) { + int expected = 0; + if (!_Py_atomic_compare_exchange_int(&gc_ownership_state, &expected, 1)) { + _Py_atomic_store_int_relaxed(&gc_ownership_state, 2); + } #define INIT_HEAD(GEN) \ do { \ GEN.head._gc_next = (uintptr_t)&GEN.head; \ @@ -124,6 +133,7 @@ _PyGC_InitState(GCState *gcstate) INIT_HEAD(gcstate->generations[i]); }; gcstate->generation0 = GEN_HEAD(gcstate, 0); + gcstate->mark_epoch = 0; INIT_HEAD(gcstate->permanent_generation); #undef INIT_HEAD @@ -160,7 +170,7 @@ _gc_prev values Between collections, _gc_prev is used for doubly linked list. -Lowest two bits of _gc_prev are used for flags. +Low bits of _gc_prev are used for flags, including a mark epoch on supported ABIs. PREV_MASK_COLLECTING is used only while collecting and cleared before GC ends or _PyObject_GC_UNTRACK() is called. @@ -400,6 +410,7 @@ update_refs(PyGC_Head *containers) PyGC_Head *next; PyGC_Head *gc = GC_NEXT(containers); Py_ssize_t candidates = 0; + uintptr_t epoch = _PyGC_PREV_MASK_EPOCH ? get_gc_state()->mark_epoch : 0; while (gc != containers) { next = GC_NEXT(gc); @@ -410,7 +421,7 @@ update_refs(PyGC_Head *containers) gc = next; continue; } - gc_reset_refs(gc, Py_REFCNT(op)); + gc_reset_refs(gc, Py_REFCNT(op), epoch); /* Python's cyclic gc should never see an incoming refcount * of 0: if something decref'ed to 0, it should have been * deallocated immediately at that time. @@ -1227,6 +1238,124 @@ deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable) { return candidates; } +typedef struct { + PyGC_Head live; + PyGC_Head *cursor; + uintptr_t epoch; + Py_ssize_t candidates; +} GCMarkQueue; + +/* Unlike gc_list_move(), preserve flags on the destination's neighbors. */ +static void +gc_list_move_after(PyGC_Head *node, PyGC_Head *cursor) +{ + assert(node != cursor); + PyGC_Head *from_prev = GC_PREV(node); + PyGC_Head *from_next = GC_NEXT(node); + _PyGCHead_SET_NEXT(from_prev, from_next); + _PyGCHead_SET_PREV(from_next, from_prev); + + PyGC_Head *to_next = GC_NEXT(cursor); + _PyGCHead_SET_PREV(node, cursor); + _PyGCHead_SET_NEXT(node, to_next); + _PyGCHead_SET_NEXT(cursor, node); + _PyGCHead_SET_PREV(to_next, node); +} + +static int +visit_add_to_container(PyObject *op, void *arg) +{ + OBJECT_STAT_INC(object_visits); + if (!_Py_IsImmortal(op) && _PyObject_IS_GC(op) && + _PyObject_GC_IS_TRACKED(op)) { + PyGC_Head *gc = AS_GC(op); + GCMarkQueue *queue = arg; + /* Recheck ownership after tp_is_gc, which can create an interpreter. + * A partial live set is safe: its outgoing references stay external. + */ + if ((gc->_gc_prev & _PyGC_PREV_MASK_EPOCH) != queue->epoch && + _Py_atomic_load_int_relaxed(&gc_ownership_state) == 1) { + gc->_gc_prev = (gc->_gc_prev & ~_PyGC_PREV_MASK_EPOCH) + | queue->epoch; + /* Prefer depth-first order within a 1 MiB address region. */ + if ((((uintptr_t)gc ^ (uintptr_t)queue->cursor) >> 20) == 0) { + gc_list_move_after(gc, queue->cursor); + } + else { + gc_list_move(gc, &queue->live); + } + queue->candidates++; + } + } + return 0; +} + +/* Avoid extension callbacks while root enumeration holds HEAD_LOCK. */ +static int +visit_known_root(PyObject *op, void *arg) +{ + if (op == NULL) { + return 0; + } + if (Py_TYPE(op)->tp_is_gc != NULL) { + OBJECT_STAT_INC(object_visits); + return 0; + } + return visit_add_to_container(op, arg); +} + +static void +mark_known_roots(GCMarkQueue *queue) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + visit_known_root(interp->sysdict, queue); + visit_known_root(interp->builtins, queue); + visit_known_root(interp->dict, queue); + visit_known_root(interp->imports.modules, queue); + _Py_FOR_EACH_TSTATE_BEGIN(interp, tstate) { + for (_PyInterpreterFrame *frame = tstate->current_frame; + frame != NULL; frame = frame->previous) { + if (frame->owner >= FRAME_OWNED_BY_INTERPRETER || + frame->stackpointer == NULL || _PyFrame_IsIncomplete(frame)) { + continue; + } + (void)_PyFrame_Traverse(frame, visit_known_root, queue); + } + } + _Py_FOR_EACH_TSTATE_END(interp); + PyGC_Head *gc = GC_NEXT(&queue->live); + while (gc != &queue->live) { + queue->cursor = gc; + PyObject *op = FROM_GC(gc); + (void)Py_TYPE(op)->tp_traverse(op, visit_add_to_container, queue); + /* Read next after traversal, which can insert children here. */ + gc = GC_NEXT(gc); + } +} + +/* Toggle the epoch to distinguish unseen objects. GC_TRACK stamps new objects + * with the new epoch. The live list doubles as the pending work queue. + */ +static Py_ssize_t +collect_from_known_roots(PyGC_Head *base, PyGC_Head *unreachable) +{ + GCState *gcstate = get_gc_state(); + gcstate->mark_epoch ^= _PyGC_PREV_MASK_EPOCH; + GCMarkQueue queue = {.epoch = gcstate->mark_epoch}; + gc_list_init(&queue.live); + queue.cursor = &queue.live; + mark_known_roots(&queue); + + /* Known-live references stay external to the residual analysis. + * update_refs stamps the remaining candidates before callbacks can run. + */ + Py_ssize_t candidates = deduce_unreachable(base, unreachable); + gc_list_merge(base, &queue.live); + gc_list_merge(&queue.live, base); + validate_list(base, collecting_clear_unreachable_clear); + return queue.candidates + candidates; +} + /* Handle objects that may have resurrected after a call to 'finalize_garbage', moving them to 'old_generation' and placing the rest on 'still_unreachable'. @@ -1521,7 +1650,19 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) } validate_list(old, collecting_clear_unreachable_clear); - stats.candidates = deduce_unreachable(young, &unreachable); + if (_PyGC_PREV_MASK_EPOCH && generation == NUM_GENERATIONS - 1 && + _Py_IsMainInterpreter(tstate->interp) && + gcstate->heap_size >= 4096 && + gc_list_is_empty(&gcstate->permanent_generation.head) && + _Py_atomic_load_int_relaxed(&gc_ownership_state) == 1) { + /* Without frozen or foreign objects, every tracked non-immortal + * object belongs to this full collection before update_refs. + */ + stats.candidates = collect_from_known_roots(young, &unreachable); + } + else { + stats.candidates = deduce_unreachable(young, &unreachable); + } untrack_tuples(young); /* Move reachable objects to next generation. */ @@ -1901,6 +2042,7 @@ finalize_unlink_gc_head(PyGC_Head *gc) { void _PyGC_Fini(PyInterpreterState *interp) { + _Py_atomic_store_int_relaxed(&gc_ownership_state, 2); GCState *gcstate = &interp->gc; Py_CLEAR(gcstate->garbage); Py_CLEAR(gcstate->callbacks); diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index cc9b389ec1a9ad4..6fa5f23002ddefe 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -22,6 +22,7 @@ Python/fileutils.c set_inheritable ioctl_works - # (set lazily, atomically, *after* first init) Modules/posixmodule.c os_dup2_impl dup3_works_atomic - Modules/posixmodule.c os_pipe_impl pipe2_works_atomic - +Python/gc.c - gc_ownership_state Atomic process-lifetime ownership guard; never reset across interpreter or runtime finalization. ## guards around resource init Python/thread_pthread.h PyThread__init_thread lib_initialized -