From 57465a9a26e0ce32dc777d03178d4dc3985f8cbb Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 11 Oct 2026 03:34:46 +0100 Subject: [PATCH 1/3] gh-159137: Copy and subtract the GC reference counts in one walk --- ...-10-11-10-00-00.gh-issue-159137.gcFuse.rst | 3 + Python/gc.c | 108 +++++++++++++++++- Tools/c-analyzer/cpython/ignored.tsv | 3 + 3 files changed, 108 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-11-10-00-00.gh-issue-159137.gcFuse.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-11-10-00-00.gh-issue-159137.gcFuse.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-11-10-00-00.gh-issue-159137.gcFuse.rst new file mode 100644 index 00000000000000..14a993cc43a8e3 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-11-10-00-00.gh-issue-159137.gcFuse.rst @@ -0,0 +1,3 @@ +Speed up full collections of the garbage collector: the reference counts are +copied and the references between the collected objects subtracted in a +single walk of the generation. diff --git a/Python/gc.c b/Python/gc.c index 5ce94819ae3f60..96da09b204331a 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -101,6 +101,12 @@ gc_decref(PyGC_Head *g) #define GEN_HEAD(gcstate, n) (&(gcstate)->generations[n].head) +/* Set once an object can be tracked outside the generations of the + * interpreter that looks at it: another interpreter exists, or one was + * finalized with objects left. + */ +static int gc_foreign_objects; + static GCState * get_gc_state(void) @@ -135,6 +141,9 @@ _PyGC_Init(PyInterpreterState *interp) { GCState *gcstate = &interp->gc; + if (!_Py_IsMainInterpreter(interp)) { + _Py_atomic_store_int_relaxed(&gc_foreign_objects, 1); + } gcstate->generation_stats = PyMem_RawCalloc(1, sizeof(struct gc_stats)); if (gcstate->generation_stats == NULL) { return _PyStatus_NO_MEMORY(); @@ -456,6 +465,30 @@ visit_decref(PyObject *op, void *parent) return 0; } +/* A traversal callback for update_and_subtract_refs(), where every tracked + * object is in the list being collected. If the walk has not reached op yet, + * its _gc_prev still holds a pointer, and the walk recovers the decrements + * from the difference with the pointer that it knows was there. + */ +static int +visit_decref_tracked(PyObject *op, void *parent) +{ + OBJECT_STAT_INC(object_visits); + _PyObject_ASSERT(_PyObject_CAST(parent), !_PyObject_IsFreed(op)); + + if (_PyObject_IS_GC(op)) { + PyGC_Head *gc = AS_GC(op); + if (gc->_gc_next != 0) { + _PyObject_ASSERT_WITH_MSG(op, + gc_get_refs(gc) > 0 + || !gc_is_collecting(gc), + "refcount is too small"); + gc->_gc_prev -= 1 << _PyGC_PREV_SHIFT; + } + } + return 0; +} + int _PyGC_VisitStackRef(_PyStackRef *ref, visitproc visit, void *arg) { @@ -463,7 +496,9 @@ _PyGC_VisitStackRef(_PyStackRef *ref, visitproc visit, void *arg) // refcounts when computing the incoming references, but otherwise treat // them like normal. assert(!PyStackRef_IsTaggedInt(*ref)); - if (!PyStackRef_RefcountOnObject(*ref) && (visit == visit_decref)) { + if (!PyStackRef_RefcountOnObject(*ref) + && (visit == visit_decref || visit == visit_decref_tracked)) + { return 0; } Py_VISIT(PyStackRef_AsPyObjectBorrow(*ref)); @@ -501,6 +536,53 @@ subtract_refs(PyGC_Head *containers) } } +/* update_refs() and subtract_refs() in one walk, for a list that holds every + * tracked object. visit_decref_tracked() then needs no flag to recognise a + * candidate, so an object can be decremented before the walk gets to it. + */ +static Py_ssize_t +update_and_subtract_refs(PyGC_Head *containers) +{ + PyGC_Head *prev = containers; /* last object left in the list */ + PyGC_Head *gc = GC_NEXT(containers); + uintptr_t orig_prev = (uintptr_t)containers; + Py_ssize_t candidates = 0; + + while (gc != containers) { + PyGC_Head *next = GC_NEXT(gc); + PyObject *op = FROM_GC(gc); + if (_Py_IsImmortal(op)) { + assert(!_Py_IsStaticImmortal(op)); + // Not _PyObject_GC_UNTRACK(): _gc_prev of next may hold + // decrements. + _PyGCHead_SET_NEXT(prev, next); + gc->_gc_next = 0; + gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED; + get_gc_state()->heap_size--; + } + else { + /* _gc_prev is orig_prev plus the flags, minus the decrements + * that came before the walk. */ + uintptr_t word = gc->_gc_prev; + gc->_gc_prev = (((uintptr_t)Py_REFCNT(op) << _PyGC_PREV_SHIFT) + + (word & _PyGC_PREV_MASK) - orig_prev) + | (word & _PyGC_PREV_MASK_FINALIZED) + | PREV_MASK_COLLECTING; + // See update_refs() for a refcount of 0. + _PyObject_ASSERT(op, Py_REFCNT(op) != 0); + _PyObject_ASSERT_WITH_MSG(op, gc_get_refs(gc) <= Py_REFCNT(op), + "refcount is too small"); + (void) Py_TYPE(op)->tp_traverse(op, visit_decref_tracked, op); + prev = gc; + candidates++; + } + orig_prev = (uintptr_t)gc; + gc = next; + } + containers->_gc_prev = (uintptr_t)prev; + return candidates; +} + /* A traversal callback for move_unreachable. */ static int visit_reachable(PyObject *op, void *arg) @@ -1175,15 +1257,21 @@ flag is cleared (for example, by using 'clear_unreachable_mask' function or by a call to 'move_legacy_finalizers'), the 'unreachable' list is not a normal list and we can not use most gc_list_* functions for it. */ static inline Py_ssize_t -deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable) { +deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable, bool all_tracked) { validate_list(base, collecting_clear_unreachable_clear); /* Using ob_refcnt and gc_refs, calculate which objects in the * container set are reachable from outside the set (i.e., have a * refcount greater than 0 when all the references within the * set are taken into account). */ - Py_ssize_t candidates = update_refs(base); // gc_prev is used for gc_refs - subtract_refs(base); + Py_ssize_t candidates; // gc_prev is used for gc_refs + if (all_tracked) { + candidates = update_and_subtract_refs(base); + } + else { + candidates = update_refs(base); + subtract_refs(base); + } /* Leave everything reachable from outside base in base, and move * everything else (in base) to unreachable. @@ -1252,7 +1340,7 @@ handle_resurrected_objects(PyGC_Head *unreachable, PyGC_Head* still_unreachable, // have the PREV_MARK_COLLECTING set, but the objects are going to be // removed so we can skip the expense of clearing the flag. PyGC_Head* resurrected = unreachable; - deduce_unreachable(resurrected, still_unreachable); + deduce_unreachable(resurrected, still_unreachable, false); clear_unreachable_mask(still_unreachable); // Move the resurrected objects to the old generation for future collection. @@ -1521,7 +1609,12 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) } validate_list(old, collecting_clear_unreachable_clear); - stats.candidates = deduce_unreachable(young, &unreachable); + /* A full collection with nothing frozen examines every tracked object, + * unless some are in no generation of this interpreter. */ + bool all_tracked = (generation == NUM_GENERATIONS - 1 + && gc_list_is_empty(&gcstate->permanent_generation.head) + && !_Py_atomic_load_int_relaxed(&gc_foreign_objects)); + stats.candidates = deduce_unreachable(young, &unreachable, all_tracked); untrack_tuples(young); /* Move reachable objects to next generation. */ @@ -1892,6 +1985,9 @@ _PyGC_DumpShutdownStats(PyInterpreterState *interp) static void finalize_unlink_gc_head(PyGC_Head *gc) { + if (!gc_list_is_empty(gc)) { + _Py_atomic_store_int_relaxed(&gc_foreign_objects, 1); + } PyGC_Head *prev = GC_PREV(gc); PyGC_Head *next = GC_NEXT(gc); _PyGCHead_SET_NEXT(prev, next); diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index cc9b389ec1a9ad..7bc8d43d73f175 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -61,6 +61,9 @@ Python/parking_lot.c - buckets - ## shared empty sentinel for the per-type method cache Python/typecache.c - empty_cache - +## set at most once (atomically), outlives runtime finalization on purpose +Python/gc.c - gc_foreign_objects - + ## data needed for introspecting asyncio state from debuggers and profilers Modules/_asynciomodule.c - _Py_AsyncioDebug - From da81986607cdfaa5c621bb8b1ab3dff5c0cb49f1 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 11 Oct 2026 03:43:54 +0100 Subject: [PATCH 2/3] Use the interpreter id counter instead of a static flag --- Python/gc.c | 16 ++-------------- Tools/c-analyzer/cpython/ignored.tsv | 3 --- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/Python/gc.c b/Python/gc.c index 96da09b204331a..422f861082cef7 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -101,12 +101,6 @@ gc_decref(PyGC_Head *g) #define GEN_HEAD(gcstate, n) (&(gcstate)->generations[n].head) -/* Set once an object can be tracked outside the generations of the - * interpreter that looks at it: another interpreter exists, or one was - * finalized with objects left. - */ -static int gc_foreign_objects; - static GCState * get_gc_state(void) @@ -141,9 +135,6 @@ _PyGC_Init(PyInterpreterState *interp) { GCState *gcstate = &interp->gc; - if (!_Py_IsMainInterpreter(interp)) { - _Py_atomic_store_int_relaxed(&gc_foreign_objects, 1); - } gcstate->generation_stats = PyMem_RawCalloc(1, sizeof(struct gc_stats)); if (gcstate->generation_stats == NULL) { return _PyStatus_NO_MEMORY(); @@ -1610,10 +1601,10 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) validate_list(old, collecting_clear_unreachable_clear); /* A full collection with nothing frozen examines every tracked object, - * unless some are in no generation of this interpreter. */ + * if no other interpreter has ever existed. */ bool all_tracked = (generation == NUM_GENERATIONS - 1 && gc_list_is_empty(&gcstate->permanent_generation.head) - && !_Py_atomic_load_int_relaxed(&gc_foreign_objects)); + && _PyRuntime.interpreters.next_id == 1); stats.candidates = deduce_unreachable(young, &unreachable, all_tracked); untrack_tuples(young); @@ -1985,9 +1976,6 @@ _PyGC_DumpShutdownStats(PyInterpreterState *interp) static void finalize_unlink_gc_head(PyGC_Head *gc) { - if (!gc_list_is_empty(gc)) { - _Py_atomic_store_int_relaxed(&gc_foreign_objects, 1); - } PyGC_Head *prev = GC_PREV(gc); PyGC_Head *next = GC_NEXT(gc); _PyGCHead_SET_NEXT(prev, next); diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 7bc8d43d73f175..cc9b389ec1a9ad 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -61,9 +61,6 @@ Python/parking_lot.c - buckets - ## shared empty sentinel for the per-type method cache Python/typecache.c - empty_cache - -## set at most once (atomically), outlives runtime finalization on purpose -Python/gc.c - gc_foreign_objects - - ## data needed for introspecting asyncio state from debuggers and profilers Modules/_asynciomodule.c - _Py_AsyncioDebug - From 899e80a43c876757f1373a799a13dd85342f8708 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 11 Oct 2026 04:02:42 +0100 Subject: [PATCH 3/3] Guard on the number of interpreters the process has had --- Lib/test/test_gdb/test_backtrace.py | 4 +-- Python/gc.c | 50 ++++++++++++++-------------- Tools/c-analyzer/cpython/ignored.tsv | 1 + 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/Lib/test/test_gdb/test_backtrace.py b/Lib/test/test_gdb/test_backtrace.py index 714853c7b4732d..2daf7a10b584c3 100644 --- a/Lib/test/test_gdb/test_backtrace.py +++ b/Lib/test/test_gdb/test_backtrace.py @@ -99,13 +99,13 @@ def test_gc(self): 'bar()\n') # Verify with "py-bt": gdb_output = self.get_stack_trace(cmd, - cmds_after_breakpoint=['break update_refs', 'continue', 'py-bt'], + cmds_after_breakpoint=['break move_unreachable', 'continue', 'py-bt'], ) self.assertIn('Garbage-collecting', gdb_output) # Verify with "py-bt-full": gdb_output = self.get_stack_trace(cmd, - cmds_after_breakpoint=['break update_refs', 'continue', 'py-bt-full'], + cmds_after_breakpoint=['break move_unreachable', 'continue', 'py-bt-full'], ) self.assertIn('Garbage-collecting', gdb_output) diff --git a/Python/gc.c b/Python/gc.c index 422f861082cef7..8306536ce49d6d 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -110,9 +110,15 @@ get_gc_state(void) } +/* The number of interpreters that the process has had. While it is 1, every + * tracked object is in a list of that interpreter. */ +static Py_ssize_t gc_states; + void _PyGC_InitState(GCState *gcstate) { + _Py_atomic_add_ssize(&gc_states, 1); + #define INIT_HEAD(GEN) \ do { \ GEN.head._gc_next = (uintptr_t)&GEN.head; \ @@ -534,9 +540,8 @@ subtract_refs(PyGC_Head *containers) static Py_ssize_t update_and_subtract_refs(PyGC_Head *containers) { - PyGC_Head *prev = containers; /* last object left in the list */ + PyGC_Head *prev = containers; PyGC_Head *gc = GC_NEXT(containers); - uintptr_t orig_prev = (uintptr_t)containers; Py_ssize_t candidates = 0; while (gc != containers) { @@ -544,33 +549,28 @@ update_and_subtract_refs(PyGC_Head *containers) PyObject *op = FROM_GC(gc); if (_Py_IsImmortal(op)) { assert(!_Py_IsStaticImmortal(op)); - // Not _PyObject_GC_UNTRACK(): _gc_prev of next may hold - // decrements. + // _PyObject_GC_UNTRACK(), keeping the decrements of next. _PyGCHead_SET_NEXT(prev, next); + next->_gc_prev += (uintptr_t)prev - (uintptr_t)gc; gc->_gc_next = 0; gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED; get_gc_state()->heap_size--; + gc = next; + continue; } - else { - /* _gc_prev is orig_prev plus the flags, minus the decrements - * that came before the walk. */ - uintptr_t word = gc->_gc_prev; - gc->_gc_prev = (((uintptr_t)Py_REFCNT(op) << _PyGC_PREV_SHIFT) - + (word & _PyGC_PREV_MASK) - orig_prev) - | (word & _PyGC_PREV_MASK_FINALIZED) - | PREV_MASK_COLLECTING; - // See update_refs() for a refcount of 0. - _PyObject_ASSERT(op, Py_REFCNT(op) != 0); - _PyObject_ASSERT_WITH_MSG(op, gc_get_refs(gc) <= Py_REFCNT(op), - "refcount is too small"); - (void) Py_TYPE(op)->tp_traverse(op, visit_decref_tracked, op); - prev = gc; - candidates++; - } - orig_prev = (uintptr_t)gc; + /* _gc_prev is prev and the flags, minus the decrements so far. */ + gc->_gc_prev = (gc->_gc_prev - (uintptr_t)prev + + ((uintptr_t)Py_REFCNT(op) << _PyGC_PREV_SHIFT)) + | PREV_MASK_COLLECTING; + // See update_refs() for a refcount of 0. + _PyObject_ASSERT(op, Py_REFCNT(op) != 0); + _PyObject_ASSERT_WITH_MSG(op, gc_get_refs(gc) <= Py_REFCNT(op), + "refcount is too small"); + (void) Py_TYPE(op)->tp_traverse(op, visit_decref_tracked, op); + prev = gc; gc = next; + candidates++; } - containers->_gc_prev = (uintptr_t)prev; return candidates; } @@ -1600,11 +1600,11 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) } validate_list(old, collecting_clear_unreachable_clear); - /* A full collection with nothing frozen examines every tracked object, - * if no other interpreter has ever existed. */ + /* A full collection examines every tracked object if none is frozen and + * the process has had no other interpreter. */ bool all_tracked = (generation == NUM_GENERATIONS - 1 && gc_list_is_empty(&gcstate->permanent_generation.head) - && _PyRuntime.interpreters.next_id == 1); + && _Py_atomic_load_ssize_relaxed(&gc_states) == 1); stats.candidates = deduce_unreachable(young, &unreachable, all_tracked); untrack_tuples(young); diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index cc9b389ec1a9ad..dfac7186ca728e 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -150,6 +150,7 @@ Python/import.c - PyImport_Inittab - ## the consolidated runtime state Python/pylifecycle.c - _PyRuntime - Python/pylifecycle.c - runtime_initialized - +Python/gc.c - gc_states - # All cases of _PyArg_Parser are handled in c-analyzr/cpython/_analyzer.py.