gh-159137: Copy and subtract the GC reference counts in one walk

Lib/test/test_gdb/test_backtrace.py

@@ -99,13 +99,13 @@ def test_gc(self):
                'bar()\n')
         # Verify with "py-bt":
         gdb_output = self.get_stack_trace(cmd,
-                                          cmds_after_breakpoint=['break update_refs', 'continue', 'py-bt'],
+                                          cmds_after_breakpoint=['break move_unreachable', 'continue', 'py-bt'],
                                           )
         self.assertIn('Garbage-collecting', gdb_output)
 
         # Verify with "py-bt-full":
         gdb_output = self.get_stack_trace(cmd,
-                                          cmds_after_breakpoint=['break update_refs', 'continue', 'py-bt-full'],
+                                          cmds_after_breakpoint=['break move_unreachable', 'continue', 'py-bt-full'],
                                           )
         self.assertIn('Garbage-collecting', gdb_output)
 

Misc/NEWS.d/next/Core_and_Builtins/2026-10-11-10-00-00.gh-issue-159137.gcFuse.rst

@@ -0,0 +1,3 @@
+Speed up full collections of the garbage collector: the reference counts are
+copied and the references between the collected objects subtracted in a
+single walk of the generation.

Python/gc.c

@@ -110,9 +110,15 @@ get_gc_state(void)
 }
 
 
+/* The number of interpreters that the process has had.  While it is 1, every
+ * tracked object is in a list of that interpreter. */
+static Py_ssize_t gc_states;
+
 void
 _PyGC_InitState(GCState *gcstate)
 {
+    _Py_atomic_add_ssize(&gc_states, 1);
+
 #define INIT_HEAD(GEN) \
     do { \
         GEN.head._gc_next = (uintptr_t)&GEN.head; \
@@ -456,14 +462,40 @@ visit_decref(PyObject *op, void *parent)
     return 0;
 }
 
+/* A traversal callback for update_and_subtract_refs(), where every tracked
+ * object is in the list being collected.  If the walk has not reached op yet,
+ * its _gc_prev still holds a pointer, and the walk recovers the decrements
+ * from the difference with the pointer that it knows was there.
+ */
+static int
+visit_decref_tracked(PyObject *op, void *parent)
+{
+    OBJECT_STAT_INC(object_visits);
+    _PyObject_ASSERT(_PyObject_CAST(parent), !_PyObject_IsFreed(op));
+
+    if (_PyObject_IS_GC(op)) {
+        PyGC_Head *gc = AS_GC(op);
+        if (gc->_gc_next != 0) {
+            _PyObject_ASSERT_WITH_MSG(op,
+                                      gc_get_refs(gc) > 0
+                                      || !gc_is_collecting(gc),
+                                      "refcount is too small");
+            gc->_gc_prev -= 1 << _PyGC_PREV_SHIFT;
+        }
+    }
+    return 0;
+}
+
 int
 _PyGC_VisitStackRef(_PyStackRef *ref, visitproc visit, void *arg)
 {
     // This is a bit tricky! We want to ignore stackrefs with embedded
     // refcounts when computing the incoming references, but otherwise treat
     // them like normal.
     assert(!PyStackRef_IsTaggedInt(*ref));
-    if (!PyStackRef_RefcountOnObject(*ref) && (visit == visit_decref)) {
+    if (!PyStackRef_RefcountOnObject(*ref)
+        && (visit == visit_decref || visit == visit_decref_tracked))
+    {
         return 0;
     }
     Py_VISIT(PyStackRef_AsPyObjectBorrow(*ref));
@@ -501,6 +533,47 @@ subtract_refs(PyGC_Head *containers)
     }
 }
 
+/* update_refs() and subtract_refs() in one walk, for a list that holds every
+ * tracked object.  visit_decref_tracked() then needs no flag to recognise a
+ * candidate, so an object can be decremented before the walk gets to it.
+ */
+static Py_ssize_t
+update_and_subtract_refs(PyGC_Head *containers)
+{
+    PyGC_Head *prev = containers;
+    PyGC_Head *gc = GC_NEXT(containers);
+    Py_ssize_t candidates = 0;
+
+    while (gc != containers) {
+        PyGC_Head *next = GC_NEXT(gc);
+        PyObject *op = FROM_GC(gc);
+        if (_Py_IsImmortal(op)) {
+            assert(!_Py_IsStaticImmortal(op));
+            // _PyObject_GC_UNTRACK(), keeping the decrements of next.
+            _PyGCHead_SET_NEXT(prev, next);
+            next->_gc_prev += (uintptr_t)prev - (uintptr_t)gc;
+            gc->_gc_next = 0;
+            gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED;
+            get_gc_state()->heap_size--;
+            gc = next;
+            continue;
+        }
+        /* _gc_prev is prev and the flags, minus the decrements so far. */
+        gc->_gc_prev = (gc->_gc_prev - (uintptr_t)prev
+                        + ((uintptr_t)Py_REFCNT(op) << _PyGC_PREV_SHIFT))
+                       | PREV_MASK_COLLECTING;
+        // See update_refs() for a refcount of 0.
+        _PyObject_ASSERT(op, Py_REFCNT(op) != 0);
+        _PyObject_ASSERT_WITH_MSG(op, gc_get_refs(gc) <= Py_REFCNT(op),
+                                  "refcount is too small");
+        (void) Py_TYPE(op)->tp_traverse(op, visit_decref_tracked, op);
+        prev = gc;
+        gc = next;
+        candidates++;
+    }
+    return candidates;
+}
+
 /* A traversal callback for move_unreachable. */
 static int
 visit_reachable(PyObject *op, void *arg)
@@ -1175,15 +1248,21 @@ flag is cleared (for example, by using 'clear_unreachable_mask' function or
 by a call to 'move_legacy_finalizers'), the 'unreachable' list is not a normal
 list and we can not use most gc_list_* functions for it. */
 static inline Py_ssize_t
-deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable) {
+deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable, bool all_tracked) {
     validate_list(base, collecting_clear_unreachable_clear);
     /* Using ob_refcnt and gc_refs, calculate which objects in the
      * container set are reachable from outside the set (i.e., have a
      * refcount greater than 0 when all the references within the
      * set are taken into account).
      */
-    Py_ssize_t candidates = update_refs(base);  // gc_prev is used for gc_refs
-    subtract_refs(base);
+    Py_ssize_t candidates;  // gc_prev is used for gc_refs
+    if (all_tracked) {
+        candidates = update_and_subtract_refs(base);
+    }
+    else {
+        candidates = update_refs(base);
+        subtract_refs(base);
+    }
 
     /* Leave everything reachable from outside base in base, and move
      * everything else (in base) to unreachable.
@@ -1252,7 +1331,7 @@ handle_resurrected_objects(PyGC_Head *unreachable, PyGC_Head* still_unreachable,
     // have the PREV_MARK_COLLECTING set, but the objects are going to be
     // removed so we can skip the expense of clearing the flag.
     PyGC_Head* resurrected = unreachable;
-    deduce_unreachable(resurrected, still_unreachable);
+    deduce_unreachable(resurrected, still_unreachable, false);
     clear_unreachable_mask(still_unreachable);
 
     // Move the resurrected objects to the old generation for future collection.
@@ -1521,7 +1600,12 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason)
     }
     validate_list(old, collecting_clear_unreachable_clear);
 
-    stats.candidates = deduce_unreachable(young, &unreachable);
+    /* A full collection examines every tracked object if none is frozen and
+     * the process has had no other interpreter. */
+    bool all_tracked = (generation == NUM_GENERATIONS - 1
+                        && gc_list_is_empty(&gcstate->permanent_generation.head)
+                        && _Py_atomic_load_ssize_relaxed(&gc_states) == 1);
+    stats.candidates = deduce_unreachable(young, &unreachable, all_tracked);
 
     untrack_tuples(young);
     /* Move reachable objects to next generation. */

Tools/c-analyzer/cpython/ignored.tsv

@@ -150,6 +150,7 @@ Python/import.c	-	PyImport_Inittab	-
 ## the consolidated runtime state
 Python/pylifecycle.c	-	_PyRuntime	-
 Python/pylifecycle.c	-	runtime_initialized	-
+Python/gc.c	-	gc_states	-
 
 # All cases of _PyArg_Parser are handled in c-analyzr/cpython/_analyzer.py.