From 247d38669d0abc815d13817c47b6bcb58bb127b8 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Tue, 2 Jun 2026 16:26:37 +0200 Subject: [PATCH 1/4] misc: add news entry --- .../next/Library/2026-06-02-14-22-05.gh-issue-150791.aQ7rNp.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2026-06-02-14-22-05.gh-issue-150791.aQ7rNp.rst diff --git a/Misc/NEWS.d/next/Library/2026-06-02-14-22-05.gh-issue-150791.aQ7rNp.rst b/Misc/NEWS.d/next/Library/2026-06-02-14-22-05.gh-issue-150791.aQ7rNp.rst new file mode 100644 index 000000000000000..b69c3d221bf5303 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-02-14-22-05.gh-issue-150791.aQ7rNp.rst @@ -0,0 +1 @@ +Fix a data race in :func:`itertools.groupby` on free-threaded builds where concurrent calls to :func:`next` could corrupt the iterator's internal state. From 54fc409769da00a42522628bda1c7521414a7a3b Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Tue, 2 Jun 2026 16:27:13 +0200 Subject: [PATCH 2/4] test: add regression test --- Lib/test/test_itertools.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index cf579d4da4e0dfb..1c3cc96ac48a64d 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -786,6 +786,43 @@ def keyfunc(element): items = list(grouper_iter) self.assertEqual(len(items), 1) + @threading_helper.requires_working_threading() + def test_groupby_concurrent_next_does_not_crash(self): + # regression test for gh-150791 + # Concurrent next calls on a shared groupby object should + # not race / corrupt state. + class K: + __slots__ = ("v",) + def __init__(self, v): + self.v = v + def __eq__(self, other): + if not isinstance(other, K): + return NotImplemented + return self.v == other.v + def __hash__(self): + return hash(self.v) + + keys = [K(i) for i in range(5_000)] + g = itertools.groupby(keys) + errors = [] + + def consume(): + try: + while True: + _, _ = next(g) + except StopIteration: + pass + except Exception as e: + errors.append(e) + + threads = [threading.Thread(target=consume) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + + self.assertEqual(errors, []) # must pass with ThreadSanitizer + def test_filter(self): self.assertEqual(list(filter(isEven, range(6))), [0,2,4]) self.assertEqual(list(filter(None, [0,1,0,2,0])), [1,2]) From ba8c0cd0f064bc4bbe294a6a34faa5117d2e37e9 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Tue, 2 Jun 2026 16:27:59 +0200 Subject: [PATCH 3/4] fix: add critical section for grouper_next --- Modules/itertoolsmodule.c | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 68ac810eaad237f..a21466a892c593a 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -529,7 +529,7 @@ groupby_step(groupbyobject *gbo) } static PyObject * -groupby_next(PyObject *op) +groupby_next_lock_held(PyObject *op) { PyObject *grouper; groupbyobject *gbo = groupbyobject_CAST(op); @@ -574,6 +574,16 @@ groupby_next(PyObject *op) return _PyTuple_FromPairSteal(Py_NewRef(gbo->currkey), grouper); } +static PyObject * +groupby_next(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = groupby_next_lock_held(op); + Py_END_CRITICAL_SECTION() + return result; +} + static PyType_Slot groupby_slots[] = { {Py_tp_dealloc, groupby_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -659,7 +669,7 @@ _grouper_traverse(PyObject *op, visitproc visit, void *arg) } static PyObject * -_grouper_next(PyObject *op) +_grouper_next_lock_held(PyObject *op) { _grouperobject *igo = _grouperobject_CAST(op); groupbyobject *gbo = groupbyobject_CAST(igo->parent); @@ -695,6 +705,16 @@ _grouper_next(PyObject *op) return r; } +static PyObject * +_grouper_next(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(_grouperobject_CAST(op)->parent); + result = _grouper_next_lock_held(op); + Py_END_CRITICAL_SECTION() + return result; +} + static PyType_Slot _grouper_slots[] = { {Py_tp_dealloc, _grouper_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, From 310dcd254c7011800b8614acd85b3f220f9791c8 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Mon, 9 Nov 2026 19:04:48 +0100 Subject: [PATCH 4/4] fix: hold strong reference to key --- Lib/test/test_itertools.py | 26 ++++++++++++++++---------- Modules/itertoolsmodule.c | 38 ++++++++++++++++++++++++-------------- 2 files changed, 40 insertions(+), 24 deletions(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 1c3cc96ac48a64d..72a4949093932e6 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -15,6 +15,7 @@ import sys import struct import threading +import time import gc maxsize = support.MAX_Py_ssize_t @@ -789,27 +790,32 @@ def keyfunc(element): @threading_helper.requires_working_threading() def test_groupby_concurrent_next_does_not_crash(self): # regression test for gh-150791 - # Concurrent next calls on a shared groupby object should - # not race / corrupt state. + # Concurrent next calls on a shared groupby object and its groupers + # should not crash, even when __eq__ suspends the critical section. class K: __slots__ = ("v",) - def __init__(self, v): + def __init__(self, v: int) -> None: self.v = v - def __eq__(self, other): + def __eq__(self, other: object) -> bool: if not isinstance(other, K): return NotImplemented + # Blocking detaches the thread state, which suspends the + # critical section held on the groupby object. + time.sleep(0) return self.v == other.v - def __hash__(self): + def __hash__(self) -> int: return hash(self.v) - keys = [K(i) for i in range(5_000)] - g = itertools.groupby(keys) - errors = [] + keys: list[K] = [K(i // 3) for i in range(5_000)] + g: groupby = itertools.groupby(keys) + errors: list[Exception] = [] - def consume(): + def consume() -> None: try: while True: - _, _ = next(g) + _, grp = next(g) + for _ in grp: + pass except StopIteration: pass except Exception as e: diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index a21466a892c593a..dfc33b58d46fa4c 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -534,14 +534,16 @@ groupby_next_lock_held(PyObject *op) PyObject *grouper; groupbyobject *gbo = groupbyobject_CAST(op); + PyObject *key = NULL; gbo->currgrouper = NULL; /* skip to next iteration group */ for (;;) { - if (gbo->currkey == NULL) + if (gbo->currkey == NULL) { /* pass */; - else if (gbo->tgtkey == NULL) + } else if (gbo->tgtkey == NULL) { + key = Py_NewRef(gbo->currkey); break; - else { + } else { /* A user-defined __eq__ can re-enter groupby and advance the iterator, mutating gbo->tgtkey / gbo->currkey while we are comparing them. Take local snapshots and hold strong references so INCREF/DECREF @@ -553,25 +555,30 @@ groupby_next_lock_held(PyObject *op) Py_INCREF(currkey); int rcmp = PyObject_RichCompareBool(tgtkey, currkey, Py_EQ); Py_DECREF(tgtkey); - Py_DECREF(currkey); - if (rcmp == -1) + if (rcmp == -1) { + Py_DECREF(currkey); return NULL; - else if (rcmp == 0) + } else if (rcmp == 0) { + key = currkey; break; + } + Py_DECREF(currkey); } - if (groupby_step(gbo) < 0) + if (groupby_step(gbo) < 0) { return NULL; + } } - Py_INCREF(gbo->currkey); - Py_XSETREF(gbo->tgtkey, gbo->currkey); + Py_XSETREF(gbo->tgtkey, Py_NewRef(key)); - grouper = _grouper_create(gbo, gbo->tgtkey); - if (grouper == NULL) + grouper = _grouper_create(gbo, key); + if (grouper == NULL) { + Py_DECREF(key); return NULL; + } - return _PyTuple_FromPairSteal(Py_NewRef(gbo->currkey), grouper); + return _PyTuple_FromPairSteal(key, grouper); } static PyObject * @@ -676,8 +683,10 @@ _grouper_next_lock_held(PyObject *op) PyObject *r; int rcmp; - if (gbo->currgrouper != igo) + if (gbo->currgrouper != igo) { return NULL; + } + if (gbo->currvalue == NULL) { if (groupby_step(gbo) < 0) return NULL; @@ -694,9 +703,10 @@ _grouper_next_lock_held(PyObject *op) Py_DECREF(tgtkey); Py_DECREF(currkey); - if (rcmp <= 0) + if (rcmp <= 0) { /* got any error or current group is end */ return NULL; + } r = gbo->currvalue; gbo->currvalue = NULL;