diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index cf579d4da4e0dfb..72a4949093932e6 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -15,6 +15,7 @@ import sys import struct import threading +import time import gc maxsize = support.MAX_Py_ssize_t @@ -786,6 +787,48 @@ def keyfunc(element): items = list(grouper_iter) self.assertEqual(len(items), 1) + @threading_helper.requires_working_threading() + def test_groupby_concurrent_next_does_not_crash(self): + # regression test for gh-150791 + # Concurrent next calls on a shared groupby object and its groupers + # should not crash, even when __eq__ suspends the critical section. + class K: + __slots__ = ("v",) + def __init__(self, v: int) -> None: + self.v = v + def __eq__(self, other: object) -> bool: + if not isinstance(other, K): + return NotImplemented + # Blocking detaches the thread state, which suspends the + # critical section held on the groupby object. + time.sleep(0) + return self.v == other.v + def __hash__(self) -> int: + return hash(self.v) + + keys: list[K] = [K(i // 3) for i in range(5_000)] + g: groupby = itertools.groupby(keys) + errors: list[Exception] = [] + + def consume() -> None: + try: + while True: + _, grp = next(g) + for _ in grp: + pass + except StopIteration: + pass + except Exception as e: + errors.append(e) + + threads = [threading.Thread(target=consume) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + + self.assertEqual(errors, []) # must pass with ThreadSanitizer + def test_filter(self): self.assertEqual(list(filter(isEven, range(6))), [0,2,4]) self.assertEqual(list(filter(None, [0,1,0,2,0])), [1,2]) diff --git a/Misc/NEWS.d/next/Library/2026-06-02-14-22-05.gh-issue-150791.aQ7rNp.rst b/Misc/NEWS.d/next/Library/2026-06-02-14-22-05.gh-issue-150791.aQ7rNp.rst new file mode 100644 index 000000000000000..b69c3d221bf5303 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-02-14-22-05.gh-issue-150791.aQ7rNp.rst @@ -0,0 +1 @@ +Fix a data race in :func:`itertools.groupby` on free-threaded builds where concurrent calls to :func:`next` could corrupt the iterator's internal state. diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 68ac810eaad237f..dfc33b58d46fa4c 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -529,19 +529,21 @@ groupby_step(groupbyobject *gbo) } static PyObject * -groupby_next(PyObject *op) +groupby_next_lock_held(PyObject *op) { PyObject *grouper; groupbyobject *gbo = groupbyobject_CAST(op); + PyObject *key = NULL; gbo->currgrouper = NULL; /* skip to next iteration group */ for (;;) { - if (gbo->currkey == NULL) + if (gbo->currkey == NULL) { /* pass */; - else if (gbo->tgtkey == NULL) + } else if (gbo->tgtkey == NULL) { + key = Py_NewRef(gbo->currkey); break; - else { + } else { /* A user-defined __eq__ can re-enter groupby and advance the iterator, mutating gbo->tgtkey / gbo->currkey while we are comparing them. Take local snapshots and hold strong references so INCREF/DECREF @@ -553,25 +555,40 @@ groupby_next(PyObject *op) Py_INCREF(currkey); int rcmp = PyObject_RichCompareBool(tgtkey, currkey, Py_EQ); Py_DECREF(tgtkey); - Py_DECREF(currkey); - if (rcmp == -1) + if (rcmp == -1) { + Py_DECREF(currkey); return NULL; - else if (rcmp == 0) + } else if (rcmp == 0) { + key = currkey; break; + } + Py_DECREF(currkey); } - if (groupby_step(gbo) < 0) + if (groupby_step(gbo) < 0) { return NULL; + } } - Py_INCREF(gbo->currkey); - Py_XSETREF(gbo->tgtkey, gbo->currkey); + Py_XSETREF(gbo->tgtkey, Py_NewRef(key)); - grouper = _grouper_create(gbo, gbo->tgtkey); - if (grouper == NULL) + grouper = _grouper_create(gbo, key); + if (grouper == NULL) { + Py_DECREF(key); return NULL; + } + + return _PyTuple_FromPairSteal(key, grouper); +} - return _PyTuple_FromPairSteal(Py_NewRef(gbo->currkey), grouper); +static PyObject * +groupby_next(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = groupby_next_lock_held(op); + Py_END_CRITICAL_SECTION() + return result; } static PyType_Slot groupby_slots[] = { @@ -659,15 +676,17 @@ _grouper_traverse(PyObject *op, visitproc visit, void *arg) } static PyObject * -_grouper_next(PyObject *op) +_grouper_next_lock_held(PyObject *op) { _grouperobject *igo = _grouperobject_CAST(op); groupbyobject *gbo = groupbyobject_CAST(igo->parent); PyObject *r; int rcmp; - if (gbo->currgrouper != igo) + if (gbo->currgrouper != igo) { return NULL; + } + if (gbo->currvalue == NULL) { if (groupby_step(gbo) < 0) return NULL; @@ -684,9 +703,10 @@ _grouper_next(PyObject *op) Py_DECREF(tgtkey); Py_DECREF(currkey); - if (rcmp <= 0) + if (rcmp <= 0) { /* got any error or current group is end */ return NULL; + } r = gbo->currvalue; gbo->currvalue = NULL; @@ -695,6 +715,16 @@ _grouper_next(PyObject *op) return r; } +static PyObject * +_grouper_next(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(_grouperobject_CAST(op)->parent); + result = _grouper_next_lock_held(op); + Py_END_CRITICAL_SECTION() + return result; +} + static PyType_Slot _grouper_slots[] = { {Py_tp_dealloc, _grouper_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr},