Repository navigation
gh-150791: add critical section for groupby.next
#150792
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -15,6 +15,7 @@ | |
| import sys | ||
| import struct | ||
| import threading | ||
| import time | ||
| import gc | ||
|
|
||
| maxsize = support.MAX_Py_ssize_t | ||
|
|
@@ -786,6 +787,48 @@ def keyfunc(element): | |
| items = list(grouper_iter) | ||
| self.assertEqual(len(items), 1) | ||
|
|
||
| @threading_helper.requires_working_threading() | ||
| def test_groupby_concurrent_next_does_not_crash(self): | ||
| # regression test for gh-150791 | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. For # gh-150791: Concurrent next calls on a shared groupby object and its groupers
# should not crash even with arbitrary code __eq__. |
||
| # Concurrent next calls on a shared groupby object and its groupers | ||
| # should not crash, even when __eq__ suspends the critical section. | ||
| class K: | ||
| __slots__ = ("v",) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is the |
||
| def __init__(self, v: int) -> None: | ||
| self.v = v | ||
| def __eq__(self, other: object) -> bool: | ||
| if not isinstance(other, K): | ||
| return NotImplemented | ||
| # Blocking detaches the thread state, which suspends the | ||
| # critical section held on the groupby object. | ||
| time.sleep(0) | ||
| return self.v == other.v | ||
| def __hash__(self) -> int: | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't think this is needed / necessary |
||
| return hash(self.v) | ||
|
|
||
| keys: list[K] = [K(i // 3) for i in range(5_000)] | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. don't do type annotations in cpython source with very a few exceptions currently. Only a few internal-only bits of cpython code use type annotations and are checked with |
||
| g: groupby = itertools.groupby(keys) | ||
| errors: list[Exception] = [] | ||
|
|
||
| def consume() -> None: | ||
| try: | ||
| while True: | ||
| _, grp = next(g) | ||
| for _ in grp: | ||
| pass | ||
| except StopIteration: | ||
| pass | ||
| except Exception as e: | ||
| errors.append(e) | ||
|
|
||
| threads = [threading.Thread(target=consume) for _ in range(8)] | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think this will be able to turn into |
||
| for t in threads: | ||
| t.start() | ||
| for t in threads: | ||
| t.join() | ||
|
|
||
| self.assertEqual(errors, []) # must pass with ThreadSanitizer | ||
|
|
||
| def test_filter(self): | ||
| self.assertEqual(list(filter(isEven, range(6))), [0,2,4]) | ||
| self.assertEqual(list(filter(None, [0,1,0,2,0])), [1,2]) | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| Fix a data race in :func:`itertools.groupby` on free-threaded builds where concurrent calls to :func:`next` could corrupt the iterator's internal state. | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fix :func: |
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -529,19 +529,21 @@ groupby_step(groupbyobject *gbo) | |
| } | ||
|
|
||
| static PyObject * | ||
| groupby_next(PyObject *op) | ||
| groupby_next_lock_held(PyObject *op) | ||
| { | ||
| PyObject *grouper; | ||
| groupbyobject *gbo = groupbyobject_CAST(op); | ||
|
|
||
| PyObject *key = NULL; | ||
| gbo->currgrouper = NULL; | ||
| /* skip to next iteration group */ | ||
| for (;;) { | ||
| if (gbo->currkey == NULL) | ||
| if (gbo->currkey == NULL) { | ||
| /* pass */; | ||
| else if (gbo->tgtkey == NULL) | ||
| } else if (gbo->tgtkey == NULL) { | ||
| key = Py_NewRef(gbo->currkey); | ||
| break; | ||
| else { | ||
| } else { | ||
| /* A user-defined __eq__ can re-enter groupby and advance the iterator, | ||
| mutating gbo->tgtkey / gbo->currkey while we are comparing them. | ||
| Take local snapshots and hold strong references so INCREF/DECREF | ||
|
|
@@ -553,25 +555,40 @@ groupby_next(PyObject *op) | |
| Py_INCREF(currkey); | ||
| int rcmp = PyObject_RichCompareBool(tgtkey, currkey, Py_EQ); | ||
| Py_DECREF(tgtkey); | ||
| Py_DECREF(currkey); | ||
|
|
||
| if (rcmp == -1) | ||
| if (rcmp == -1) { | ||
| Py_DECREF(currkey); | ||
| return NULL; | ||
| else if (rcmp == 0) | ||
| } else if (rcmp == 0) { | ||
| key = currkey; | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Default to |
||
| break; | ||
| } | ||
| Py_DECREF(currkey); | ||
| } | ||
|
|
||
| if (groupby_step(gbo) < 0) | ||
| if (groupby_step(gbo) < 0) { | ||
| return NULL; | ||
| } | ||
| } | ||
| Py_INCREF(gbo->currkey); | ||
| Py_XSETREF(gbo->tgtkey, gbo->currkey); | ||
| Py_XSETREF(gbo->tgtkey, Py_NewRef(key)); | ||
|
|
||
| grouper = _grouper_create(gbo, gbo->tgtkey); | ||
| if (grouper == NULL) | ||
| grouper = _grouper_create(gbo, key); | ||
| if (grouper == NULL) { | ||
| Py_DECREF(key); | ||
| return NULL; | ||
| } | ||
|
|
||
| return _PyTuple_FromPairSteal(key, grouper); | ||
| } | ||
|
|
||
| return _PyTuple_FromPairSteal(Py_NewRef(gbo->currkey), grouper); | ||
| static PyObject * | ||
| groupby_next(PyObject *op) | ||
| { | ||
| PyObject *result; | ||
| Py_BEGIN_CRITICAL_SECTION(op); | ||
| result = groupby_next_lock_held(op); | ||
| Py_END_CRITICAL_SECTION() | ||
| return result; | ||
| } | ||
|
|
||
| static PyType_Slot groupby_slots[] = { | ||
|
|
@@ -659,15 +676,17 @@ _grouper_traverse(PyObject *op, visitproc visit, void *arg) | |
| } | ||
|
|
||
| static PyObject * | ||
| _grouper_next(PyObject *op) | ||
| _grouper_next_lock_held(PyObject *op) | ||
| { | ||
| _grouperobject *igo = _grouperobject_CAST(op); | ||
| groupbyobject *gbo = groupbyobject_CAST(igo->parent); | ||
| PyObject *r; | ||
| int rcmp; | ||
|
|
||
| if (gbo->currgrouper != igo) | ||
| if (gbo->currgrouper != igo) { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This can change while the critical section is released during items = [("a", 1), ("a", 2), ("b", 3), ("b", 4)]
g = itertools.groupby(items, key=lambda it: Key(it[0])) # Key.__eq__ can block on an Event
ka, grp_a = next(g)
# thread t1:
next(grp_a) # ('a', 1)
next(grp_a) # compares Key('a')#1 == Key('a')#2 -> blocks inside __eq__
# main thread, while t1 is blocked:
kb, grp_b = next(g)
# then release t1 |
||
| return NULL; | ||
| } | ||
|
|
||
| if (gbo->currvalue == NULL) { | ||
| if (groupby_step(gbo) < 0) | ||
| return NULL; | ||
|
|
@@ -684,9 +703,10 @@ _grouper_next(PyObject *op) | |
| Py_DECREF(tgtkey); | ||
| Py_DECREF(currkey); | ||
|
|
||
| if (rcmp <= 0) | ||
| if (rcmp <= 0) { | ||
| /* got any error or current group is end */ | ||
| return NULL; | ||
| } | ||
|
|
||
| r = gbo->currvalue; | ||
| gbo->currvalue = NULL; | ||
|
|
@@ -695,6 +715,16 @@ _grouper_next(PyObject *op) | |
| return r; | ||
| } | ||
|
|
||
| static PyObject * | ||
| _grouper_next(PyObject *op) | ||
| { | ||
| PyObject *result; | ||
| Py_BEGIN_CRITICAL_SECTION(_grouperobject_CAST(op)->parent); | ||
| result = _grouper_next_lock_held(op); | ||
| Py_END_CRITICAL_SECTION() | ||
| return result; | ||
| } | ||
|
|
||
| static PyType_Slot _grouper_slots[] = { | ||
| {Py_tp_dealloc, _grouper_dealloc}, | ||
| {Py_tp_getattro, PyObject_GenericGetAttr}, | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This should probably go in https://github.com/python/cpython/blob/main/Lib/test/test_free_threading/test_itertools.py