Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions Lib/test/test_itertools.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import sys
import struct
import threading
import time
import gc

maxsize = support.MAX_Py_ssize_t
Expand Down Expand Up @@ -786,6 +787,48 @@ def keyfunc(element):
items = list(grouper_iter)
self.assertEqual(len(items), 1)

@threading_helper.requires_working_threading()
def test_groupby_concurrent_next_does_not_crash(self):

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

# regression test for gh-150791

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For gh- references typically do at the start,

# gh-150791: Concurrent next calls on a shared groupby object and its groupers
# should not crash even with arbitrary code __eq__.

# Concurrent next calls on a shared groupby object and its groupers
# should not crash, even when __eq__ suspends the critical section.
class K:
__slots__ = ("v",)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the __slots__ critical to the test?

def __init__(self, v: int) -> None:
self.v = v
def __eq__(self, other: object) -> bool:
if not isinstance(other, K):
return NotImplemented
# Blocking detaches the thread state, which suspends the
# critical section held on the groupby object.
time.sleep(0)
return self.v == other.v
def __hash__(self) -> int:

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think this is needed / necessary

return hash(self.v)

keys: list[K] = [K(i // 3) for i in range(5_000)]

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

don't do type annotations in cpython source with very a few exceptions currently. Only a few internal-only bits of cpython code use type annotations and are checked with mypy, elsewhere don't add currently.

g: groupby = itertools.groupby(keys)
errors: list[Exception] = []

def consume() -> None:
try:
while True:
_, grp = next(g)
for _ in grp:
pass
except StopIteration:
pass
except Exception as e:
errors.append(e)

threads = [threading.Thread(target=consume) for _ in range(8)]

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this will be able to turn into threading_helper.run_concurrently

for t in threads:
t.start()
for t in threads:
t.join()

self.assertEqual(errors, []) # must pass with ThreadSanitizer

def test_filter(self):
self.assertEqual(list(filter(isEven, range(6))), [0,2,4])
self.assertEqual(list(filter(None, [0,1,0,2,0])), [1,2])
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fix a data race in :func:`itertools.groupby` on free-threaded builds where concurrent calls to :func:`next` could corrupt the iterator's internal state.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fix :func:itertools.groupby data races in :term:free-threading.

62 changes: 46 additions & 16 deletions Modules/itertoolsmodule.c
Original file line number Diff line number Diff line change
Expand Up @@ -529,19 +529,21 @@ groupby_step(groupbyobject *gbo)
}

static PyObject *
groupby_next(PyObject *op)
groupby_next_lock_held(PyObject *op)
{
PyObject *grouper;
groupbyobject *gbo = groupbyobject_CAST(op);

PyObject *key = NULL;
gbo->currgrouper = NULL;
/* skip to next iteration group */
for (;;) {
if (gbo->currkey == NULL)
if (gbo->currkey == NULL) {
/* pass */;
else if (gbo->tgtkey == NULL)
} else if (gbo->tgtkey == NULL) {
key = Py_NewRef(gbo->currkey);
break;
else {
} else {
/* A user-defined __eq__ can re-enter groupby and advance the iterator,
mutating gbo->tgtkey / gbo->currkey while we are comparing them.
Take local snapshots and hold strong references so INCREF/DECREF
Expand All @@ -553,25 +555,40 @@ groupby_next(PyObject *op)
Py_INCREF(currkey);
int rcmp = PyObject_RichCompareBool(tgtkey, currkey, Py_EQ);
Py_DECREF(tgtkey);
Py_DECREF(currkey);

if (rcmp == -1)
if (rcmp == -1) {
Py_DECREF(currkey);
return NULL;
else if (rcmp == 0)
} else if (rcmp == 0) {
key = currkey;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Default to Py_XSETREF and/or Py_SETREF for cases like this, there are lots of edge cases in other options.

break;
}
Py_DECREF(currkey);
}

if (groupby_step(gbo) < 0)
if (groupby_step(gbo) < 0) {
return NULL;
}
}
Py_INCREF(gbo->currkey);
Py_XSETREF(gbo->tgtkey, gbo->currkey);
Py_XSETREF(gbo->tgtkey, Py_NewRef(key));

grouper = _grouper_create(gbo, gbo->tgtkey);
if (grouper == NULL)
grouper = _grouper_create(gbo, key);
if (grouper == NULL) {
Py_DECREF(key);
return NULL;
}

return _PyTuple_FromPairSteal(key, grouper);
}

return _PyTuple_FromPairSteal(Py_NewRef(gbo->currkey), grouper);
static PyObject *
groupby_next(PyObject *op)
{
PyObject *result;
Py_BEGIN_CRITICAL_SECTION(op);
result = groupby_next_lock_held(op);
Py_END_CRITICAL_SECTION()
return result;
}

static PyType_Slot groupby_slots[] = {
Expand Down Expand Up @@ -659,15 +676,17 @@ _grouper_traverse(PyObject *op, visitproc visit, void *arg)
}

static PyObject *
_grouper_next(PyObject *op)
_grouper_next_lock_held(PyObject *op)
{
_grouperobject *igo = _grouperobject_CAST(op);
groupbyobject *gbo = groupbyobject_CAST(igo->parent);
PyObject *r;
int rcmp;

if (gbo->currgrouper != igo)
if (gbo->currgrouper != igo) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This can change while the critical section is released during __eq__ if another thread calls next() on the groupby object. I think re-checking it would be good? (although doesn't crash at least)

items = [("a", 1), ("a", 2), ("b", 3), ("b", 4)]
g = itertools.groupby(items, key=lambda it: Key(it[0]))   # Key.__eq__ can block on an Event
ka, grp_a = next(g)

# thread t1:
next(grp_a)        # ('a', 1)
next(grp_a)        # compares Key('a')#1 == Key('a')#2 -> blocks inside __eq__

# main thread, while t1 is blocked:
kb, grp_b = next(g)
# then release t1

return NULL;
}

if (gbo->currvalue == NULL) {
if (groupby_step(gbo) < 0)
return NULL;
Expand All @@ -684,9 +703,10 @@ _grouper_next(PyObject *op)
Py_DECREF(tgtkey);
Py_DECREF(currkey);

if (rcmp <= 0)
if (rcmp <= 0) {
/* got any error or current group is end */
return NULL;
}

r = gbo->currvalue;
gbo->currvalue = NULL;
Expand All @@ -695,6 +715,16 @@ _grouper_next(PyObject *op)
return r;
}

static PyObject *
_grouper_next(PyObject *op)
{
PyObject *result;
Py_BEGIN_CRITICAL_SECTION(_grouperobject_CAST(op)->parent);
result = _grouper_next_lock_held(op);
Py_END_CRITICAL_SECTION()
return result;
}

static PyType_Slot _grouper_slots[] = {
{Py_tp_dealloc, _grouper_dealloc},
{Py_tp_getattro, PyObject_GenericGetAttr},
Expand Down
Loading