Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Include/internal/pycore_crossinterp.h
Original file line number Diff line number Diff line change
Expand Up @@ -368,7 +368,7 @@ PyAPI_FUNC(void) _PyXI_ClearResult(_PyXI_session_result *);

PyAPI_FUNC(int) _PyXI_Enter(
_PyXI_session *session,
PyInterpreterState *interp,
PyInterpreterView *view,
PyObject *nsupdates,
_PyXI_session_result *);
PyAPI_FUNC(int) _PyXI_Exit(
Expand Down
3 changes: 3 additions & 0 deletions Include/internal/pycore_pystate.h
Original file line number Diff line number Diff line change
Expand Up @@ -387,6 +387,9 @@ extern int _PyInterpreterGuard_TryAcquire(PyInterpreterState *interp,
PyInterpreterGuard *guard);
extern void _PyInterpreterGuard_Release(PyInterpreterGuard *guard);

// Export for '_interpreters' shared extension
PyAPI_FUNC(PyInterpreterView *) _PyInterpreterView_FromID(int64_t interp_id);

#ifdef __cplusplus
}
#endif
Expand Down
34 changes: 34 additions & 0 deletions Lib/test/test__interpreters.py
Original file line number Diff line number Diff line change
Expand Up @@ -466,6 +466,40 @@ def test_still_running(self):
_interpreters.destroy(interp)
self.assertTrue(_interpreters.is_running(interp))

def test_concurrent_run_and_destroy(self):
# gh-127041: An interpreter could start running during destroy().
for _ in range(50):
interp = _interpreters.create()
barrier = threading.Barrier(2)

def runner():
barrier.wait()
try:
_interpreters.run_string(interp, "x = 1")
except (_interpreters.InterpreterError,
PythonFinalizationError):
pass

def destroyer():
barrier.wait()
try:
_interpreters.destroy(interp)
except _interpreters.InterpreterError:
pass

threads = [
threading.Thread(target=runner),
threading.Thread(target=destroyer),
]
for t in threads:
t.start()
for t in threads:
t.join()
try:
_interpreters.destroy(interp)
except _interpreters.InterpreterError:
pass


class CommonTests(TestBase):
def setUp(self):
Expand Down
125 changes: 103 additions & 22 deletions Modules/_interpretersmodule.c
Original file line number Diff line number Diff line change
Expand Up @@ -655,7 +655,7 @@ _run_result_clear(struct run_result *runres)
}

static int
_run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp,
_run_in_interpreter(PyThreadState *tstate, PyInterpreterView *view,
_PyXIData_t *script, struct interp_call *call,
PyObject *shareables, struct run_result *runres)
{
Expand All @@ -673,7 +673,7 @@ _run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp,
_PyXI_session_result result = {0};

// Prep and switch interpreters.
if (_PyXI_Enter(session, interp, shareables, &result) < 0) {
if (_PyXI_Enter(session, view, shareables, &result) < 0) {
// If an error occurred at this step, it means that interp
// was not prepared and switched.
_PyXI_FreeSession(session);
Expand Down Expand Up @@ -784,6 +784,71 @@ resolve_interp(PyObject *idobj, int restricted, int reqready, const char *op)
return interp;
}

static PyInterpreterView *
resolve_interp_view(PyObject *idobj, int restricted, int reqready,
const char *op)
{
PyInterpreterView *view;
if (idobj == NULL) {
view = PyInterpreterView_FromCurrent();
if (view == NULL) {
return NULL;
}
}
else {
int64_t id = _PyInterpreterState_ObjectToID(idobj);
if (id < 0) {
return NULL;
}
view = _PyInterpreterView_FromID(id);
if (view == NULL) {
return NULL;
}
}

PyInterpreterGuard *guard = PyInterpreterGuard_FromView(view);
if (guard == NULL) {
PyInterpreterView_Close(view);
if (!PyErr_Occurred()) {
PyErr_Format(PyExc_InterpreterNotFoundError,
"unrecognized interpreter ID %R", idobj);
}
return NULL;
}
PyInterpreterState *interp = _PyInterpreterGuard_GetInterpreter(guard);

if (reqready && !_PyInterpreterState_IsReady(interp)) {
PyInterpreterGuard_Close(guard);
PyInterpreterView_Close(view);
if (idobj == NULL) {
PyErr_Format(PyExc_InterpreterError,
"cannot %s current interpreter (not ready)", op);
}
else {
PyErr_Format(PyExc_InterpreterError,
"cannot %s interpreter %R (not ready)", op, idobj);
}
return NULL;
}

if (restricted && get_whence(interp) != _PyInterpreterState_WHENCE_STDLIB) {
PyInterpreterGuard_Close(guard);
PyInterpreterView_Close(view);
if (idobj == NULL) {
PyErr_Format(PyExc_InterpreterError,
"cannot %s unrecognized current interpreter", op);
}
else {
PyErr_Format(PyExc_InterpreterError,
"cannot %s unrecognized interpreter %R", op, idobj);
}
return NULL;
}

PyInterpreterGuard_Close(guard);
return view;
}


static PyObject *
get_summary(PyInterpreterState *interp)
Expand Down Expand Up @@ -1043,30 +1108,35 @@ _interpreters_set___main___attrs_impl(PyObject *module, PyObject *id,
{
// Look up the interpreter.
int reqready = 1;
PyInterpreterState *interp = \
resolve_interp(id, restricted, reqready, "update __main__ for");
if (interp == NULL) {
PyInterpreterView *view = \
resolve_interp_view(id, restricted, reqready, "update __main__ for");
if (view == NULL) {
return NULL;
}

// Check the updates.
Py_ssize_t size = PyDict_Size(updates);
if (size < 0) {
PyInterpreterView_Close(view);
return NULL;
}
if (size == 0) {
PyInterpreterView_Close(view);
PyErr_SetString(PyExc_ValueError,
"arg 2 must be a non-empty dict");
return NULL;
}

_PyXI_session *session = _PyXI_NewSession();
if (session == NULL) {
PyInterpreterView_Close(view);
return NULL;
}

// Prep and switch interpreters, including apply the updates.
if (_PyXI_Enter(session, interp, updates, NULL) < 0) {
int entered = _PyXI_Enter(session, view, updates, NULL);
PyInterpreterView_Close(view);
if (entered < 0) {
_PyXI_FreeSession(session);
return NULL;
}
Expand Down Expand Up @@ -1131,23 +1201,25 @@ _interpreters_exec_impl(PyObject *module, PyObject *id, PyObject *code,
{
PyThreadState *tstate = _PyThreadState_GET();
int reqready = 1;
PyInterpreterState *interp = \
resolve_interp(id, restricted, reqready, "exec code for");
if (interp == NULL) {
PyInterpreterView *view = \
resolve_interp_view(id, restricted, reqready, "exec code for");
if (view == NULL) {
return NULL;
}

// We don't need the script to be "pure", which means it can use
// global variables. They will be resolved against __main__.
_PyXIData_t xidata = {0};
if (_PyCode_GetScriptXIData(tstate, code, &xidata) < 0) {
PyInterpreterView_Close(view);
unwrap_not_shareable(tstate, NULL);
return NULL;
}

struct run_result runres = {0};
int res = _run_in_interpreter(
tstate, interp, &xidata, NULL, shared, &runres);
tstate, view, &xidata, NULL, shared, &runres);
PyInterpreterView_Close(view);
_PyXIData_Release(&xidata);
if (res < 0) {
return _handle_script_error(&runres);
Expand Down Expand Up @@ -1178,26 +1250,29 @@ _interpreters_run_string_impl(PyObject *module, PyObject *id,
#define FUNCNAME MODULE_NAME_STR ".run_string"
PyThreadState *tstate = _PyThreadState_GET();
int reqready = 1;
PyInterpreterState *interp = \
resolve_interp(id, restricted, reqready, "run a string in");
if (interp == NULL) {
PyInterpreterView *view = \
resolve_interp_view(id, restricted, reqready, "run a string in");
if (view == NULL) {
return NULL;
}

if (PyFunction_Check(script) || PyCode_Check(script)) {
PyInterpreterView_Close(view);
_PyArg_BadArgument(FUNCNAME, "argument 2", "a string", script);
return NULL;
}

_PyXIData_t xidata = {0};
if (_PyCode_GetScriptXIData(tstate, script, &xidata) < 0) {
PyInterpreterView_Close(view);
unwrap_not_shareable(tstate, NULL);
return NULL;
}

struct run_result runres = {0};
int res = _run_in_interpreter(
tstate, interp, &xidata, NULL, shared, &runres);
tstate, view, &xidata, NULL, shared, &runres);
PyInterpreterView_Close(view);
_PyXIData_Release(&xidata);
if (res < 0) {
return _handle_script_error(&runres);
Expand Down Expand Up @@ -1232,9 +1307,9 @@ _interpreters_run_func_impl(PyObject *module, PyObject *id, PyObject *func,
#define FUNCNAME MODULE_NAME_STR ".run_func"
PyThreadState *tstate = _PyThreadState_GET();
int reqready = 1;
PyInterpreterState *interp = \
resolve_interp(id, restricted, reqready, "run a function in");
if (interp == NULL) {
PyInterpreterView *view = \
resolve_interp_view(id, restricted, reqready, "run a function in");
if (view == NULL) {
return NULL;
}

Expand All @@ -1248,19 +1323,22 @@ _interpreters_run_func_impl(PyObject *module, PyObject *id, PyObject *func,
code = func;
}
else {
PyInterpreterView_Close(view);
_PyArg_BadArgument(FUNCNAME, "argument 2", "a function", func);
return NULL;
}

_PyXIData_t xidata = {0};
if (_PyCode_GetScriptXIData(tstate, code, &xidata) < 0) {
PyInterpreterView_Close(view);
unwrap_not_shareable(tstate, NULL);
return NULL;
}

struct run_result runres = {0};
int res = _run_in_interpreter(
tstate, interp, &xidata, NULL, shared, &runres);
tstate, view, &xidata, NULL, shared, &runres);
PyInterpreterView_Close(view);
_PyXIData_Release(&xidata);
if (res < 0) {
return _handle_script_error(&runres);
Expand Down Expand Up @@ -1293,20 +1371,23 @@ _interpreters_call_impl(PyObject *module, PyObject *id, PyObject *callable,
{
PyThreadState *tstate = _PyThreadState_GET();
int reqready = 1;
PyInterpreterState *interp = \
resolve_interp(id, restricted, reqready, "make a call in");
if (interp == NULL) {
PyInterpreterView *view = \
resolve_interp_view(id, restricted, reqready, "make a call in");
if (view == NULL) {
return NULL;
}

struct interp_call call = {0};
if (_interp_call_pack(tstate, &call, callable, args, kwargs) < 0) {
PyInterpreterView_Close(view);
return NULL;
}

PyObject *res_and_exc = NULL;
struct run_result runres = {0};
if (_run_in_interpreter(tstate, interp, NULL, &call, NULL, &runres) < 0) {
int res = _run_in_interpreter(tstate, view, NULL, &call, NULL, &runres);
PyInterpreterView_Close(view);
if (res < 0) {
if (runres.excinfo == NULL) {
assert(_PyErr_Occurred(tstate));
goto finally;
Expand Down
26 changes: 25 additions & 1 deletion Python/crossinterp.c
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include "pycore_import.h" // _PyImport_SetModule()
#include "pycore_initconfig.h" // _PyStatus_OK()
#include "pycore_namespace.h" // _PyNamespace_New()
#include "pycore_pystate.h" // _PyInterpreterGuard_GetInterpreter()
#include "pycore_pythonrun.h" // _Py_SourceAsString()
#include "pycore_runtime.h" // _PyRuntime
#include "pycore_setobject.h" // _PySet_NextEntry()
Expand Down Expand Up @@ -2546,6 +2547,10 @@ struct xi_session {
// once the session exits. Do not access this directly; use
// _PyXI_Preserve() and _PyXI_GetPreserved() instead;
PyObject *_preserved;

// PEP 788 finalization guard to prevent the interpreter from being
// destroyed/finalized while this session is active.
PyInterpreterGuard *guard;
};

_PyXI_session *
Expand Down Expand Up @@ -2606,6 +2611,7 @@ _enter_session(_PyXI_session *session, PyInterpreterState *interp)
.init_tstate = tstate,
.prev_tstate = prev,
.own_init_tstate = !same_interp,
.guard = session->guard,
};
}

Expand Down Expand Up @@ -2641,6 +2647,12 @@ _exit_session(_PyXI_session *session)
assert(!session->own_init_tstate);
}

// Release PEP 788 finalization guard after thread state cleanup & swap back.
if (session->guard != NULL) {
PyInterpreterGuard_Close(session->guard);
session->guard = NULL;
}

*session = (_PyXI_session){0};
}

Expand Down Expand Up @@ -2670,7 +2682,7 @@ static const char * capture_session_error(_PyXI_session *, _PyXI_error *,

int
_PyXI_Enter(_PyXI_session *session,
PyInterpreterState *interp, PyObject *nsupdates,
PyInterpreterView *view, PyObject *nsupdates,
_PyXI_session_result *result)
{
#ifndef NDEBUG
Expand Down Expand Up @@ -2714,6 +2726,18 @@ _PyXI_Enter(_PyXI_session *session,
}
}

// Acquire PEP 788 finalization guard before switching or running.
session->guard = PyInterpreterGuard_FromView(view);
if (session->guard == NULL) {
if (sharedns != NULL) {
_destroy_sharedns(sharedns);
}
PyErr_SetString(PyExc_PythonFinalizationError,
"cannot enter an interpreter that is being finalized");
return -1;
}
PyInterpreterState *interp = _PyInterpreterGuard_GetInterpreter(session->guard);

// Switch to the requested interpreter (if necessary).
_enter_session(session, interp);
_PyXI_failure override = XI_FAILURE_INIT;
Expand Down
Loading
Loading