diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index bed966681fa1f0c..d5cec059a7c971b 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -368,7 +368,7 @@ PyAPI_FUNC(void) _PyXI_ClearResult(_PyXI_session_result *); PyAPI_FUNC(int) _PyXI_Enter( _PyXI_session *session, - PyInterpreterState *interp, + PyInterpreterView *view, PyObject *nsupdates, _PyXI_session_result *); PyAPI_FUNC(int) _PyXI_Exit( diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 0a764d886c6a31c..dd360d597d79a7b 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -387,6 +387,9 @@ extern int _PyInterpreterGuard_TryAcquire(PyInterpreterState *interp, PyInterpreterGuard *guard); extern void _PyInterpreterGuard_Release(PyInterpreterGuard *guard); +// Export for '_interpreters' shared extension +PyAPI_FUNC(PyInterpreterView *) _PyInterpreterView_FromID(int64_t interp_id); + #ifdef __cplusplus } #endif diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 1b66f9c7646eb72..111996817d552c6 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -466,6 +466,40 @@ def test_still_running(self): _interpreters.destroy(interp) self.assertTrue(_interpreters.is_running(interp)) + def test_concurrent_run_and_destroy(self): + # gh-127041: An interpreter could start running during destroy(). + for _ in range(50): + interp = _interpreters.create() + barrier = threading.Barrier(2) + + def runner(): + barrier.wait() + try: + _interpreters.run_string(interp, "x = 1") + except (_interpreters.InterpreterError, + PythonFinalizationError): + pass + + def destroyer(): + barrier.wait() + try: + _interpreters.destroy(interp) + except _interpreters.InterpreterError: + pass + + threads = [ + threading.Thread(target=runner), + threading.Thread(target=destroyer), + ] + for t in threads: + t.start() + for t in threads: + t.join() + try: + _interpreters.destroy(interp) + except _interpreters.InterpreterError: + pass + class CommonTests(TestBase): def setUp(self): diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 454db063d18b6a0..8e64864399c821b 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -655,7 +655,7 @@ _run_result_clear(struct run_result *runres) } static int -_run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, +_run_in_interpreter(PyThreadState *tstate, PyInterpreterView *view, _PyXIData_t *script, struct interp_call *call, PyObject *shareables, struct run_result *runres) { @@ -673,7 +673,7 @@ _run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, _PyXI_session_result result = {0}; // Prep and switch interpreters. - if (_PyXI_Enter(session, interp, shareables, &result) < 0) { + if (_PyXI_Enter(session, view, shareables, &result) < 0) { // If an error occurred at this step, it means that interp // was not prepared and switched. _PyXI_FreeSession(session); @@ -784,6 +784,71 @@ resolve_interp(PyObject *idobj, int restricted, int reqready, const char *op) return interp; } +static PyInterpreterView * +resolve_interp_view(PyObject *idobj, int restricted, int reqready, + const char *op) +{ + PyInterpreterView *view; + if (idobj == NULL) { + view = PyInterpreterView_FromCurrent(); + if (view == NULL) { + return NULL; + } + } + else { + int64_t id = _PyInterpreterState_ObjectToID(idobj); + if (id < 0) { + return NULL; + } + view = _PyInterpreterView_FromID(id); + if (view == NULL) { + return NULL; + } + } + + PyInterpreterGuard *guard = PyInterpreterGuard_FromView(view); + if (guard == NULL) { + PyInterpreterView_Close(view); + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_InterpreterNotFoundError, + "unrecognized interpreter ID %R", idobj); + } + return NULL; + } + PyInterpreterState *interp = _PyInterpreterGuard_GetInterpreter(guard); + + if (reqready && !_PyInterpreterState_IsReady(interp)) { + PyInterpreterGuard_Close(guard); + PyInterpreterView_Close(view); + if (idobj == NULL) { + PyErr_Format(PyExc_InterpreterError, + "cannot %s current interpreter (not ready)", op); + } + else { + PyErr_Format(PyExc_InterpreterError, + "cannot %s interpreter %R (not ready)", op, idobj); + } + return NULL; + } + + if (restricted && get_whence(interp) != _PyInterpreterState_WHENCE_STDLIB) { + PyInterpreterGuard_Close(guard); + PyInterpreterView_Close(view); + if (idobj == NULL) { + PyErr_Format(PyExc_InterpreterError, + "cannot %s unrecognized current interpreter", op); + } + else { + PyErr_Format(PyExc_InterpreterError, + "cannot %s unrecognized interpreter %R", op, idobj); + } + return NULL; + } + + PyInterpreterGuard_Close(guard); + return view; +} + static PyObject * get_summary(PyInterpreterState *interp) @@ -1043,18 +1108,20 @@ _interpreters_set___main___attrs_impl(PyObject *module, PyObject *id, { // Look up the interpreter. int reqready = 1; - PyInterpreterState *interp = \ - resolve_interp(id, restricted, reqready, "update __main__ for"); - if (interp == NULL) { + PyInterpreterView *view = \ + resolve_interp_view(id, restricted, reqready, "update __main__ for"); + if (view == NULL) { return NULL; } // Check the updates. Py_ssize_t size = PyDict_Size(updates); if (size < 0) { + PyInterpreterView_Close(view); return NULL; } if (size == 0) { + PyInterpreterView_Close(view); PyErr_SetString(PyExc_ValueError, "arg 2 must be a non-empty dict"); return NULL; @@ -1062,11 +1129,14 @@ _interpreters_set___main___attrs_impl(PyObject *module, PyObject *id, _PyXI_session *session = _PyXI_NewSession(); if (session == NULL) { + PyInterpreterView_Close(view); return NULL; } // Prep and switch interpreters, including apply the updates. - if (_PyXI_Enter(session, interp, updates, NULL) < 0) { + int entered = _PyXI_Enter(session, view, updates, NULL); + PyInterpreterView_Close(view); + if (entered < 0) { _PyXI_FreeSession(session); return NULL; } @@ -1131,9 +1201,9 @@ _interpreters_exec_impl(PyObject *module, PyObject *id, PyObject *code, { PyThreadState *tstate = _PyThreadState_GET(); int reqready = 1; - PyInterpreterState *interp = \ - resolve_interp(id, restricted, reqready, "exec code for"); - if (interp == NULL) { + PyInterpreterView *view = \ + resolve_interp_view(id, restricted, reqready, "exec code for"); + if (view == NULL) { return NULL; } @@ -1141,13 +1211,15 @@ _interpreters_exec_impl(PyObject *module, PyObject *id, PyObject *code, // global variables. They will be resolved against __main__. _PyXIData_t xidata = {0}; if (_PyCode_GetScriptXIData(tstate, code, &xidata) < 0) { + PyInterpreterView_Close(view); unwrap_not_shareable(tstate, NULL); return NULL; } struct run_result runres = {0}; int res = _run_in_interpreter( - tstate, interp, &xidata, NULL, shared, &runres); + tstate, view, &xidata, NULL, shared, &runres); + PyInterpreterView_Close(view); _PyXIData_Release(&xidata); if (res < 0) { return _handle_script_error(&runres); @@ -1178,26 +1250,29 @@ _interpreters_run_string_impl(PyObject *module, PyObject *id, #define FUNCNAME MODULE_NAME_STR ".run_string" PyThreadState *tstate = _PyThreadState_GET(); int reqready = 1; - PyInterpreterState *interp = \ - resolve_interp(id, restricted, reqready, "run a string in"); - if (interp == NULL) { + PyInterpreterView *view = \ + resolve_interp_view(id, restricted, reqready, "run a string in"); + if (view == NULL) { return NULL; } if (PyFunction_Check(script) || PyCode_Check(script)) { + PyInterpreterView_Close(view); _PyArg_BadArgument(FUNCNAME, "argument 2", "a string", script); return NULL; } _PyXIData_t xidata = {0}; if (_PyCode_GetScriptXIData(tstate, script, &xidata) < 0) { + PyInterpreterView_Close(view); unwrap_not_shareable(tstate, NULL); return NULL; } struct run_result runres = {0}; int res = _run_in_interpreter( - tstate, interp, &xidata, NULL, shared, &runres); + tstate, view, &xidata, NULL, shared, &runres); + PyInterpreterView_Close(view); _PyXIData_Release(&xidata); if (res < 0) { return _handle_script_error(&runres); @@ -1232,9 +1307,9 @@ _interpreters_run_func_impl(PyObject *module, PyObject *id, PyObject *func, #define FUNCNAME MODULE_NAME_STR ".run_func" PyThreadState *tstate = _PyThreadState_GET(); int reqready = 1; - PyInterpreterState *interp = \ - resolve_interp(id, restricted, reqready, "run a function in"); - if (interp == NULL) { + PyInterpreterView *view = \ + resolve_interp_view(id, restricted, reqready, "run a function in"); + if (view == NULL) { return NULL; } @@ -1248,19 +1323,22 @@ _interpreters_run_func_impl(PyObject *module, PyObject *id, PyObject *func, code = func; } else { + PyInterpreterView_Close(view); _PyArg_BadArgument(FUNCNAME, "argument 2", "a function", func); return NULL; } _PyXIData_t xidata = {0}; if (_PyCode_GetScriptXIData(tstate, code, &xidata) < 0) { + PyInterpreterView_Close(view); unwrap_not_shareable(tstate, NULL); return NULL; } struct run_result runres = {0}; int res = _run_in_interpreter( - tstate, interp, &xidata, NULL, shared, &runres); + tstate, view, &xidata, NULL, shared, &runres); + PyInterpreterView_Close(view); _PyXIData_Release(&xidata); if (res < 0) { return _handle_script_error(&runres); @@ -1293,20 +1371,23 @@ _interpreters_call_impl(PyObject *module, PyObject *id, PyObject *callable, { PyThreadState *tstate = _PyThreadState_GET(); int reqready = 1; - PyInterpreterState *interp = \ - resolve_interp(id, restricted, reqready, "make a call in"); - if (interp == NULL) { + PyInterpreterView *view = \ + resolve_interp_view(id, restricted, reqready, "make a call in"); + if (view == NULL) { return NULL; } struct interp_call call = {0}; if (_interp_call_pack(tstate, &call, callable, args, kwargs) < 0) { + PyInterpreterView_Close(view); return NULL; } PyObject *res_and_exc = NULL; struct run_result runres = {0}; - if (_run_in_interpreter(tstate, interp, NULL, &call, NULL, &runres) < 0) { + int res = _run_in_interpreter(tstate, view, NULL, &call, NULL, &runres); + PyInterpreterView_Close(view); + if (res < 0) { if (runres.excinfo == NULL) { assert(_PyErr_Occurred(tstate)); goto finally; diff --git a/Python/crossinterp.c b/Python/crossinterp.c index b8dd5f11c2a44c7..7f04500e30defa2 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -11,6 +11,7 @@ #include "pycore_import.h" // _PyImport_SetModule() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_namespace.h" // _PyNamespace_New() +#include "pycore_pystate.h" // _PyInterpreterGuard_GetInterpreter() #include "pycore_pythonrun.h" // _Py_SourceAsString() #include "pycore_runtime.h" // _PyRuntime #include "pycore_setobject.h" // _PySet_NextEntry() @@ -2546,6 +2547,10 @@ struct xi_session { // once the session exits. Do not access this directly; use // _PyXI_Preserve() and _PyXI_GetPreserved() instead; PyObject *_preserved; + + // PEP 788 finalization guard to prevent the interpreter from being + // destroyed/finalized while this session is active. + PyInterpreterGuard *guard; }; _PyXI_session * @@ -2606,6 +2611,7 @@ _enter_session(_PyXI_session *session, PyInterpreterState *interp) .init_tstate = tstate, .prev_tstate = prev, .own_init_tstate = !same_interp, + .guard = session->guard, }; } @@ -2641,6 +2647,12 @@ _exit_session(_PyXI_session *session) assert(!session->own_init_tstate); } + // Release PEP 788 finalization guard after thread state cleanup & swap back. + if (session->guard != NULL) { + PyInterpreterGuard_Close(session->guard); + session->guard = NULL; + } + *session = (_PyXI_session){0}; } @@ -2670,7 +2682,7 @@ static const char * capture_session_error(_PyXI_session *, _PyXI_error *, int _PyXI_Enter(_PyXI_session *session, - PyInterpreterState *interp, PyObject *nsupdates, + PyInterpreterView *view, PyObject *nsupdates, _PyXI_session_result *result) { #ifndef NDEBUG @@ -2714,6 +2726,18 @@ _PyXI_Enter(_PyXI_session *session, } } + // Acquire PEP 788 finalization guard before switching or running. + session->guard = PyInterpreterGuard_FromView(view); + if (session->guard == NULL) { + if (sharedns != NULL) { + _destroy_sharedns(sharedns); + } + PyErr_SetString(PyExc_PythonFinalizationError, + "cannot enter an interpreter that is being finalized"); + return -1; + } + PyInterpreterState *interp = _PyInterpreterGuard_GetInterpreter(session->guard); + // Switch to the requested interpreter (if necessary). _enter_session(session, interp); _PyXI_failure override = XI_FAILURE_INIT; diff --git a/Python/pystate.c b/Python/pystate.c index ce7197c4e054283..6401fa0f21d8c1e 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -3626,6 +3626,21 @@ PyInterpreterView_FromCurrent(void) return view; } +PyInterpreterView * +_PyInterpreterView_FromID(int64_t interp_id) +{ + assert(interp_id >= 0); + + PyInterpreterView *view = PyMem_RawMalloc(sizeof(PyInterpreterView)); + if (view == NULL) { + PyErr_NoMemory(); + return NULL; + } + + view->id = interp_id; + return view; +} + void PyInterpreterView_Close(PyInterpreterView *view) {