From f8eaede62e0133a803ef3fce578860d4d9264760 Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Tue, 10 Nov 2026 14:26:39 -0700 Subject: [PATCH] gh-128942: Make array.array thread-safe in the free-threaded build Based on gh-130771. Keep single-item reads and writes lock-free, serialize mutating and multi-item operations, and reclaim shared buffers with QSBR. Preserve the object layout and default-build performance. Validate the buffer and length together for lock-free access, and convert values before loading the buffer when assignment can run user code. Concurrent assignments racing with a resize may still be lost, and 16-byte complex items can tear. Add free-threading tests and document thread-safety guarantees. Co-authored-by: Tomasz Pytel --- Doc/library/array.rst | 57 + Include/internal/pycore_pymem.h | 2 +- Lib/test/test_array.py | 39 + Lib/test/test_free_threading/test_array.py | 279 +++ ...-03-02-21-08-05.gh-issue-128942.4MTI6s.rst | 1 + Modules/arraymodule.c | 1937 +++++++++++------ Modules/clinic/arraymodule.c.h | 99 +- 7 files changed, 1789 insertions(+), 625 deletions(-) create mode 100644 Lib/test/test_free_threading/test_array.py create mode 100644 Misc/NEWS.d/next/Library/2025-03-02-21-08-05.gh-issue-128942.4MTI6s.rst diff --git a/Doc/library/array.rst b/Doc/library/array.rst index c688d54318e707..60d9f2c141b301 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -300,6 +300,63 @@ Examples:: array('d', [1.0, 2.0, 3.14, -inf, nan]) +.. _array-thread-safety: + +Thread safety +------------- + +In the :term:`free-threaded build`, array operations are safe to call from +multiple threads and will not corrupt the array. Most operations hold the +:term:`per-object lock`. Operations that take a second array, such as +:meth:`~array.array.extend`, concatenation, comparison and slice assignment, +lock both arrays. + +The :func:`len` function is :term:`lock-free` and +:term:`atomic `. + +The following operations are also :term:`lock-free`. They do not block +concurrent modifications and may observe intermediate states from +operations that hold the :term:`per-object lock`: + +.. code-block:: + :class: maybe + + a[i] # read a single item + a[i] = x # write a single item + for x in a: # iteration + ... + +These operations have the following limitations: + +* Reading a single item uses an :term:`atomic ` read, + except for type code ``'Zd'``. The read may observe intermediate states + from concurrent modifications, such as :meth:`~array.array.reverse` or + :meth:`~array.array.insert`. + +* Writing a single item is not :term:`atomic ` with + respect to concurrent operations that resize or rearrange the array + (for example :meth:`~array.array.append`, :meth:`~array.array.extend`, + :meth:`~array.array.insert`, :meth:`~array.array.pop`, + :meth:`~array.array.reverse` or slice assignment). The write can be + lost: ``a[i] = x`` returns normally but the array may keep the previous + item. Writes are not lost if the other threads only read the array or + write single items. + +* Reading or writing an item of type code ``'Zd'`` is not + :term:`atomic `. The real and imaginary parts are + accessed separately, so a concurrent read and write of the same item + may return the real part of one value and the imaginary part of the other. + +Operations that involve multiple accesses, as well as iteration, are not +:term:`atomic `. Use external synchronization, such as a +:class:`threading.Lock`, when these operations must be atomic or to avoid +the limitations above. + +Objects that use the buffer interface of an array, such as +:class:`memoryview`, access its memory without holding the array's +:term:`per-object lock`. Concurrent reads and writes to the same memory +region through the buffer interface require external synchronization. + .. seealso:: Module :mod:`struct` diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 05484e847f1195..04c41a5141bd29 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -111,7 +111,7 @@ extern wchar_t *_PyMem_DefaultRawWcsdup(const wchar_t *str); extern int _PyMem_DebugEnabled(void); // Enqueue a pointer to be freed possibly after some delay. -extern void _PyMem_FreeDelayed(void *ptr, size_t size); +PyAPI_FUNC(void) _PyMem_FreeDelayed(void *ptr, size_t size); // Periodically process delayed free requests. extern void _PyMem_ProcessDelayed(PyThreadState *tstate); diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 580966589fdd6b..6e06c7b640d581 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1272,6 +1272,29 @@ def test_obsolete_write_lock(self): a = array.array('B', b"") self.assertRaises(BufferError, _testcapi.getbuffer_with_null_view, a) + @support.cpython_only + def test_sequence_delitem(self): + # PySequence_DelItem() goes through the sq_ass_item slot. + _testlimitedcapi = import_helper.import_module('_testlimitedcapi') + delitem = _testlimitedcapi.sequence_delitem + a = array.array(self.typecode, self.example) + delitem(a, 0) + self.assertEqual(a, array.array(self.typecode, self.example[1:])) + self.assertRaises(IndexError, delitem, a, len(a)) + self.assertRaises(IndexError, delitem, a, -len(a) - 1) + self.assertEqual(a, array.array(self.typecode, self.example[1:])) + + def test_create_from_shared_iterator(self): + # The iterator returned by __iter__() can have other references. + class Iterable: + def __init__(self, it): + self.it = it + def __iter__(self): + return self.it + + a = array.array(self.typecode, Iterable(iter(self.example))) + self.assertEqual(a, array.array(self.typecode, self.example)) + def test_free_after_iterating(self): support.check_free_after_iterating(self, iter, array.array, (self.typecode,)) @@ -1316,6 +1339,22 @@ def test_unicode(self): self.assertRaises(TypeError, a.fromunicode) + def test_invalid_code_point(self): + # An item that is not a valid code point raises ValueError, also + # for the following items. + a = array.array(self.typecode, 'ab') + a.frombytes((0x110000).to_bytes(a.itemsize, sys.byteorder)) + a.fromunicode('c') + self.assertRaises(ValueError, a.__getitem__, 2) + self.assertRaises(ValueError, a.__getitem__, -2) + self.assertRaises(ValueError, a.tolist) + it = iter(a) + self.assertEqual(next(it), 'a') + self.assertEqual(next(it), 'b') + self.assertRaises(ValueError, next, it) + self.assertEqual(next(it), 'c') + self.assertRaises(StopIteration, next, it) + class NumberTest(BaseTest): diff --git a/Lib/test/test_free_threading/test_array.py b/Lib/test/test_free_threading/test_array.py new file mode 100644 index 00000000000000..28629966efe134 --- /dev/null +++ b/Lib/test/test_free_threading/test_array.py @@ -0,0 +1,279 @@ +import array +import io +import threading +import unittest + +from test.support import threading_helper +from test.support.threading_helper import run_concurrently + +threading_helper.requires_working_threading(module=True) + + +# The workers below take the shared array as their first argument. In a +# debug build freed memory is filled with 0xdd, so that byte must never show +# up in a copy of an array of b'0'. + +def pop1(a, *args): + try: + a.pop() + except IndexError: + pass + +def append1(a, *args): + a.append(2) + +def insert1(a, *args): + a.insert(0, 2) + +def extend(a, *args): + a.extend(array.array('i', [2])) + +def inplace_concat(a, *args): + a += array.array('i', [2]) + +def inplace_repeat(a, *args): + a *= 2 + +def clear(a, *args): + a.clear() + +def remove1(a, *args): + try: + a.remove(1) + except ValueError: + pass + +def fromunicode(a, *args): + a.fromunicode('test') + +def frombytes(a, *args): + a.frombytes(b'0000') + +def fromlist(a, *args): + a.fromlist([2] * 50) + +def setitem0(a, *args): + try: + a[0] = 0 + except IndexError: + pass + +def byteswap(a, *args): + a.byteswap() + +def reverse(a, *args): + a.reverse() + +def tounicode(a, *args): + a.tounicode() + +def tobytes(a, *args): + a.tobytes() + +def tolist(a, *args): + a.tolist() + +def tofile(a, *args): + a.tofile(io.BytesIO()) + +def reduce_ex2(a, *args): + a.__reduce_ex__(2) + +def reduce_ex3(a, *args): + c = a.__reduce_ex__(3) + assert not c[1] or 0xdd not in c[1][3] + +def copy(a, *args): + c = a.__copy__() + assert not c or 0xdd not in c + +def repr1(a, *args): + repr(a) + +def repeat2(a, *args): + a * 2 + +def count1(a, *args): + a.count(1) + +def index1(a, *args): + try: + a.index(1) + except ValueError: + pass + +def contains1(a, *args): + 1 in a + +def getitem0(a, *args): + try: + a[0] + except IndexError: + pass + +def getslice(a, *args): + a[:] + a[::2] + +def concat(a, *args): + a + a + +def richcmplhs(a, *args): + a == a[:] + +def richcmprhs(a, *args): + a[:] == a + +def new(a, *args): + array.array(a.typecode, a) + +# Workers that take a second object. + +def clear2(a, c): + try: + c.clear() + except BufferError: + pass + +def concat2(a, c): + a + c + +def inplace_concat2(a, c): + a += c + +def extend2(a, c): + a.extend(c) + +def setslice2(a, c): + a[:] = c + +def frombytes2(a, c): + a.frombytes(c) + +def new2(a, c): + array.array('i', c) + +def fromlist2(a, c): + a.fromlist(c) + +def iter_next(a, it): + list(it) + +def iter_reduce(a, it): + c = it.__reduce__() + assert not c[1] or 0xdd not in c[1][0] + + +class TestArray(unittest.TestCase): + + def check(self, funcs, a=None, *args): + if a is None: + a = array.array('i', [1]) + run_concurrently(funcs, args=(a, *args)) + + def test_mutate(self): + self.check([pop1] * 10) + self.check([append1] * 10) + self.check([insert1] * 10) + self.check([remove1] * 10) + self.check([inplace_repeat] * 10) + self.check([byteswap] * 10) + self.check([reverse] * 10, array.array('i', range(1000))) + self.check([frombytes] * 10) + self.check([fromlist] * 10) + self.check([fromunicode] * 10, array.array('w', 'a')) + self.check([insert1] + [clear] * 10) + self.check([clear] + [extend] * 10) + self.check([clear] + [inplace_concat] * 10) + self.check([clear, setitem0] * 10, array.array('i', [1] * 10000)) + self.check([clear] + [inplace_repeat] * 10, + array.array('B', b'0' * 0x400)) + + def test_read_and_mutate(self): + self.check([pop1] + [getitem0] * 10) + self.check([pop1] + [index1] * 10) + self.check([pop1] + [contains1] * 10) + self.check([pop1] + [count1] * 10) + self.check([pop1] + [repr1] * 10) + self.check([pop1] + [reduce_ex2] * 10) + self.check([pop1] + [tobytes] * 10) + self.check([pop1] + [tolist] * 10) + self.check([insert1] + [repeat2] * 10) + self.check([clear] + [copy] * 10, array.array('B', b'0' * 0x10000)) + self.check([clear] + [reduce_ex3] * 10, + array.array('B', b'0' * 0x10000)) + self.check([clear] + [repr1] * 10, array.array('B', b'0' * 0x1000)) + self.check([clear] + [getslice] * 10, array.array('i', [1] * 10000)) + self.check([clear] + [richcmplhs] * 10, array.array('i', [1] * 10000)) + self.check([clear] + [richcmprhs] * 10, array.array('i', [1] * 10000)) + self.check([clear, tounicode] * 10, array.array('w', 'a' * 10000)) + self.check([clear, tofile] * 10, array.array('w', 'a' * 10000)) + self.check([clear] + [concat] * 10, array.array('w', 'a' * 10000)) + self.check([clear] + [new] * 10, array.array('w', 'a' * 10000)) + + def test_two_objects(self): + def unicode_array(): + return array.array('w', 'a' * 10000) + + self.check([clear2] + [concat2] * 10, + unicode_array(), unicode_array()) + self.check([clear2] + [inplace_concat2] * 10, + unicode_array(), unicode_array()) + self.check([clear2] + [extend2] * 10, + unicode_array(), unicode_array()) + self.check([clear2] + [setslice2] * 10, + unicode_array(), unicode_array()) + self.check([clear2] + [frombytes2] * 10, + unicode_array(), array.array('B', b'a' * 10000)) + self.check([clear2] + [new2] * 10, None, [2] * 100) + self.check([clear2] + [fromlist2] * 10, None, [2] * 100) + + def test_iterator(self): + a = array.array('i', [1] * 10) + self.check([clear] + [iter_next] * 10, a, iter(a)) + a = array.array('B', b'0' * 0x400) + self.check([clear] + [iter_reduce] * 10, a, iter(a)) + + def test_item_access_while_resizing(self): + # Items are read and written without the lock. A reader must only + # see values that were stored, also if the buffer is replaced. + n = 100 + for typecode, value in [('b', 1), ('H', 1), ('i', 1), ('q', 1), + ('d', 1.0), ('Zd', 1+2j), ('w', 'x')]: + with self.subTest(typecode=typecode): + a = array.array(typecode, [value] * n) + tail = a[1:] + done = threading.Event() + + def resize(): + try: + for _ in range(200): + del a[1:] + a.extend(tail) + finally: + done.set() + + def read(): + while not done.is_set(): + try: + item = a[n - 1] + except IndexError: + pass + else: + self.assertEqual(item, value) + for item in a: + self.assertEqual(item, value) + + def write(): + while not done.is_set(): + try: + a[n - 1] = value + except IndexError: + pass + + run_concurrently([resize] + [read] * 4 + [write] * 2) + self.assertEqual(a, array.array(typecode, [value] * n)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/Library/2025-03-02-21-08-05.gh-issue-128942.4MTI6s.rst b/Misc/NEWS.d/next/Library/2025-03-02-21-08-05.gh-issue-128942.4MTI6s.rst new file mode 100644 index 00000000000000..2b5eb581f1ffa1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-02-21-08-05.gh-issue-128942.4MTI6s.rst @@ -0,0 +1 @@ +Make the :mod:`array` module safe under :term:`free threading`. diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 64dacd1b8f1f61..e25bacc6084a06 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -12,9 +12,12 @@ #include "pycore_call.h" // _PyObject_CallMethod() #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_floatobject.h" // _PY_FLOAT_BIG_ENDIAN +#include "pycore_gc.h" // _PyObject_GC_IS_SHARED() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_pyatomic_ft_wrappers.h" +#include "pycore_pymem.h" // _PyMem_FreeDelayed() #include "pycore_tuple.h" // _PyTuple_FromPairSteal #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() @@ -36,8 +39,8 @@ static struct PyModuleDef arraymodule; struct arraydescr { char typecode[3]; // big enough to store "Zd\0" int itemsize; - PyObject * (*getitem)(struct arrayobject *, Py_ssize_t); - int (*setitem)(struct arrayobject *, Py_ssize_t, PyObject *); + PyObject * (*getitem)(const char *, Py_ssize_t); + int (*pack)(PyObject *, char *); int (*compareitems)(const void *, const void *, Py_ssize_t); int is_integer_type; int is_signed; @@ -52,11 +55,31 @@ typedef struct arrayobject { Py_ssize_t ob_exports; /* Number of exported buffers */ } arrayobject; +#ifdef Py_GIL_DISABLED +// As in listobject.c, the capacity is also stored in front of the items. A +// thread that reads without the lock checks its index against the capacity +// of the buffer that it loaded. The items keep the alignment that +// PyMem_Malloc() gives them in the default build. +#if SIZEOF_VOID_P > 4 +# define ARRAY_ITEMS_ALIGNMENT 16 +#else +# define ARRAY_ITEMS_ALIGNMENT 8 +#endif + +typedef struct { + Py_ssize_t allocated; + _Py_ALIGNED_DEF(ARRAY_ITEMS_ALIGNMENT, char) ob_item[]; +} _PyArrayArray; + +#define ARRAY_HEADER_SIZE sizeof(_PyArrayArray) +#else +#define ARRAY_HEADER_SIZE 0 +#endif + typedef struct { PyObject_HEAD Py_ssize_t index; arrayobject *ao; - PyObject* (*getitem)(struct arrayobject *, Py_ssize_t); } arrayiterobject; typedef struct { @@ -72,6 +95,16 @@ typedef struct { PyObject *typecodes; } array_state; +static inline Py_ssize_t +array_get_size(PyObject *op) +{ + return FT_ATOMIC_LOAD_SSIZE_ACQUIRE(_PyVarObject_CAST(op)->ob_size); +} +#define array_get_size(op) array_get_size(_PyObject_CAST(op)) + +/* Forward declaration. */ +static PyObject *array_array_frombytes(PyObject *self, PyObject *bytes); + static array_state * get_array_state(PyObject *module) { @@ -137,13 +170,260 @@ enum machine_format_code { #define array_Check(op, state) PyObject_TypeCheck(op, state->ArrayType) +// True if a buffer for `size` items can be allocated. +static inline bool +array_size_valid(size_t size, int itemsize) +{ + return size <= ((size_t)PY_SSIZE_T_MAX - ARRAY_HEADER_SIZE) / itemsize; +} + +// Allocate an uninitialized buffer for `capacity` items. +static char * +array_items_alloc(Py_ssize_t capacity, int itemsize) +{ + assert(capacity > 0); + assert(array_size_valid(capacity, itemsize)); +#ifdef Py_GIL_DISABLED + _PyArrayArray *array = PyMem_Malloc( + sizeof(_PyArrayArray) + (size_t)capacity * itemsize); + if (array == NULL) { + return NULL; + } + array->allocated = capacity; + return array->ob_item; +#else + return PyMem_Malloc((size_t)capacity * itemsize); +#endif +} + +static void +array_items_free(char *items, int itemsize, bool use_qsbr) +{ +#ifdef Py_GIL_DISABLED + if (items == NULL) { + return; + } + _PyArrayArray *array = _Py_CONTAINER_OF(items, _PyArrayArray, ob_item); + if (use_qsbr) { + size_t size = (sizeof(_PyArrayArray) + + (size_t)array->allocated * itemsize); + _PyMem_FreeDelayed(array, size); + } + else { + PyMem_Free(array); + } +#else + PyMem_Free(items); +#endif +} + +#ifdef Py_GIL_DISABLED +static inline Py_ssize_t +array_capacity(char *items) +{ + _PyArrayArray *array = _Py_CONTAINER_OF(items, _PyArrayArray, ob_item); + return array->allocated; +} +#endif + +// Memory ordering in the free-threaded build. +// +// Single items are read without the lock (getarrayitem_threadsafe(), the +// iterator) and, for arrays that are shared or owned by the current +// thread, also written without it (setarrayitem_threadsafe()). So: +// +// - `ob_item` is published with a release store and read with an acquire +// load. Old buffers of a shared array are freed with QSBR. +// - `ob_size` uses release stores (array_set_size()) and acquire loads +// (array_get_size()). Items are written before a larger size is +// published, so a reader that sees the new size also sees the items. +// - At any time the first `ob_size` items of the current buffer are +// initialized. A lock-free access loads `ob_item`, then `ob_size`, and +// then checks that `ob_item` is unchanged, so its size belongs to its +// buffer. QSBR keeps the address of the old buffer from being reused in +// between. +// - Each item is loaded with acquire and stored with release. Items of +// 16 bytes ('Zd') are accessed as two 8-byte halves and can tear. +// - Bulk copies use per-item atomic accesses unless the array is owned +// by the current thread and not shared (array_is_exclusive()). +// - `allocated` is only accessed with the lock held. +// +// A lock-free item store that races with a resize can be lost: it may land +// in the old buffer after it was copied. This is accepted for speed. + +// Load or store one item of the given size. The switch gives the compiler +// a constant size for each case, so no call to memcpy() is made. + +static inline void +array_load_raw(const char *src, void *dst, int size) +{ + switch (size) { +#ifdef Py_GIL_DISABLED + // There are no 8- and 16-bit acquire loads; a relaxed load followed + // by an acquire fence gives the same ordering. + case 1: { + uint8_t v = _Py_atomic_load_uint8_relaxed((const uint8_t *)src); + _Py_atomic_fence_acquire(); + memcpy(dst, &v, 1); + return; + } + case 2: { + uint16_t v = _Py_atomic_load_uint16_relaxed((const uint16_t *)src); + _Py_atomic_fence_acquire(); + memcpy(dst, &v, 2); + return; + } + case 4: { + uint32_t v = _Py_atomic_load_uint32_acquire((const uint32_t *)src); + memcpy(dst, &v, 4); + return; + } + case 8: { + uint64_t v = _Py_atomic_load_uint64_acquire((const uint64_t *)src); + memcpy(dst, &v, 8); + return; + } + case 16: { + uint64_t v[2]; + v[0] = _Py_atomic_load_uint64_acquire((const uint64_t *)src); + v[1] = _Py_atomic_load_uint64_acquire((const uint64_t *)src + 1); + memcpy(dst, v, 16); + return; + } +#else + case 1: memcpy(dst, src, 1); return; + case 2: memcpy(dst, src, 2); return; + case 4: memcpy(dst, src, 4); return; + case 8: memcpy(dst, src, 8); return; + case 16: memcpy(dst, src, 16); return; +#endif + } + Py_UNREACHABLE(); +} + +static inline void +array_store_raw(char *dst, const void *src, int size) +{ + switch (size) { +#ifdef Py_GIL_DISABLED + case 1: { + uint8_t v; + memcpy(&v, src, 1); + _Py_atomic_store_uint8_release((uint8_t *)dst, v); + return; + } + case 2: { + // No 16-bit release store; use a release fence and a relaxed store. + uint16_t v; + memcpy(&v, src, 2); + _Py_atomic_fence_release(); + _Py_atomic_store_uint16_relaxed((uint16_t *)dst, v); + return; + } + case 4: { + uint32_t v; + memcpy(&v, src, 4); + _Py_atomic_store_uint32_release((uint32_t *)dst, v); + return; + } + case 8: { + uint64_t v; + memcpy(&v, src, 8); + _Py_atomic_store_uint64_release((uint64_t *)dst, v); + return; + } + case 16: { + uint64_t v[2]; + memcpy(v, src, 16); + _Py_atomic_store_uint64_release((uint64_t *)dst, v[0]); + _Py_atomic_store_uint64_release((uint64_t *)dst + 1, v[1]); + return; + } +#else + case 1: memcpy(dst, src, 1); return; + case 2: memcpy(dst, src, 2); return; + case 4: memcpy(dst, src, 4); return; + case 8: memcpy(dst, src, 8); return; + case 16: memcpy(dst, src, 16); return; +#endif + } + Py_UNREACHABLE(); +} + +// True if no other thread can access the items without the lock. +static inline bool +array_is_exclusive(arrayobject *a) +{ +#ifdef Py_GIL_DISABLED + return (_Py_IsOwnedByCurrentThread((PyObject *)a) && + !_PyObject_GC_IS_SHARED(a)); +#else + return true; +#endif +} + +// Copy n items of the given size; dst and src may overlap. Use per-item +// atomic accesses unless `exclusive` is true. +static void +array_copy_raw(char *dst, const char *src, Py_ssize_t n, int itemsize, + bool exclusive) +{ + if (n <= 0 || dst == src) { + return; + } +#ifdef Py_GIL_DISABLED + if (!exclusive) { + char buf[16]; + if (dst < src) { + for (Py_ssize_t i = 0; i < n; i++) { + array_load_raw(src + i * itemsize, buf, itemsize); + array_store_raw(dst + i * itemsize, buf, itemsize); + } + } + else { + for (Py_ssize_t i = n; --i >= 0; ) { + array_load_raw(src + i * itemsize, buf, itemsize); + array_store_raw(dst + i * itemsize, buf, itemsize); + } + } + return; + } +#endif + memmove(dst, src, n * itemsize); +} + +static inline void +array_set_size(arrayobject *self, Py_ssize_t size) +{ + FT_ATOMIC_STORE_SSIZE_RELEASE(((PyVarObject *)self)->ob_size, size); +} + +static void +ensure_shared_on_resize(arrayobject *self) +{ +#ifdef Py_GIL_DISABLED + // Ensure that the array data is freed using QSBR if we are not the + // owning thread. + if (!_Py_IsOwnedByCurrentThread((PyObject *)self) && + !_PyObject_GC_IS_SHARED(self)) + { + _PyObject_GC_SET_SHARED(self); + } +#endif +} + +// Resize the storage for newsize items. If publish is false and the array +// grows, ob_size is left unchanged: the caller writes the new items and +// then calls array_set_size(). static int -array_resize(arrayobject *self, Py_ssize_t newsize) +array_resize_ex(arrayobject *self, Py_ssize_t newsize, bool publish) { - char *items; - size_t _new_size; + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + assert(newsize >= 0); + Py_ssize_t size = Py_SIZE(self); + assert(publish || newsize >= size); - if (self->ob_exports > 0 && newsize != Py_SIZE(self)) { + if (self->ob_exports > 0 && newsize != size) { PyErr_SetString(PyExc_BufferError, "cannot resize an array that is exporting buffers"); return -1; @@ -154,18 +434,25 @@ array_resize(arrayobject *self, Py_ssize_t newsize) current size, then proceed with the realloc() to shrink the array. */ - if (self->allocated >= newsize && - Py_SIZE(self) < newsize + 16 && - self->ob_item != NULL) { - Py_SET_SIZE(self, newsize); + char *items = self->ob_item; + int itemsize = self->ob_descr->itemsize; + + if (items != NULL && + self->allocated >= newsize && + size < newsize + 16) { + if (publish) { + array_set_size(self, newsize); + } return 0; } + ensure_shared_on_resize(self); + if (newsize == 0) { - PyMem_Free(self->ob_item); - self->ob_item = NULL; - Py_SET_SIZE(self, 0); + array_set_size(self, 0); + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item, NULL); self->allocated = 0; + array_items_free(items, itemsize, !array_is_exclusive(self)); return 0; } @@ -181,72 +468,100 @@ array_resize(arrayobject *self, Py_ssize_t newsize) * memory critical. */ - _new_size = (newsize >> 4) + (Py_SIZE(self) < 8 ? 3 : 7) + newsize; - items = self->ob_item; - /* XXX The following multiplication and division does not optimize away - like it does for lists since the size is not known at compile time */ - if (_new_size <= ((~(size_t)0) / self->ob_descr->itemsize)) - PyMem_RESIZE(items, char, (_new_size * self->ob_descr->itemsize)); - else - items = NULL; + size_t _new_size = (size_t)(newsize >> 4) + (size < 8 ? 3 : 7) + newsize; + + if (!array_size_valid(_new_size, itemsize)) { + PyErr_NoMemory(); + return -1; + } + +#ifdef Py_GIL_DISABLED + char *newitems = array_items_alloc(_new_size, itemsize); + if (newitems == NULL) { + PyErr_NoMemory(); + return -1; + } + bool exclusive = array_is_exclusive(self); + if (newsize < size) { + // Publish the smaller size before the smaller buffer, so that the + // first `ob_size` items of the current buffer are always valid. + array_set_size(self, newsize); + } + array_copy_raw(newitems, items, Py_MIN(size, newsize), itemsize, + exclusive); + _Py_atomic_store_ptr_release(&self->ob_item, newitems); + array_items_free(items, itemsize, !exclusive); +#else + items = PyMem_Realloc(items, _new_size * itemsize); if (items == NULL) { PyErr_NoMemory(); return -1; } self->ob_item = items; - Py_SET_SIZE(self, newsize); +#endif self->allocated = _new_size; + + if (publish) { + array_set_size(self, newsize); + } return 0; } -/**************************************************************************** -Get and Set functions for each type. -A Get function takes an arrayobject* and an integer index, returning the -array value at that index wrapped in an appropriate PyObject*. -A Set function takes an arrayobject, integer index, and PyObject*; sets -the array value at that index to the raw C data extracted from the PyObject*, -and returns 0 if successful, else nonzero on failure (PyObject* not of an -appropriate type or value). -Note that the basic Get and Set functions do NOT check that the index is -in bounds; that's the responsibility of the caller. -****************************************************************************/ +static int +array_resize(arrayobject *self, Py_ssize_t newsize) +{ + return array_resize_ex(self, newsize, true); +} + +// Make room for newsize >= Py_SIZE(self) items without publishing the new +// size. +static int +array_reserve(arrayobject *self, Py_ssize_t newsize) +{ + return array_resize_ex(self, newsize, false); +} + +// Get and Pack functions for each type. +// A Get function takes a char* to the items and an integer index, returning +// the array value at that index wrapped in an appropriate PyObject*. It does +// not check that the index is in bounds. +// A Pack function converts a PyObject* to the raw C value and writes it to a +// local buffer of itemsize bytes. It returns 0 on success, or -1 with an +// exception set. It may call user code (__index__, __float__, ...), so it is +// called before the array buffer is loaded; the caller then checks the index +// and stores the item with array_store_raw(). + +#define DEFINE_ITEM_LOAD(NAME, T) \ + static inline T \ + NAME##_load(const char *items, Py_ssize_t i) \ + { \ + T x; \ + array_load_raw(items + i * (Py_ssize_t)sizeof(T), &x, sizeof(T)); \ + return x; \ + } + +DEFINE_ITEM_LOAD(schar, signed char) +DEFINE_ITEM_LOAD(uchar, unsigned char) +DEFINE_ITEM_LOAD(ucs4, Py_UCS4) +DEFINE_ITEM_LOAD(short, short) +DEFINE_ITEM_LOAD(ushort, unsigned short) +DEFINE_ITEM_LOAD(int, int) +DEFINE_ITEM_LOAD(uint, unsigned int) +DEFINE_ITEM_LOAD(long, long) +DEFINE_ITEM_LOAD(ulong, unsigned long) +DEFINE_ITEM_LOAD(longlong, long long) +DEFINE_ITEM_LOAD(ulonglong, unsigned long long) +DEFINE_ITEM_LOAD(float, float) +DEFINE_ITEM_LOAD(double, double) -/* Macro to check array buffer validity and bounds after calling - user-defined methods (like __index__ or __float__) that might modify - the array during the call. -*/ -#define CHECK_ARRAY_BOUNDS(OP, IDX) \ - do { \ - if ((IDX) >= 0 && ((OP)->ob_item == NULL || \ - (IDX) >= Py_SIZE((OP)))) { \ - PyErr_SetString(PyExc_IndexError, \ - "array assignment index out of range"); \ - return -1; \ - } \ - } while (0) - -#define CHECK_ARRAY_BOUNDS_WITH_CLEANUP(OP, IDX, VAL, CLEANUP) \ - do { \ - if ((IDX) >= 0 && ((OP)->ob_item == NULL || \ - (IDX) >= Py_SIZE((OP)))) { \ - PyErr_SetString(PyExc_IndexError, \ - "array assignment index out of range"); \ - if (CLEANUP) { \ - Py_DECREF(VAL); \ - } \ - return -1; \ - } \ - } while (0) - -static PyObject * -b_getitem(arrayobject *ap, Py_ssize_t i) -{ - long x = ((signed char *)ap->ob_item)[i]; - return PyLong_FromLong(x); +static PyObject * +b_getitem(const char *items, Py_ssize_t i) +{ + return PyLong_FromLong(schar_load(items, i)); } static int -b_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +b_pack(PyObject *v, char *buf) { short x; /* PyArg_Parse's 'b' formatter is for an unsigned char, therefore @@ -254,9 +569,6 @@ b_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) the overflow checking */ if (!PyArg_Parse(v, "h;array item must be integer", &x)) return -1; - - CHECK_ARRAY_BOUNDS(ap, i); - if (x < -128) { PyErr_SetString(PyExc_OverflowError, "signed char is less than minimum"); @@ -267,41 +579,35 @@ b_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) "signed char is greater than maximum"); return -1; } - if (i >= 0) - ((char *)ap->ob_item)[i] = (char)x; + *(signed char *)buf = (signed char)x; return 0; } static PyObject * -BB_getitem(arrayobject *ap, Py_ssize_t i) +BB_getitem(const char *items, Py_ssize_t i) { - long x = ((unsigned char *)ap->ob_item)[i]; - return PyLong_FromLong(x); + return PyLong_FromLong(uchar_load(items, i)); } static int -BB_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +BB_pack(PyObject *v, char *buf) { unsigned char x; /* 'B' == unsigned char, maps to PyArg_Parse's 'b' formatter */ if (!PyArg_Parse(v, "b;array item must be integer", &x)) return -1; - - CHECK_ARRAY_BOUNDS(ap, i); - - if (i >= 0) - ((unsigned char *)ap->ob_item)[i] = x; + *(unsigned char *)buf = x; return 0; } static PyObject * -w_getitem(arrayobject *ap, Py_ssize_t i) +w_getitem(const char *items, Py_ssize_t i) { - return PyUnicode_FromOrdinal(((Py_UCS4 *) ap->ob_item)[i]); + return PyUnicode_FromOrdinal(ucs4_load(items, i)); } static int -w_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +w_pack(PyObject *v, char *buf) { if (!PyUnicode_Check(v)) { PyErr_Format(PyExc_TypeError, @@ -318,42 +624,36 @@ w_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) return -1; } - if (i >= 0) { - ((Py_UCS4 *)ap->ob_item)[i] = PyUnicode_READ_CHAR(v, 0); - } + Py_UCS4 x = PyUnicode_READ_CHAR(v, 0); + memcpy(buf, &x, sizeof(x)); return 0; } static PyObject * -h_getitem(arrayobject *ap, Py_ssize_t i) +h_getitem(const char *items, Py_ssize_t i) { - return PyLong_FromLong((long) ((short *)ap->ob_item)[i]); + return PyLong_FromLong(short_load(items, i)); } - static int -h_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +h_pack(PyObject *v, char *buf) { short x; /* 'h' == signed short, maps to PyArg_Parse's 'h' formatter */ if (!PyArg_Parse(v, "h;array item must be integer", &x)) return -1; - - CHECK_ARRAY_BOUNDS(ap, i); - - if (i >= 0) - ((short *)ap->ob_item)[i] = x; + memcpy(buf, &x, sizeof(x)); return 0; } static PyObject * -HH_getitem(arrayobject *ap, Py_ssize_t i) +HH_getitem(const char *items, Py_ssize_t i) { - return PyLong_FromLong((long) ((unsigned short *)ap->ob_item)[i]); + return PyLong_FromLong(ushort_load(items, i)); } static int -HH_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +HH_pack(PyObject *v, char *buf) { int x; /* PyArg_Parse's 'h' formatter is for a signed short, therefore @@ -370,44 +670,36 @@ HH_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) "unsigned short is greater than maximum"); return -1; } - - CHECK_ARRAY_BOUNDS(ap, i); - - if (i >= 0) - ((short *)ap->ob_item)[i] = (short)x; + unsigned short y = (unsigned short)x; + memcpy(buf, &y, sizeof(y)); return 0; } static PyObject * -i_getitem(arrayobject *ap, Py_ssize_t i) +i_getitem(const char *items, Py_ssize_t i) { - return PyLong_FromLong((long) ((int *)ap->ob_item)[i]); + return PyLong_FromLong(int_load(items, i)); } static int -i_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +i_pack(PyObject *v, char *buf) { int x; /* 'i' == signed int, maps to PyArg_Parse's 'i' formatter */ if (!PyArg_Parse(v, "i;array item must be integer", &x)) return -1; - - CHECK_ARRAY_BOUNDS(ap, i); - - if (i >= 0) - ((int *)ap->ob_item)[i] = x; + memcpy(buf, &x, sizeof(x)); return 0; } static PyObject * -II_getitem(arrayobject *ap, Py_ssize_t i) +II_getitem(const char *items, Py_ssize_t i) { - return PyLong_FromUnsignedLong( - (unsigned long) ((unsigned int *)ap->ob_item)[i]); + return PyLong_FromUnsignedLong(uint_load(items, i)); } static int -II_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +II_pack(PyObject *v, char *buf) { unsigned long x; int do_decref = 0; /* if nb_int was called */ @@ -423,60 +715,46 @@ II_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) do_decref = 1; } x = PyLong_AsUnsignedLong(v); + if (do_decref) { + Py_DECREF(v); + } if (x == (unsigned long)-1 && PyErr_Occurred()) { - if (do_decref) { - Py_DECREF(v); - } return -1; } if (x > UINT_MAX) { PyErr_SetString(PyExc_OverflowError, "unsigned int is greater than maximum"); - if (do_decref) { - Py_DECREF(v); - } return -1; } - - CHECK_ARRAY_BOUNDS_WITH_CLEANUP(ap, i, v, do_decref); - - if (i >= 0) - ((unsigned int *)ap->ob_item)[i] = (unsigned int)x; - - if (do_decref) { - Py_DECREF(v); - } + unsigned int y = (unsigned int)x; + memcpy(buf, &y, sizeof(y)); return 0; } static PyObject * -l_getitem(arrayobject *ap, Py_ssize_t i) +l_getitem(const char *items, Py_ssize_t i) { - return PyLong_FromLong(((long *)ap->ob_item)[i]); + return PyLong_FromLong(long_load(items, i)); } static int -l_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +l_pack(PyObject *v, char *buf) { long x; if (!PyArg_Parse(v, "l;array item must be integer", &x)) return -1; - - CHECK_ARRAY_BOUNDS(ap, i); - - if (i >= 0) - ((long *)ap->ob_item)[i] = x; + memcpy(buf, &x, sizeof(x)); return 0; } static PyObject * -LL_getitem(arrayobject *ap, Py_ssize_t i) +LL_getitem(const char *items, Py_ssize_t i) { - return PyLong_FromUnsignedLong(((unsigned long *)ap->ob_item)[i]); + return PyLong_FromUnsignedLong(ulong_load(items, i)); } static int -LL_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +LL_pack(PyObject *v, char *buf) { unsigned long x; int do_decref = 0; /* if nb_int was called */ @@ -492,53 +770,40 @@ LL_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) do_decref = 1; } x = PyLong_AsUnsignedLong(v); - if (x == (unsigned long)-1 && PyErr_Occurred()) { - if (do_decref) { - Py_DECREF(v); - } - return -1; - } - - CHECK_ARRAY_BOUNDS_WITH_CLEANUP(ap, i, v, do_decref); - - if (i >= 0) - ((unsigned long *)ap->ob_item)[i] = x; - if (do_decref) { Py_DECREF(v); } + if (x == (unsigned long)-1 && PyErr_Occurred()) { + return -1; + } + memcpy(buf, &x, sizeof(x)); return 0; } static PyObject * -q_getitem(arrayobject *ap, Py_ssize_t i) +q_getitem(const char *items, Py_ssize_t i) { - return PyLong_FromLongLong(((long long *)ap->ob_item)[i]); + return PyLong_FromLongLong(longlong_load(items, i)); } static int -q_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +q_pack(PyObject *v, char *buf) { long long x; if (!PyArg_Parse(v, "L;array item must be integer", &x)) return -1; - - CHECK_ARRAY_BOUNDS(ap, i); - - if (i >= 0) - ((long long *)ap->ob_item)[i] = x; + memcpy(buf, &x, sizeof(x)); return 0; } static PyObject * -QQ_getitem(arrayobject *ap, Py_ssize_t i) +QQ_getitem(const char *items, Py_ssize_t i) { - return PyLong_FromUnsignedLongLong( - ((unsigned long long *)ap->ob_item)[i]); + return PyLong_FromUnsignedLongLong(ulonglong_load(items, i)); } static int -QQ_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +QQ_pack(PyObject *v, char *buf) { unsigned long long x; int do_decref = 0; /* if nb_int was called */ @@ -554,139 +819,100 @@ QQ_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) do_decref = 1; } x = PyLong_AsUnsignedLongLong(v); - if (x == (unsigned long long)-1 && PyErr_Occurred()) { - if (do_decref) { - Py_DECREF(v); - } - return -1; - } - - CHECK_ARRAY_BOUNDS_WITH_CLEANUP(ap, i, v, do_decref); - - if (i >= 0) - ((unsigned long long *)ap->ob_item)[i] = x; - if (do_decref) { Py_DECREF(v); } + if (x == (unsigned long long)-1 && PyErr_Occurred()) { + return -1; + } + memcpy(buf, &x, sizeof(x)); return 0; } static PyObject * -e_getitem(arrayobject *ap, Py_ssize_t i) +e_getitem(const char *items, Py_ssize_t i) { - double x = PyFloat_Unpack2(ap->ob_item + sizeof(short)*i, - PY_LITTLE_ENDIAN); - + char b[2]; + array_load_raw(items + sizeof(short)*i, b, 2); + double x = PyFloat_Unpack2(b, PY_LITTLE_ENDIAN); return PyFloat_FromDouble(x); } static int -e_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +e_pack(PyObject *v, char *buf) { double x; if (!PyArg_Parse(v, "d;array item must be float", &x)) { return -1; } - - CHECK_ARRAY_BOUNDS(ap, i); - - if (i >= 0) { - return PyFloat_Pack2(x, ap->ob_item + sizeof(short)*i, - PY_LITTLE_ENDIAN); - } - return 0; + return PyFloat_Pack2(x, buf, PY_LITTLE_ENDIAN); } static PyObject * -f_getitem(arrayobject *ap, Py_ssize_t i) +f_getitem(const char *items, Py_ssize_t i) { - return PyFloat_FromDouble((double) ((float *)ap->ob_item)[i]); + return PyFloat_FromDouble((double)float_load(items, i)); } static int -f_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +f_pack(PyObject *v, char *buf) { double x; if (!PyArg_Parse(v, "d;array item must be float", &x)) return -1; - - CHECK_ARRAY_BOUNDS(ap, i); - - if (i >= 0) { - return PyFloat_Pack4(x, ap->ob_item + sizeof(float)*i, - PY_LITTLE_ENDIAN); - } - return 0; + return PyFloat_Pack4(x, buf, PY_LITTLE_ENDIAN); } static PyObject * -d_getitem(arrayobject *ap, Py_ssize_t i) +d_getitem(const char *items, Py_ssize_t i) { - return PyFloat_FromDouble(((double *)ap->ob_item)[i]); + return PyFloat_FromDouble(double_load(items, i)); } static int -d_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +d_pack(PyObject *v, char *buf) { double x; if (!PyArg_Parse(v, "d;array item must be float", &x)) return -1; - - CHECK_ARRAY_BOUNDS(ap, i); - - if (i >= 0) - ((double *)ap->ob_item)[i] = x; + memcpy(buf, &x, sizeof(x)); return 0; } static PyObject * -cf_getitem(arrayobject *ap, Py_ssize_t i) +cf_getitem(const char *items, Py_ssize_t i) { float f[2]; - memcpy(&f, ap->ob_item + i*sizeof(f), sizeof(f)); + array_load_raw(items + i*sizeof(f), f, sizeof(f)); return PyComplex_FromDoubles(f[0], f[1]); } static int -cf_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +cf_pack(PyObject *v, char *buf) { Py_complex x; if (!PyArg_Parse(v, "D;array item must be complex", &x)) { return -1; } - - CHECK_ARRAY_BOUNDS(ap, i); - - if (i >= 0) { - char f[8]; - int ret = PyFloat_Pack4(x.real, f, PY_LITTLE_ENDIAN); - - if (ret) { - return ret; - } - ret = PyFloat_Pack4(x.imag, f + sizeof(float), PY_LITTLE_ENDIAN); - if (!ret) { - memcpy(ap->ob_item + i*sizeof(f), &f, sizeof(f)); - } - return ret; + if (PyFloat_Pack4(x.real, buf, PY_LITTLE_ENDIAN)) { + return -1; } - return 0; + return PyFloat_Pack4(x.imag, buf + sizeof(float), PY_LITTLE_ENDIAN); } static PyObject * -cd_getitem(arrayobject *ap, Py_ssize_t i) +cd_getitem(const char *items, Py_ssize_t i) { double f[2]; - memcpy(&f, ap->ob_item + i*sizeof(f), sizeof(f)); + array_load_raw(items + i*sizeof(f), f, sizeof(f)); return PyComplex_FromDoubles(f[0], f[1]); } static int -cd_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +cd_pack(PyObject *v, char *buf) { Py_complex x; double f[2]; @@ -694,39 +920,36 @@ cd_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) if (!PyArg_Parse(v, "D;array item must be complex", &x)) { return -1; } - - CHECK_ARRAY_BOUNDS(ap, i); - f[0] = x.real; f[1] = x.imag; - if (i >= 0) { - memcpy(ap->ob_item + i*sizeof(f), &f, sizeof(f)); - } + memcpy(buf, f, sizeof(f)); return 0; } -#define DEFINE_COMPAREITEMS(code, type) \ +#define DEFINE_COMPAREITEMS(code, name, type) \ static int \ code##_compareitems(const void *lhs, const void *rhs, Py_ssize_t length) \ { \ - const type *a = lhs, *b = rhs; \ - for (Py_ssize_t i = 0; i < length; ++i) \ - if (a[i] != b[i]) \ - return a[i] < b[i] ? -1 : 1; \ + for (Py_ssize_t i = 0; i < length; ++i) { \ + type a = name##_load(lhs, i); \ + type b = name##_load(rhs, i); \ + if (a != b) \ + return a < b ? -1 : 1; \ + } \ return 0; \ } -DEFINE_COMPAREITEMS(b, signed char) -DEFINE_COMPAREITEMS(BB, unsigned char) -DEFINE_COMPAREITEMS(w, Py_UCS4) -DEFINE_COMPAREITEMS(h, short) -DEFINE_COMPAREITEMS(HH, unsigned short) -DEFINE_COMPAREITEMS(i, int) -DEFINE_COMPAREITEMS(II, unsigned int) -DEFINE_COMPAREITEMS(l, long) -DEFINE_COMPAREITEMS(LL, unsigned long) -DEFINE_COMPAREITEMS(q, long long) -DEFINE_COMPAREITEMS(QQ, unsigned long long) +DEFINE_COMPAREITEMS(b, schar, signed char) +DEFINE_COMPAREITEMS(BB, uchar, unsigned char) +DEFINE_COMPAREITEMS(w, ucs4, Py_UCS4) +DEFINE_COMPAREITEMS(h, short, short) +DEFINE_COMPAREITEMS(HH, ushort, unsigned short) +DEFINE_COMPAREITEMS(i, int, int) +DEFINE_COMPAREITEMS(II, uint, unsigned int) +DEFINE_COMPAREITEMS(l, long, long) +DEFINE_COMPAREITEMS(LL, ulong, unsigned long) +DEFINE_COMPAREITEMS(q, longlong, long long) +DEFINE_COMPAREITEMS(QQ, ulonglong, unsigned long long) /* Description of types. * @@ -734,22 +957,22 @@ DEFINE_COMPAREITEMS(QQ, unsigned long long) * typecode. */ static const struct arraydescr descriptors[] = { - {"b", 1, b_getitem, b_setitem, b_compareitems, 1, 1}, - {"B", 1, BB_getitem, BB_setitem, BB_compareitems, 1, 0}, - {"w", sizeof(Py_UCS4), w_getitem, w_setitem, w_compareitems, 0, 0,}, - {"h", sizeof(short), h_getitem, h_setitem, h_compareitems, 1, 1}, - {"H", sizeof(short), HH_getitem, HH_setitem, HH_compareitems, 1, 0}, - {"i", sizeof(int), i_getitem, i_setitem, i_compareitems, 1, 1}, - {"I", sizeof(int), II_getitem, II_setitem, II_compareitems, 1, 0}, - {"l", sizeof(long), l_getitem, l_setitem, l_compareitems, 1, 1}, - {"L", sizeof(long), LL_getitem, LL_setitem, LL_compareitems, 1, 0}, - {"q", sizeof(long long), q_getitem, q_setitem, q_compareitems, 1, 1}, - {"Q", sizeof(long long), QQ_getitem, QQ_setitem, QQ_compareitems, 1, 0}, - {"e", sizeof(short), e_getitem, e_setitem, NULL, 0, 0}, - {"f", sizeof(float), f_getitem, f_setitem, NULL, 0, 0}, - {"d", sizeof(double), d_getitem, d_setitem, NULL, 0, 0}, - {"Zf", 2*sizeof(float), cf_getitem, cf_setitem, NULL, 0, 0}, - {"Zd", 2*sizeof(double), cd_getitem, cd_setitem, NULL, 0, 0}, + {"b", 1, b_getitem, b_pack, b_compareitems, 1, 1}, + {"B", 1, BB_getitem, BB_pack, BB_compareitems, 1, 0}, + {"w", sizeof(Py_UCS4), w_getitem, w_pack, w_compareitems, 0, 0,}, + {"h", sizeof(short), h_getitem, h_pack, h_compareitems, 1, 1}, + {"H", sizeof(short), HH_getitem, HH_pack, HH_compareitems, 1, 0}, + {"i", sizeof(int), i_getitem, i_pack, i_compareitems, 1, 1}, + {"I", sizeof(int), II_getitem, II_pack, II_compareitems, 1, 0}, + {"l", sizeof(long), l_getitem, l_pack, l_compareitems, 1, 1}, + {"L", sizeof(long), LL_getitem, LL_pack, LL_compareitems, 1, 0}, + {"q", sizeof(long long), q_getitem, q_pack, q_compareitems, 1, 1}, + {"Q", sizeof(long long), QQ_getitem, QQ_pack, QQ_compareitems, 1, 0}, + {"e", sizeof(short), e_getitem, e_pack, NULL, 0, 0}, + {"f", sizeof(float), f_getitem, f_pack, NULL, 0, 0}, + {"d", sizeof(double), d_getitem, d_pack, NULL, 0, 0}, + {"Zf", 2*sizeof(float), cf_getitem, cf_pack, NULL, 0, 0}, + {"Zd", 2*sizeof(double), cd_getitem, cd_pack, NULL, 0, 0}, {"", 0, 0, 0, 0, 0, 0} /* Sentinel */ }; @@ -766,7 +989,6 @@ newarrayobject_untracked(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *descr) { arrayobject *op; - size_t nbytes; if (size < 0) { PyErr_BadInternalCall(); @@ -774,10 +996,9 @@ newarrayobject_untracked(PyTypeObject *type, Py_ssize_t size, } /* Check for overflow */ - if (size > PY_SSIZE_T_MAX / descr->itemsize) { + if (!array_size_valid(size, descr->itemsize)) { return PyErr_NoMemory(); } - nbytes = size * descr->itemsize; op = (arrayobject *) _PyType_AllocNoTrack(type, 0); if (op == NULL) { return NULL; @@ -790,7 +1011,7 @@ newarrayobject_untracked(PyTypeObject *type, Py_ssize_t size, op->ob_item = NULL; } else { - op->ob_item = PyMem_NEW(char, nbytes); + op->ob_item = array_items_alloc(size, descr->itemsize); if (op->ob_item == NULL) { Py_DECREF(op); return PyErr_NoMemory(); @@ -800,6 +1021,12 @@ newarrayobject_untracked(PyTypeObject *type, Py_ssize_t size, return (PyObject *) op; } +static int +valid_index(Py_ssize_t i, Py_ssize_t limit) +{ + return (size_t) i < (size_t) limit; +} + static PyObject * newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *descr) { @@ -810,34 +1037,171 @@ newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *des return op; } +/* Get item i; the caller holds the lock or owns a private array. */ static PyObject * -getarrayitem(PyObject *op, Py_ssize_t i) +getarrayitem(arrayobject *ap, Py_ssize_t i) { #ifndef NDEBUG - array_state *state = find_array_state_by_type(Py_TYPE(op)); - assert(array_Check(op, state)); + array_state *state = find_array_state_by_type(Py_TYPE(ap)); + assert(array_Check(ap, state)); + assert(valid_index(i, Py_SIZE(ap))); +#endif + return (*ap->ob_descr->getitem)(ap->ob_item, i); +} + +static void +array_mark_shared(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + if (!_PyObject_GC_IS_SHARED(op)) { + _PyObject_GC_SET_SHARED(op); + } +#endif +} + +// The getarrayitem_*() functions below return a new reference to item i. +// They return NULL without an exception set if i is out of range, and NULL +// with an exception set if the item object could not be created. + +#ifdef Py_GIL_DISABLED + +static PyObject * +getarrayitem_locked(PyObject *op, Py_ssize_t i) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + array_mark_shared(op); + if (!valid_index(i, Py_SIZE(op))) { + ret = NULL; + } + else { + ret = getarrayitem(arrayobject_CAST(op), i); + } + Py_END_CRITICAL_SECTION(); + return ret; +} + +// Return true if item i of `items` can be accessed without the lock. +// `items` and then `size` were loaded from the array, in that order. The +// size only belongs to the buffer if the buffer was not replaced in between; +// otherwise item i may be past the end or not yet written. +static inline bool +array_items_valid(arrayobject *ap, char *items, Py_ssize_t size, Py_ssize_t i) +{ + assert(valid_index(i, size)); + return (items != NULL && + valid_index(i, array_capacity(items)) && + items == _Py_atomic_load_ptr_acquire(&ap->ob_item)); +} + +static PyObject * +getarrayitem_threadsafe(PyObject *op, Py_ssize_t i) +{ + if (!_Py_IsOwnedByCurrentThread(op) && !_PyObject_GC_IS_SHARED(op)) { + return getarrayitem_locked(op, i); + } + arrayobject *ap = arrayobject_CAST(op); + char *items = _Py_atomic_load_ptr_acquire(&ap->ob_item); + Py_ssize_t size = array_get_size(op); + if (!valid_index(i, size)) { + return NULL; + } + if (!array_items_valid(ap, items, size, i)) { + return getarrayitem_locked(op, i); + } + return (*ap->ob_descr->getitem)(items, i); +} + +#else // Py_GIL_DISABLED + +static inline PyObject * +getarrayitem_threadsafe(PyObject *op, Py_ssize_t i) +{ + if (!valid_index(i, Py_SIZE(op))) { + return NULL; + } + return getarrayitem(arrayobject_CAST(op), i); +} + +#endif // Py_GIL_DISABLED + +// Store a packed item at index i; the caller checked the index. +static inline void +array_store_item(arrayobject *ap, Py_ssize_t i, const char *buf) +{ + int itemsize = ap->ob_descr->itemsize; + array_store_raw(ap->ob_item + i * itemsize, buf, itemsize); +} + +static int +setarrayitem_packed_locked(PyObject *op, Py_ssize_t i, const char *buf) +{ + int ret = 0; + Py_BEGIN_CRITICAL_SECTION(op); + array_mark_shared(op); + if (!valid_index(i, Py_SIZE(op))) { + PyErr_SetString(PyExc_IndexError, + "array assignment index out of range"); + ret = -1; + } + else { + array_store_item(arrayobject_CAST(op), i, buf); + } + Py_END_CRITICAL_SECTION(); + return ret; +} + +// a[i] = v. The item is converted before the buffer is loaded, since the +// conversion can run user code that resizes the array. +static int +setarrayitem_threadsafe(PyObject *op, Py_ssize_t i, PyObject *v) +{ + arrayobject *ap = arrayobject_CAST(op); + if (!valid_index(i, array_get_size(op))) { + PyErr_SetString(PyExc_IndexError, + "array assignment index out of range"); + return -1; + } + _Py_ALIGNED_DEF(8, char) buf[16]; + if ((*ap->ob_descr->pack)(v, buf) < 0) { + return -1; + } +#ifdef Py_GIL_DISABLED + if (_Py_IsOwnedByCurrentThread(op) || _PyObject_GC_IS_SHARED(op)) { + char *items = _Py_atomic_load_ptr_acquire(&ap->ob_item); + Py_ssize_t size = array_get_size(op); + if (!valid_index(i, size)) { + PyErr_SetString(PyExc_IndexError, + "array assignment index out of range"); + return -1; + } + if (array_items_valid(ap, items, size, i)) { + int itemsize = ap->ob_descr->itemsize; + array_store_raw(items + i * itemsize, buf, itemsize); + return 0; + } + } #endif - arrayobject *ap; - ap = (arrayobject *)op; - assert(i>=0 && iob_descr->getitem)(ap, i); + return setarrayitem_packed_locked(op, i, buf); } static int ins1(arrayobject *self, Py_ssize_t where, PyObject *v) { - char *items; - Py_ssize_t n = Py_SIZE(self); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); if (v == NULL) { PyErr_BadInternalCall(); return -1; } - if ((*self->ob_descr->setitem)(self, -1, v) < 0) + _Py_ALIGNED_DEF(8, char) buf[16]; + if ((*self->ob_descr->pack)(v, buf) < 0) { return -1; + } - if (array_resize(self, n+1) == -1) + // pack() can run user code, so read the size after it. + Py_ssize_t n = Py_SIZE(self); + if (array_reserve(self, n+1) == -1) return -1; - items = self->ob_item; if (where < 0) { where += n; if (where < 0) @@ -845,12 +1209,16 @@ ins1(arrayobject *self, Py_ssize_t where, PyObject *v) } if (where > n) where = n; - /* appends don't need to call memmove() */ - if (where != n) - memmove(items + (where+1)*self->ob_descr->itemsize, - items + where*self->ob_descr->itemsize, - (n-where)*self->ob_descr->itemsize); - return (*self->ob_descr->setitem)(self, where, v); + int itemsize = self->ob_descr->itemsize; + char *items = self->ob_item; + /* appends don't need to move items */ + if (where != n) { + array_copy_raw(items + (where+1)*itemsize, items + where*itemsize, + n - where, itemsize, array_is_exclusive(self)); + } + array_store_item(self, where, buf); + array_set_size(self, n+1); + return 0; } /* Methods */ @@ -861,17 +1229,20 @@ array_dealloc(PyObject *op) PyObject_GC_UnTrack(op); arrayobject *self = arrayobject_CAST(op); - FT_CLEAR_WEAKREFS(op, self->weakreflist); - if (self->ob_item != NULL) { - PyMem_Free(self->ob_item); + if (self->ob_exports > 0) { + PyErr_FormatUnraisable("deallocated array object has exported buffers"); } + FT_CLEAR_WEAKREFS(op, self->weakreflist); + array_items_free(self->ob_item, self->ob_descr->itemsize, false); tp->tp_free(op); Py_DECREF(tp); } static PyObject * -array_richcompare(PyObject *v, PyObject *w, int op) +array_richcompare_lock_held(PyObject *v, PyObject *w, int op) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(v); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(w); array_state *state = find_array_state_by_type(Py_TYPE(v)); arrayobject *va, *wa; PyObject *vi = NULL; @@ -921,11 +1292,11 @@ array_richcompare(PyObject *v, PyObject *w, int op) /* Search for the first index where items are different */ k = 1; for (i = 0; i < Py_SIZE(va) && i < Py_SIZE(wa); i++) { - vi = getarrayitem(v, i); + vi = getarrayitem(va, i); if (vi == NULL) { return NULL; } - wi = getarrayitem(w, i); + wi = getarrayitem(wa, i); if (wi == NULL) { Py_DECREF(vi); return NULL; @@ -979,25 +1350,39 @@ array_richcompare(PyObject *v, PyObject *w, int op) return res; } +static PyObject * +array_richcompare(PyObject *v, PyObject *w, int op) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION2(v, w); + ret = array_richcompare_lock_held(v, w, op); + Py_END_CRITICAL_SECTION2(); + return ret; +} + static Py_ssize_t array_length(PyObject *op) { - return Py_SIZE(op); + arrayobject *self = arrayobject_CAST(op); + return array_get_size(self); } + static PyObject * array_item(PyObject *op, Py_ssize_t i) { - if (i < 0 || i >= Py_SIZE(op)) { + PyObject *item = getarrayitem_threadsafe(op, i); + if (item == NULL && !PyErr_Occurred()) { PyErr_SetString(PyExc_IndexError, "array index out of range"); - return NULL; } - return getarrayitem(op, i); + return item; } + static PyObject * array_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(a); array_state *state = find_array_state_by_type(Py_TYPE(a)); arrayobject *np; @@ -1015,14 +1400,16 @@ array_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) if (np == NULL) return NULL; if (ihigh > ilow) { - memcpy(np->ob_item, a->ob_item + ilow * a->ob_descr->itemsize, - (ihigh-ilow) * a->ob_descr->itemsize); + int itemsize = a->ob_descr->itemsize; + array_copy_raw(np->ob_item, a->ob_item + ilow * itemsize, + ihigh - ilow, itemsize, array_is_exclusive(a)); } _PyObject_GC_TRACK(np); return (PyObject *)np; } /*[clinic input] +@critical_section array.array.clear Remove all items from the array. @@ -1030,7 +1417,7 @@ Remove all items from the array. static PyObject * array_array_clear_impl(arrayobject *self) -/*[clinic end generated code: output=5efe0417062210a9 input=5dffa30e94e717a4]*/ +/*[clinic end generated code: output=5efe0417062210a9 input=1c9dfcc80f5b6731]*/ { if (array_resize(self, 0) == -1) { return NULL; @@ -1039,6 +1426,7 @@ array_array_clear_impl(arrayobject *self) } /*[clinic input] +@critical_section array.array.__copy__ Return a copy of the array. @@ -1046,12 +1434,13 @@ Return a copy of the array. static PyObject * array_array___copy___impl(arrayobject *self) -/*[clinic end generated code: output=dec7c3f925d9619e input=ad1ee5b086965f09]*/ +/*[clinic end generated code: output=dec7c3f925d9619e input=7622f8f9489472d5]*/ { return array_slice(self, 0, Py_SIZE(self)); } /*[clinic input] +@critical_section array.array.__deepcopy__ unused: object @@ -1062,14 +1451,16 @@ Return a copy of the array. static PyObject * array_array___deepcopy___impl(arrayobject *self, PyObject *unused) -/*[clinic end generated code: output=703b4c412feaaf31 input=2405ecb4933748c4]*/ +/*[clinic end generated code: output=703b4c412feaaf31 input=1a29f718f5b8a1dc]*/ { return array_array___copy___impl(self); } static PyObject * -array_concat(PyObject *op, PyObject *bb) +array_concat_lock_held(PyObject *op, PyObject *bb) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(bb); arrayobject *a = arrayobject_CAST(op); array_state *state = find_array_state_by_type(Py_TYPE(a)); Py_ssize_t size; @@ -1093,12 +1484,15 @@ array_concat(PyObject *op, PyObject *bb) if (np == NULL) { return NULL; } + int itemsize = a->ob_descr->itemsize; if (Py_SIZE(a) > 0) { - memcpy(np->ob_item, a->ob_item, Py_SIZE(a)*a->ob_descr->itemsize); + array_copy_raw(np->ob_item, a->ob_item, Py_SIZE(a), + itemsize, array_is_exclusive(a)); } if (Py_SIZE(b) > 0) { - memcpy(np->ob_item + Py_SIZE(a)*a->ob_descr->itemsize, - b->ob_item, Py_SIZE(b)*b->ob_descr->itemsize); + array_copy_raw(np->ob_item + Py_SIZE(a) * itemsize, + b->ob_item, Py_SIZE(b), + itemsize, array_is_exclusive(b)); } _PyObject_GC_TRACK(np); return (PyObject *)np; @@ -1106,8 +1500,19 @@ array_concat(PyObject *op, PyObject *bb) } static PyObject * -array_repeat(PyObject *op, Py_ssize_t n) +array_concat(PyObject *op, PyObject *bb) { + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION2(op, bb); + ret = array_concat_lock_held(op, bb); + Py_END_CRITICAL_SECTION2(); + return ret; +} + +static PyObject * +array_repeat_lock_held(PyObject *op, Py_ssize_t n) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); arrayobject *a = arrayobject_CAST(op); array_state *state = find_array_state_by_type(Py_TYPE(a)); @@ -1128,15 +1533,30 @@ array_repeat(PyObject *op, Py_ssize_t n) const Py_ssize_t oldbytes = array_length * a->ob_descr->itemsize; const Py_ssize_t newbytes = oldbytes * n; - _PyBytes_RepeatBuffer(np->ob_item, newbytes, a->ob_item, oldbytes); + // Copy one repetition from a, then repeat within the private np. + array_copy_raw(np->ob_item, a->ob_item, array_length, + a->ob_descr->itemsize, array_is_exclusive(a)); + _PyBytes_RepeatBuffer(np->ob_item, newbytes, np->ob_item, + oldbytes); _PyObject_GC_TRACK(np); return (PyObject *)np; } +static PyObject * +array_repeat(PyObject *op, Py_ssize_t n) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_repeat_lock_held(op, n); + Py_END_CRITICAL_SECTION(); + return ret; +} + static int array_del_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(a); char *item; Py_ssize_t d; /* Change in size */ if (ilow < 0) @@ -1160,9 +1580,9 @@ array_del_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) return -1; } if (d > 0) { /* Delete d items */ - memmove(item + (ihigh-d)*a->ob_descr->itemsize, - item + ihigh*a->ob_descr->itemsize, - (Py_SIZE(a)-ihigh)*a->ob_descr->itemsize); + int itemsize = a->ob_descr->itemsize; + array_copy_raw(item + ilow * itemsize, item + ihigh * itemsize, + Py_SIZE(a) - ihigh, itemsize, array_is_exclusive(a)); if (array_resize(a, Py_SIZE(a) - d) == -1) return -1; } @@ -1172,30 +1592,27 @@ array_del_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) static int array_ass_item(PyObject *op, Py_ssize_t i, PyObject *v) { - arrayobject *a = arrayobject_CAST(op); - if (i < 0 || i >= Py_SIZE(a)) { + if (v != NULL) { + return setarrayitem_threadsafe(op, i, v); + } + int ret; + Py_BEGIN_CRITICAL_SECTION(op); + if (!valid_index(i, Py_SIZE(op))) { PyErr_SetString(PyExc_IndexError, - "array assignment index out of range"); - return -1; + "array assignment index out of range"); + ret = -1; } - if (v == NULL) - return array_del_slice(a, i, i+1); - return (*a->ob_descr->setitem)(a, i, v); -} - -static int -setarrayitem(PyObject *a, Py_ssize_t i, PyObject *v) -{ -#ifndef NDEBUG - array_state *state = find_array_state_by_type(Py_TYPE(a)); - assert(array_Check(a, state)); -#endif - return array_ass_item(a, i, v); + else { + ret = array_del_slice(arrayobject_CAST(op), i, i+1); + } + Py_END_CRITICAL_SECTION(); + return ret; } static int array_iter_extend(arrayobject *self, PyObject *bb) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); PyObject *it, *v; it = PyObject_GetIter(bb); @@ -1217,8 +1634,10 @@ array_iter_extend(arrayobject *self, PyObject *bb) } static int -array_do_extend(array_state *state, arrayobject *self, PyObject *bb) +array_do_extend_lock_held(array_state *state, arrayobject *self, PyObject *bb) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(bb); Py_ssize_t size, oldsize, bbsize; if (!array_Check(bb, state)) @@ -1230,25 +1649,40 @@ array_do_extend(array_state *state, arrayobject *self, PyObject *bb) return -1; } if ((Py_SIZE(self) > PY_SSIZE_T_MAX - Py_SIZE(b)) || - ((Py_SIZE(self) + Py_SIZE(b)) > PY_SSIZE_T_MAX / self->ob_descr->itemsize)) { + !array_size_valid(Py_SIZE(self) + Py_SIZE(b), + self->ob_descr->itemsize)) + { PyErr_NoMemory(); return -1; } oldsize = Py_SIZE(self); /* Get the size of bb before resizing the array since bb could be self. */ bbsize = Py_SIZE(bb); - size = oldsize + Py_SIZE(b); - if (array_resize(self, size) == -1) + size = oldsize + bbsize; + if (array_reserve(self, size) == -1) return -1; if (bbsize > 0) { - memcpy(self->ob_item + oldsize * self->ob_descr->itemsize, - b->ob_item, bbsize * b->ob_descr->itemsize); + int itemsize = self->ob_descr->itemsize; + array_copy_raw(self->ob_item + oldsize * itemsize, + b->ob_item, bbsize, itemsize, + array_is_exclusive(self) && array_is_exclusive(b)); } + array_set_size(self, size); return 0; #undef b } +static int +array_do_extend(array_state *state, arrayobject *self, PyObject *bb) +{ + int ret; + Py_BEGIN_CRITICAL_SECTION2(self, bb); + ret = array_do_extend_lock_held(state, self, bb); + Py_END_CRITICAL_SECTION2(); + return ret; +} + static PyObject * array_inplace_concat(PyObject *op, PyObject *bb) { @@ -1267,14 +1701,16 @@ array_inplace_concat(PyObject *op, PyObject *bb) } static PyObject * -array_inplace_repeat(PyObject *op, Py_ssize_t n) +array_inplace_repeat_lock_held(PyObject *op, Py_ssize_t n) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); arrayobject *self = arrayobject_CAST(op); const Py_ssize_t array_size = Py_SIZE(self); if (array_size > 0 && n != 1 ) { if (n < 0) n = 0; + // XXX This check below does nothing useful??? if ((self->ob_descr->itemsize != 0) && (array_size > PY_SSIZE_T_MAX / self->ob_descr->itemsize)) { return PyErr_NoMemory(); @@ -1283,24 +1719,43 @@ array_inplace_repeat(PyObject *op, Py_ssize_t n) if (n > 0 && size > PY_SSIZE_T_MAX / n) { return PyErr_NoMemory(); } - if (array_resize(self, n * array_size) == -1) + if (n == 0) { + if (array_resize(self, 0) == -1) + return NULL; + return Py_NewRef(self); + } + if (array_reserve(self, n * array_size) == -1) return NULL; - _PyBytes_RepeatBuffer(self->ob_item, n*size, self->ob_item, size); + char *items = self->ob_item; + if (array_is_exclusive(self)) { + _PyBytes_RepeatBuffer(items, n*size, items, size); + } + else { + int itemsize = self->ob_descr->itemsize; + for (Py_ssize_t k = 1; k < n; k++) { + array_copy_raw(items + k * size, items, array_size, + itemsize, false); + } + } + array_set_size(self, n * array_size); } return Py_NewRef(self); } - static PyObject * -ins(arrayobject *self, Py_ssize_t where, PyObject *v) +array_inplace_repeat(PyObject *op, Py_ssize_t n) { - if (ins1(self, where, v) != 0) - return NULL; - Py_RETURN_NONE; + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_inplace_repeat_lock_held(op, n); + Py_END_CRITICAL_SECTION(); + return ret; } + /*[clinic input] +@critical_section array.array.count v: object @@ -1311,7 +1766,7 @@ Return number of occurrences of v in the array. static PyObject * array_array_count_impl(arrayobject *self, PyObject *v) -/*[clinic end generated code: output=93ead26a2affb739 input=d9bce9d65e39d1f5]*/ +/*[clinic end generated code: output=93ead26a2affb739 input=c12c0042c1d0e27e]*/ { Py_ssize_t count = 0; Py_ssize_t i; @@ -1320,7 +1775,7 @@ array_array_count_impl(arrayobject *self, PyObject *v) PyObject *selfi; int cmp; - selfi = getarrayitem((PyObject *)self, i); + selfi = getarrayitem(self, i); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1335,6 +1790,7 @@ array_array_count_impl(arrayobject *self, PyObject *v) /*[clinic input] +@critical_section array.array.index v: object @@ -1350,7 +1806,7 @@ Raise ValueError if the value is not present. static PyObject * array_array_index_impl(arrayobject *self, PyObject *v, Py_ssize_t start, Py_ssize_t stop) -/*[clinic end generated code: output=c45e777880c99f52 input=089dff7baa7e5a7e]*/ +/*[clinic end generated code: output=c45e777880c99f52 input=fa32ac8ec22175d6]*/ { if (start < 0) { start += Py_SIZE(self); @@ -1367,7 +1823,7 @@ array_array_index_impl(arrayobject *self, PyObject *v, Py_ssize_t start, PyObject *selfi; int cmp; - selfi = getarrayitem((PyObject *)self, i); + selfi = getarrayitem(self, i); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1383,22 +1839,35 @@ array_array_index_impl(arrayobject *self, PyObject *v, Py_ssize_t start, } static int -array_contains(PyObject *self, PyObject *v) +array_contains_lock_held(PyObject *op, PyObject *v) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); Py_ssize_t i; int cmp; - for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(self); i++) { - PyObject *selfi = getarrayitem(self, i); - if (selfi == NULL) + for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(op); i++) { + arrayobject *ap = (arrayobject *)op; + PyObject *opi = getarrayitem(ap, i); + if (opi == NULL) return -1; - cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); - Py_DECREF(selfi); + cmp = PyObject_RichCompareBool(opi, v, Py_EQ); + Py_DECREF(opi); } return cmp; } +static int +array_contains(PyObject *op, PyObject *v) +{ + int ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_contains_lock_held(op, v); + Py_END_CRITICAL_SECTION(); + return ret; +} + /*[clinic input] +@critical_section array.array.remove v: object @@ -1409,7 +1878,7 @@ Remove the first occurrence of v in the array. static PyObject * array_array_remove_impl(arrayobject *self, PyObject *v) -/*[clinic end generated code: output=f2a24e288ecb2a35 input=0b1e5aed25590027]*/ +/*[clinic end generated code: output=f2a24e288ecb2a35 input=78bef3fd40e62f7a]*/ { Py_ssize_t i; @@ -1417,7 +1886,7 @@ array_array_remove_impl(arrayobject *self, PyObject *v) PyObject *selfi; int cmp; - selfi = getarrayitem((PyObject *)self,i); + selfi = getarrayitem(self, i); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1435,6 +1904,7 @@ array_array_remove_impl(arrayobject *self, PyObject *v) } /*[clinic input] +@critical_section array.array.pop i: Py_ssize_t = -1 @@ -1447,7 +1917,7 @@ i defaults to -1. static PyObject * array_array_pop_impl(arrayobject *self, Py_ssize_t i) -/*[clinic end generated code: output=bc1f0c54fe5308e4 input=8e5feb4c1a11cd44]*/ +/*[clinic end generated code: output=bc1f0c54fe5308e4 input=c69a7f1f8c570e2f]*/ { PyObject *v; @@ -1462,7 +1932,7 @@ array_array_pop_impl(arrayobject *self, Py_ssize_t i) PyErr_SetString(PyExc_IndexError, "pop index out of range"); return NULL; } - v = getarrayitem((PyObject *)self, i); + v = getarrayitem(self, i); if (v == NULL) return NULL; if (array_del_slice(self, i, i+1) != 0) { @@ -1494,6 +1964,7 @@ array_array_extend_impl(arrayobject *self, PyTypeObject *cls, PyObject *bb) } /*[clinic input] +@critical_section array.array.insert i: Py_ssize_t @@ -1505,12 +1976,15 @@ Insert a new item v into the array before position i. static PyObject * array_array_insert_impl(arrayobject *self, Py_ssize_t i, PyObject *v) -/*[clinic end generated code: output=5a3648e278348564 input=5577d1b4383e9313]*/ +/*[clinic end generated code: output=5a3648e278348564 input=3c922bbd81462978]*/ { - return ins(self, i, v); + if (ins1(self, i, v) != 0) + return NULL; + Py_RETURN_NONE; } /*[clinic input] +@critical_section @permit_long_summary array.array.buffer_info @@ -1522,7 +1996,7 @@ calculate the buffer length in bytes. static PyObject * array_array_buffer_info_impl(arrayobject *self) -/*[clinic end generated code: output=9b2a4ec3ae7e98e7 input=c2771b9f6a8e1c86]*/ +/*[clinic end generated code: output=9b2a4ec3ae7e98e7 input=05c248e0009ce284]*/ { PyObject* item1 = PyLong_FromVoidPtr(self->ob_item); if (item1 == NULL) { @@ -1538,6 +2012,7 @@ array_array_buffer_info_impl(arrayobject *self) } /*[clinic input] +@critical_section array.array.append v: object @@ -1548,12 +2023,46 @@ Append new value v to the end of the array. static PyObject * array_array_append_impl(arrayobject *self, PyObject *v) -/*[clinic end generated code: output=2f1e8cbad70c2a8b input=0b98d9d78e78f0fa]*/ +/*[clinic end generated code: output=2f1e8cbad70c2a8b input=9cdd897c66a40c3f]*/ +{ + if (ins1(self, Py_SIZE(self), v) != 0) + return NULL; + Py_RETURN_NONE; +} + +#ifdef Py_GIL_DISABLED +// Byteswap an array that other threads can read without the lock. +static void +array_byteswap_shared(arrayobject *self) { - return ins(self, Py_SIZE(self), v); + int itemsize = self->ob_descr->itemsize; + // Complex types swap the real and imaginary parts separately. + int part = itemsize; + if (itemsize == 1) { + return; + } + if (self->ob_descr->typecode[0] == 'Z') { + part = itemsize / 2; + } + char *items = self->ob_item; + for (Py_ssize_t i = 0; i < Py_SIZE(self); i++) { + _Py_ALIGNED_DEF(8, char) buf[16]; + char *p = items + i * itemsize; + array_load_raw(p, buf, itemsize); + for (int k = 0; k < itemsize; k += part) { + for (int lo = k, hi = k + part - 1; lo < hi; lo++, hi--) { + char t = buf[lo]; + buf[lo] = buf[hi]; + buf[hi] = t; + } + } + array_store_raw(p, buf, itemsize); + } } +#endif /*[clinic input] +@critical_section array.array.byteswap Byteswap all items of the array. @@ -1565,11 +2074,18 @@ components (the real part, followed by imaginary part) is preserved. static PyObject * array_array_byteswap_impl(arrayobject *self) -/*[clinic end generated code: output=5f8236cbdf0d90b5 input=8732f800e1b47bac]*/ +/*[clinic end generated code: output=5f8236cbdf0d90b5 input=91e114b82e32c2b4]*/ { char *p; Py_ssize_t i; +#ifdef Py_GIL_DISABLED + if (!array_is_exclusive(self)) { + array_byteswap_shared(self); + Py_RETURN_NONE; + } +#endif + switch (self->ob_descr->itemsize) { case 1: break; @@ -1662,6 +2178,7 @@ array_array_byteswap_impl(arrayobject *self) } /*[clinic input] +@critical_section array.array.reverse Reverse the order of the items in the array. @@ -1669,26 +2186,21 @@ Reverse the order of the items in the array. static PyObject * array_array_reverse_impl(arrayobject *self) -/*[clinic end generated code: output=c04868b36f6f4089 input=cd904f01b27d966a]*/ +/*[clinic end generated code: output=c04868b36f6f4089 input=e3947e98aed068ed]*/ { - Py_ssize_t itemsize = self->ob_descr->itemsize; - char *p, *q; - /* little buffer to hold items while swapping */ - char tmp[256]; /* 8 is probably enough -- but why skimp */ - assert((size_t)itemsize <= sizeof(tmp)); + int itemsize = self->ob_descr->itemsize; + Py_ssize_t n = Py_SIZE(self); + char *items = self->ob_item; + /* little buffers to hold items while swapping */ + _Py_ALIGNED_DEF(8, char) a[16]; + _Py_ALIGNED_DEF(8, char) b[16]; + assert((size_t)itemsize <= sizeof(a)); - if (Py_SIZE(self) > 1) { - for (p = self->ob_item, - q = self->ob_item + (Py_SIZE(self) - 1)*itemsize; - p < q; - p += itemsize, q -= itemsize) { - /* memory areas guaranteed disjoint, so memcpy - * is safe (& memmove may be slower). - */ - memcpy(tmp, p, itemsize); - memcpy(p, q, itemsize); - memcpy(q, tmp, itemsize); - } + for (Py_ssize_t i = 0, j = n - 1; i < j; i++, j--) { + array_load_raw(items + i * itemsize, a, itemsize); + array_load_raw(items + j * itemsize, b, itemsize); + array_store_raw(items + i * itemsize, b, itemsize); + array_store_raw(items + j * itemsize, a, itemsize); } Py_RETURN_NONE; @@ -1712,16 +2224,15 @@ array_array_fromfile_impl(arrayobject *self, PyTypeObject *cls, PyObject *f, /*[clinic end generated code: output=83a667080b345ebc input=db46b06ac1b6de87]*/ { PyObject *b, *res; - Py_ssize_t itemsize = self->ob_descr->itemsize; + int itemsize = self->ob_descr->itemsize; Py_ssize_t nbytes; int not_enough_bytes; - if (n > PY_SSIZE_T_MAX / itemsize) { + if (!array_size_valid(n, itemsize)) { PyErr_NoMemory(); return NULL; } - array_state *state = get_array_state_by_class(cls); assert(state != NULL); @@ -1756,6 +2267,7 @@ array_array_fromfile_impl(arrayobject *self, PyTypeObject *cls, PyObject *f, } /*[clinic input] +@critical_section array.array.tofile cls: defining_class @@ -1767,32 +2279,28 @@ Write all items (as machine values) to the file object f. static PyObject * array_array_tofile_impl(arrayobject *self, PyTypeObject *cls, PyObject *f) -/*[clinic end generated code: output=4560c628d9c18bc2 input=5a24da7a7b407b52]*/ +/*[clinic end generated code: output=4560c628d9c18bc2 input=a26bc66df57864dd]*/ { - Py_ssize_t nbytes = Py_SIZE(self) * self->ob_descr->itemsize; - /* Write 64K blocks at a time */ + int itemsize = self->ob_descr->itemsize; + /* Write about 64K at a time, in whole items */ /* XXX Make the block size settable */ - int BLOCKSIZE = 64*1024; - Py_ssize_t nblocks = (nbytes + BLOCKSIZE - 1) / BLOCKSIZE; - Py_ssize_t i; - - if (Py_SIZE(self) == 0) - goto done; - + Py_ssize_t BLOCKITEMS = Py_MAX(1, 64*1024 / itemsize); array_state *state = get_array_state_by_class(cls); assert(state != NULL); - for (i = 0; i < nblocks; i++) { - char* ptr = self->ob_item + i*BLOCKSIZE; - Py_ssize_t size = BLOCKSIZE; + // f.write() can run user code that resizes the array, so recheck the + // size for each block. + for (Py_ssize_t start = 0; start < Py_SIZE(self); start += BLOCKITEMS) { + Py_ssize_t n = Py_MIN(BLOCKITEMS, Py_SIZE(self) - start); PyObject *bytes, *res; - if (i*BLOCKSIZE + size > nbytes) - size = nbytes - i*BLOCKSIZE; - bytes = PyBytes_FromStringAndSize(ptr, size); + bytes = PyBytes_FromStringAndSize(NULL, n * itemsize); if (bytes == NULL) return NULL; + array_copy_raw(PyBytes_AS_STRING(bytes), + self->ob_item + start * itemsize, + n, itemsize, array_is_exclusive(self)); res = PyObject_CallMethodOneArg(f, state->str_write, bytes); Py_DECREF(bytes); if (res == NULL) @@ -1800,11 +2308,11 @@ array_array_tofile_impl(arrayobject *self, PyTypeObject *cls, PyObject *f) Py_DECREF(res); /* drop write result */ } - done: Py_RETURN_NONE; } /*[clinic input] +@critical_section self list array.array.fromlist list: object @@ -1815,7 +2323,7 @@ Append items to array from list. static PyObject * array_array_fromlist_impl(arrayobject *self, PyObject *list) -/*[clinic end generated code: output=6c23733a68dd68df input=be2605a96c49680f]*/ +/*[clinic end generated code: output=6c23733a68dd68df input=c7c056aaf85d997a]*/ { Py_ssize_t n; @@ -1823,31 +2331,52 @@ array_array_fromlist_impl(arrayobject *self, PyObject *list) PyErr_SetString(PyExc_TypeError, "arg must be list"); return NULL; } - n = PyList_Size(list); - if (n > 0) { - Py_ssize_t i, old_size; - old_size = Py_SIZE(self); - if (array_resize(self, old_size + n) == -1) - return NULL; - for (i = 0; i < n; i++) { - PyObject *v = PyList_GET_ITEM(list, i); - if ((*self->ob_descr->setitem)(self, - Py_SIZE(self) - n + i, v) != 0) { - array_resize(self, old_size); - return NULL; - } - if (n != PyList_GET_SIZE(list)) { - PyErr_SetString(PyExc_RuntimeError, - "list changed size during iteration"); - array_resize(self, old_size); - return NULL; - } + n = PyList_GET_SIZE(list); + if (n == 0) { + Py_RETURN_NONE; + } + int itemsize = self->ob_descr->itemsize; + if (!array_size_valid(n, itemsize)) { + return PyErr_NoMemory(); + } + /* Convert all items first: pack() can run user code. */ + char *buf = PyMem_Malloc(n * itemsize); + if (buf == NULL) { + return PyErr_NoMemory(); + } + for (Py_ssize_t i = 0; i < n; i++) { + PyObject *v = PyList_GET_ITEM(list, i); + if ((*self->ob_descr->pack)(v, buf + i * itemsize) != 0) { + goto error; + } + if (n != PyList_GET_SIZE(list)) { + PyErr_SetString(PyExc_RuntimeError, + "list changed size during iteration"); + goto error; } } + Py_ssize_t old_size = Py_SIZE(self); + if (n > PY_SSIZE_T_MAX - old_size || + !array_size_valid(old_size + n, itemsize)) { + PyErr_NoMemory(); + goto error; + } + if (array_reserve(self, old_size + n) == -1) { + goto error; + } + array_copy_raw(self->ob_item + old_size * itemsize, buf, n, + itemsize, array_is_exclusive(self)); + array_set_size(self, old_size + n); + PyMem_Free(buf); Py_RETURN_NONE; + +error: + PyMem_Free(buf); + return NULL; } /*[clinic input] +@critical_section array.array.tolist Convert array to an ordinary list with the same items. @@ -1855,7 +2384,7 @@ Convert array to an ordinary list with the same items. static PyObject * array_array_tolist_impl(arrayobject *self) -/*[clinic end generated code: output=00b60cc9eab8ef89 input=a8d7784a94f86b53]*/ +/*[clinic end generated code: output=00b60cc9eab8ef89 input=4543fdbac475c52c]*/ { PyObject *list = PyList_New(Py_SIZE(self)); Py_ssize_t i; @@ -1863,7 +2392,7 @@ array_array_tolist_impl(arrayobject *self) if (list == NULL) return NULL; for (i = 0; i < Py_SIZE(self); i++) { - PyObject *v = getarrayitem((PyObject *)self, i); + PyObject *v = getarrayitem(self, i); if (v == NULL) goto error; PyList_SET_ITEM(list, i, v); @@ -1875,19 +2404,30 @@ array_array_tolist_impl(arrayobject *self) return NULL; } + +/*[clinic input] +@critical_section +@permit_long_summary +array.array.frombytes + + buffer: Py_buffer + / + +Appends items from the string, interpreting it as an array of machine values, as if it had been read from a file using the fromfile() method. +[clinic start generated code]*/ + static PyObject * -frombytes(arrayobject *self, Py_buffer *buffer) +array_array_frombytes_impl(arrayobject *self, Py_buffer *buffer) +/*[clinic end generated code: output=d9842c8f7510a516 input=db8cf498a38b0210]*/ { int itemsize = self->ob_descr->itemsize; Py_ssize_t n; if (buffer->itemsize != 1) { - PyBuffer_Release(buffer); PyErr_SetString(PyExc_TypeError, "a bytes-like object is required"); return NULL; } n = buffer->len; if (n % itemsize != 0) { - PyBuffer_Release(buffer); PyErr_SetString(PyExc_ValueError, "bytes length not a multiple of item size"); return NULL; @@ -1896,39 +2436,33 @@ frombytes(arrayobject *self, Py_buffer *buffer) if (n > 0) { Py_ssize_t old_size = Py_SIZE(self); if ((n > PY_SSIZE_T_MAX - old_size) || - ((old_size + n) > PY_SSIZE_T_MAX / itemsize)) { - PyBuffer_Release(buffer); + !array_size_valid(old_size + n, itemsize)) { return PyErr_NoMemory(); } - if (array_resize(self, old_size + n) == -1) { - PyBuffer_Release(buffer); + if (array_reserve(self, old_size + n) == -1) { return NULL; } - memcpy(self->ob_item + old_size * itemsize, - buffer->buf, n * itemsize); + char *dst = self->ob_item + old_size * itemsize; + if (array_is_exclusive(self)) { + memcpy(dst, buffer->buf, n * itemsize); + } + else { + // buffer->buf may not be aligned; copy through a temporary. + char *tmp = PyMem_Malloc(n * itemsize); + if (tmp == NULL) { + return PyErr_NoMemory(); + } + memcpy(tmp, buffer->buf, n * itemsize); + array_copy_raw(dst, tmp, n, itemsize, false); + PyMem_Free(tmp); + } + array_set_size(self, old_size + n); } - PyBuffer_Release(buffer); Py_RETURN_NONE; } /*[clinic input] -@permit_long_summary -array.array.frombytes - - buffer: Py_buffer - / - -Appends items from the string, interpreting it as an array of machine values, as if it had been read from a file using the fromfile() method. -[clinic start generated code]*/ - -static PyObject * -array_array_frombytes_impl(arrayobject *self, Py_buffer *buffer) -/*[clinic end generated code: output=d9842c8f7510a516 input=6b90ce5895f677a4]*/ -{ - return frombytes(self, buffer); -} - -/*[clinic input] +@critical_section @permit_long_summary array.array.tobytes @@ -1937,17 +2471,23 @@ Convert the array to an array of machine values and return the bytes representat static PyObject * array_array_tobytes_impl(arrayobject *self) -/*[clinic end generated code: output=87318e4edcdc2bb6 input=41081256b97f9d7a]*/ +/*[clinic end generated code: output=87318e4edcdc2bb6 input=15e6fd44fed32743]*/ { - if (Py_SIZE(self) <= PY_SSIZE_T_MAX / self->ob_descr->itemsize) { - return PyBytes_FromStringAndSize(self->ob_item, - Py_SIZE(self) * self->ob_descr->itemsize); - } else { - return PyErr_NoMemory(); + Py_ssize_t n = Py_SIZE(self); + int itemsize = self->ob_descr->itemsize; + PyObject *bytes = PyBytes_FromStringAndSize(NULL, n * itemsize); + if (bytes == NULL) { + return NULL; + } + if (n > 0) { + array_copy_raw(PyBytes_AS_STRING(bytes), self->ob_item, n, + itemsize, array_is_exclusive(self)); } + return bytes; } /*[clinic input] +@critical_section array.array.fromunicode ustr: unicode @@ -1962,7 +2502,7 @@ data to an array of some other type. static PyObject * array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) -/*[clinic end generated code: output=24359f5e001a7f2b input=01fa592ec7b948b6]*/ +/*[clinic end generated code: output=24359f5e001a7f2b input=f92d1b5af53561b0]*/ { const char *typecode = self->ob_descr->typecode; if (strcmp(typecode, "w") != 0) { @@ -1979,20 +2519,36 @@ array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) if (new_size < 0 || (size_t)new_size > PY_SSIZE_T_MAX / sizeof(Py_UCS4)) { return PyErr_NoMemory(); } - if (array_resize(self, new_size) == -1) { + if (ustr_length == 0) { + Py_RETURN_NONE; + } + if (array_reserve(self, new_size) == -1) { return NULL; } - // must not fail - Py_UCS4 *u = PyUnicode_AsUCS4(ustr, ((Py_UCS4*)self->ob_item) + old_size, - ustr_length, 0); - assert(u != NULL); - (void)u; // Suppress unused_variable warning. + Py_UCS4 *dst = ((Py_UCS4 *)self->ob_item) + old_size; + if (array_is_exclusive(self)) { + // must not fail + Py_UCS4 *u = PyUnicode_AsUCS4(ustr, dst, ustr_length, 0); + assert(u != NULL); + (void)u; // Suppress unused_variable warning. + } + else { + Py_UCS4 *tmp = PyUnicode_AsUCS4Copy(ustr); + if (tmp == NULL) { + return NULL; + } + array_copy_raw((char *)dst, (char *)tmp, ustr_length, + sizeof(Py_UCS4), false); + PyMem_Free(tmp); + } + array_set_size(self, new_size); Py_RETURN_NONE; } /*[clinic input] +@critical_section array.array.tounicode Extends this array with data from the unicode string ustr. @@ -2005,7 +2561,7 @@ some other type. static PyObject * array_array_tounicode_impl(arrayobject *self) -/*[clinic end generated code: output=08e442378336e1ef input=d4d5f398aa71a2be]*/ +/*[clinic end generated code: output=08e442378336e1ef input=0bcded7ac2cde2af]*/ { const char *typecode = self->ob_descr->typecode; if (strcmp(typecode, "w") != 0) { @@ -2014,11 +2570,23 @@ array_array_tounicode_impl(arrayobject *self) return NULL; } int byteorder = 0; // native byteorder - return PyUnicode_DecodeUTF32((const char *) self->ob_item, Py_SIZE(self) * 4, - NULL, &byteorder); + Py_ssize_t n = Py_SIZE(self); + if (n == 0 || array_is_exclusive(self)) { + return PyUnicode_DecodeUTF32((const char *)self->ob_item, + n * 4, NULL, &byteorder); + } + char *tmp = PyMem_Malloc(n * 4); + if (tmp == NULL) { + return PyErr_NoMemory(); + } + array_copy_raw(tmp, self->ob_item, n, 4, false); + PyObject *res = PyUnicode_DecodeUTF32(tmp, n * 4, NULL, &byteorder); + PyMem_Free(tmp); + return res; } /*[clinic input] +@critical_section array.array.__sizeof__ Size of the array in memory, in bytes. @@ -2026,7 +2594,7 @@ Size of the array in memory, in bytes. static PyObject * array_array___sizeof___impl(arrayobject *self) -/*[clinic end generated code: output=d8e1c61ebbe3eaed input=805586565bf2b3c6]*/ +/*[clinic end generated code: output=d8e1c61ebbe3eaed input=e88bfd334166dcb7]*/ { size_t res = _PyObject_SIZE(Py_TYPE(self)); res += (size_t)self->allocated * (size_t)self->ob_descr->itemsize; @@ -2492,6 +3060,7 @@ array__array_reconstructor_impl(PyObject *module, PyTypeObject *arraytype, } /*[clinic input] +@critical_section array.array.__reduce_ex__ cls: defining_class @@ -2504,7 +3073,7 @@ Return state information for pickling. static PyObject * array_array___reduce_ex___impl(arrayobject *self, PyTypeObject *cls, PyObject *value) -/*[clinic end generated code: output=4958ee5d79452ad5 input=19968cf0f91d3eea]*/ +/*[clinic end generated code: output=4958ee5d79452ad5 input=18c90a4cad7ac527]*/ { PyObject *dict; PyObject *result; @@ -2516,12 +3085,25 @@ array_array___reduce_ex___impl(arrayobject *self, PyTypeObject *cls, array_state *state = get_array_state_by_class(cls); assert(state != NULL); - if (state->array_reconstructor == NULL) { - state->array_reconstructor = PyImport_ImportModuleAttrString( - "array", "_array_reconstructor"); - if (state->array_reconstructor == NULL) { + PyObject *array_reconstructor = + FT_ATOMIC_LOAD_PTR_ACQUIRE(state->array_reconstructor); + if (array_reconstructor == NULL) { + array_reconstructor = PyImport_ImportModuleAttrString( + "array", "_array_reconstructor"); + if (array_reconstructor == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyObject *expected = NULL; + if (!_Py_atomic_compare_exchange_ptr(&state->array_reconstructor, + &expected, array_reconstructor)) { + // Another thread stored it first. + Py_DECREF(array_reconstructor); + array_reconstructor = expected; + } +#else + state->array_reconstructor = array_reconstructor; +#endif } if (!PyLong_Check(value)) { @@ -2572,9 +3154,9 @@ array_array___reduce_ex___impl(arrayobject *self, PyTypeObject *cls, return NULL; } - assert(state->array_reconstructor != NULL); + assert(array_reconstructor != NULL); result = Py_BuildValue( - "O(OsiN)O", state->array_reconstructor, Py_TYPE(self), typecode, + "O(OsiN)O", array_reconstructor, Py_TYPE(self), typecode, mformat_code, array_str, dict); Py_DECREF(dict); return result; @@ -2633,8 +3215,9 @@ static PyMethodDef array_methods[] = { }; static PyObject * -array_repr(PyObject *op) +array_repr_lock_held(PyObject *op) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); const char *typecode; PyObject *s, *v = NULL; Py_ssize_t len; @@ -2660,22 +3243,24 @@ array_repr(PyObject *op) return s; } +static PyObject * +array_repr(PyObject *op) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_repr_lock_held(op); + Py_END_CRITICAL_SECTION(); + return ret; +} + static PyObject* -array_subscr(PyObject *op, PyObject *item) +array_subscr_slice_lock_held(PyObject *op, PyObject *item) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); arrayobject *self = arrayobject_CAST(op); array_state *state = find_array_state_by_type(Py_TYPE(self)); - if (PyIndex_Check(item)) { - Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); - if (i==-1 && PyErr_Occurred()) { - return NULL; - } - if (i < 0) - i += Py_SIZE(self); - return array_item(op, i); - } - else if (PySlice_Check(item)) { + if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelength, i; size_t cur; PyObject* result; @@ -2695,9 +3280,9 @@ array_subscr(PyObject *op, PyObject *item) PyObject *result = newarrayobject_untracked(state->ArrayType, slicelength, self->ob_descr); if (result == NULL) return NULL; - memcpy(((arrayobject *)result)->ob_item, - self->ob_item + start * itemsize, - slicelength * itemsize); + array_copy_raw(((arrayobject *)result)->ob_item, + self->ob_item + start * itemsize, + slicelength, itemsize, array_is_exclusive(self)); _PyObject_GC_TRACK(result); return result; } @@ -2707,11 +3292,12 @@ array_subscr(PyObject *op, PyObject *item) ar = (arrayobject*)result; + bool exclusive = array_is_exclusive(self); for (cur = start, i = 0; i < slicelength; cur += step, i++) { - memcpy(ar->ob_item + i*itemsize, - self->ob_item + cur*itemsize, - itemsize); + array_copy_raw(ar->ob_item + i*itemsize, + self->ob_item + cur*itemsize, + 1, itemsize, exclusive); } _PyObject_GC_TRACK(result); @@ -2725,12 +3311,40 @@ array_subscr(PyObject *op, PyObject *item) } } +static PyObject * +array_subscr(PyObject *op, PyObject *item) +{ + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i==-1 && PyErr_Occurred()) { + return NULL; + } + Py_ssize_t size = array_get_size(op); + if (i < 0) { + i += size; + } + return array_item(op, i); + } + + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_subscr_slice_lock_held(op, item); + Py_END_CRITICAL_SECTION(); + return ret; +} + static int -array_ass_subscr(PyObject *op, PyObject *item, PyObject *value) +array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) { - Py_ssize_t start, stop, step, slicelength, needed; + array_state* state = find_array_state_by_type(Py_TYPE(op)); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); +#ifdef Py_DEBUG + if (value != NULL && array_Check(value, state)) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(value); + } +#endif arrayobject *self = arrayobject_CAST(op); - array_state* state = find_array_state_by_type(Py_TYPE(self)); + Py_ssize_t start, stop, step, slicelength, needed; arrayobject* other; int itemsize; @@ -2746,15 +3360,13 @@ array_ass_subscr(PyObject *op, PyObject *item, PyObject *value) "array assignment index out of range"); return -1; } - if (value == NULL) { - /* Fall through to slice assignment */ - start = i; - stop = i + 1; - step = 1; - slicelength = 1; - } - else - return (*self->ob_descr->setitem)(self, i, value); + // array_ass_subscr() handles a[i] = value without the lock. + assert(value == NULL); + /* Fall through to slice assignment */ + start = i; + stop = i + 1; + step = 1; + slicelength = 1; } else if (PySlice_Check(item)) { if (PySlice_Unpack(item, &start, &stop, &step) < 0) { @@ -2781,7 +3393,7 @@ array_ass_subscr(PyObject *op, PyObject *item, PyObject *value) value = array_slice(other, 0, needed); if (value == NULL) return -1; - ret = array_ass_subscr(op, item, value); + ret = array_ass_subscr_lock_held(op, item, value); Py_DECREF(value); return ret; } @@ -2811,26 +3423,32 @@ array_ass_subscr(PyObject *op, PyObject *item, PyObject *value) return -1; } + bool exclusive = array_is_exclusive(self); if (step == 1) { + Py_ssize_t size = Py_SIZE(self); + Py_ssize_t newsize = size + needed - slicelength; if (slicelength > needed) { - memmove(self->ob_item + (start + needed) * itemsize, - self->ob_item + stop * itemsize, - (Py_SIZE(self) - stop) * itemsize); - if (array_resize(self, Py_SIZE(self) + - needed - slicelength) < 0) + array_copy_raw(self->ob_item + (start + needed) * itemsize, + self->ob_item + stop * itemsize, + size - stop, itemsize, exclusive); + if (array_resize(self, newsize) < 0) return -1; } else if (slicelength < needed) { - if (array_resize(self, Py_SIZE(self) + - needed - slicelength) < 0) + if (array_reserve(self, newsize) < 0) return -1; - memmove(self->ob_item + (start + needed) * itemsize, - self->ob_item + stop * itemsize, - (Py_SIZE(self) - start - needed) * itemsize); + array_copy_raw(self->ob_item + (start + needed) * itemsize, + self->ob_item + stop * itemsize, + size - stop, itemsize, exclusive); + } + if (needed > 0) { + array_copy_raw(self->ob_item + start * itemsize, + other->ob_item, needed, itemsize, + exclusive && array_is_exclusive(other)); + } + if (newsize > size) { + array_set_size(self, newsize); } - if (needed > 0) - memcpy(self->ob_item + start * itemsize, - other->ob_item, needed * itemsize); return 0; } else if (needed == 0) { @@ -2849,15 +3467,15 @@ array_ass_subscr(PyObject *op, PyObject *item, PyObject *value) if (cur + step >= (size_t)Py_SIZE(self)) lim = Py_SIZE(self) - cur - 1; - memmove(self->ob_item + (cur - i) * itemsize, - self->ob_item + (cur + 1) * itemsize, - lim * itemsize); + array_copy_raw(self->ob_item + (cur - i) * itemsize, + self->ob_item + (cur + 1) * itemsize, + lim, itemsize, exclusive); } cur = start + (size_t)slicelength * step; if (cur < (size_t)Py_SIZE(self)) { - memmove(self->ob_item + (cur-slicelength) * itemsize, - self->ob_item + cur * itemsize, - (Py_SIZE(self) - cur) * itemsize); + array_copy_raw(self->ob_item + (cur-slicelength) * itemsize, + self->ob_item + cur * itemsize, + Py_SIZE(self) - cur, itemsize, exclusive); } if (array_resize(self, Py_SIZE(self) - slicelength) < 0) return -1; @@ -2874,29 +3492,60 @@ array_ass_subscr(PyObject *op, PyObject *item, PyObject *value) needed, slicelength); return -1; } + exclusive = exclusive && array_is_exclusive(other); for (cur = start, i = 0; i < slicelength; cur += step, i++) { - memcpy(self->ob_item + cur * itemsize, - other->ob_item + i * itemsize, - itemsize); + array_copy_raw(self->ob_item + cur * itemsize, + other->ob_item + i * itemsize, + 1, itemsize, exclusive); } return 0; } } -static const _Py_ALIGNED_DEF(ALIGNOF_MAX_ALIGN_T, char) emptybuf[] = ""; +static int +array_ass_subscr(PyObject *op, PyObject* item, PyObject* value) +{ + int ret; + + if (value != NULL) { + if (PyIndex_Check(item)) { + // a[i] = value + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i == -1 && PyErr_Occurred()) + return -1; + if (i < 0) + i += array_get_size(op); + return setarrayitem_threadsafe(op, i, value); + } + array_state* state = find_array_state_by_type(Py_TYPE(op)); + if (array_Check(value, state)) { + Py_BEGIN_CRITICAL_SECTION2(op, value); + ret = array_ass_subscr_lock_held(op, item, value); + Py_END_CRITICAL_SECTION2(); + return ret; + } + } + + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_ass_subscr_lock_held(op, item, value); + Py_END_CRITICAL_SECTION(); + return ret; +} +static const _Py_ALIGNED_DEF(ALIGNOF_MAX_ALIGN_T, char) emptybuf[] = ""; static int -array_buffer_getbuf(PyObject *op, Py_buffer *view, int flags) +array_buffer_getbuf_lock_held(PyObject *op, Py_buffer *view, int flags) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + arrayobject *self = arrayobject_CAST(op); if (view == NULL) { PyErr_SetString(PyExc_BufferError, "array_buffer_getbuf: view==NULL argument is obsolete"); return -1; } - arrayobject *self = arrayobject_CAST(op); view->buf = (void *)self->ob_item; view->obj = Py_NewRef(self); if (view->buf == NULL) @@ -2923,52 +3572,40 @@ array_buffer_getbuf(PyObject *op, Py_buffer *view, int flags) return 0; } +static int +array_buffer_getbuf(PyObject *op, Py_buffer *view, int flags) +{ + int ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_buffer_getbuf_lock_held(op, view, flags); + Py_END_CRITICAL_SECTION(); + return ret; +} + static void array_buffer_relbuf(PyObject *op, Py_buffer *Py_UNUSED(view)) { + Py_BEGIN_CRITICAL_SECTION(op); arrayobject *self = arrayobject_CAST(op); self->ob_exports--; + assert(self->ob_exports >= 0); + Py_END_CRITICAL_SECTION(); } static PyObject * -array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, + const char *s) { +#ifdef Py_DEBUG + if (initial != NULL) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(initial); + } +#endif array_state *state = find_array_state_by_type(type); - const char *s; - PyObject *initial = NULL, *it = NULL; + PyObject *it = NULL; const struct arraydescr *descr; - - if ((type == state->ArrayType || - type->tp_init == state->ArrayType->tp_init) && - !_PyArg_NoKeywords("array.array", kwds)) - return NULL; - - if (!PyArg_ParseTuple(args, "s|O:array", &s, &initial)) - return NULL; - - if (PySys_Audit("array.__new__", "sO", - s, initial ? initial : Py_None) < 0) { - return NULL; - } - bool is_unicode = (strcmp(s, "w") == 0); - if (initial && !is_unicode) { - if (PyUnicode_Check(initial)) { - PyErr_Format(PyExc_TypeError, "cannot use a str to initialize " - "an array with typecode '%s'", s); - return NULL; - } - else if (array_Check(initial, state)) { - const char *is = ((arrayobject*)initial)->ob_descr->typecode; - if (strcmp(is, "w") == 0) { - PyErr_Format(PyExc_TypeError, "cannot use a unicode array to " - "initialize an array with typecode '%s'", s); - return NULL; - } - } - } - if (!(initial == NULL || PyList_Check(initial) || PyByteArray_Check(initial) || PyBytes_Check(initial) @@ -3008,6 +3645,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (len > 0 && !array_Check(initial, state)) { Py_ssize_t i; + arrayobject *ap = arrayobject_CAST(a); for (i = 0; i < len; i++) { PyObject *v = PySequence_GetItem(initial, i); @@ -3016,12 +3654,15 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_XDECREF(it); return NULL; } - if (setarrayitem(a, i, v) != 0) { + _Py_ALIGNED_DEF(8, char) buf[16]; + if ((*descr->pack)(v, buf) != 0) { Py_DECREF(v); Py_DECREF(a); Py_XDECREF(it); return NULL; } + // ap is private, so no lock is needed. + array_store_item(ap, i, buf); Py_DECREF(v); } } @@ -3039,24 +3680,30 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) else if (initial != NULL && PyUnicode_Check(initial)) { assert(strcmp(descr->typecode, "w") == 0); Py_ssize_t n = PyUnicode_GET_LENGTH(initial); - Py_UCS4 *ustr = PyUnicode_AsUCS4Copy(initial); - if (ustr == NULL) { - Py_DECREF(a); - Py_XDECREF(it); - return NULL; + if (n > 0) { + arrayobject *self = (arrayobject *)a; + assert(self->ob_item == NULL); + self->ob_item = array_items_alloc(n, sizeof(Py_UCS4)); + if (self->ob_item == NULL) { + Py_DECREF(a); + Py_XDECREF(it); + return PyErr_NoMemory(); + } + Py_UCS4 *u = PyUnicode_AsUCS4(initial, + (Py_UCS4 *)self->ob_item, + n, 0); + assert(u != NULL); + (void)u; + Py_SET_SIZE(self, n); + self->allocated = n; } - - arrayobject *self = (arrayobject *)a; - // self->ob_item may be NULL but it is safe. - PyMem_Free(self->ob_item); - self->ob_item = (char *)ustr; - Py_SET_SIZE(self, n); - self->allocated = n; } else if (initial != NULL && array_Check(initial, state) && len > 0) { arrayobject *self = (arrayobject *)a; arrayobject *other = (arrayobject *)initial; - memcpy(self->ob_item, other->ob_item, len * other->ob_descr->itemsize); + array_copy_raw(self->ob_item, other->ob_item, len, + other->ob_descr->itemsize, + array_is_exclusive(other)); } if (it != NULL) { if (array_iter_extend((arrayobject *)a, it) == -1) { @@ -3087,6 +3734,56 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } +static PyObject * +array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + array_state *state = find_array_state_by_type(type); + const char *s; + PyObject *initial = NULL; + + if ((type == state->ArrayType || + type->tp_init == state->ArrayType->tp_init) && + !_PyArg_NoKeywords("array.array", kwds)) + return NULL; + + if (!PyArg_ParseTuple(args, "s|O:array", &s, &initial)) + return NULL; + + if (PySys_Audit("array.__new__", "sO", + s, initial ? initial : Py_None) < 0) { + return NULL; + } + + bool is_unicode = (strcmp(s, "w") == 0); + + if (initial && !is_unicode) { + if (PyUnicode_Check(initial)) { + PyErr_Format(PyExc_TypeError, "cannot use a str to initialize " + "an array with typecode '%s'", s); + return NULL; + } + else if (array_Check(initial, state)) { + const char *is = ((arrayobject*)initial)->ob_descr->typecode; + if (strcmp(is, "w") == 0) { + PyErr_Format(PyExc_TypeError, "cannot use a unicode array to " + "initialize an array with typecode '%s'", s); + return NULL; + } + } + } + + PyObject *ret; + if (initial == NULL) { + ret = array_new_internal_lock_held(type, initial, s); + } + else { + Py_BEGIN_CRITICAL_SECTION(initial); + ret = array_new_internal_lock_held(type, initial, s); + Py_END_CRITICAL_SECTION(); + } + return ret; +} + PyDoc_STRVAR(module_doc, "This module defines an object type which can efficiently represent\n\ @@ -3229,8 +3926,7 @@ array_iter(PyObject *op) return NULL; it->ao = (arrayobject*)Py_NewRef(ao); - it->index = 0; - it->getitem = ao->ob_descr->getitem; + it->index = 0; // -1 indicates exhausted PyObject_GC_Track(it); return (PyObject *)it; } @@ -3240,22 +3936,32 @@ arrayiter_next(PyObject *op) { arrayiterobject *it = arrayiterobject_CAST(op); assert(it != NULL); -#ifndef NDEBUG - array_state *state = find_array_state_by_type(Py_TYPE(it)); - assert(PyObject_TypeCheck(it, state->ArrayIterType)); -#endif - arrayobject *ao = it->ao; - if (ao == NULL) { + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->index); + if (index < 0) { return NULL; } + arrayobject *ao = it->ao; #ifndef NDEBUG + array_state *state = find_array_state_by_type(Py_TYPE(it)); + assert(PyObject_TypeCheck(it, state->ArrayIterType)); assert(array_Check(ao, state)); #endif - if (it->index < Py_SIZE(ao)) { - return (*it->getitem)(ao, it->index++); +#ifdef Py_GIL_DISABLED + PyObject *ret = getarrayitem_threadsafe((PyObject *)ao, index); + if (ret != NULL || PyErr_Occurred()) { + // A failed conversion skips the item, it does not end the iteration. + _Py_atomic_store_ssize_relaxed(&it->index, index + 1); + return ret; } - it->ao = NULL; - Py_DECREF(ao); + _Py_atomic_store_ssize_relaxed(&it->index, -1); +#else + if (index < Py_SIZE(ao)) { + it->index = index + 1; + return getarrayitem(ao, index); + } + it->index = -1; + Py_CLEAR(it->ao); +#endif return NULL; } @@ -3292,14 +3998,14 @@ static PyObject * array_arrayiterator___reduce___impl(arrayiterobject *self, PyTypeObject *cls) /*[clinic end generated code: output=4b032417a2c8f5e6 input=ac64e65a87ad452e]*/ { - array_state *state = get_array_state_by_class(cls); assert(state != NULL); PyObject *func = _PyEval_GetBuiltin(state->str_iter); - if (self->ao == NULL) { - return Py_BuildValue("N(())", func); + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->index); + if (index >= 0) { + return Py_BuildValue("N(O)n", func, self->ao, index); } - return Py_BuildValue("N(O)n", func, self->ao, self->index); + return Py_BuildValue("N(())", func); } /*[clinic input] @@ -3316,17 +4022,20 @@ array_arrayiterator___setstate___impl(arrayiterobject *self, PyObject *state) /*[clinic end generated code: output=d7837ae4ac1fd8b9 input=f47d5ceda19e787b]*/ { Py_ssize_t index = PyLong_AsSsize_t(state); - if (index == -1 && PyErr_Occurred()) + if (index == -1 && PyErr_Occurred()) { return NULL; - arrayobject *ao = self->ao; - if (ao != NULL) { - if (index < 0) { - index = 0; + } + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->index) >= 0) { + if (index < -1) { + index = -1; } - else if (index > Py_SIZE(ao)) { - index = Py_SIZE(ao); /* iterator exhausted */ + else { + Py_ssize_t size = array_get_size(self->ao); + if (index > size) { + index = size; /* iterator at end */ + } } - self->index = index; + FT_ATOMIC_STORE_SSIZE_RELAXED(self->index, index); } Py_RETURN_NONE; } @@ -3359,7 +4068,7 @@ static PyType_Spec arrayiter_spec = { /*********************** Install Module **************************/ static int -array_traverse(PyObject *module, visitproc visit, void *arg) +arraymodule_traverse(PyObject *module, visitproc visit, void *arg) { array_state *state = get_array_state(module); Py_VISIT(state->ArrayType); @@ -3370,7 +4079,7 @@ array_traverse(PyObject *module, visitproc visit, void *arg) } static int -array_clear(PyObject *module) +arraymodule_clear(PyObject *module) { array_state *state = get_array_state(module); Py_CLEAR(state->ArrayType); @@ -3385,9 +4094,9 @@ array_clear(PyObject *module) } static void -array_free(void *module) +arraymodule_free(void *module) { - (void)array_clear((PyObject *)module); + (void)arraymodule_clear((PyObject *)module); } /* No functions in array module. */ @@ -3414,7 +4123,7 @@ do { \ } while (0) static int -array_modexec(PyObject *m) +arraymodule_modexec(PyObject *m) { array_state *state = get_array_state(m); PyObject *typecodes; @@ -3486,7 +4195,7 @@ array_modexec(PyObject *m) static PyModuleDef_Slot arrayslots[] = { _Py_ABI_SLOT, - {Py_mod_exec, array_modexec}, + {Py_mod_exec, arraymodule_modexec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} @@ -3500,9 +4209,9 @@ static struct PyModuleDef arraymodule = { .m_doc = module_doc, .m_methods = a_methods, .m_slots = arrayslots, - .m_traverse = array_traverse, - .m_clear = array_clear, - .m_free = array_free, + .m_traverse = arraymodule_traverse, + .m_clear = arraymodule_clear, + .m_free = arraymodule_free, }; diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index 41798e098389c2..7d6ccfe596d5e9 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -3,6 +3,7 @@ preserve [clinic start generated code]*/ #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() @@ -21,7 +22,13 @@ array_array_clear_impl(arrayobject *self); static PyObject * array_array_clear(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_clear_impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_clear_impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array___copy____doc__, @@ -39,7 +46,13 @@ array_array___copy___impl(arrayobject *self); static PyObject * array_array___copy__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array___copy___impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array___copy___impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array___deepcopy____doc__, @@ -59,7 +72,9 @@ array_array___deepcopy__(PyObject *self, PyObject *unused) { PyObject *return_value = NULL; + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array___deepcopy___impl((arrayobject *)self, unused); + Py_END_CRITICAL_SECTION(); return return_value; } @@ -81,7 +96,9 @@ array_array_count(PyObject *self, PyObject *v) { PyObject *return_value = NULL; + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array_count_impl((arrayobject *)self, v); + Py_END_CRITICAL_SECTION(); return return_value; } @@ -126,7 +143,9 @@ array_array_index(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array_index_impl((arrayobject *)self, v, start, stop); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -149,7 +168,9 @@ array_array_remove(PyObject *self, PyObject *v) { PyObject *return_value = NULL; + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array_remove_impl((arrayobject *)self, v); + Py_END_CRITICAL_SECTION(); return return_value; } @@ -193,7 +214,9 @@ array_array_pop(PyObject *self, PyObject *const *args, Py_ssize_t nargs) i = ival; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array_pop_impl((arrayobject *)self, i); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -265,7 +288,9 @@ array_array_insert(PyObject *self, PyObject *const *args, Py_ssize_t nargs) i = ival; } v = args[1]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array_insert_impl((arrayobject *)self, i, v); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -289,7 +314,13 @@ array_array_buffer_info_impl(arrayobject *self); static PyObject * array_array_buffer_info(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_buffer_info_impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_buffer_info_impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array_append__doc__, @@ -309,7 +340,9 @@ array_array_append(PyObject *self, PyObject *v) { PyObject *return_value = NULL; + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array_append_impl((arrayobject *)self, v); + Py_END_CRITICAL_SECTION(); return return_value; } @@ -333,7 +366,13 @@ array_array_byteswap_impl(arrayobject *self); static PyObject * array_array_byteswap(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_byteswap_impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_byteswap_impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array_reverse__doc__, @@ -351,7 +390,13 @@ array_array_reverse_impl(arrayobject *self); static PyObject * array_array_reverse(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_reverse_impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_reverse_impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array_fromfile__doc__, @@ -429,7 +474,9 @@ array_array_tofile(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ goto exit; } f = args[0]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array_tofile_impl((arrayobject *)self, cls, f); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -452,7 +499,9 @@ array_array_fromlist(PyObject *self, PyObject *list) { PyObject *return_value = NULL; + Py_BEGIN_CRITICAL_SECTION2(self, list); return_value = array_array_fromlist_impl((arrayobject *)self, list); + Py_END_CRITICAL_SECTION2(); return return_value; } @@ -472,7 +521,13 @@ array_array_tolist_impl(arrayobject *self); static PyObject * array_array_tolist(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_tolist_impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_tolist_impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array_frombytes__doc__, @@ -496,7 +551,9 @@ array_array_frombytes(PyObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &buffer, PyBUF_SIMPLE) != 0) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array_frombytes_impl((arrayobject *)self, &buffer); + Py_END_CRITICAL_SECTION(); exit: /* Cleanup for buffer */ @@ -522,7 +579,13 @@ array_array_tobytes_impl(arrayobject *self); static PyObject * array_array_tobytes(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_tobytes_impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_tobytes_impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array_fromunicode__doc__, @@ -552,7 +615,9 @@ array_array_fromunicode(PyObject *self, PyObject *arg) goto exit; } ustr = arg; + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array_fromunicode_impl((arrayobject *)self, ustr); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -578,7 +643,13 @@ array_array_tounicode_impl(arrayobject *self); static PyObject * array_array_tounicode(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_tounicode_impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_tounicode_impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array___sizeof____doc__, @@ -596,7 +667,13 @@ array_array___sizeof___impl(arrayobject *self); static PyObject * array_array___sizeof__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array___sizeof___impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array___sizeof___impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array__array_reconstructor__doc__, @@ -673,7 +750,9 @@ array_array___reduce_ex__(PyObject *self, PyTypeObject *cls, PyObject *const *ar goto exit; } value = args[0]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array___reduce_ex___impl((arrayobject *)self, cls, value); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -722,4 +801,4 @@ array_arrayiterator___setstate__(PyObject *self, PyObject *state) return return_value; } -/*[clinic end generated code: output=172a9af0316afc14 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0b97c49fee5c6d36 input=a9049054013a1b77]*/