Issue #12834: Fix PyBuffer_ToContiguous() for non-contiguous arrays.
diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py
index b6cb3ac..c91f3c4 100644
--- a/Lib/test/test_buffer.py
+++ b/Lib/test/test_buffer.py
@@ -53,6 +53,11 @@
'f':0, 'd':0, 'P':0
}
+# NumPy does not have 'n' or 'N':
+if numpy_array:
+ del NATIVE['n']
+ del NATIVE['N']
+
if struct:
try:
# Add "qQ" if present in native mode.
@@ -855,11 +860,49 @@
is_contiguous(result, 'F') and order == 'C':
# The flattened list is already in C-order.
expected = ndarray(flattened, shape=shape, format=ff)
- contig = get_contiguous(result, PyBUF_READ, order)
+
contig = get_contiguous(result, PyBUF_READ, order)
self.assertEqual(contig.tobytes(), b)
self.assertTrue(cmp_contig(contig, expected))
+ if ndim == 0:
+ continue
+
+ nmemb = len(flattened)
+ ro = 0 if readonly else ND_WRITABLE
+
+ ### See comment in test_py_buffer_to_contiguous for an
+ ### explanation why these tests are valid.
+
+ # To 'C'
+ contig = py_buffer_to_contiguous(result, 'C', PyBUF_FULL_RO)
+ self.assertEqual(len(contig), nmemb * itemsize)
+ initlst = [struct.unpack_from(fmt, contig, n*itemsize)[0]
+ for n in range(nmemb)]
+
+ y = ndarray(initlst, shape=shape, flags=ro, format=fmt)
+ self.assertEqual(memoryview(y), memoryview(result))
+
+ # To 'F'
+ contig = py_buffer_to_contiguous(result, 'F', PyBUF_FULL_RO)
+ self.assertEqual(len(contig), nmemb * itemsize)
+ initlst = [struct.unpack_from(fmt, contig, n*itemsize)[0]
+ for n in range(nmemb)]
+
+ y = ndarray(initlst, shape=shape, flags=ro|ND_FORTRAN,
+ format=fmt)
+ self.assertEqual(memoryview(y), memoryview(result))
+
+ # To 'A'
+ contig = py_buffer_to_contiguous(result, 'A', PyBUF_FULL_RO)
+ self.assertEqual(len(contig), nmemb * itemsize)
+ initlst = [struct.unpack_from(fmt, contig, n*itemsize)[0]
+ for n in range(nmemb)]
+
+ f = ND_FORTRAN if is_contiguous(result, 'F') else 0
+ y = ndarray(initlst, shape=shape, flags=f|ro, format=fmt)
+ self.assertEqual(memoryview(y), memoryview(result))
+
if is_memoryview_format(fmt):
try:
m = memoryview(result)
@@ -1805,6 +1848,9 @@
self.assertEqual(mvlist, ylist)
if numpy_array:
+ # XXX NumPy (as far as it compiles with 3.3) currently
+ # segfaults here. Wait for a stable 3.3 compatible version.
+ continue
shape = t[3]
if 0 in shape:
continue # http://projects.scipy.org/numpy/ticket/1910
@@ -1884,6 +1930,9 @@
self.assertEqual(mr.tolist(), yrlist)
if numpy_array:
+ # XXX NumPy (as far as it compiles with 3.3) currently
+ # segfaults here. Wait for a stable 3.3 compatible version.
+ continue
if 0 in lshape or 0 in rshape:
continue # http://projects.scipy.org/numpy/ticket/1910
@@ -2020,6 +2069,246 @@
nd = ndarray(list(range(12)), shape=[2,2,3], format='L')
self.assertEqual(hash(nd), hash(nd.tobytes()))
+ def test_py_buffer_to_contiguous(self):
+
+ # The requests are used in _testbuffer.c:py_buffer_to_contiguous
+ # to generate buffers without full information for testing.
+ requests = (
+ # distinct flags
+ PyBUF_INDIRECT, PyBUF_STRIDES, PyBUF_ND, PyBUF_SIMPLE,
+ # compound requests
+ PyBUF_FULL, PyBUF_FULL_RO,
+ PyBUF_RECORDS, PyBUF_RECORDS_RO,
+ PyBUF_STRIDED, PyBUF_STRIDED_RO,
+ PyBUF_CONTIG, PyBUF_CONTIG_RO,
+ )
+
+ # no buffer interface
+ self.assertRaises(TypeError, py_buffer_to_contiguous, {}, 'F',
+ PyBUF_FULL_RO)
+
+ # scalar, read-only request
+ nd = ndarray(9, shape=(), format="L", flags=ND_WRITABLE)
+ for order in ['C', 'F', 'A']:
+ for request in requests:
+ b = py_buffer_to_contiguous(nd, order, request)
+ self.assertEqual(b, nd.tobytes())
+
+ # zeros in shape
+ nd = ndarray([1], shape=[0], format="L", flags=ND_WRITABLE)
+ for order in ['C', 'F', 'A']:
+ for request in requests:
+ b = py_buffer_to_contiguous(nd, order, request)
+ self.assertEqual(b, b'')
+
+ nd = ndarray(list(range(8)), shape=[2, 0, 7], format="L",
+ flags=ND_WRITABLE)
+ for order in ['C', 'F', 'A']:
+ for request in requests:
+ b = py_buffer_to_contiguous(nd, order, request)
+ self.assertEqual(b, b'')
+
+ ### One-dimensional arrays are trivial, since Fortran and C order
+ ### are the same.
+
+ # one-dimensional
+ for f in [0, ND_FORTRAN]:
+ nd = ndarray([1], shape=[1], format="h", flags=f|ND_WRITABLE)
+ ndbytes = nd.tobytes()
+ for order in ['C', 'F', 'A']:
+ for request in requests:
+ b = py_buffer_to_contiguous(nd, order, request)
+ self.assertEqual(b, ndbytes)
+
+ nd = ndarray([1, 2, 3], shape=[3], format="b", flags=f|ND_WRITABLE)
+ ndbytes = nd.tobytes()
+ for order in ['C', 'F', 'A']:
+ for request in requests:
+ b = py_buffer_to_contiguous(nd, order, request)
+ self.assertEqual(b, ndbytes)
+
+ # one-dimensional, non-contiguous input
+ nd = ndarray([1, 2, 3], shape=[2], strides=[2], flags=ND_WRITABLE)
+ ndbytes = nd.tobytes()
+ for order in ['C', 'F', 'A']:
+ for request in [PyBUF_STRIDES, PyBUF_FULL]:
+ b = py_buffer_to_contiguous(nd, order, request)
+ self.assertEqual(b, ndbytes)
+
+ nd = nd[::-1]
+ ndbytes = nd.tobytes()
+ for order in ['C', 'F', 'A']:
+ for request in requests:
+ try:
+ b = py_buffer_to_contiguous(nd, order, request)
+ except BufferError:
+ continue
+ self.assertEqual(b, ndbytes)
+
+ ###
+ ### Multi-dimensional arrays:
+ ###
+ ### The goal here is to preserve the logical representation of the
+ ### input array but change the physical representation if necessary.
+ ###
+ ### _testbuffer example:
+ ### ====================
+ ###
+ ### C input array:
+ ### --------------
+ ### >>> nd = ndarray(list(range(12)), shape=[3, 4])
+ ### >>> nd.tolist()
+ ### [[0, 1, 2, 3],
+ ### [4, 5, 6, 7],
+ ### [8, 9, 10, 11]]
+ ###
+ ### Fortran output:
+ ### ---------------
+ ### >>> py_buffer_to_contiguous(nd, 'F', PyBUF_FULL_RO)
+ ### >>> b'\x00\x04\x08\x01\x05\t\x02\x06\n\x03\x07\x0b'
+ ###
+ ### The return value corresponds to this input list for
+ ### _testbuffer's ndarray:
+ ### >>> nd = ndarray([0,4,8,1,5,9,2,6,10,3,7,11], shape=[3,4],
+ ### flags=ND_FORTRAN)
+ ### >>> nd.tolist()
+ ### [[0, 1, 2, 3],
+ ### [4, 5, 6, 7],
+ ### [8, 9, 10, 11]]
+ ###
+ ### The logical array is the same, but the values in memory are now
+ ### in Fortran order.
+ ###
+ ### NumPy example:
+ ### ==============
+ ### _testbuffer's ndarray takes lists to initialize the memory.
+ ### Here's the same sequence in NumPy:
+ ###
+ ### C input:
+ ### --------
+ ### >>> nd = ndarray(buffer=bytearray(list(range(12))),
+ ### shape=[3, 4], dtype='B')
+ ### >>> nd
+ ### array([[ 0, 1, 2, 3],
+ ### [ 4, 5, 6, 7],
+ ### [ 8, 9, 10, 11]], dtype=uint8)
+ ###
+ ### Fortran output:
+ ### ---------------
+ ### >>> fortran_buf = nd.tostring(order='F')
+ ### >>> fortran_buf
+ ### b'\x00\x04\x08\x01\x05\t\x02\x06\n\x03\x07\x0b'
+ ###
+ ### >>> nd = ndarray(buffer=fortran_buf, shape=[3, 4],
+ ### dtype='B', order='F')
+ ###
+ ### >>> nd
+ ### array([[ 0, 1, 2, 3],
+ ### [ 4, 5, 6, 7],
+ ### [ 8, 9, 10, 11]], dtype=uint8)
+ ###
+
+ # multi-dimensional, contiguous input
+ lst = list(range(12))
+ for f in [0, ND_FORTRAN]:
+ nd = ndarray(lst, shape=[3, 4], flags=f|ND_WRITABLE)
+ if numpy_array:
+ na = numpy_array(buffer=bytearray(lst),
+ shape=[3, 4], dtype='B',
+ order='C' if f == 0 else 'F')
+
+ # 'C' request
+ if f == ND_FORTRAN: # 'F' to 'C'
+ x = ndarray(transpose(lst, [4, 3]), shape=[3, 4],
+ flags=ND_WRITABLE)
+ expected = x.tobytes()
+ else:
+ expected = nd.tobytes()
+ for request in requests:
+ try:
+ b = py_buffer_to_contiguous(nd, 'C', request)
+ except BufferError:
+ continue
+
+ self.assertEqual(b, expected)
+
+ # Check that output can be used as the basis for constructing
+ # a C array that is logically identical to the input array.
+ y = ndarray([v for v in b], shape=[3, 4], flags=ND_WRITABLE)
+ self.assertEqual(memoryview(y), memoryview(nd))
+
+ if numpy_array:
+ self.assertEqual(b, na.tostring(order='C'))
+
+ # 'F' request
+ if f == 0: # 'C' to 'F'
+ x = ndarray(transpose(lst, [3, 4]), shape=[4, 3],
+ flags=ND_WRITABLE)
+ else:
+ x = ndarray(lst, shape=[3, 4], flags=ND_WRITABLE)
+ expected = x.tobytes()
+ for request in [PyBUF_FULL, PyBUF_FULL_RO, PyBUF_INDIRECT,
+ PyBUF_STRIDES, PyBUF_ND]:
+ try:
+ b = py_buffer_to_contiguous(nd, 'F', request)
+ except BufferError:
+ continue
+ self.assertEqual(b, expected)
+
+ # Check that output can be used as the basis for constructing
+ # a Fortran array that is logically identical to the input array.
+ y = ndarray([v for v in b], shape=[3, 4], flags=ND_FORTRAN|ND_WRITABLE)
+ self.assertEqual(memoryview(y), memoryview(nd))
+
+ if numpy_array:
+ self.assertEqual(b, na.tostring(order='F'))
+
+ # 'A' request
+ if f == ND_FORTRAN:
+ x = ndarray(lst, shape=[3, 4], flags=ND_WRITABLE)
+ expected = x.tobytes()
+ else:
+ expected = nd.tobytes()
+ for request in [PyBUF_FULL, PyBUF_FULL_RO, PyBUF_INDIRECT,
+ PyBUF_STRIDES, PyBUF_ND]:
+ try:
+ b = py_buffer_to_contiguous(nd, 'A', request)
+ except BufferError:
+ continue
+
+ self.assertEqual(b, expected)
+
+ # Check that output can be used as the basis for constructing
+ # an array with order=f that is logically identical to the input
+ # array.
+ y = ndarray([v for v in b], shape=[3, 4], flags=f|ND_WRITABLE)
+ self.assertEqual(memoryview(y), memoryview(nd))
+
+ if numpy_array:
+ self.assertEqual(b, na.tostring(order='A'))
+
+ # multi-dimensional, non-contiguous input
+ nd = ndarray(list(range(12)), shape=[3, 4], flags=ND_WRITABLE|ND_PIL)
+
+ # 'C'
+ b = py_buffer_to_contiguous(nd, 'C', PyBUF_FULL_RO)
+ self.assertEqual(b, nd.tobytes())
+ y = ndarray([v for v in b], shape=[3, 4], flags=ND_WRITABLE)
+ self.assertEqual(memoryview(y), memoryview(nd))
+
+ # 'F'
+ b = py_buffer_to_contiguous(nd, 'F', PyBUF_FULL_RO)
+ x = ndarray(transpose(lst, [3, 4]), shape=[4, 3], flags=ND_WRITABLE)
+ self.assertEqual(b, x.tobytes())
+ y = ndarray([v for v in b], shape=[3, 4], flags=ND_FORTRAN|ND_WRITABLE)
+ self.assertEqual(memoryview(y), memoryview(nd))
+
+ # 'A'
+ b = py_buffer_to_contiguous(nd, 'A', PyBUF_FULL_RO)
+ self.assertEqual(b, nd.tobytes())
+ y = ndarray([v for v in b], shape=[3, 4], flags=ND_WRITABLE)
+ self.assertEqual(memoryview(y), memoryview(nd))
+
def test_memoryview_construction(self):
items_shape = [(9, []), ([1,2,3], [3]), (list(range(2*3*5)), [2,3,5])]