bpo-24658: Fix read/write greater than 2 GiB on macOS (GH-1705)
On macOS, fix reading from and writing into a file with a size larger than 2 GiB.
diff --git a/Include/fileutils.h b/Include/fileutils.h
index f0a8e2c..232d966 100644
--- a/Include/fileutils.h
+++ b/Include/fileutils.h
@@ -81,6 +81,19 @@
#ifndef Py_LIMITED_API
PyAPI_FUNC(PyObject *) _Py_device_encoding(int);
+#if defined(MS_WINDOWS) || defined(__APPLE__)
+ /* On Windows, the count parameter of read() is an int (bpo-9015, bpo-9611).
+ On macOS 10.13, read() and write() with more than INT_MAX bytes
+ fail with EINVAL (bpo-24658). */
+# define _PY_READ_MAX INT_MAX
+# define _PY_WRITE_MAX INT_MAX
+#else
+ /* write() should truncate the input to PY_SSIZE_T_MAX bytes,
+ but it's safer to do it ourself to have a portable behaviour */
+# define _PY_READ_MAX PY_SSIZE_T_MAX
+# define _PY_WRITE_MAX PY_SSIZE_T_MAX
+#endif
+
#ifdef MS_WINDOWS
struct _Py_stat_struct {
unsigned long st_dev;
diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py
index 21296cc..8870c72 100644
--- a/Lib/test/test_largefile.py
+++ b/Lib/test/test_largefile.py
@@ -5,12 +5,12 @@
import stat
import sys
import unittest
-from test.support import TESTFN, requires, unlink
+from test.support import TESTFN, requires, unlink, bigmemtest
import io # C implementation of io
import _pyio as pyio # Python implementation of io
# size of file to create (>2 GiB; 2 GiB == 2,147,483,648 bytes)
-size = 2500000000
+size = 2_500_000_000
class LargeFileTest:
"""Test that each file function works as expected for large
@@ -45,6 +45,15 @@
raise cls.failureException('File was not truncated by opening '
'with mode "wb"')
+ # _pyio.FileIO.readall() uses a temporary bytearray then casted to bytes,
+ # so memuse=2 is needed
+ @bigmemtest(size=size, memuse=2, dry_run=False)
+ def test_large_read(self, _size):
+ # bpo-24658: Test that a read greater than 2GB does not fail.
+ with self.open(TESTFN, "rb") as f:
+ self.assertEqual(len(f.read()), size + 1)
+ self.assertEqual(f.tell(), size + 1)
+
def test_osstat(self):
self.assertEqual(os.stat(TESTFN)[stat.ST_SIZE], size+1)
diff --git a/Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst b/Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst
new file mode 100644
index 0000000..ff660a1
--- /dev/null
+++ b/Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst
@@ -0,0 +1 @@
+On macOS, fix reading from and writing into a file with a size larger than 2 GiB.
\ No newline at end of file
diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c
index c0e43e0..44d51c9 100644
--- a/Modules/_io/fileio.c
+++ b/Modules/_io/fileio.c
@@ -791,11 +791,9 @@
if (size < 0)
return _io_FileIO_readall_impl(self);
-#ifdef MS_WINDOWS
- /* On Windows, the count parameter of read() is an int */
- if (size > INT_MAX)
- size = INT_MAX;
-#endif
+ if (size > _PY_READ_MAX) {
+ size = _PY_READ_MAX;
+ }
bytes = PyBytes_FromStringAndSize(NULL, size);
if (bytes == NULL)
diff --git a/Python/fileutils.c b/Python/fileutils.c
index 0486f86..2c4061e 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -1471,18 +1471,9 @@
* handler raised an exception. */
assert(!PyErr_Occurred());
-#ifdef MS_WINDOWS
- if (count > INT_MAX) {
- /* On Windows, the count parameter of read() is an int */
- count = INT_MAX;
+ if (count > _PY_READ_MAX) {
+ count = _PY_READ_MAX;
}
-#else
- if (count > PY_SSIZE_T_MAX) {
- /* if count is greater than PY_SSIZE_T_MAX,
- * read() result is undefined */
- count = PY_SSIZE_T_MAX;
- }
-#endif
_Py_BEGIN_SUPPRESS_IPH
do {
@@ -1533,15 +1524,10 @@
depending on heap usage). */
count = 32767;
}
- else if (count > INT_MAX)
- count = INT_MAX;
-#else
- if (count > PY_SSIZE_T_MAX) {
- /* write() should truncate count to PY_SSIZE_T_MAX, but it's safer
- * to do it ourself to have a portable behaviour. */
- count = PY_SSIZE_T_MAX;
- }
#endif
+ if (count > _PY_WRITE_MAX) {
+ count = _PY_WRITE_MAX;
+ }
if (gil_held) {
do {