Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 1 | """Test largefile support on system where this makes sense. |
| 2 | """ |
Trent Mick | f29f47b | 2000-08-11 19:02:59 +0000 | [diff] [blame] | 3 | |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 4 | import os |
| 5 | import stat |
| 6 | import sys |
| 7 | import unittest |
Stéphane Wirtel | 74a8b6e | 2018-10-18 01:05:04 +0200 | [diff] [blame] | 8 | from test.support import TESTFN, requires, unlink, bigmemtest |
Benjamin Peterson | 4fa88fa | 2009-03-04 00:14:51 +0000 | [diff] [blame] | 9 | import io # C implementation of io |
| 10 | import _pyio as pyio # Python implementation of io |
Trent Mick | f29f47b | 2000-08-11 19:02:59 +0000 | [diff] [blame] | 11 | |
Victor Stinner | 8c663fd | 2017-11-08 14:44:44 -0800 | [diff] [blame] | 12 | # size of file to create (>2 GiB; 2 GiB == 2,147,483,648 bytes) |
Stéphane Wirtel | 74a8b6e | 2018-10-18 01:05:04 +0200 | [diff] [blame] | 13 | size = 2_500_000_000 |
Guido van Rossum | a31ddbb | 2001-09-10 15:03:18 +0000 | [diff] [blame] | 14 | |
Serhiy Storchaka | c406a12 | 2013-07-17 13:42:24 +0300 | [diff] [blame] | 15 | class LargeFileTest: |
| 16 | """Test that each file function works as expected for large |
Victor Stinner | 8c663fd | 2017-11-08 14:44:44 -0800 | [diff] [blame] | 17 | (i.e. > 2 GiB) files. |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 18 | """ |
| 19 | |
Serhiy Storchaka | c406a12 | 2013-07-17 13:42:24 +0300 | [diff] [blame] | 20 | def setUp(self): |
| 21 | if os.path.exists(TESTFN): |
| 22 | mode = 'r+b' |
| 23 | else: |
| 24 | mode = 'w+b' |
| 25 | |
| 26 | with self.open(TESTFN, mode) as f: |
| 27 | current_size = os.fstat(f.fileno())[stat.ST_SIZE] |
| 28 | if current_size == size+1: |
| 29 | return |
| 30 | |
| 31 | if current_size == 0: |
| 32 | f.write(b'z') |
| 33 | |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 34 | f.seek(0) |
| 35 | f.seek(size) |
Alexandre Vassalotti | a351f77 | 2008-03-03 02:59:49 +0000 | [diff] [blame] | 36 | f.write(b'a') |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 37 | f.flush() |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 38 | self.assertEqual(os.fstat(f.fileno())[stat.ST_SIZE], size+1) |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 39 | |
Serhiy Storchaka | c406a12 | 2013-07-17 13:42:24 +0300 | [diff] [blame] | 40 | @classmethod |
| 41 | def tearDownClass(cls): |
| 42 | with cls.open(TESTFN, 'wb'): |
| 43 | pass |
| 44 | if not os.stat(TESTFN)[stat.ST_SIZE] == 0: |
| 45 | raise cls.failureException('File was not truncated by opening ' |
| 46 | 'with mode "wb"') |
| 47 | |
Stéphane Wirtel | 74a8b6e | 2018-10-18 01:05:04 +0200 | [diff] [blame] | 48 | # _pyio.FileIO.readall() uses a temporary bytearray then casted to bytes, |
| 49 | # so memuse=2 is needed |
| 50 | @bigmemtest(size=size, memuse=2, dry_run=False) |
| 51 | def test_large_read(self, _size): |
| 52 | # bpo-24658: Test that a read greater than 2GB does not fail. |
| 53 | with self.open(TESTFN, "rb") as f: |
| 54 | self.assertEqual(len(f.read()), size + 1) |
| 55 | self.assertEqual(f.tell(), size + 1) |
| 56 | |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 57 | def test_osstat(self): |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 58 | self.assertEqual(os.stat(TESTFN)[stat.ST_SIZE], size+1) |
| 59 | |
| 60 | def test_seek_read(self): |
Benjamin Peterson | 4fa88fa | 2009-03-04 00:14:51 +0000 | [diff] [blame] | 61 | with self.open(TESTFN, 'rb') as f: |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 62 | self.assertEqual(f.tell(), 0) |
Alexandre Vassalotti | a351f77 | 2008-03-03 02:59:49 +0000 | [diff] [blame] | 63 | self.assertEqual(f.read(1), b'z') |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 64 | self.assertEqual(f.tell(), 1) |
| 65 | f.seek(0) |
| 66 | self.assertEqual(f.tell(), 0) |
| 67 | f.seek(0, 0) |
| 68 | self.assertEqual(f.tell(), 0) |
| 69 | f.seek(42) |
| 70 | self.assertEqual(f.tell(), 42) |
| 71 | f.seek(42, 0) |
| 72 | self.assertEqual(f.tell(), 42) |
| 73 | f.seek(42, 1) |
| 74 | self.assertEqual(f.tell(), 84) |
| 75 | f.seek(0, 1) |
| 76 | self.assertEqual(f.tell(), 84) |
| 77 | f.seek(0, 2) # seek from the end |
| 78 | self.assertEqual(f.tell(), size + 1 + 0) |
| 79 | f.seek(-10, 2) |
| 80 | self.assertEqual(f.tell(), size + 1 - 10) |
| 81 | f.seek(-size-1, 2) |
| 82 | self.assertEqual(f.tell(), 0) |
| 83 | f.seek(size) |
| 84 | self.assertEqual(f.tell(), size) |
| 85 | # the 'a' that was written at the end of file above |
Alexandre Vassalotti | a351f77 | 2008-03-03 02:59:49 +0000 | [diff] [blame] | 86 | self.assertEqual(f.read(1), b'a') |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 87 | f.seek(-size-1, 1) |
Alexandre Vassalotti | a351f77 | 2008-03-03 02:59:49 +0000 | [diff] [blame] | 88 | self.assertEqual(f.read(1), b'z') |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 89 | self.assertEqual(f.tell(), 1) |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 90 | |
| 91 | def test_lseek(self): |
Benjamin Peterson | 4fa88fa | 2009-03-04 00:14:51 +0000 | [diff] [blame] | 92 | with self.open(TESTFN, 'rb') as f: |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 93 | self.assertEqual(os.lseek(f.fileno(), 0, 0), 0) |
| 94 | self.assertEqual(os.lseek(f.fileno(), 42, 0), 42) |
| 95 | self.assertEqual(os.lseek(f.fileno(), 42, 1), 84) |
| 96 | self.assertEqual(os.lseek(f.fileno(), 0, 1), 84) |
| 97 | self.assertEqual(os.lseek(f.fileno(), 0, 2), size+1+0) |
| 98 | self.assertEqual(os.lseek(f.fileno(), -10, 2), size+1-10) |
| 99 | self.assertEqual(os.lseek(f.fileno(), -size-1, 2), 0) |
| 100 | self.assertEqual(os.lseek(f.fileno(), size, 0), size) |
| 101 | # the 'a' that was written at the end of file above |
Alexandre Vassalotti | a351f77 | 2008-03-03 02:59:49 +0000 | [diff] [blame] | 102 | self.assertEqual(f.read(1), b'a') |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 103 | |
| 104 | def test_truncate(self): |
Benjamin Peterson | 4fa88fa | 2009-03-04 00:14:51 +0000 | [diff] [blame] | 105 | with self.open(TESTFN, 'r+b') as f: |
Christian Heimes | 180510d | 2008-03-03 19:15:45 +0000 | [diff] [blame] | 106 | if not hasattr(f, 'truncate'): |
Serhiy Storchaka | c406a12 | 2013-07-17 13:42:24 +0300 | [diff] [blame] | 107 | raise unittest.SkipTest("open().truncate() not available " |
| 108 | "on this system") |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 109 | f.seek(0, 2) |
| 110 | # else we've lost track of the true size |
| 111 | self.assertEqual(f.tell(), size+1) |
| 112 | # Cut it back via seek + truncate with no argument. |
| 113 | newsize = size - 10 |
| 114 | f.seek(newsize) |
| 115 | f.truncate() |
| 116 | self.assertEqual(f.tell(), newsize) # else pointer moved |
| 117 | f.seek(0, 2) |
| 118 | self.assertEqual(f.tell(), newsize) # else wasn't truncated |
| 119 | # Ensure that truncate(smaller than true size) shrinks |
| 120 | # the file. |
| 121 | newsize -= 1 |
| 122 | f.seek(42) |
| 123 | f.truncate(newsize) |
Antoine Pitrou | 905a2ff | 2010-01-31 22:47:27 +0000 | [diff] [blame] | 124 | self.assertEqual(f.tell(), 42) |
Alexandre Vassalotti | 77250f4 | 2008-05-06 19:48:38 +0000 | [diff] [blame] | 125 | f.seek(0, 2) |
| 126 | self.assertEqual(f.tell(), newsize) |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 127 | # XXX truncate(larger than true size) is ill-defined |
| 128 | # across platform; cut it waaaaay back |
| 129 | f.seek(0) |
| 130 | f.truncate(1) |
Antoine Pitrou | 905a2ff | 2010-01-31 22:47:27 +0000 | [diff] [blame] | 131 | self.assertEqual(f.tell(), 0) # else pointer moved |
Alexandre Vassalotti | 77250f4 | 2008-05-06 19:48:38 +0000 | [diff] [blame] | 132 | f.seek(0) |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 133 | self.assertEqual(len(f.read()), 1) # else wasn't truncated |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 134 | |
Antoine Pitrou | a28fcfd | 2009-03-13 23:42:55 +0000 | [diff] [blame] | 135 | def test_seekable(self): |
| 136 | # Issue #5016; seekable() can return False when the current position |
| 137 | # is negative when truncated to an int. |
| 138 | for pos in (2**31-1, 2**31, 2**31+1): |
| 139 | with self.open(TESTFN, 'rb') as f: |
| 140 | f.seek(pos) |
Benjamin Peterson | c9c0f20 | 2009-06-30 23:06:06 +0000 | [diff] [blame] | 141 | self.assertTrue(f.seekable()) |
Antoine Pitrou | a28fcfd | 2009-03-13 23:42:55 +0000 | [diff] [blame] | 142 | |
Serhiy Storchaka | c406a12 | 2013-07-17 13:42:24 +0300 | [diff] [blame] | 143 | def setUpModule(): |
| 144 | try: |
| 145 | import signal |
| 146 | # The default handler for SIGXFSZ is to abort the process. |
| 147 | # By ignoring it, system calls exceeding the file size resource |
| 148 | # limit will raise OSError instead of crashing the interpreter. |
| 149 | signal.signal(signal.SIGXFSZ, signal.SIG_IGN) |
| 150 | except (ImportError, AttributeError): |
| 151 | pass |
Antoine Pitrou | a28fcfd | 2009-03-13 23:42:55 +0000 | [diff] [blame] | 152 | |
Mike | 53f7a7c | 2017-12-14 14:04:53 +0300 | [diff] [blame] | 153 | # On Windows and Mac OSX this test consumes large resources; It |
Victor Stinner | 8c663fd | 2017-11-08 14:44:44 -0800 | [diff] [blame] | 154 | # takes a long time to build the >2 GiB file and takes >2 GiB of disk |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 155 | # space therefore the resource must be enabled to run this test. |
| 156 | # If not, nothing after this line stanza will be executed. |
Victor Stinner | 937ee9e | 2018-06-26 02:11:06 +0200 | [diff] [blame] | 157 | if sys.platform[:3] == 'win' or sys.platform == 'darwin': |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 158 | requires('largefile', |
| 159 | 'test requires %s bytes and a long time to run' % str(size)) |
Guido van Rossum | 47f4034 | 2001-09-10 13:34:12 +0000 | [diff] [blame] | 160 | else: |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 161 | # Only run if the current filesystem supports large files. |
| 162 | # (Skip this test on Windows, since we now always support |
| 163 | # large files.) |
Benjamin Peterson | 4fa88fa | 2009-03-04 00:14:51 +0000 | [diff] [blame] | 164 | f = open(TESTFN, 'wb', buffering=0) |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 165 | try: |
| 166 | # 2**31 == 2147483648 |
| 167 | f.seek(2147483649) |
Serhiy Storchaka | c406a12 | 2013-07-17 13:42:24 +0300 | [diff] [blame] | 168 | # Seeking is not enough of a test: you must write and flush, too! |
Alexandre Vassalotti | a351f77 | 2008-03-03 02:59:49 +0000 | [diff] [blame] | 169 | f.write(b'x') |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 170 | f.flush() |
Andrew Svetlov | f7a17b4 | 2012-12-25 16:47:37 +0200 | [diff] [blame] | 171 | except (OSError, OverflowError): |
Serhiy Storchaka | c406a12 | 2013-07-17 13:42:24 +0300 | [diff] [blame] | 172 | raise unittest.SkipTest("filesystem does not have " |
| 173 | "largefile support") |
| 174 | finally: |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 175 | f.close() |
| 176 | unlink(TESTFN) |
Serhiy Storchaka | c406a12 | 2013-07-17 13:42:24 +0300 | [diff] [blame] | 177 | |
| 178 | |
| 179 | class CLargeFileTest(LargeFileTest, unittest.TestCase): |
| 180 | open = staticmethod(io.open) |
| 181 | |
| 182 | class PyLargeFileTest(LargeFileTest, unittest.TestCase): |
| 183 | open = staticmethod(pyio.open) |
| 184 | |
| 185 | def tearDownModule(): |
| 186 | unlink(TESTFN) |
Guido van Rossum | 47f4034 | 2001-09-10 13:34:12 +0000 | [diff] [blame] | 187 | |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 188 | if __name__ == '__main__': |
Serhiy Storchaka | c406a12 | 2013-07-17 13:42:24 +0300 | [diff] [blame] | 189 | unittest.main() |