Mike Frysinger | f601376 | 2019-06-13 02:30:51 -0400 | [diff] [blame] | 1 | # -*- coding:utf-8 -*- |
Renaud Paquay | 2e70291 | 2016-11-01 11:23:38 -0700 | [diff] [blame] | 2 | # |
| 3 | # Copyright (C) 2016 The Android Open Source Project |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
| 16 | |
Renaud Paquay | ad1abcb | 2016-11-01 11:34:55 -0700 | [diff] [blame] | 17 | import errno |
Renaud Paquay | 2e70291 | 2016-11-01 11:23:38 -0700 | [diff] [blame] | 18 | import os |
| 19 | import platform |
| 20 | import select |
Renaud Paquay | a65adf7 | 2016-11-03 10:37:53 -0700 | [diff] [blame] | 21 | import shutil |
| 22 | import stat |
Renaud Paquay | 2e70291 | 2016-11-01 11:23:38 -0700 | [diff] [blame] | 23 | |
Dylan Deng | e469a0c | 2018-06-23 15:02:26 +0800 | [diff] [blame] | 24 | from pyversion import is_python3 |
| 25 | if is_python3(): |
| 26 | from queue import Queue |
| 27 | else: |
| 28 | from Queue import Queue |
| 29 | |
Renaud Paquay | 2e70291 | 2016-11-01 11:23:38 -0700 | [diff] [blame] | 30 | from threading import Thread |
| 31 | |
| 32 | |
| 33 | def isWindows(): |
| 34 | """ Returns True when running with the native port of Python for Windows, |
| 35 | False when running on any other platform (including the Cygwin port of |
| 36 | Python). |
| 37 | """ |
| 38 | # Note: The cygwin port of Python returns "CYGWIN_NT_xxx" |
| 39 | return platform.system() == "Windows" |
| 40 | |
| 41 | |
| 42 | class FileDescriptorStreams(object): |
| 43 | """ Platform agnostic abstraction enabling non-blocking I/O over a |
| 44 | collection of file descriptors. This abstraction is required because |
| 45 | fctnl(os.O_NONBLOCK) is not supported on Windows. |
| 46 | """ |
| 47 | @classmethod |
| 48 | def create(cls): |
| 49 | """ Factory method: instantiates the concrete class according to the |
| 50 | current platform. |
| 51 | """ |
| 52 | if isWindows(): |
| 53 | return _FileDescriptorStreamsThreads() |
| 54 | else: |
| 55 | return _FileDescriptorStreamsNonBlocking() |
| 56 | |
| 57 | def __init__(self): |
| 58 | self.streams = [] |
| 59 | |
| 60 | def add(self, fd, dest, std_name): |
| 61 | """ Wraps an existing file descriptor as a stream. |
| 62 | """ |
| 63 | self.streams.append(self._create_stream(fd, dest, std_name)) |
| 64 | |
| 65 | def remove(self, stream): |
| 66 | """ Removes a stream, when done with it. |
| 67 | """ |
| 68 | self.streams.remove(stream) |
| 69 | |
| 70 | @property |
| 71 | def is_done(self): |
| 72 | """ Returns True when all streams have been processed. |
| 73 | """ |
| 74 | return len(self.streams) == 0 |
| 75 | |
| 76 | def select(self): |
| 77 | """ Returns the set of streams that have data available to read. |
| 78 | The returned streams each expose a read() and a close() method. |
| 79 | When done with a stream, call the remove(stream) method. |
| 80 | """ |
| 81 | raise NotImplementedError |
| 82 | |
Rostislav Krasny | ec0ba27 | 2020-01-25 14:32:37 +0200 | [diff] [blame] | 83 | def _create_stream(self, fd, dest, std_name): |
Renaud Paquay | 2e70291 | 2016-11-01 11:23:38 -0700 | [diff] [blame] | 84 | """ Creates a new stream wrapping an existing file descriptor. |
| 85 | """ |
| 86 | raise NotImplementedError |
| 87 | |
| 88 | |
| 89 | class _FileDescriptorStreamsNonBlocking(FileDescriptorStreams): |
| 90 | """ Implementation of FileDescriptorStreams for platforms that support |
| 91 | non blocking I/O. |
| 92 | """ |
| 93 | class Stream(object): |
| 94 | """ Encapsulates a file descriptor """ |
| 95 | def __init__(self, fd, dest, std_name): |
| 96 | self.fd = fd |
| 97 | self.dest = dest |
| 98 | self.std_name = std_name |
| 99 | self.set_non_blocking() |
| 100 | |
| 101 | def set_non_blocking(self): |
| 102 | import fcntl |
| 103 | flags = fcntl.fcntl(self.fd, fcntl.F_GETFL) |
| 104 | fcntl.fcntl(self.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) |
| 105 | |
| 106 | def fileno(self): |
| 107 | return self.fd.fileno() |
| 108 | |
| 109 | def read(self): |
| 110 | return self.fd.read(4096) |
| 111 | |
| 112 | def close(self): |
| 113 | self.fd.close() |
| 114 | |
| 115 | def _create_stream(self, fd, dest, std_name): |
Mike Frysinger | 91d9587 | 2020-02-03 22:11:19 +0000 | [diff] [blame] | 116 | return self.Stream(fd, dest, std_name) |
Renaud Paquay | 2e70291 | 2016-11-01 11:23:38 -0700 | [diff] [blame] | 117 | |
| 118 | def select(self): |
Mike Frysinger | 91d9587 | 2020-02-03 22:11:19 +0000 | [diff] [blame] | 119 | ready_streams, _, _ = select.select(self.streams, [], []) |
| 120 | return ready_streams |
Renaud Paquay | 2e70291 | 2016-11-01 11:23:38 -0700 | [diff] [blame] | 121 | |
| 122 | |
| 123 | class _FileDescriptorStreamsThreads(FileDescriptorStreams): |
| 124 | """ Implementation of FileDescriptorStreams for platforms that don't support |
| 125 | non blocking I/O. This implementation requires creating threads issuing |
| 126 | blocking read operations on file descriptors. |
| 127 | """ |
| 128 | def __init__(self): |
| 129 | super(_FileDescriptorStreamsThreads, self).__init__() |
| 130 | # The queue is shared accross all threads so we can simulate the |
| 131 | # behavior of the select() function |
| 132 | self.queue = Queue(10) # Limit incoming data from streams |
| 133 | |
| 134 | def _create_stream(self, fd, dest, std_name): |
| 135 | return self.Stream(fd, dest, std_name, self.queue) |
| 136 | |
| 137 | def select(self): |
| 138 | # Return only one stream at a time, as it is the most straighforward |
| 139 | # thing to do and it is compatible with the select() function. |
| 140 | item = self.queue.get() |
| 141 | stream = item.stream |
| 142 | stream.data = item.data |
| 143 | return [stream] |
| 144 | |
| 145 | class QueueItem(object): |
| 146 | """ Item put in the shared queue """ |
| 147 | def __init__(self, stream, data): |
| 148 | self.stream = stream |
| 149 | self.data = data |
| 150 | |
| 151 | class Stream(object): |
| 152 | """ Encapsulates a file descriptor """ |
| 153 | def __init__(self, fd, dest, std_name, queue): |
| 154 | self.fd = fd |
| 155 | self.dest = dest |
| 156 | self.std_name = std_name |
| 157 | self.queue = queue |
| 158 | self.data = None |
| 159 | self.thread = Thread(target=self.read_to_queue) |
| 160 | self.thread.daemon = True |
| 161 | self.thread.start() |
| 162 | |
| 163 | def close(self): |
| 164 | self.fd.close() |
| 165 | |
| 166 | def read(self): |
| 167 | data = self.data |
| 168 | self.data = None |
| 169 | return data |
| 170 | |
| 171 | def read_to_queue(self): |
| 172 | """ The thread function: reads everything from the file descriptor into |
| 173 | the shared queue and terminates when reaching EOF. |
| 174 | """ |
| 175 | for line in iter(self.fd.readline, b''): |
| 176 | self.queue.put(_FileDescriptorStreamsThreads.QueueItem(self, line)) |
| 177 | self.fd.close() |
| 178 | self.queue.put(_FileDescriptorStreamsThreads.QueueItem(self, None)) |
Renaud Paquay | d5cec5e | 2016-11-01 11:24:03 -0700 | [diff] [blame] | 179 | |
| 180 | |
| 181 | def symlink(source, link_name): |
| 182 | """Creates a symbolic link pointing to source named link_name. |
| 183 | Note: On Windows, source must exist on disk, as the implementation needs |
| 184 | to know whether to create a "File" or a "Directory" symbolic link. |
| 185 | """ |
| 186 | if isWindows(): |
| 187 | import platform_utils_win32 |
| 188 | source = _validate_winpath(source) |
| 189 | link_name = _validate_winpath(link_name) |
| 190 | target = os.path.join(os.path.dirname(link_name), source) |
Renaud Paquay | bed8b62 | 2018-09-27 10:46:58 -0700 | [diff] [blame] | 191 | if isdir(target): |
| 192 | platform_utils_win32.create_dirsymlink(_makelongpath(source), link_name) |
Renaud Paquay | d5cec5e | 2016-11-01 11:24:03 -0700 | [diff] [blame] | 193 | else: |
Renaud Paquay | bed8b62 | 2018-09-27 10:46:58 -0700 | [diff] [blame] | 194 | platform_utils_win32.create_filesymlink(_makelongpath(source), link_name) |
Renaud Paquay | d5cec5e | 2016-11-01 11:24:03 -0700 | [diff] [blame] | 195 | else: |
| 196 | return os.symlink(source, link_name) |
| 197 | |
| 198 | |
| 199 | def _validate_winpath(path): |
| 200 | path = os.path.normpath(path) |
| 201 | if _winpath_is_valid(path): |
| 202 | return path |
| 203 | raise ValueError("Path \"%s\" must be a relative path or an absolute " |
| 204 | "path starting with a drive letter".format(path)) |
| 205 | |
| 206 | |
| 207 | def _winpath_is_valid(path): |
| 208 | """Windows only: returns True if path is relative (e.g. ".\\foo") or is |
| 209 | absolute including a drive letter (e.g. "c:\\foo"). Returns False if path |
| 210 | is ambiguous (e.g. "x:foo" or "\\foo"). |
| 211 | """ |
| 212 | assert isWindows() |
| 213 | path = os.path.normpath(path) |
| 214 | drive, tail = os.path.splitdrive(path) |
| 215 | if tail: |
| 216 | if not drive: |
| 217 | return tail[0] != os.sep # "\\foo" is invalid |
| 218 | else: |
| 219 | return tail[0] == os.sep # "x:foo" is invalid |
| 220 | else: |
| 221 | return not drive # "x:" is invalid |
Renaud Paquay | a65adf7 | 2016-11-03 10:37:53 -0700 | [diff] [blame] | 222 | |
| 223 | |
Renaud Paquay | bed8b62 | 2018-09-27 10:46:58 -0700 | [diff] [blame] | 224 | def _makelongpath(path): |
| 225 | """Return the input path normalized to support the Windows long path syntax |
| 226 | ("\\\\?\\" prefix) if needed, i.e. if the input path is longer than the |
| 227 | MAX_PATH limit. |
| 228 | """ |
Renaud Paquay | a65adf7 | 2016-11-03 10:37:53 -0700 | [diff] [blame] | 229 | if isWindows(): |
Renaud Paquay | bed8b62 | 2018-09-27 10:46:58 -0700 | [diff] [blame] | 230 | # Note: MAX_PATH is 260, but, for directories, the maximum value is actually 246. |
| 231 | if len(path) < 246: |
| 232 | return path |
| 233 | if path.startswith(u"\\\\?\\"): |
| 234 | return path |
| 235 | if not os.path.isabs(path): |
| 236 | return path |
| 237 | # Append prefix and ensure unicode so that the special longpath syntax |
| 238 | # is supported by underlying Win32 API calls |
| 239 | return u"\\\\?\\" + os.path.normpath(path) |
| 240 | else: |
| 241 | return path |
| 242 | |
| 243 | |
Mike Frysinger | f454512 | 2019-11-11 04:34:16 -0500 | [diff] [blame] | 244 | def rmtree(path, ignore_errors=False): |
Renaud Paquay | bed8b62 | 2018-09-27 10:46:58 -0700 | [diff] [blame] | 245 | """shutil.rmtree(path) wrapper with support for long paths on Windows. |
| 246 | |
| 247 | Availability: Unix, Windows.""" |
Mike Frysinger | f454512 | 2019-11-11 04:34:16 -0500 | [diff] [blame] | 248 | onerror = None |
Renaud Paquay | bed8b62 | 2018-09-27 10:46:58 -0700 | [diff] [blame] | 249 | if isWindows(): |
Mike Frysinger | f454512 | 2019-11-11 04:34:16 -0500 | [diff] [blame] | 250 | path = _makelongpath(path) |
| 251 | onerror = handle_rmtree_error |
| 252 | shutil.rmtree(path, ignore_errors=ignore_errors, onerror=onerror) |
Renaud Paquay | a65adf7 | 2016-11-03 10:37:53 -0700 | [diff] [blame] | 253 | |
| 254 | |
| 255 | def handle_rmtree_error(function, path, excinfo): |
| 256 | # Allow deleting read-only files |
| 257 | os.chmod(path, stat.S_IWRITE) |
| 258 | function(path) |
Renaud Paquay | ad1abcb | 2016-11-01 11:34:55 -0700 | [diff] [blame] | 259 | |
| 260 | |
| 261 | def rename(src, dst): |
Renaud Paquay | bed8b62 | 2018-09-27 10:46:58 -0700 | [diff] [blame] | 262 | """os.rename(src, dst) wrapper with support for long paths on Windows. |
| 263 | |
| 264 | Availability: Unix, Windows.""" |
Renaud Paquay | ad1abcb | 2016-11-01 11:34:55 -0700 | [diff] [blame] | 265 | if isWindows(): |
| 266 | # On Windows, rename fails if destination exists, see |
| 267 | # https://docs.python.org/2/library/os.html#os.rename |
| 268 | try: |
Renaud Paquay | bed8b62 | 2018-09-27 10:46:58 -0700 | [diff] [blame] | 269 | os.rename(_makelongpath(src), _makelongpath(dst)) |
Renaud Paquay | ad1abcb | 2016-11-01 11:34:55 -0700 | [diff] [blame] | 270 | except OSError as e: |
| 271 | if e.errno == errno.EEXIST: |
Renaud Paquay | bed8b62 | 2018-09-27 10:46:58 -0700 | [diff] [blame] | 272 | os.remove(_makelongpath(dst)) |
| 273 | os.rename(_makelongpath(src), _makelongpath(dst)) |
Renaud Paquay | ad1abcb | 2016-11-01 11:34:55 -0700 | [diff] [blame] | 274 | else: |
| 275 | raise |
| 276 | else: |
| 277 | os.rename(src, dst) |
Renaud Paquay | 227ad2e | 2016-11-01 14:37:13 -0700 | [diff] [blame] | 278 | |
| 279 | |
Renaud Paquay | 010fed7 | 2016-11-11 14:25:29 -0800 | [diff] [blame] | 280 | def remove(path): |
Renaud Paquay | bed8b62 | 2018-09-27 10:46:58 -0700 | [diff] [blame] | 281 | """Remove (delete) the file path. This is a replacement for os.remove that |
| 282 | allows deleting read-only files on Windows, with support for long paths and |
| 283 | for deleting directory symbolic links. |
| 284 | |
| 285 | Availability: Unix, Windows.""" |
Renaud Paquay | 010fed7 | 2016-11-11 14:25:29 -0800 | [diff] [blame] | 286 | if isWindows(): |
Renaud Paquay | bed8b62 | 2018-09-27 10:46:58 -0700 | [diff] [blame] | 287 | longpath = _makelongpath(path) |
Renaud Paquay | 010fed7 | 2016-11-11 14:25:29 -0800 | [diff] [blame] | 288 | try: |
Renaud Paquay | bed8b62 | 2018-09-27 10:46:58 -0700 | [diff] [blame] | 289 | os.remove(longpath) |
Renaud Paquay | 010fed7 | 2016-11-11 14:25:29 -0800 | [diff] [blame] | 290 | except OSError as e: |
| 291 | if e.errno == errno.EACCES: |
Renaud Paquay | bed8b62 | 2018-09-27 10:46:58 -0700 | [diff] [blame] | 292 | os.chmod(longpath, stat.S_IWRITE) |
| 293 | # Directory symbolic links must be deleted with 'rmdir'. |
| 294 | if islink(longpath) and isdir(longpath): |
| 295 | os.rmdir(longpath) |
| 296 | else: |
| 297 | os.remove(longpath) |
Renaud Paquay | 010fed7 | 2016-11-11 14:25:29 -0800 | [diff] [blame] | 298 | else: |
| 299 | raise |
| 300 | else: |
| 301 | os.remove(path) |
| 302 | |
| 303 | |
Renaud Paquay | bed8b62 | 2018-09-27 10:46:58 -0700 | [diff] [blame] | 304 | def walk(top, topdown=True, onerror=None, followlinks=False): |
| 305 | """os.walk(path) wrapper with support for long paths on Windows. |
| 306 | |
| 307 | Availability: Windows, Unix. |
| 308 | """ |
| 309 | if isWindows(): |
| 310 | return _walk_windows_impl(top, topdown, onerror, followlinks) |
| 311 | else: |
| 312 | return os.walk(top, topdown, onerror, followlinks) |
| 313 | |
| 314 | |
| 315 | def _walk_windows_impl(top, topdown, onerror, followlinks): |
| 316 | try: |
| 317 | names = listdir(top) |
David Pursehouse | d26146d | 2018-11-01 11:54:10 +0900 | [diff] [blame] | 318 | except Exception as err: |
Renaud Paquay | bed8b62 | 2018-09-27 10:46:58 -0700 | [diff] [blame] | 319 | if onerror is not None: |
| 320 | onerror(err) |
| 321 | return |
| 322 | |
| 323 | dirs, nondirs = [], [] |
| 324 | for name in names: |
| 325 | if isdir(os.path.join(top, name)): |
| 326 | dirs.append(name) |
| 327 | else: |
| 328 | nondirs.append(name) |
| 329 | |
| 330 | if topdown: |
| 331 | yield top, dirs, nondirs |
| 332 | for name in dirs: |
| 333 | new_path = os.path.join(top, name) |
| 334 | if followlinks or not islink(new_path): |
| 335 | for x in _walk_windows_impl(new_path, topdown, onerror, followlinks): |
| 336 | yield x |
| 337 | if not topdown: |
| 338 | yield top, dirs, nondirs |
| 339 | |
| 340 | |
| 341 | def listdir(path): |
| 342 | """os.listdir(path) wrapper with support for long paths on Windows. |
| 343 | |
| 344 | Availability: Windows, Unix. |
| 345 | """ |
| 346 | return os.listdir(_makelongpath(path)) |
| 347 | |
| 348 | |
| 349 | def rmdir(path): |
| 350 | """os.rmdir(path) wrapper with support for long paths on Windows. |
| 351 | |
| 352 | Availability: Windows, Unix. |
| 353 | """ |
| 354 | os.rmdir(_makelongpath(path)) |
| 355 | |
| 356 | |
| 357 | def isdir(path): |
| 358 | """os.path.isdir(path) wrapper with support for long paths on Windows. |
| 359 | |
| 360 | Availability: Windows, Unix. |
| 361 | """ |
| 362 | return os.path.isdir(_makelongpath(path)) |
| 363 | |
| 364 | |
Renaud Paquay | 227ad2e | 2016-11-01 14:37:13 -0700 | [diff] [blame] | 365 | def islink(path): |
Renaud Paquay | bed8b62 | 2018-09-27 10:46:58 -0700 | [diff] [blame] | 366 | """os.path.islink(path) wrapper with support for long paths on Windows. |
Renaud Paquay | 227ad2e | 2016-11-01 14:37:13 -0700 | [diff] [blame] | 367 | |
| 368 | Availability: Windows, Unix. |
| 369 | """ |
| 370 | if isWindows(): |
| 371 | import platform_utils_win32 |
Renaud Paquay | bed8b62 | 2018-09-27 10:46:58 -0700 | [diff] [blame] | 372 | return platform_utils_win32.islink(_makelongpath(path)) |
Renaud Paquay | 227ad2e | 2016-11-01 14:37:13 -0700 | [diff] [blame] | 373 | else: |
| 374 | return os.path.islink(path) |
| 375 | |
| 376 | |
| 377 | def readlink(path): |
| 378 | """Return a string representing the path to which the symbolic link |
| 379 | points. The result may be either an absolute or relative pathname; |
| 380 | if it is relative, it may be converted to an absolute pathname using |
| 381 | os.path.join(os.path.dirname(path), result). |
| 382 | |
| 383 | Availability: Windows, Unix. |
| 384 | """ |
| 385 | if isWindows(): |
| 386 | import platform_utils_win32 |
Renaud Paquay | bed8b62 | 2018-09-27 10:46:58 -0700 | [diff] [blame] | 387 | return platform_utils_win32.readlink(_makelongpath(path)) |
Renaud Paquay | 227ad2e | 2016-11-01 14:37:13 -0700 | [diff] [blame] | 388 | else: |
| 389 | return os.readlink(path) |
| 390 | |
| 391 | |
| 392 | def realpath(path): |
| 393 | """Return the canonical path of the specified filename, eliminating |
| 394 | any symbolic links encountered in the path. |
| 395 | |
| 396 | Availability: Windows, Unix. |
| 397 | """ |
| 398 | if isWindows(): |
| 399 | current_path = os.path.abspath(path) |
| 400 | path_tail = [] |
| 401 | for c in range(0, 100): # Avoid cycles |
| 402 | if islink(current_path): |
| 403 | target = readlink(current_path) |
| 404 | current_path = os.path.join(os.path.dirname(current_path), target) |
| 405 | else: |
| 406 | basename = os.path.basename(current_path) |
| 407 | if basename == '': |
| 408 | path_tail.append(current_path) |
| 409 | break |
| 410 | path_tail.append(basename) |
| 411 | current_path = os.path.dirname(current_path) |
| 412 | path_tail.reverse() |
| 413 | result = os.path.normpath(os.path.join(*path_tail)) |
| 414 | return result |
| 415 | else: |
| 416 | return os.path.realpath(path) |