mussa | fd5b805 | 2014-05-06 10:04:54 -0700 | [diff] [blame] | 1 | # Copyright (c) 2014 The Chromium OS Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
| 5 | import errno |
Dan Shi | 96c3bdc | 2017-05-24 11:34:30 -0700 | [diff] [blame] | 6 | import logging |
mussa | fd5b805 | 2014-05-06 10:04:54 -0700 | [diff] [blame] | 7 | import os |
| 8 | import shutil |
Dan Shi | 96c3bdc | 2017-05-24 11:34:30 -0700 | [diff] [blame] | 9 | import subprocess |
Mussa | 45745d8 | 2014-08-26 16:06:29 -0700 | [diff] [blame] | 10 | import urllib2 |
mussa | fd5b805 | 2014-05-06 10:04:54 -0700 | [diff] [blame] | 11 | |
Ed Baker | 34fdca4 | 2015-06-01 09:06:06 -0700 | [diff] [blame] | 12 | from autotest_lib.client.common_lib import global_config |
mussa | fd5b805 | 2014-05-06 10:04:54 -0700 | [diff] [blame] | 13 | |
| 14 | def rm_dir_if_exists(dir_to_remove): |
| 15 | """ |
| 16 | Removes a directory. Does not fail if the directory does NOT exist. |
| 17 | |
| 18 | @param dir_to_remove: path, directory to be removed. |
| 19 | |
| 20 | """ |
| 21 | try: |
| 22 | shutil.rmtree(dir_to_remove) |
| 23 | except OSError as e: |
mussa | 5fec82c | 2014-05-27 14:35:41 -0700 | [diff] [blame] | 24 | if e.errno != errno.ENOENT: |
mussa | fd5b805 | 2014-05-06 10:04:54 -0700 | [diff] [blame] | 25 | raise |
| 26 | |
| 27 | |
| 28 | def rm_dirs_if_exist(dirs_to_remove): |
| 29 | """ |
| 30 | Removes multiple directories. Does not fail if directories do NOT exist. |
| 31 | |
| 32 | @param dirs_to_remove: list of directory paths to be removed. |
| 33 | |
| 34 | """ |
| 35 | for dr in dirs_to_remove: |
| 36 | rm_dir_if_exists(dr) |
| 37 | |
| 38 | |
| 39 | def ensure_file_exists(filepath): |
| 40 | """ |
| 41 | Verifies path given points to an existing file. |
| 42 | |
| 43 | @param filepath: path, path to check. |
| 44 | |
| 45 | @raises IOError if the path given does not point to a valid file. |
| 46 | |
| 47 | """ |
| 48 | error_msg = 'File %s does not exist.' % filepath |
| 49 | if not os.path.isfile(filepath): |
| 50 | raise IOError(error_msg) |
| 51 | |
| 52 | |
| 53 | def ensure_all_files_exist(filepaths): |
| 54 | """ |
| 55 | Verifies all paths given point to existing files. |
| 56 | |
| 57 | @param filepaths: List of paths to check. |
| 58 | |
| 59 | @raises IOError if given paths do not point to existing files. |
| 60 | |
| 61 | """ |
| 62 | for filepath in filepaths: |
| 63 | ensure_file_exists(filepath) |
| 64 | |
| 65 | |
| 66 | def ensure_dir_exists(dirpath): |
| 67 | """ |
| 68 | Verifies path given points to an existing directory. |
| 69 | |
| 70 | @param dirpath: path, dir to check. |
| 71 | |
| 72 | @raises IOError if path does not point to an existing directory. |
| 73 | |
| 74 | """ |
| 75 | error_msg = 'Directory %s does not exist.' % dirpath |
| 76 | if not os.path.isdir(dirpath): |
| 77 | raise IOError(error_msg) |
| 78 | |
| 79 | |
| 80 | def ensure_all_dirs_exist(dirpaths): |
| 81 | """ |
| 82 | Verifies all paths given point to existing directories. |
| 83 | |
| 84 | @param dirpaths: list of directory paths to check. |
| 85 | |
| 86 | @raises IOError if given paths do not point to existing directories. |
| 87 | |
| 88 | """ |
| 89 | for dirpath in dirpaths: |
| 90 | ensure_dir_exists(dirpath) |
| 91 | |
| 92 | |
| 93 | def make_leaf_dir(dirpath): |
| 94 | """ |
| 95 | Creates a directory, also creating parent directories if they do not exist. |
| 96 | |
| 97 | @param dirpath: path, directory to create. |
| 98 | |
| 99 | @raises whatever exception raised other than "path already exist". |
| 100 | |
| 101 | """ |
| 102 | try: |
| 103 | os.makedirs(dirpath) |
| 104 | except OSError as e: |
| 105 | if e.errno != errno.EEXIST: |
| 106 | raise |
| 107 | |
| 108 | |
| 109 | def make_leaf_dirs(dirpaths): |
| 110 | """ |
| 111 | Creates multiple directories building all respective parent directories if |
| 112 | they do not exist. |
| 113 | |
| 114 | @param dirpaths: list of directory paths to create. |
| 115 | |
| 116 | @raises whatever exception raised other than "path already exists". |
| 117 | """ |
| 118 | for dirpath in dirpaths: |
Mussa | 45745d8 | 2014-08-26 16:06:29 -0700 | [diff] [blame] | 119 | make_leaf_dir(dirpath) |
| 120 | |
| 121 | |
| 122 | def download_file(remote_path, local_path): |
| 123 | """ |
| 124 | Download file from a remote resource. |
| 125 | |
| 126 | @param remote_path: path, complete path to the remote file. |
| 127 | @param local_path: path, complete path to save downloaded file. |
| 128 | |
| 129 | @raises: urllib2.HTTPError or urlib2.URLError exception. Both with added |
| 130 | debug information |
| 131 | |
| 132 | """ |
Ed Baker | 34fdca4 | 2015-06-01 09:06:06 -0700 | [diff] [blame] | 133 | client_config = global_config.global_config.get_section_values('CLIENT') |
| 134 | proxies = {} |
| 135 | |
| 136 | for name, value in client_config.items('CLIENT'): |
| 137 | if value and name.endswith('_proxy'): |
| 138 | proxies[name[:-6]] = value |
| 139 | |
| 140 | if proxies: |
| 141 | proxy_handler = urllib2.ProxyHandler(proxies) |
| 142 | opener = urllib2.build_opener(proxy_handler) |
| 143 | urllib2.install_opener(opener) |
| 144 | |
Mussa | 45745d8 | 2014-08-26 16:06:29 -0700 | [diff] [blame] | 145 | # Unlike urllib.urlopen urllib2.urlopen will immediately throw on error |
| 146 | # If we could not find the file pointed by remote_path we will get an |
| 147 | # exception, catch the exception to log useful information then re-raise |
| 148 | |
| 149 | try: |
| 150 | remote_file = urllib2.urlopen(remote_path) |
| 151 | |
| 152 | # Catch exceptions, extract exception properties and then re-raise |
| 153 | # This helps us with debugging what went wrong quickly as we get to see |
| 154 | # test_that output immediately |
| 155 | |
| 156 | except urllib2.HTTPError as e: |
| 157 | e.msg = (("""HTTPError raised while retrieving file %s\n. |
| 158 | Http Code = %s.\n. Reason = %s\n. Headers = %s.\n |
| 159 | Original Message = %s.\n""") |
| 160 | % (remote_path, e.code, e.reason, e.headers, e.msg)) |
| 161 | raise |
| 162 | |
| 163 | except urllib2.URLError as e: |
| 164 | e.msg = (("""URLError raised while retrieving file %s\n. |
| 165 | Reason = %s\n. Original Message = %s\n.""") |
Todd Broch | 3d97d21 | 2016-06-23 15:08:18 -0700 | [diff] [blame] | 166 | % (remote_path, e.reason, e.message)) |
Mussa | 45745d8 | 2014-08-26 16:06:29 -0700 | [diff] [blame] | 167 | raise |
| 168 | |
| 169 | with open(local_path, 'wb') as local_file: |
Kuang-che Wu | 221caec | 2017-05-05 11:53:08 +0800 | [diff] [blame] | 170 | while True: |
| 171 | block = remote_file.read(128 * 1024) |
| 172 | if not block: |
| 173 | break |
| 174 | local_file.write(block) |
Dan Shi | 96c3bdc | 2017-05-24 11:34:30 -0700 | [diff] [blame] | 175 | |
| 176 | |
Dan Shi | 96c3bdc | 2017-05-24 11:34:30 -0700 | [diff] [blame] | 177 | def get_directory_size_kibibytes(directory): |
| 178 | """Calculate the total size of a directory with all its contents. |
| 179 | |
| 180 | @param directory: Path to the directory |
| 181 | |
| 182 | @return Size of the directory in kibibytes. |
| 183 | """ |
Allen Li | b41527d | 2017-06-22 17:28:00 -0700 | [diff] [blame] | 184 | cmd = ['du', '-sk', directory] |
Dan Shi | 96c3bdc | 2017-05-24 11:34:30 -0700 | [diff] [blame] | 185 | process = subprocess.Popen(cmd, |
| 186 | stdout=subprocess.PIPE, |
| 187 | stderr=subprocess.PIPE) |
| 188 | stdout_data, stderr_data = process.communicate() |
| 189 | |
| 190 | if process.returncode != 0: |
| 191 | # This function is used for statistics only, if it fails, |
| 192 | # nothing else should crash. |
| 193 | logging.warning('Getting size of %s failed. Stderr:', directory) |
| 194 | logging.warning(stderr_data) |
| 195 | return 0 |
| 196 | |
Allen Li | b41527d | 2017-06-22 17:28:00 -0700 | [diff] [blame] | 197 | return int(stdout_data.split('\t', 1)[0]) |
Dean Liao | 1ea28c6 | 2018-06-27 14:29:24 +0800 | [diff] [blame] | 198 | |
| 199 | |
| 200 | def recursive_path_permission(path): |
| 201 | """ |
| 202 | Recursively lists a path and its parent's permission. |
| 203 | |
| 204 | For a path, it emits a path_perm string "<path> <permission>" where |
| 205 | <permission> is an octal representation of the path's st_mode. After that, |
| 206 | it emits its parent's path_perm string(s) recursively. |
| 207 | For example, recursive_path_permission('/foo/bar/buz') returns |
| 208 | ['/foo/bar/buz 0100644', '/foo/bar 040755', '/foo 040755', '/ 040755'] |
| 209 | |
| 210 | If |path| is invalid, it returns empty list. |
| 211 | If |path| does not have parent directory, it returns a list with only its |
| 212 | own path_perm string. |
| 213 | |
| 214 | @param path: file path. |
| 215 | |
| 216 | @return list of "<path> <permission>" string. |
| 217 | """ |
| 218 | def path_st_mode(p): |
| 219 | """ |
| 220 | Gets path's permission. |
| 221 | |
| 222 | @param p: file path. |
| 223 | |
| 224 | @return "<path> <permission>" where <permission> is an octal |
| 225 | representation of the path's st_mode. |
| 226 | """ |
| 227 | return '%s %s' % (p, oct(os.stat(p).st_mode)) |
| 228 | |
| 229 | if not path or not os.path.exists(path): |
| 230 | return [] |
| 231 | if path == '/': |
| 232 | return [path_st_mode(path)] |
| 233 | dirname, basename = os.path.split(path) |
| 234 | if not basename: |
| 235 | return recursive_path_permission(dirname) |
| 236 | result = [path_st_mode(path)] |
| 237 | if dirname: |
| 238 | result.extend(recursive_path_permission(dirname)) |
| 239 | return result |