showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 1 | #!/usr/bin/python -u |
| 2 | |
| 3 | """ |
| 4 | Utility to upload or remove the packages from the packages repository. |
| 5 | """ |
| 6 | |
Dale Curtis | 6ad3319 | 2011-07-06 18:04:50 -0700 | [diff] [blame] | 7 | import logging, optparse, os, shutil, sys, tempfile |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 8 | import common |
| 9 | from autotest_lib.client.common_lib import utils as client_utils |
jadmanski | c27c231 | 2009-08-05 20:58:51 +0000 | [diff] [blame] | 10 | from autotest_lib.client.common_lib import global_config, error |
| 11 | from autotest_lib.client.common_lib import base_packages, packages |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 12 | from autotest_lib.server import utils as server_utils |
| 13 | |
| 14 | c = global_config.global_config |
showard | f6547fe | 2009-07-08 22:58:53 +0000 | [diff] [blame] | 15 | logging.basicConfig(level=logging.DEBUG) |
mbligh | dbfc4e3 | 2008-08-22 18:08:07 +0000 | [diff] [blame] | 16 | |
| 17 | def get_exclude_string(client_dir): |
| 18 | ''' |
| 19 | Get the exclude string for the tar command to exclude specific |
| 20 | subdirectories inside client_dir. |
| 21 | For profilers we need to exclude everything except the __ini__.py |
| 22 | file so that the profilers can be imported. |
| 23 | ''' |
| 24 | exclude_string = ('--exclude=deps/* --exclude=tests/* ' |
| 25 | '--exclude=site_tests/*') |
| 26 | |
| 27 | # Get the profilers directory |
| 28 | prof_dir = os.path.join(client_dir, 'profilers') |
| 29 | |
| 30 | # Include the __init__.py file for the profilers and exclude all its |
| 31 | # subdirectories |
| 32 | for f in os.listdir(prof_dir): |
| 33 | if os.path.isdir(os.path.join(prof_dir, f)): |
| 34 | exclude_string += ' --exclude=profilers/%s' % f |
| 35 | |
| 36 | # The '.' here is needed to zip the files in the current |
| 37 | # directory. We use '-C' for tar to change to the required |
| 38 | # directory i.e. src_dir and then zip up the files in that |
| 39 | # directory(which is '.') excluding the ones in the exclude_dirs |
| 40 | exclude_string += " ." |
| 41 | |
| 42 | return exclude_string |
| 43 | |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 44 | |
| 45 | def parse_args(): |
| 46 | parser = optparse.OptionParser() |
| 47 | parser.add_option("-d", "--dependency", help="package the dependency" |
| 48 | " from client/deps directory and upload to the repo", |
| 49 | dest="dep") |
| 50 | parser.add_option("-p", "--profiler", help="package the profiler " |
| 51 | "from client/profilers directory and upload to the repo", |
| 52 | dest="prof") |
| 53 | parser.add_option("-t", "--test", help="package the test from client/tests" |
| 54 | " or client/site_tests and upload to the repo.", |
| 55 | dest="test") |
| 56 | parser.add_option("-c", "--client", help="package the client " |
| 57 | "directory alone without the tests, deps and profilers", |
| 58 | dest="client", action="store_true", default=False) |
| 59 | parser.add_option("-f", "--file", help="simply uploads the specified" |
| 60 | "file on to the repo", dest="file") |
| 61 | parser.add_option("-r", "--repository", help="the URL of the packages" |
| 62 | "repository location to upload the packages to.", |
| 63 | dest="repo", default=None) |
| 64 | parser.add_option("--all", help="Upload all the files locally " |
| 65 | "to all the repos specified in global_config.ini. " |
| 66 | "(includes the client, tests, deps and profilers)", |
| 67 | dest="all", action="store_true", default=False) |
| 68 | |
| 69 | options, args = parser.parse_args() |
| 70 | return options, args |
| 71 | |
| 72 | |
| 73 | # Method to upload or remove package depending on the flag passed to it. |
Eric Li | d656d56 | 2011-04-20 11:48:29 -0700 | [diff] [blame] | 74 | def process_packages(pkgmgr, pkg_type, pkg_names, src_dir, |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 75 | remove=False): |
mbligh | dbfc4e3 | 2008-08-22 18:08:07 +0000 | [diff] [blame] | 76 | exclude_string = ' .' |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 77 | names = [p.strip() for p in pkg_names.split(',')] |
| 78 | for name in names: |
| 79 | print "Processing %s ... " % name |
Dale Curtis | 6ad3319 | 2011-07-06 18:04:50 -0700 | [diff] [blame] | 80 | if pkg_type == 'client': |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 81 | pkg_dir = src_dir |
Dale Curtis | 6ad3319 | 2011-07-06 18:04:50 -0700 | [diff] [blame] | 82 | exclude_string = get_exclude_string(pkg_dir) |
| 83 | elif pkg_type == 'test': |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 84 | # if the package is a test then look whether it is in client/tests |
| 85 | # or client/site_tests |
| 86 | pkg_dir = os.path.join(get_test_dir(name, src_dir), name) |
| 87 | else: |
| 88 | # for the profilers and deps |
| 89 | pkg_dir = os.path.join(src_dir, name) |
| 90 | |
| 91 | pkg_name = pkgmgr.get_tarball_name(name, pkg_type) |
| 92 | if not remove: |
| 93 | # Tar the source and upload |
mbligh | 982d13f | 2009-09-03 20:46:00 +0000 | [diff] [blame] | 94 | temp_dir = tempfile.mkdtemp() |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 95 | try: |
mbligh | e183681 | 2009-04-17 18:25:14 +0000 | [diff] [blame] | 96 | try: |
jadmanski | c27c231 | 2009-08-05 20:58:51 +0000 | [diff] [blame] | 97 | base_packages.check_diskspace(temp_dir) |
mbligh | 6d7c565 | 2009-11-06 03:00:55 +0000 | [diff] [blame] | 98 | except error.RepoDiskFullError, e: |
| 99 | msg = ("Temporary directory for packages %s does not have " |
| 100 | "enough space available: %s" % (temp_dir, e)) |
jadmanski | c27c231 | 2009-08-05 20:58:51 +0000 | [diff] [blame] | 101 | raise error.RepoDiskFullError(msg) |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 102 | tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir, |
mbligh | dbfc4e3 | 2008-08-22 18:08:07 +0000 | [diff] [blame] | 103 | temp_dir, exclude_string) |
Eric Li | d656d56 | 2011-04-20 11:48:29 -0700 | [diff] [blame] | 104 | pkgmgr.upload_pkg(tarball_path, update_checksum=True) |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 105 | finally: |
| 106 | # remove the temporary directory |
| 107 | shutil.rmtree(temp_dir) |
| 108 | else: |
Eric Li | d656d56 | 2011-04-20 11:48:29 -0700 | [diff] [blame] | 109 | pkgmgr.remove_pkg(pkg_name, remove_checksum=True) |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 110 | print "Done." |
| 111 | |
| 112 | |
mbligh | 9fc7797 | 2008-10-02 20:32:09 +0000 | [diff] [blame] | 113 | def tar_packages(pkgmgr, pkg_type, pkg_names, src_dir, temp_dir): |
| 114 | """Tar all packages up and return a list of each tar created""" |
| 115 | tarballs = [] |
| 116 | exclude_string = ' .' |
| 117 | names = [p.strip() for p in pkg_names.split(',')] |
| 118 | for name in names: |
| 119 | print "Processing %s ... " % name |
Dale Curtis | 6ad3319 | 2011-07-06 18:04:50 -0700 | [diff] [blame] | 120 | if pkg_type == 'client': |
mbligh | 9fc7797 | 2008-10-02 20:32:09 +0000 | [diff] [blame] | 121 | pkg_dir = src_dir |
Dale Curtis | 6ad3319 | 2011-07-06 18:04:50 -0700 | [diff] [blame] | 122 | exclude_string = get_exclude_string(pkg_dir) |
| 123 | elif pkg_type == 'test': |
mbligh | 9fc7797 | 2008-10-02 20:32:09 +0000 | [diff] [blame] | 124 | # if the package is a test then look whether it is in client/tests |
| 125 | # or client/site_tests |
| 126 | pkg_dir = os.path.join(get_test_dir(name, src_dir), name) |
| 127 | else: |
| 128 | # for the profilers and deps |
| 129 | pkg_dir = os.path.join(src_dir, name) |
| 130 | |
| 131 | pkg_name = pkgmgr.get_tarball_name(name, pkg_type) |
| 132 | tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir, |
| 133 | temp_dir, exclude_string) |
| 134 | |
| 135 | tarballs.append(tarball_path) |
| 136 | |
| 137 | return tarballs |
| 138 | |
| 139 | |
Eric Li | d656d56 | 2011-04-20 11:48:29 -0700 | [diff] [blame] | 140 | def process_all_packages(pkgmgr, client_dir, remove=False): |
mbligh | 9fc7797 | 2008-10-02 20:32:09 +0000 | [diff] [blame] | 141 | """Process a full upload of packages as a directory upload.""" |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 142 | dep_dir = os.path.join(client_dir, "deps") |
| 143 | prof_dir = os.path.join(client_dir, "profilers") |
mbligh | 9fc7797 | 2008-10-02 20:32:09 +0000 | [diff] [blame] | 144 | # Directory where all are kept |
| 145 | temp_dir = tempfile.mkdtemp() |
mbligh | e183681 | 2009-04-17 18:25:14 +0000 | [diff] [blame] | 146 | try: |
jadmanski | c27c231 | 2009-08-05 20:58:51 +0000 | [diff] [blame] | 147 | base_packages.check_diskspace(temp_dir) |
mbligh | 6d7c565 | 2009-11-06 03:00:55 +0000 | [diff] [blame] | 148 | except error.RepoDiskFullError, e: |
| 149 | print ("Temp destination for packages is full %s, aborting upload: %s" |
| 150 | % (temp_dir, e)) |
mbligh | e183681 | 2009-04-17 18:25:14 +0000 | [diff] [blame] | 151 | os.rmdir(temp_dir) |
| 152 | sys.exit(1) |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 153 | |
| 154 | # process tests |
| 155 | tests_list = get_subdir_list('tests', client_dir) |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 156 | tests = ','.join(tests_list) |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 157 | |
| 158 | # process site_tests |
| 159 | site_tests_list = get_subdir_list('site_tests', client_dir) |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 160 | site_tests = ','.join(site_tests_list) |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 161 | |
| 162 | # process deps |
| 163 | deps_list = get_subdir_list('deps', client_dir) |
| 164 | deps = ','.join(deps_list) |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 165 | |
| 166 | # process profilers |
| 167 | profilers_list = get_subdir_list('profilers', client_dir) |
| 168 | profilers = ','.join(profilers_list) |
mbligh | 9fc7797 | 2008-10-02 20:32:09 +0000 | [diff] [blame] | 169 | |
| 170 | # Update md5sum |
| 171 | if not remove: |
| 172 | tar_packages(pkgmgr, 'profiler', profilers, prof_dir, temp_dir) |
| 173 | tar_packages(pkgmgr, 'dep', deps, dep_dir, temp_dir) |
| 174 | tar_packages(pkgmgr, 'test', site_tests, client_dir, temp_dir) |
| 175 | tar_packages(pkgmgr, 'test', tests, client_dir, temp_dir) |
| 176 | tar_packages(pkgmgr, 'client', 'autotest', client_dir, temp_dir) |
| 177 | cwd = os.getcwd() |
| 178 | os.chdir(temp_dir) |
| 179 | client_utils.system('md5sum * > packages.checksum') |
| 180 | os.chdir(cwd) |
Eric Li | d656d56 | 2011-04-20 11:48:29 -0700 | [diff] [blame] | 181 | pkgmgr.upload_pkg(temp_dir) |
mbligh | 9fc7797 | 2008-10-02 20:32:09 +0000 | [diff] [blame] | 182 | client_utils.run('rm -rf ' + temp_dir) |
| 183 | else: |
Dale Curtis | 6ad3319 | 2011-07-06 18:04:50 -0700 | [diff] [blame] | 184 | process_packages(pkgmgr, 'test', tests, client_dir, remove=remove) |
Eric Li | d656d56 | 2011-04-20 11:48:29 -0700 | [diff] [blame] | 185 | process_packages(pkgmgr, 'test', site_tests, client_dir, remove=remove) |
| 186 | process_packages(pkgmgr, 'client', 'autotest', client_dir, |
| 187 | remove=remove) |
| 188 | process_packages(pkgmgr, 'dep', deps, dep_dir, remove=remove) |
| 189 | process_packages(pkgmgr, 'profiler', profilers, prof_dir, |
| 190 | remove=remove) |
mbligh | 9fc7797 | 2008-10-02 20:32:09 +0000 | [diff] [blame] | 191 | |
| 192 | |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 193 | # Get the list of sub directories present in a directory |
| 194 | def get_subdir_list(name, client_dir): |
| 195 | dir_name = os.path.join(client_dir, name) |
| 196 | return [f for f in |
| 197 | os.listdir(dir_name) |
| 198 | if os.path.isdir(os.path.join(dir_name, f)) ] |
| 199 | |
| 200 | |
| 201 | # Look whether the test is present in client/tests and client/site_tests dirs |
| 202 | def get_test_dir(name, client_dir): |
| 203 | names_test = os.listdir(os.path.join(client_dir, 'tests')) |
| 204 | names_site_test = os.listdir(os.path.join(client_dir, 'site_tests')) |
| 205 | if name in names_test: |
| 206 | src_dir = os.path.join(client_dir, 'tests') |
| 207 | elif name in names_site_test: |
| 208 | src_dir = os.path.join(client_dir, 'site_tests') |
| 209 | else: |
| 210 | print "Test %s not found" % name |
| 211 | sys.exit(0) |
| 212 | return src_dir |
| 213 | |
| 214 | |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 215 | def main(): |
mbligh | de613a9 | 2009-02-03 21:55:34 +0000 | [diff] [blame] | 216 | # get options and args |
| 217 | options, args = parse_args() |
| 218 | |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 219 | server_dir = server_utils.get_server_dir() |
| 220 | autotest_dir = os.path.abspath(os.path.join(server_dir, '..')) |
| 221 | |
| 222 | # extract the pkg locations from global config |
| 223 | repo_urls = c.get_config_value('PACKAGES', 'fetch_location', |
| 224 | type=list, default=[]) |
| 225 | upload_paths = c.get_config_value('PACKAGES', 'upload_location', |
| 226 | type=list, default=[]) |
Dale Curtis | 6ad3319 | 2011-07-06 18:04:50 -0700 | [diff] [blame] | 227 | |
| 228 | if options.repo: |
| 229 | upload_paths.append(options.repo) |
| 230 | |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 231 | # Having no upload paths basically means you're not using packaging. |
Dale Curtis | 6ad3319 | 2011-07-06 18:04:50 -0700 | [diff] [blame] | 232 | if not upload_paths: |
| 233 | print("No upload locations found. Please set upload_location under" |
| 234 | " PACKAGES in the global_config.ini or provide a location using" |
| 235 | " the --repository option.") |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 236 | return |
| 237 | |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 238 | client_dir = os.path.join(autotest_dir, "client") |
| 239 | |
| 240 | # Bail out if the client directory does not exist |
| 241 | if not os.path.exists(client_dir): |
| 242 | sys.exit(0) |
| 243 | |
| 244 | dep_dir = os.path.join(client_dir, "deps") |
| 245 | prof_dir = os.path.join(client_dir, "profilers") |
| 246 | |
Dale Curtis | 6ad3319 | 2011-07-06 18:04:50 -0700 | [diff] [blame] | 247 | if len(args) == 0 or args[0] not in ['upload', 'remove']: |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 248 | print("Either 'upload' or 'remove' needs to be specified " |
| 249 | "for the package") |
| 250 | sys.exit(0) |
| 251 | |
Dale Curtis | 6ad3319 | 2011-07-06 18:04:50 -0700 | [diff] [blame] | 252 | if args[0] == 'upload': |
| 253 | remove_flag = False |
| 254 | elif args[0] == 'remove': |
| 255 | remove_flag = True |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 256 | else: |
| 257 | # we should not be getting here |
| 258 | assert(False) |
| 259 | |
Eric Li | d656d56 | 2011-04-20 11:48:29 -0700 | [diff] [blame] | 260 | pkgmgr = packages.PackageManager(autotest_dir, repo_urls=repo_urls, |
Dale Curtis | 6ad3319 | 2011-07-06 18:04:50 -0700 | [diff] [blame] | 261 | upload_paths=upload_paths, |
Eric Li | d656d56 | 2011-04-20 11:48:29 -0700 | [diff] [blame] | 262 | run_function_dargs={'timeout':600}) |
| 263 | |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 264 | if options.all: |
Eric Li | d656d56 | 2011-04-20 11:48:29 -0700 | [diff] [blame] | 265 | process_all_packages(pkgmgr, client_dir, remove=remove_flag) |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 266 | |
| 267 | if options.client: |
| 268 | process_packages(pkgmgr, 'client', 'autotest', client_dir, |
Eric Li | d656d56 | 2011-04-20 11:48:29 -0700 | [diff] [blame] | 269 | remove=remove_flag) |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 270 | |
| 271 | if options.dep: |
| 272 | process_packages(pkgmgr, 'dep', options.dep, dep_dir, |
Eric Li | d656d56 | 2011-04-20 11:48:29 -0700 | [diff] [blame] | 273 | remove=remove_flag) |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 274 | |
| 275 | if options.test: |
| 276 | process_packages(pkgmgr, 'test', options.test, client_dir, |
Eric Li | d656d56 | 2011-04-20 11:48:29 -0700 | [diff] [blame] | 277 | remove=remove_flag) |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 278 | |
| 279 | if options.prof: |
| 280 | process_packages(pkgmgr, 'profiler', options.prof, prof_dir, |
Eric Li | d656d56 | 2011-04-20 11:48:29 -0700 | [diff] [blame] | 281 | remove=remove_flag) |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 282 | |
| 283 | if options.file: |
| 284 | if remove_flag: |
Eric Li | d656d56 | 2011-04-20 11:48:29 -0700 | [diff] [blame] | 285 | pkgmgr.remove_pkg(options.file, remove_checksum=True) |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 286 | else: |
Eric Li | d656d56 | 2011-04-20 11:48:29 -0700 | [diff] [blame] | 287 | pkgmgr.upload_pkg(options.file, update_checksum=True) |
showard | 9d02fb5 | 2008-08-08 18:20:37 +0000 | [diff] [blame] | 288 | |
| 289 | |
| 290 | if __name__ == "__main__": |
| 291 | main() |