mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | # |
| 3 | # Please keep this code python 2.4 compatible and stand alone. |
| 4 | |
| 5 | """ |
| 6 | Fetch, build and install external Python library dependancies. |
| 7 | |
| 8 | This fetches external python libraries, builds them using your host's |
| 9 | python and installs them under our own autotest/site-packages/ directory. |
| 10 | |
| 11 | Usage? Just run it. |
| 12 | utils/build_externals.py |
| 13 | """ |
| 14 | |
Shuqian Zhao | 697f7ed | 2016-10-21 14:58:28 -0700 | [diff] [blame] | 15 | import argparse |
| 16 | import compileall |
| 17 | import logging |
| 18 | import os |
| 19 | import sys |
| 20 | |
mbligh | e5699e2 | 2009-09-18 16:44:12 +0000 | [diff] [blame] | 21 | import common |
| 22 | from autotest_lib.client.common_lib import logging_config, logging_manager |
lmr | 6f80e7a | 2010-02-04 03:18:28 +0000 | [diff] [blame] | 23 | from autotest_lib.client.common_lib import utils |
jamesren | ff6e5aa | 2010-02-12 00:46:40 +0000 | [diff] [blame] | 24 | from autotest_lib.utils import external_packages |
| 25 | |
| 26 | # bring in site packages as well |
| 27 | utils.import_site_module(__file__, 'autotest_lib.utils.site_external_packages') |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 28 | |
| 29 | # Where package source be fetched to relative to the top of the autotest tree. |
| 30 | PACKAGE_DIR = 'ExternalSource' |
| 31 | |
| 32 | # Where packages will be installed to relative to the top of the autotest tree. |
| 33 | INSTALL_DIR = 'site-packages' |
| 34 | |
mbligh | 489e0e0 | 2009-06-08 16:46:11 +0000 | [diff] [blame] | 35 | # Installs all packages, even if the system already has the version required |
| 36 | INSTALL_ALL = False |
| 37 | |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 38 | |
| 39 | # Want to add more packages to fetch, build and install? See the class |
jamesren | ff6e5aa | 2010-02-12 00:46:40 +0000 | [diff] [blame] | 40 | # definitions at the end of external_packages.py for examples of how to do it. |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 41 | |
| 42 | |
mbligh | e5699e2 | 2009-09-18 16:44:12 +0000 | [diff] [blame] | 43 | class BuildExternalsLoggingConfig(logging_config.LoggingConfig): |
| 44 | def configure_logging(self, results_dir=None, verbose=False): |
| 45 | super(BuildExternalsLoggingConfig, self).configure_logging( |
| 46 | use_console=True, |
| 47 | verbose=verbose) |
| 48 | |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 49 | |
| 50 | def main(): |
| 51 | """ |
| 52 | Find all ExternalPackage classes defined in this file and ask them to |
| 53 | fetch, build and install themselves. |
| 54 | """ |
Shuqian Zhao | 697f7ed | 2016-10-21 14:58:28 -0700 | [diff] [blame] | 55 | options = parse_arguments(sys.argv[1:]) |
mbligh | e5699e2 | 2009-09-18 16:44:12 +0000 | [diff] [blame] | 56 | logging_manager.configure_logging(BuildExternalsLoggingConfig(), |
| 57 | verbose=True) |
mbligh | b155d0c | 2009-07-02 19:04:55 +0000 | [diff] [blame] | 58 | os.umask(022) |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 59 | |
jamesren | ff6e5aa | 2010-02-12 00:46:40 +0000 | [diff] [blame] | 60 | top_of_tree = external_packages.find_top_of_autotest_tree() |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 61 | package_dir = os.path.join(top_of_tree, PACKAGE_DIR) |
| 62 | install_dir = os.path.join(top_of_tree, INSTALL_DIR) |
| 63 | |
lmr | 7ebba66 | 2009-10-23 12:55:09 +0000 | [diff] [blame] | 64 | # Make sure the install_dir is in our python module search path |
| 65 | # as well as the PYTHONPATH being used by all our setup.py |
| 66 | # install subprocesses. |
mbligh | 623c518 | 2009-02-07 01:31:31 +0000 | [diff] [blame] | 67 | if install_dir not in sys.path: |
mbligh | 623c518 | 2009-02-07 01:31:31 +0000 | [diff] [blame] | 68 | sys.path.insert(0, install_dir) |
lmr | 7ebba66 | 2009-10-23 12:55:09 +0000 | [diff] [blame] | 69 | env_python_path_varname = 'PYTHONPATH' |
| 70 | env_python_path = os.environ.get(env_python_path_varname, '') |
| 71 | if install_dir+':' not in env_python_path: |
| 72 | os.environ[env_python_path_varname] = ':'.join([ |
| 73 | install_dir, env_python_path]) |
mbligh | 623c518 | 2009-02-07 01:31:31 +0000 | [diff] [blame] | 74 | |
Shuqian Zhao | 697f7ed | 2016-10-21 14:58:28 -0700 | [diff] [blame] | 75 | fetched_packages, fetch_errors = fetch_necessary_packages( |
| 76 | package_dir, install_dir, set(options.names_to_check)) |
| 77 | install_errors = build_and_install_packages( |
| 78 | fetched_packages, install_dir, options.use_chromite_master) |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 79 | |
| 80 | # Byte compile the code after it has been installed in its final |
| 81 | # location as .pyc files contain the path passed to compile_dir(). |
| 82 | # When printing exception tracebacks, python uses that path first to look |
| 83 | # for the source code before checking the directory of the .pyc file. |
| 84 | # Don't leave references to our temporary build dir in the files. |
mbligh | 623c518 | 2009-02-07 01:31:31 +0000 | [diff] [blame] | 85 | logging.info('compiling .py files in %s to .pyc', install_dir) |
| 86 | compileall.compile_dir(install_dir, quiet=True) |
| 87 | |
| 88 | # Some things install with whacky permissions, fix that. |
jamesren | ff6e5aa | 2010-02-12 00:46:40 +0000 | [diff] [blame] | 89 | external_packages.system("chmod -R a+rX '%s'" % install_dir) |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 90 | |
| 91 | errors = fetch_errors + install_errors |
| 92 | for error_msg in errors: |
lmr | 65bec4f | 2010-04-08 02:30:35 +0000 | [diff] [blame] | 93 | logging.error(error_msg) |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 94 | |
| 95 | return len(errors) |
| 96 | |
| 97 | |
Shuqian Zhao | 697f7ed | 2016-10-21 14:58:28 -0700 | [diff] [blame] | 98 | def parse_arguments(args): |
| 99 | """Parse command line arguments. |
| 100 | |
| 101 | @param args: The command line arguments to parse. (ususally sys.argsv[1:]) |
| 102 | |
| 103 | @returns An argparse.Namespace populated with argument values. |
| 104 | """ |
| 105 | parser = argparse.ArgumentParser( |
| 106 | description='Command to build third party dependencies required ' |
| 107 | 'for autotest.') |
| 108 | parser.add_argument('--use_chromite_master', action='store_true', |
| 109 | help='Update chromite to master branch, rather than ' |
| 110 | 'prod.') |
| 111 | parser.add_argument('--names_to_check', nargs='*', type=str, default=set(), |
| 112 | help='Package names to check whether they are needed ' |
| 113 | 'in current system.') |
| 114 | return parser.parse_args(args) |
| 115 | |
| 116 | |
| 117 | def fetch_necessary_packages(dest_dir, install_dir, names_to_check=set()): |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 118 | """ |
| 119 | Fetches all ExternalPackages into dest_dir. |
| 120 | |
mbligh | b155d0c | 2009-07-02 19:04:55 +0000 | [diff] [blame] | 121 | @param dest_dir: Directory the packages should be fetched into. |
| 122 | @param install_dir: Directory where packages will later installed. |
Shuqian Zhao | 697f7ed | 2016-10-21 14:58:28 -0700 | [diff] [blame] | 123 | @param names_to_check: A set of package names to check whether they are |
| 124 | needed on current system. Default is empty. |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 125 | |
| 126 | @returns A tuple containing two lists: |
| 127 | * A list of ExternalPackage instances that were fetched and |
| 128 | need to be installed. |
| 129 | * A list of error messages for any failed fetches. |
| 130 | """ |
| 131 | errors = [] |
| 132 | fetched_packages = [] |
jamesren | ff6e5aa | 2010-02-12 00:46:40 +0000 | [diff] [blame] | 133 | for package_class in external_packages.ExternalPackage.subclasses: |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 134 | package = package_class() |
mbligh | 623c518 | 2009-02-07 01:31:31 +0000 | [diff] [blame] | 135 | if names_to_check and package.name.lower() not in names_to_check: |
| 136 | continue |
mbligh | b155d0c | 2009-07-02 19:04:55 +0000 | [diff] [blame] | 137 | if not package.is_needed(install_dir): |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 138 | logging.info('A new %s is not needed on this system.', |
| 139 | package.name) |
mbligh | 489e0e0 | 2009-06-08 16:46:11 +0000 | [diff] [blame] | 140 | if INSTALL_ALL: |
| 141 | logging.info('Installing anyways...') |
| 142 | else: |
| 143 | continue |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 144 | if not package.fetch(dest_dir): |
lmr | 65bec4f | 2010-04-08 02:30:35 +0000 | [diff] [blame] | 145 | msg = 'Unable to download %s' % package.name |
| 146 | logging.error(msg) |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 147 | errors.append(msg) |
| 148 | else: |
| 149 | fetched_packages.append(package) |
| 150 | |
| 151 | return fetched_packages, errors |
| 152 | |
| 153 | |
Shuqian Zhao | 697f7ed | 2016-10-21 14:58:28 -0700 | [diff] [blame] | 154 | def build_and_install_packages(packages, install_dir, |
| 155 | use_chromite_master=False): |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 156 | """ |
| 157 | Builds and installs all packages into install_dir. |
| 158 | |
| 159 | @param packages - A list of already fetched ExternalPackage instances. |
| 160 | @param install_dir - Directory the packages should be installed into. |
Shuqian Zhao | 697f7ed | 2016-10-21 14:58:28 -0700 | [diff] [blame] | 161 | @param use_chromite_master: True if updating chromite to master branch. |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 162 | |
| 163 | @returns A list of error messages for any installs that failed. |
| 164 | """ |
| 165 | errors = [] |
| 166 | for package in packages: |
Shuqian Zhao | 697f7ed | 2016-10-21 14:58:28 -0700 | [diff] [blame] | 167 | if use_chromite_master and package.name.lower() == 'chromiterepo': |
| 168 | result = package.build_and_install(install_dir, master_branch=True) |
| 169 | else: |
| 170 | result = package.build_and_install(install_dir) |
Dan Shi | 16c0a50 | 2015-07-14 17:29:48 -0700 | [diff] [blame] | 171 | if isinstance(result, bool): |
| 172 | success = result |
| 173 | message = None |
| 174 | else: |
| 175 | success = result[0] |
| 176 | message = result[1] |
| 177 | if not success: |
| 178 | msg = ('Unable to build and install %s.\nError: %s' % |
| 179 | (package.name, message)) |
lmr | 65bec4f | 2010-04-08 02:30:35 +0000 | [diff] [blame] | 180 | logging.error(msg) |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 181 | errors.append(msg) |
| 182 | return errors |
| 183 | |
| 184 | |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 185 | if __name__ == '__main__': |
| 186 | sys.exit(main()) |