mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | # |
| 3 | # Please keep this code python 2.4 compatible and stand alone. |
| 4 | |
| 5 | """ |
| 6 | Fetch, build and install external Python library dependancies. |
| 7 | |
| 8 | This fetches external python libraries, builds them using your host's |
| 9 | python and installs them under our own autotest/site-packages/ directory. |
| 10 | |
| 11 | Usage? Just run it. |
| 12 | utils/build_externals.py |
| 13 | """ |
| 14 | |
jamesren | ff6e5aa | 2010-02-12 00:46:40 +0000 | [diff] [blame^] | 15 | import compileall, logging, os, sha, shutil, sys, tempfile, time, urllib2 |
mbligh | 489e0e0 | 2009-06-08 16:46:11 +0000 | [diff] [blame] | 16 | import subprocess, re |
mbligh | e5699e2 | 2009-09-18 16:44:12 +0000 | [diff] [blame] | 17 | import common |
| 18 | from autotest_lib.client.common_lib import logging_config, logging_manager |
lmr | 6f80e7a | 2010-02-04 03:18:28 +0000 | [diff] [blame] | 19 | from autotest_lib.client.common_lib import utils |
jamesren | ff6e5aa | 2010-02-12 00:46:40 +0000 | [diff] [blame^] | 20 | from autotest_lib.utils import external_packages |
| 21 | |
| 22 | # bring in site packages as well |
| 23 | utils.import_site_module(__file__, 'autotest_lib.utils.site_external_packages') |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 24 | |
| 25 | # Where package source be fetched to relative to the top of the autotest tree. |
| 26 | PACKAGE_DIR = 'ExternalSource' |
| 27 | |
| 28 | # Where packages will be installed to relative to the top of the autotest tree. |
| 29 | INSTALL_DIR = 'site-packages' |
| 30 | |
mbligh | 489e0e0 | 2009-06-08 16:46:11 +0000 | [diff] [blame] | 31 | # Installs all packages, even if the system already has the version required |
| 32 | INSTALL_ALL = False |
| 33 | |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 34 | |
| 35 | # Want to add more packages to fetch, build and install? See the class |
jamesren | ff6e5aa | 2010-02-12 00:46:40 +0000 | [diff] [blame^] | 36 | # definitions at the end of external_packages.py for examples of how to do it. |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 37 | |
| 38 | |
mbligh | e5699e2 | 2009-09-18 16:44:12 +0000 | [diff] [blame] | 39 | class BuildExternalsLoggingConfig(logging_config.LoggingConfig): |
| 40 | def configure_logging(self, results_dir=None, verbose=False): |
| 41 | super(BuildExternalsLoggingConfig, self).configure_logging( |
| 42 | use_console=True, |
| 43 | verbose=verbose) |
| 44 | |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 45 | |
| 46 | def main(): |
| 47 | """ |
| 48 | Find all ExternalPackage classes defined in this file and ask them to |
| 49 | fetch, build and install themselves. |
| 50 | """ |
mbligh | e5699e2 | 2009-09-18 16:44:12 +0000 | [diff] [blame] | 51 | logging_manager.configure_logging(BuildExternalsLoggingConfig(), |
| 52 | verbose=True) |
mbligh | b155d0c | 2009-07-02 19:04:55 +0000 | [diff] [blame] | 53 | os.umask(022) |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 54 | |
jamesren | ff6e5aa | 2010-02-12 00:46:40 +0000 | [diff] [blame^] | 55 | top_of_tree = external_packages.find_top_of_autotest_tree() |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 56 | package_dir = os.path.join(top_of_tree, PACKAGE_DIR) |
| 57 | install_dir = os.path.join(top_of_tree, INSTALL_DIR) |
| 58 | |
lmr | 7ebba66 | 2009-10-23 12:55:09 +0000 | [diff] [blame] | 59 | # Make sure the install_dir is in our python module search path |
| 60 | # as well as the PYTHONPATH being used by all our setup.py |
| 61 | # install subprocesses. |
mbligh | 623c518 | 2009-02-07 01:31:31 +0000 | [diff] [blame] | 62 | if install_dir not in sys.path: |
mbligh | 623c518 | 2009-02-07 01:31:31 +0000 | [diff] [blame] | 63 | sys.path.insert(0, install_dir) |
lmr | 7ebba66 | 2009-10-23 12:55:09 +0000 | [diff] [blame] | 64 | env_python_path_varname = 'PYTHONPATH' |
| 65 | env_python_path = os.environ.get(env_python_path_varname, '') |
| 66 | if install_dir+':' not in env_python_path: |
| 67 | os.environ[env_python_path_varname] = ':'.join([ |
| 68 | install_dir, env_python_path]) |
mbligh | 623c518 | 2009-02-07 01:31:31 +0000 | [diff] [blame] | 69 | |
mbligh | b155d0c | 2009-07-02 19:04:55 +0000 | [diff] [blame] | 70 | fetched_packages, fetch_errors = fetch_necessary_packages(package_dir, |
| 71 | install_dir) |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 72 | install_errors = build_and_install_packages(fetched_packages, install_dir) |
| 73 | |
| 74 | # Byte compile the code after it has been installed in its final |
| 75 | # location as .pyc files contain the path passed to compile_dir(). |
| 76 | # When printing exception tracebacks, python uses that path first to look |
| 77 | # for the source code before checking the directory of the .pyc file. |
| 78 | # Don't leave references to our temporary build dir in the files. |
mbligh | 623c518 | 2009-02-07 01:31:31 +0000 | [diff] [blame] | 79 | logging.info('compiling .py files in %s to .pyc', install_dir) |
| 80 | compileall.compile_dir(install_dir, quiet=True) |
| 81 | |
| 82 | # Some things install with whacky permissions, fix that. |
jamesren | ff6e5aa | 2010-02-12 00:46:40 +0000 | [diff] [blame^] | 83 | external_packages.system("chmod -R a+rX '%s'" % install_dir) |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 84 | |
| 85 | errors = fetch_errors + install_errors |
| 86 | for error_msg in errors: |
| 87 | print >>sys.stderr, error_msg |
| 88 | |
| 89 | return len(errors) |
| 90 | |
| 91 | |
mbligh | b155d0c | 2009-07-02 19:04:55 +0000 | [diff] [blame] | 92 | def fetch_necessary_packages(dest_dir, install_dir): |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 93 | """ |
| 94 | Fetches all ExternalPackages into dest_dir. |
| 95 | |
mbligh | b155d0c | 2009-07-02 19:04:55 +0000 | [diff] [blame] | 96 | @param dest_dir: Directory the packages should be fetched into. |
| 97 | @param install_dir: Directory where packages will later installed. |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 98 | |
| 99 | @returns A tuple containing two lists: |
| 100 | * A list of ExternalPackage instances that were fetched and |
| 101 | need to be installed. |
| 102 | * A list of error messages for any failed fetches. |
| 103 | """ |
mbligh | 623c518 | 2009-02-07 01:31:31 +0000 | [diff] [blame] | 104 | names_to_check = sys.argv[1:] |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 105 | errors = [] |
| 106 | fetched_packages = [] |
jamesren | ff6e5aa | 2010-02-12 00:46:40 +0000 | [diff] [blame^] | 107 | for package_class in external_packages.ExternalPackage.subclasses: |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 108 | package = package_class() |
mbligh | 623c518 | 2009-02-07 01:31:31 +0000 | [diff] [blame] | 109 | if names_to_check and package.name.lower() not in names_to_check: |
| 110 | continue |
mbligh | b155d0c | 2009-07-02 19:04:55 +0000 | [diff] [blame] | 111 | if not package.is_needed(install_dir): |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 112 | logging.info('A new %s is not needed on this system.', |
| 113 | package.name) |
mbligh | 489e0e0 | 2009-06-08 16:46:11 +0000 | [diff] [blame] | 114 | if INSTALL_ALL: |
| 115 | logging.info('Installing anyways...') |
| 116 | else: |
| 117 | continue |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 118 | if not package.fetch(dest_dir): |
| 119 | msg = '!!! Unable to download %s' % package.name |
| 120 | print msg |
| 121 | errors.append(msg) |
| 122 | else: |
| 123 | fetched_packages.append(package) |
| 124 | |
| 125 | return fetched_packages, errors |
| 126 | |
| 127 | |
| 128 | def build_and_install_packages(packages, install_dir): |
| 129 | """ |
| 130 | Builds and installs all packages into install_dir. |
| 131 | |
| 132 | @param packages - A list of already fetched ExternalPackage instances. |
| 133 | @param install_dir - Directory the packages should be installed into. |
| 134 | |
| 135 | @returns A list of error messages for any installs that failed. |
| 136 | """ |
| 137 | errors = [] |
| 138 | for package in packages: |
| 139 | if not package.build_and_install(install_dir): |
| 140 | msg = '!!! Unable to build and install %s' % package.name |
| 141 | print msg |
| 142 | errors.append(msg) |
| 143 | return errors |
| 144 | |
| 145 | |
mbligh | fa29d66 | 2009-02-05 00:44:26 +0000 | [diff] [blame] | 146 | if __name__ == '__main__': |
| 147 | sys.exit(main()) |