Rishabh Bhatnagar | f661942 | 2018-04-05 14:19:20 -0700 | [diff] [blame] | 1 | #! /usr/bin/env python2 |
Rishabh Bhatnagar | e9a05bb | 2018-12-10 11:09:45 -0800 | [diff] [blame] | 2 | # SPDX-License-Identifier: GPL-2.0-only |
Rishabh Bhatnagar | f661942 | 2018-04-05 14:19:20 -0700 | [diff] [blame] | 3 | # Copyright (c) 2009-2015, 2017-18, The Linux Foundation. All rights reserved. |
| 4 | |
| 5 | # Build the kernel for all targets using the Android build environment. |
| 6 | |
| 7 | from collections import namedtuple |
| 8 | import glob |
| 9 | from optparse import OptionParser |
| 10 | import os |
| 11 | import re |
| 12 | import shutil |
| 13 | import subprocess |
| 14 | import sys |
| 15 | import threading |
| 16 | import Queue |
| 17 | |
| 18 | version = 'build-all.py, version 1.99' |
| 19 | |
| 20 | build_dir = '../all-kernels' |
| 21 | make_command = ["vmlinux", "modules", "dtbs"] |
| 22 | all_options = {} |
| 23 | compile64 = os.environ.get('CROSS_COMPILE64') |
| 24 | clang_bin = os.environ.get('CLANG_BIN') |
| 25 | |
| 26 | def error(msg): |
| 27 | sys.stderr.write("error: %s\n" % msg) |
| 28 | |
| 29 | def fail(msg): |
| 30 | """Fail with a user-printed message""" |
| 31 | error(msg) |
| 32 | sys.exit(1) |
| 33 | |
| 34 | if not os.environ.get('CROSS_COMPILE'): |
| 35 | fail("CROSS_COMPILE must be set in the environment") |
| 36 | |
| 37 | def check_kernel(): |
| 38 | """Ensure that PWD is a kernel directory""" |
| 39 | if not os.path.isfile('MAINTAINERS'): |
| 40 | fail("This doesn't seem to be a kernel dir") |
| 41 | |
| 42 | def check_build(): |
| 43 | """Ensure that the build directory is present.""" |
| 44 | if not os.path.isdir(build_dir): |
| 45 | try: |
| 46 | os.makedirs(build_dir) |
| 47 | except OSError as exc: |
| 48 | if exc.errno == errno.EEXIST: |
| 49 | pass |
| 50 | else: |
| 51 | raise |
| 52 | |
| 53 | failed_targets = [] |
| 54 | |
| 55 | BuildResult = namedtuple('BuildResult', ['status', 'messages']) |
| 56 | |
| 57 | class BuildSequence(namedtuple('BuildSequence', ['log_name', 'short_name', 'steps'])): |
| 58 | |
| 59 | def set_width(self, width): |
| 60 | self.width = width |
| 61 | |
| 62 | def __enter__(self): |
| 63 | self.log = open(self.log_name, 'w') |
| 64 | def __exit__(self, type, value, traceback): |
| 65 | self.log.close() |
| 66 | |
| 67 | def run(self): |
| 68 | self.status = None |
| 69 | messages = ["Building: " + self.short_name] |
| 70 | def printer(line): |
| 71 | text = "[%-*s] %s" % (self.width, self.short_name, line) |
| 72 | messages.append(text) |
| 73 | self.log.write(text) |
| 74 | self.log.write('\n') |
| 75 | for step in self.steps: |
| 76 | st = step.run(printer) |
| 77 | if st: |
| 78 | self.status = BuildResult(self.short_name, messages) |
| 79 | break |
| 80 | if not self.status: |
| 81 | self.status = BuildResult(None, messages) |
| 82 | |
| 83 | class BuildTracker: |
| 84 | """Manages all of the steps necessary to perform a build. The |
| 85 | build consists of one or more sequences of steps. The different |
| 86 | sequences can be processed independently, while the steps within a |
| 87 | sequence must be done in order.""" |
| 88 | |
| 89 | def __init__(self, parallel_builds): |
| 90 | self.sequence = [] |
| 91 | self.lock = threading.Lock() |
| 92 | self.parallel_builds = parallel_builds |
| 93 | |
| 94 | def add_sequence(self, log_name, short_name, steps): |
| 95 | self.sequence.append(BuildSequence(log_name, short_name, steps)) |
| 96 | |
| 97 | def longest_name(self): |
| 98 | longest = 0 |
| 99 | for seq in self.sequence: |
| 100 | longest = max(longest, len(seq.short_name)) |
| 101 | return longest |
| 102 | |
| 103 | def __repr__(self): |
| 104 | return "BuildTracker(%s)" % self.sequence |
| 105 | |
| 106 | def run_child(self, seq): |
| 107 | seq.set_width(self.longest) |
| 108 | tok = self.build_tokens.get() |
| 109 | with self.lock: |
| 110 | print "Building:", seq.short_name |
| 111 | with seq: |
| 112 | seq.run() |
| 113 | self.results.put(seq.status) |
| 114 | self.build_tokens.put(tok) |
| 115 | |
| 116 | def run(self): |
| 117 | self.longest = self.longest_name() |
| 118 | self.results = Queue.Queue() |
| 119 | children = [] |
| 120 | errors = [] |
| 121 | self.build_tokens = Queue.Queue() |
| 122 | nthreads = self.parallel_builds |
| 123 | print "Building with", nthreads, "threads" |
| 124 | for i in range(nthreads): |
| 125 | self.build_tokens.put(True) |
| 126 | for seq in self.sequence: |
| 127 | child = threading.Thread(target=self.run_child, args=[seq]) |
| 128 | children.append(child) |
| 129 | child.start() |
| 130 | for child in children: |
| 131 | stats = self.results.get() |
| 132 | if all_options.verbose: |
| 133 | with self.lock: |
| 134 | for line in stats.messages: |
| 135 | print line |
| 136 | sys.stdout.flush() |
| 137 | if stats.status: |
| 138 | errors.append(stats.status) |
| 139 | for child in children: |
| 140 | child.join() |
| 141 | if errors: |
| 142 | fail("\n ".join(["Failed targets:"] + errors)) |
| 143 | |
| 144 | class PrintStep: |
| 145 | """A step that just prints a message""" |
| 146 | def __init__(self, message): |
| 147 | self.message = message |
| 148 | |
| 149 | def run(self, outp): |
| 150 | outp(self.message) |
| 151 | |
| 152 | class MkdirStep: |
| 153 | """A step that makes a directory""" |
| 154 | def __init__(self, direc): |
| 155 | self.direc = direc |
| 156 | |
| 157 | def run(self, outp): |
| 158 | outp("mkdir %s" % self.direc) |
| 159 | os.mkdir(self.direc) |
| 160 | |
| 161 | class RmtreeStep: |
| 162 | def __init__(self, direc): |
| 163 | self.direc = direc |
| 164 | |
| 165 | def run(self, outp): |
| 166 | outp("rmtree %s" % self.direc) |
| 167 | shutil.rmtree(self.direc, ignore_errors=True) |
| 168 | |
| 169 | class CopyfileStep: |
| 170 | def __init__(self, src, dest): |
| 171 | self.src = src |
| 172 | self.dest = dest |
| 173 | |
| 174 | def run(self, outp): |
| 175 | outp("cp %s %s" % (self.src, self.dest)) |
| 176 | shutil.copyfile(self.src, self.dest) |
| 177 | |
| 178 | class ExecStep: |
| 179 | def __init__(self, cmd, **kwargs): |
| 180 | self.cmd = cmd |
| 181 | self.kwargs = kwargs |
| 182 | |
| 183 | def run(self, outp): |
| 184 | outp("exec: %s" % (" ".join(self.cmd),)) |
| 185 | with open('/dev/null', 'r') as devnull: |
| 186 | proc = subprocess.Popen(self.cmd, stdin=devnull, |
| 187 | stdout=subprocess.PIPE, |
| 188 | stderr=subprocess.STDOUT, |
| 189 | **self.kwargs) |
| 190 | stdout = proc.stdout |
| 191 | while True: |
| 192 | line = stdout.readline() |
| 193 | if not line: |
| 194 | break |
| 195 | line = line.rstrip('\n') |
| 196 | outp(line) |
| 197 | result = proc.wait() |
| 198 | if result != 0: |
| 199 | return ('error', result) |
| 200 | else: |
| 201 | return None |
| 202 | |
| 203 | class Builder(): |
| 204 | |
| 205 | def __init__(self, name, defconfig): |
| 206 | self.name = name |
| 207 | self.defconfig = defconfig |
| 208 | |
| 209 | self.confname = re.sub('arch/arm[64]*/configs/', '', self.defconfig) |
| 210 | |
| 211 | # Determine if this is a 64-bit target based on the location |
| 212 | # of the defconfig. |
| 213 | self.make_env = os.environ.copy() |
| 214 | if "/arm64/" in defconfig: |
| 215 | if compile64: |
| 216 | self.make_env['CROSS_COMPILE'] = compile64 |
| 217 | else: |
| 218 | fail("Attempting to build 64-bit, without setting CROSS_COMPILE64") |
| 219 | self.make_env['ARCH'] = 'arm64' |
| 220 | else: |
| 221 | self.make_env['ARCH'] = 'arm' |
| 222 | self.make_env['KCONFIG_NOTIMESTAMP'] = 'true' |
| 223 | self.log_name = "%s/log-%s.log" % (build_dir, self.name) |
| 224 | |
| 225 | def build(self): |
| 226 | steps = [] |
| 227 | dest_dir = os.path.join(build_dir, self.name) |
| 228 | log_name = "%s/log-%s.log" % (build_dir, self.name) |
| 229 | steps.append(PrintStep('Building %s in %s log %s' % |
| 230 | (self.name, dest_dir, log_name))) |
| 231 | if not os.path.isdir(dest_dir): |
| 232 | steps.append(MkdirStep(dest_dir)) |
| 233 | defconfig = self.defconfig |
| 234 | dotconfig = '%s/.config' % dest_dir |
| 235 | savedefconfig = '%s/defconfig' % dest_dir |
| 236 | |
| 237 | staging_dir = 'install_staging' |
| 238 | modi_dir = '%s' % staging_dir |
| 239 | hdri_dir = '%s/usr' % staging_dir |
| 240 | steps.append(RmtreeStep(os.path.join(dest_dir, staging_dir))) |
| 241 | |
| 242 | steps.append(ExecStep(['make', 'O=%s' % dest_dir, |
| 243 | self.confname], env=self.make_env)) |
| 244 | |
| 245 | # Build targets can be dependent upon the completion of |
| 246 | # previous build targets, so build them one at a time. |
| 247 | cmd_line = ['make', |
| 248 | 'INSTALL_HDR_PATH=%s' % hdri_dir, |
| 249 | 'INSTALL_MOD_PATH=%s' % modi_dir, |
| 250 | 'O=%s' % dest_dir, |
| 251 | 'REAL_CC=%s' % clang_bin] |
| 252 | build_targets = [] |
| 253 | for c in make_command: |
| 254 | if re.match(r'^-{1,2}\w', c): |
| 255 | cmd_line.append(c) |
| 256 | else: |
| 257 | build_targets.append(c) |
| 258 | for t in build_targets: |
| 259 | steps.append(ExecStep(cmd_line + [t], env=self.make_env)) |
| 260 | |
| 261 | return steps |
| 262 | |
| 263 | def scan_configs(): |
| 264 | """Get the full list of defconfigs appropriate for this tree.""" |
| 265 | names = [] |
| 266 | for defconfig in glob.glob('arch/arm*/configs/vendor/*_defconfig'): |
| 267 | target = os.path.basename(defconfig)[:-10] |
| 268 | name = target + "-llvm" |
| 269 | if 'arch/arm64' in defconfig: |
| 270 | name = name + "-64" |
| 271 | names.append(Builder(name, defconfig)) |
| 272 | |
| 273 | return names |
| 274 | |
| 275 | def build_many(targets): |
| 276 | print "Building %d target(s)" % len(targets) |
| 277 | |
| 278 | # To try and make up for the link phase being serial, try to do |
| 279 | # two full builds in parallel. Don't do too many because lots of |
| 280 | # parallel builds tends to use up available memory rather quickly. |
| 281 | parallel = 2 |
| 282 | if all_options.jobs and all_options.jobs > 1: |
| 283 | j = max(all_options.jobs / parallel, 2) |
| 284 | make_command.append("-j" + str(j)) |
| 285 | |
| 286 | tracker = BuildTracker(parallel) |
| 287 | for target in targets: |
| 288 | steps = target.build() |
| 289 | tracker.add_sequence(target.log_name, target.name, steps) |
| 290 | tracker.run() |
| 291 | |
| 292 | def main(): |
| 293 | global make_command |
| 294 | |
| 295 | check_kernel() |
| 296 | check_build() |
| 297 | |
| 298 | configs = scan_configs() |
| 299 | |
| 300 | usage = (""" |
| 301 | %prog [options] all -- Build all targets |
| 302 | %prog [options] target target ... -- List specific targets |
| 303 | """) |
| 304 | parser = OptionParser(usage=usage, version=version) |
| 305 | parser.add_option('--list', action='store_true', |
| 306 | dest='list', |
| 307 | help='List available targets') |
| 308 | parser.add_option('-v', '--verbose', action='store_true', |
| 309 | dest='verbose', |
| 310 | help='Output to stdout in addition to log file') |
| 311 | parser.add_option('-j', '--jobs', type='int', dest="jobs", |
| 312 | help="Number of simultaneous jobs") |
| 313 | parser.add_option('-l', '--load-average', type='int', |
| 314 | dest='load_average', |
| 315 | help="Don't start multiple jobs unless load is below LOAD_AVERAGE") |
| 316 | parser.add_option('-k', '--keep-going', action='store_true', |
| 317 | dest='keep_going', default=False, |
| 318 | help="Keep building other targets if a target fails") |
| 319 | parser.add_option('-m', '--make-target', action='append', |
| 320 | help='Build the indicated make target (default: %s)' % |
| 321 | ' '.join(make_command)) |
| 322 | |
| 323 | (options, args) = parser.parse_args() |
| 324 | global all_options |
| 325 | all_options = options |
| 326 | |
| 327 | if options.list: |
| 328 | print "Available targets:" |
| 329 | for target in configs: |
| 330 | print " %s" % target.name |
| 331 | sys.exit(0) |
| 332 | |
| 333 | if options.make_target: |
| 334 | make_command = options.make_target |
| 335 | |
| 336 | if args == ['all']: |
| 337 | build_many(configs) |
| 338 | elif len(args) > 0: |
| 339 | all_configs = {} |
| 340 | for t in configs: |
| 341 | all_configs[t.name] = t |
| 342 | targets = [] |
| 343 | for t in args: |
| 344 | if t not in all_configs: |
| 345 | parser.error("Target '%s' not one of %s" % (t, all_configs.keys())) |
| 346 | targets.append(all_configs[t]) |
| 347 | build_many(targets) |
| 348 | else: |
| 349 | parser.error("Must specify a target to build, or 'all'") |
| 350 | |
| 351 | if __name__ == "__main__": |
| 352 | main() |