scripts/build-all.py

#! /usr/bin/env python2

# Copyright (c) 2009-2015, 2017, 2019, The Linux Foundation. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#     * Neither the name of The Linux Foundation nor
#       the names of its contributors may be used to endorse or promote
#       products derived from this software without specific prior written
#       permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NON-INFRINGEMENT ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# Build the kernel for all targets using the Android build environment.

from collections import namedtuple
import glob
from optparse import OptionParser
import os
import re
import shutil
import subprocess
import sys
import threading
import Queue

version = 'build-all.py, version 1.99'

build_dir = '../all-kernels'
make_command = ["vmlinux", "modules", "dtbs"]
all_options = {}
compile64 = os.environ.get('CROSS_COMPILE64')
clang_bin = os.environ.get('CLANG_BIN')

def error(msg):
    sys.stderr.write("error: %s\n" % msg)

def fail(msg):
    """Fail with a user-printed message"""
    error(msg)
    sys.exit(1)

if not os.environ.get('CROSS_COMPILE'):
    fail("CROSS_COMPILE must be set in the environment")

def check_kernel():
    """Ensure that PWD is a kernel directory"""
    if not os.path.isfile('MAINTAINERS'):
        fail("This doesn't seem to be a kernel dir")

def check_build():
    """Ensure that the build directory is present."""
    if not os.path.isdir(build_dir):
        try:
            os.makedirs(build_dir)
        except OSError as exc:
            if exc.errno == errno.EEXIST:
                pass
            else:
                raise

failed_targets = []

BuildResult = namedtuple('BuildResult', ['status', 'messages'])

class BuildSequence(namedtuple('BuildSequence', ['log_name', 'short_name', 'steps'])):

    def set_width(self, width):
        self.width = width

    def __enter__(self):
        self.log = open(self.log_name, 'w')
    def __exit__(self, type, value, traceback):
        self.log.close()

    def run(self):
        self.status = None
        messages = ["Building: " + self.short_name]
        def printer(line):
            text = "[%-*s] %s" % (self.width, self.short_name, line)
            messages.append(text)
            self.log.write(text)
            self.log.write('\n')
        for step in self.steps:
            st = step.run(printer)
            if st:
                self.status = BuildResult(self.short_name, messages)
                break
        if not self.status:
            self.status = BuildResult(None, messages)

class BuildTracker:
    """Manages all of the steps necessary to perform a build.  The
    build consists of one or more sequences of steps.  The different
    sequences can be processed independently, while the steps within a
    sequence must be done in order."""

    def __init__(self, parallel_builds):
        self.sequence = []
        self.lock = threading.Lock()
        self.parallel_builds = parallel_builds

    def add_sequence(self, log_name, short_name, steps):
        self.sequence.append(BuildSequence(log_name, short_name, steps))

    def longest_name(self):
        longest = 0
        for seq in self.sequence:
            longest = max(longest, len(seq.short_name))
        return longest

    def __repr__(self):
        return "BuildTracker(%s)" % self.sequence

    def run_child(self, seq):
        seq.set_width(self.longest)
        tok = self.build_tokens.get()
        with self.lock:
            print "Building:", seq.short_name
        with seq:
            seq.run()
            self.results.put(seq.status)
        self.build_tokens.put(tok)

    def run(self):
        self.longest = self.longest_name()
        self.results = Queue.Queue()
        children = []
        errors = []
        self.build_tokens = Queue.Queue()
        nthreads = self.parallel_builds
        print "Building with", nthreads, "threads"
        for i in range(nthreads):
            self.build_tokens.put(True)
        for seq in self.sequence:
            child = threading.Thread(target=self.run_child, args=[seq])
            children.append(child)
            child.start()
        for child in children:
            stats = self.results.get()
            if all_options.verbose:
                with self.lock:
                    for line in stats.messages:
                        print line
                    sys.stdout.flush()
            if stats.status:
                errors.append(stats.status)
        for child in children:
            child.join()
        if errors:
            fail("\n  ".join(["Failed targets:"] + errors))

class PrintStep:
    """A step that just prints a message"""
    def __init__(self, message):
        self.message = message

    def run(self, outp):
        outp(self.message)

class MkdirStep:
    """A step that makes a directory"""
    def __init__(self, direc):
        self.direc = direc

    def run(self, outp):
        outp("mkdir %s" % self.direc)
        os.mkdir(self.direc)

class RmtreeStep:
    def __init__(self, direc):
        self.direc = direc

    def run(self, outp):
        outp("rmtree %s" % self.direc)
        shutil.rmtree(self.direc, ignore_errors=True)

class CopyfileStep:
    def __init__(self, src, dest):
        self.src = src
        self.dest = dest

    def run(self, outp):
        outp("cp %s %s" % (self.src, self.dest))
        shutil.copyfile(self.src, self.dest)

class ExecStep:
    def __init__(self, cmd, **kwargs):
        self.cmd = cmd
        self.kwargs = kwargs

    def run(self, outp):
        outp("exec: %s" % (" ".join(self.cmd),))
        with open('/dev/null', 'r') as devnull:
            proc = subprocess.Popen(self.cmd, stdin=devnull,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.STDOUT,
                    **self.kwargs)
            stdout = proc.stdout
            while True:
                line = stdout.readline()
                if not line:
                    break
                line = line.rstrip('\n')
                outp(line)
            result = proc.wait()
            if result != 0:
                return ('error', result)
            else:
                return None

class Builder():

    def __init__(self, name, defconfig):
        self.name = name
        self.defconfig = defconfig

        self.confname = self.defconfig.split('/')[-1]

        # Determine if this is a 64-bit target based on the location
        # of the defconfig.
        self.make_env = os.environ.copy()
        if "/arm64/" in defconfig:
            if compile64:
                self.make_env['CROSS_COMPILE'] = compile64
            else:
                fail("Attempting to build 64-bit, without setting CROSS_COMPILE64")
            self.make_env['ARCH'] = 'arm64'
        else:
            self.make_env['ARCH'] = 'arm'
        self.make_env['KCONFIG_NOTIMESTAMP'] = 'true'
        self.log_name = "%s/log-%s.log" % (build_dir, self.name)

    def build(self):
        steps = []
        dest_dir = os.path.join(build_dir, self.name)
        log_name = "%s/log-%s.log" % (build_dir, self.name)
        steps.append(PrintStep('Building %s in %s log %s' %
            (self.name, dest_dir, log_name)))
        if not os.path.isdir(dest_dir):
            steps.append(MkdirStep(dest_dir))
        defconfig = self.defconfig
        dotconfig = '%s/.config' % dest_dir
        savedefconfig = '%s/defconfig' % dest_dir

        staging_dir = 'install_staging'
        modi_dir = '%s' % staging_dir
        hdri_dir = '%s/usr' % staging_dir
        steps.append(RmtreeStep(os.path.join(dest_dir, staging_dir)))

        steps.append(ExecStep(['make', 'O=%s' % dest_dir,
            self.confname], env=self.make_env))

        if not all_options.updateconfigs:
            # Build targets can be dependent upon the completion of
            # previous build targets, so build them one at a time.
            if os.environ.get('ARCH') == "arm64":
                cmd_line = ['make',
                    'INSTALL_HDR_PATH=%s' % hdri_dir,
                    'INSTALL_MOD_PATH=%s' % modi_dir,
                    'O=%s' % dest_dir,
                    'REAL_CC=%s' % clang_bin]
            else:
                cmd_line = ['make',
                    'INSTALL_HDR_PATH=%s' % hdri_dir,
                    'INSTALL_MOD_PATH=%s' % modi_dir,
                    'O=%s' % dest_dir]

            build_targets = []
            for c in make_command:
                if re.match(r'^-{1,2}\w', c):
                    cmd_line.append(c)
                else:
                    build_targets.append(c)
            for t in build_targets:
                steps.append(ExecStep(cmd_line + [t], env=self.make_env))

        # Copy the defconfig back.
        if all_options.configs or all_options.updateconfigs:
            steps.append(ExecStep(['make', 'O=%s' % dest_dir,
                'savedefconfig'], env=self.make_env))
            steps.append(CopyfileStep(savedefconfig, defconfig))

        return steps

def update_config(file, str):
    print 'Updating %s with \'%s\'\n' % (file, str)
    with open(file, 'a') as defconfig:
        defconfig.write(str + '\n')

def scan_configs():
    """Get the full list of defconfigs appropriate for this tree."""
    names = []
    arch_pats = (
        r'[fm]sm[0-9]*_defconfig',
        r'apq*_defconfig',
        r'qsd*_defconfig',
        r'mpq*_defconfig',
        r'sdm[0-9]*_defconfig',
        r'sdx*_defconfig',
        )
    arch64_pats = (
        r'msm*_defconfig',
        r'sdm[0-9]*_defconfig',
        r'sdx*_defconfig',
        )
    for p in arch_pats:
        for n in glob.glob('arch/arm/configs/' + p):
            name = os.path.basename(n)[:-10]
            names.append(Builder(name, n))
    if 'CROSS_COMPILE64' in os.environ:
        for p in arch64_pats:
            for n in glob.glob('arch/arm64/configs/' + p):
                name = os.path.basename(n)[:-10] + "-64"
                names.append(Builder(name, n))
    return names

def build_many(targets):
    print "Building %d target(s)" % len(targets)

    # To try and make up for the link phase being serial, try to do
    # two full builds in parallel.  Don't do too many because lots of
    # parallel builds tends to use up available memory rather quickly.
    parallel = 2
    if all_options.jobs and all_options.jobs > 1:
        j = max(all_options.jobs / parallel, 2)
        make_command.append("-j" + str(j))

    tracker = BuildTracker(parallel)
    for target in targets:
        if all_options.updateconfigs:
            update_config(target.defconfig, all_options.updateconfigs)
        steps = target.build()
        tracker.add_sequence(target.log_name, target.name, steps)
    tracker.run()

def main():
    global make_command

    check_kernel()
    check_build()

    configs = scan_configs()

    usage = ("""
           %prog [options] all                 -- Build all targets
           %prog [options] target target ...   -- List specific targets
           %prog [options] perf                -- Build all perf targets
           %prog [options] noperf              -- Build all non-perf targets""")
    parser = OptionParser(usage=usage, version=version)
    parser.add_option('--configs', action='store_true',
            dest='configs',
            help="Copy configs back into tree")
    parser.add_option('--list', action='store_true',
            dest='list',
            help='List available targets')
    parser.add_option('-v', '--verbose', action='store_true',
            dest='verbose',
            help='Output to stdout in addition to log file')
    parser.add_option('--oldconfig', action='store_true',
            dest='oldconfig',
            help='Only process "make oldconfig"')
    parser.add_option('--updateconfigs',
            dest='updateconfigs',
            help="Update defconfigs with provided option setting, "
                 "e.g. --updateconfigs=\'CONFIG_USE_THING=y\'")
    parser.add_option('-j', '--jobs', type='int', dest="jobs",
            help="Number of simultaneous jobs")
    parser.add_option('-l', '--load-average', type='int',
            dest='load_average',
            help="Don't start multiple jobs unless load is below LOAD_AVERAGE")
    parser.add_option('-k', '--keep-going', action='store_true',
            dest='keep_going', default=False,
            help="Keep building other targets if a target fails")
    parser.add_option('-m', '--make-target', action='append',
            help='Build the indicated make target (default: %s)' %
                 ' '.join(make_command))

    (options, args) = parser.parse_args()
    global all_options
    all_options = options

    if options.list:
        print "Available targets:"
        for target in configs:
            print "   %s" % target.name
        sys.exit(0)

    if options.oldconfig:
        make_command = ["oldconfig"]
    elif options.make_target:
        make_command = options.make_target

    if args == ['all']:
        build_many(configs)
    elif args == ['perf']:
        targets = []
        for t in configs:
            if "perf" in t.name:
                targets.append(t)
        build_many(targets)
    elif args == ['noperf']:
        targets = []
        for t in configs:
            if "perf" not in t.name:
                targets.append(t)
        build_many(targets)
    elif len(args) > 0:
        all_configs = {}
        for t in configs:
            all_configs[t.name] = t
        targets = []
        for t in args:
            if t not in all_configs:
                parser.error("Target '%s' not one of %s" % (t, all_configs.keys()))
            targets.append(all_configs[t])
        build_many(targets)
    else:
        parser.error("Must specify a target to build, or 'all'")

if __name__ == "__main__":
    main()