scripts: snapshot of kernel build utility scripts
This is a snapshot of build scripts taken as of msm-4.4 commit
<400520a6e2f06cc> ("ASoC: msm: qdsp6v2: Fix FE id check in routing
driver").
CRs-Fixed: 1040265
Change-Id: If5774f946d41f2c15503a6641938412c5d3aefb3
Signed-off-by: Channagoud Kadabi <ckadabi@codeaurora.org>
diff --git a/scripts/build-all.py b/scripts/build-all.py
new file mode 100755
index 0000000..0f8babf
--- /dev/null
+++ b/scripts/build-all.py
@@ -0,0 +1,428 @@
+#! /usr/bin/env python2
+
+# Copyright (c) 2009-2015, The Linux Foundation. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of The Linux Foundation nor
+# the names of its contributors may be used to endorse or promote
+# products derived from this software without specific prior written
+# permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NON-INFRINGEMENT ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Build the kernel for all targets using the Android build environment.
+
+from collections import namedtuple
+import glob
+from optparse import OptionParser
+import os
+import re
+import shutil
+import subprocess
+import sys
+import threading
+import Queue
+
+version = 'build-all.py, version 1.99'
+
+build_dir = '../all-kernels'
+make_command = ["vmlinux", "modules", "dtbs"]
+all_options = {}
+compile64 = os.environ.get('CROSS_COMPILE64')
+
+def error(msg):
+ sys.stderr.write("error: %s\n" % msg)
+
+def fail(msg):
+ """Fail with a user-printed message"""
+ error(msg)
+ sys.exit(1)
+
+if not os.environ.get('CROSS_COMPILE'):
+ fail("CROSS_COMPILE must be set in the environment")
+
+def check_kernel():
+ """Ensure that PWD is a kernel directory"""
+ have_defconfig = any([
+ os.path.isfile('arch/arm64/configs/msm_defconfig'),
+ os.path.isfile('arch/arm64/configs/msmskunk_defconfig')])
+
+ if not all([os.path.isfile('MAINTAINERS'), have_defconfig]):
+ fail("This doesn't seem to be an MSM kernel dir")
+
+def check_build():
+ """Ensure that the build directory is present."""
+ if not os.path.isdir(build_dir):
+ try:
+ os.makedirs(build_dir)
+ except OSError as exc:
+ if exc.errno == errno.EEXIST:
+ pass
+ else:
+ raise
+
+failed_targets = []
+
+BuildResult = namedtuple('BuildResult', ['status', 'messages'])
+
+class BuildSequence(namedtuple('BuildSequence', ['log_name', 'short_name', 'steps'])):
+
+ def set_width(self, width):
+ self.width = width
+
+ def __enter__(self):
+ self.log = open(self.log_name, 'w')
+ def __exit__(self, type, value, traceback):
+ self.log.close()
+
+ def run(self):
+ self.status = None
+ messages = ["Building: " + self.short_name]
+ def printer(line):
+ text = "[%-*s] %s" % (self.width, self.short_name, line)
+ messages.append(text)
+ self.log.write(text)
+ self.log.write('\n')
+ for step in self.steps:
+ st = step.run(printer)
+ if st:
+ self.status = BuildResult(self.short_name, messages)
+ break
+ if not self.status:
+ self.status = BuildResult(None, messages)
+
+class BuildTracker:
+ """Manages all of the steps necessary to perform a build. The
+ build consists of one or more sequences of steps. The different
+ sequences can be processed independently, while the steps within a
+ sequence must be done in order."""
+
+ def __init__(self, parallel_builds):
+ self.sequence = []
+ self.lock = threading.Lock()
+ self.parallel_builds = parallel_builds
+
+ def add_sequence(self, log_name, short_name, steps):
+ self.sequence.append(BuildSequence(log_name, short_name, steps))
+
+ def longest_name(self):
+ longest = 0
+ for seq in self.sequence:
+ longest = max(longest, len(seq.short_name))
+ return longest
+
+ def __repr__(self):
+ return "BuildTracker(%s)" % self.sequence
+
+ def run_child(self, seq):
+ seq.set_width(self.longest)
+ tok = self.build_tokens.get()
+ with self.lock:
+ print "Building:", seq.short_name
+ with seq:
+ seq.run()
+ self.results.put(seq.status)
+ self.build_tokens.put(tok)
+
+ def run(self):
+ self.longest = self.longest_name()
+ self.results = Queue.Queue()
+ children = []
+ errors = []
+ self.build_tokens = Queue.Queue()
+ nthreads = self.parallel_builds
+ print "Building with", nthreads, "threads"
+ for i in range(nthreads):
+ self.build_tokens.put(True)
+ for seq in self.sequence:
+ child = threading.Thread(target=self.run_child, args=[seq])
+ children.append(child)
+ child.start()
+ for child in children:
+ stats = self.results.get()
+ if all_options.verbose:
+ with self.lock:
+ for line in stats.messages:
+ print line
+ sys.stdout.flush()
+ if stats.status:
+ errors.append(stats.status)
+ for child in children:
+ child.join()
+ if errors:
+ fail("\n ".join(["Failed targets:"] + errors))
+
+class PrintStep:
+ """A step that just prints a message"""
+ def __init__(self, message):
+ self.message = message
+
+ def run(self, outp):
+ outp(self.message)
+
+class MkdirStep:
+ """A step that makes a directory"""
+ def __init__(self, direc):
+ self.direc = direc
+
+ def run(self, outp):
+ outp("mkdir %s" % self.direc)
+ os.mkdir(self.direc)
+
+class RmtreeStep:
+ def __init__(self, direc):
+ self.direc = direc
+
+ def run(self, outp):
+ outp("rmtree %s" % self.direc)
+ shutil.rmtree(self.direc, ignore_errors=True)
+
+class CopyfileStep:
+ def __init__(self, src, dest):
+ self.src = src
+ self.dest = dest
+
+ def run(self, outp):
+ outp("cp %s %s" % (self.src, self.dest))
+ shutil.copyfile(self.src, self.dest)
+
+class ExecStep:
+ def __init__(self, cmd, **kwargs):
+ self.cmd = cmd
+ self.kwargs = kwargs
+
+ def run(self, outp):
+ outp("exec: %s" % (" ".join(self.cmd),))
+ with open('/dev/null', 'r') as devnull:
+ proc = subprocess.Popen(self.cmd, stdin=devnull,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ **self.kwargs)
+ stdout = proc.stdout
+ while True:
+ line = stdout.readline()
+ if not line:
+ break
+ line = line.rstrip('\n')
+ outp(line)
+ result = proc.wait()
+ if result != 0:
+ return ('error', result)
+ else:
+ return None
+
+class Builder():
+
+ def __init__(self, name, defconfig):
+ self.name = name
+ self.defconfig = defconfig
+
+ self.confname = self.defconfig.split('/')[-1]
+
+ # Determine if this is a 64-bit target based on the location
+ # of the defconfig.
+ self.make_env = os.environ.copy()
+ if "/arm64/" in defconfig:
+ if compile64:
+ self.make_env['CROSS_COMPILE'] = compile64
+ else:
+ fail("Attempting to build 64-bit, without setting CROSS_COMPILE64")
+ self.make_env['ARCH'] = 'arm64'
+ else:
+ self.make_env['ARCH'] = 'arm'
+ self.make_env['KCONFIG_NOTIMESTAMP'] = 'true'
+ self.log_name = "%s/log-%s.log" % (build_dir, self.name)
+
+ def build(self):
+ steps = []
+ dest_dir = os.path.join(build_dir, self.name)
+ log_name = "%s/log-%s.log" % (build_dir, self.name)
+ steps.append(PrintStep('Building %s in %s log %s' %
+ (self.name, dest_dir, log_name)))
+ if not os.path.isdir(dest_dir):
+ steps.append(MkdirStep(dest_dir))
+ defconfig = self.defconfig
+ dotconfig = '%s/.config' % dest_dir
+ savedefconfig = '%s/defconfig' % dest_dir
+
+ staging_dir = 'install_staging'
+ modi_dir = '%s' % staging_dir
+ hdri_dir = '%s/usr' % staging_dir
+ steps.append(RmtreeStep(os.path.join(dest_dir, staging_dir)))
+
+ steps.append(ExecStep(['make', 'O=%s' % dest_dir,
+ self.confname], env=self.make_env))
+
+ if not all_options.updateconfigs:
+ # Build targets can be dependent upon the completion of
+ # previous build targets, so build them one at a time.
+ cmd_line = ['make',
+ 'INSTALL_HDR_PATH=%s' % hdri_dir,
+ 'INSTALL_MOD_PATH=%s' % modi_dir,
+ 'O=%s' % dest_dir]
+ build_targets = []
+ for c in make_command:
+ if re.match(r'^-{1,2}\w', c):
+ cmd_line.append(c)
+ else:
+ build_targets.append(c)
+ for t in build_targets:
+ steps.append(ExecStep(cmd_line + [t], env=self.make_env))
+
+ # Copy the defconfig back.
+ if all_options.configs or all_options.updateconfigs:
+ steps.append(ExecStep(['make', 'O=%s' % dest_dir,
+ 'savedefconfig'], env=self.make_env))
+ steps.append(CopyfileStep(savedefconfig, defconfig))
+
+ return steps
+
+def update_config(file, str):
+ print 'Updating %s with \'%s\'\n' % (file, str)
+ with open(file, 'a') as defconfig:
+ defconfig.write(str + '\n')
+
+def scan_configs():
+ """Get the full list of defconfigs appropriate for this tree."""
+ names = []
+ arch_pats = (
+ r'[fm]sm[0-9]*_defconfig',
+ r'apq*_defconfig',
+ r'qsd*_defconfig',
+ r'mpq*_defconfig',
+ )
+ arch64_pats = (
+ r'msm*_defconfig',
+ )
+ for p in arch_pats:
+ for n in glob.glob('arch/arm/configs/' + p):
+ name = os.path.basename(n)[:-10]
+ names.append(Builder(name, n))
+ if 'CROSS_COMPILE64' in os.environ:
+ for p in arch64_pats:
+ for n in glob.glob('arch/arm64/configs/' + p):
+ name = os.path.basename(n)[:-10] + "-64"
+ names.append(Builder(name, n))
+ return names
+
+def build_many(targets):
+ print "Building %d target(s)" % len(targets)
+
+ # To try and make up for the link phase being serial, try to do
+ # two full builds in parallel. Don't do too many because lots of
+ # parallel builds tends to use up available memory rather quickly.
+ parallel = 2
+ if all_options.jobs and all_options.jobs > 1:
+ j = max(all_options.jobs / parallel, 2)
+ make_command.append("-j" + str(j))
+
+ tracker = BuildTracker(parallel)
+ for target in targets:
+ if all_options.updateconfigs:
+ update_config(target.defconfig, all_options.updateconfigs)
+ steps = target.build()
+ tracker.add_sequence(target.log_name, target.name, steps)
+ tracker.run()
+
+def main():
+ global make_command
+
+ check_kernel()
+ check_build()
+
+ configs = scan_configs()
+
+ usage = ("""
+ %prog [options] all -- Build all targets
+ %prog [options] target target ... -- List specific targets
+ %prog [options] perf -- Build all perf targets
+ %prog [options] noperf -- Build all non-perf targets""")
+ parser = OptionParser(usage=usage, version=version)
+ parser.add_option('--configs', action='store_true',
+ dest='configs',
+ help="Copy configs back into tree")
+ parser.add_option('--list', action='store_true',
+ dest='list',
+ help='List available targets')
+ parser.add_option('-v', '--verbose', action='store_true',
+ dest='verbose',
+ help='Output to stdout in addition to log file')
+ parser.add_option('--oldconfig', action='store_true',
+ dest='oldconfig',
+ help='Only process "make oldconfig"')
+ parser.add_option('--updateconfigs',
+ dest='updateconfigs',
+ help="Update defconfigs with provided option setting, "
+ "e.g. --updateconfigs=\'CONFIG_USE_THING=y\'")
+ parser.add_option('-j', '--jobs', type='int', dest="jobs",
+ help="Number of simultaneous jobs")
+ parser.add_option('-l', '--load-average', type='int',
+ dest='load_average',
+ help="Don't start multiple jobs unless load is below LOAD_AVERAGE")
+ parser.add_option('-k', '--keep-going', action='store_true',
+ dest='keep_going', default=False,
+ help="Keep building other targets if a target fails")
+ parser.add_option('-m', '--make-target', action='append',
+ help='Build the indicated make target (default: %s)' %
+ ' '.join(make_command))
+
+ (options, args) = parser.parse_args()
+ global all_options
+ all_options = options
+
+ if options.list:
+ print "Available targets:"
+ for target in configs:
+ print " %s" % target.name
+ sys.exit(0)
+
+ if options.oldconfig:
+ make_command = ["oldconfig"]
+ elif options.make_target:
+ make_command = options.make_target
+
+ if args == ['all']:
+ build_many(configs)
+ elif args == ['perf']:
+ targets = []
+ for t in configs:
+ if "perf" in t.name:
+ targets.append(t)
+ build_many(targets)
+ elif args == ['noperf']:
+ targets = []
+ for t in configs:
+ if "perf" not in t.name:
+ targets.append(t)
+ build_many(targets)
+ elif len(args) > 0:
+ all_configs = {}
+ for t in configs:
+ all_configs[t.name] = t
+ targets = []
+ for t in args:
+ if t not in all_configs:
+ parser.error("Target '%s' not one of %s" % (t, all_configs.keys()))
+ targets.append(all_configs[t])
+ build_many(targets)
+ else:
+ parser.error("Must specify a target to build, or 'all'")
+
+if __name__ == "__main__":
+ main()