Craig Tiller | 1a71811 | 2015-07-03 12:07:06 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python2.7 |
| 2 | |
Craig Tiller | 6169d5f | 2016-03-31 07:46:18 -0700 | [diff] [blame] | 3 | # Copyright 2015, Google Inc. |
Craig Tiller | 1a71811 | 2015-07-03 12:07:06 -0700 | [diff] [blame] | 4 | # All rights reserved. |
| 5 | # |
| 6 | # Redistribution and use in source and binary forms, with or without |
| 7 | # modification, are permitted provided that the following conditions are |
| 8 | # met: |
| 9 | # |
| 10 | # * Redistributions of source code must retain the above copyright |
| 11 | # notice, this list of conditions and the following disclaimer. |
| 12 | # * Redistributions in binary form must reproduce the above |
| 13 | # copyright notice, this list of conditions and the following disclaimer |
| 14 | # in the documentation and/or other materials provided with the |
| 15 | # distribution. |
| 16 | # * Neither the name of Google Inc. nor the names of its |
| 17 | # contributors may be used to endorse or promote products derived from |
| 18 | # this software without specific prior written permission. |
| 19 | # |
| 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 31 | |
Craig Tiller | 0fe5ee7 | 2015-12-22 12:50:36 -0800 | [diff] [blame] | 32 | import argparse |
Craig Tiller | 1a71811 | 2015-07-03 12:07:06 -0700 | [diff] [blame] | 33 | import glob |
| 34 | import os |
Craig Tiller | 259e627 | 2015-08-31 16:58:18 -0700 | [diff] [blame] | 35 | import shutil |
Craig Tiller | 1a71811 | 2015-07-03 12:07:06 -0700 | [diff] [blame] | 36 | import sys |
| 37 | import tempfile |
David Garcia Quintas | 985f22e | 2015-10-04 23:14:37 -0700 | [diff] [blame] | 38 | import multiprocessing |
Craig Tiller | 1a71811 | 2015-07-03 12:07:06 -0700 | [diff] [blame] | 39 | sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', 'run_tests')) |
| 40 | |
| 41 | assert sys.argv[1:], 'run generate_projects.sh instead of this directly' |
| 42 | |
| 43 | import jobset |
| 44 | |
| 45 | os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '..', '..')) |
Craig Tiller | 0fe5ee7 | 2015-12-22 12:50:36 -0800 | [diff] [blame] | 46 | |
| 47 | argp = argparse.ArgumentParser() |
Craig Tiller | 560c901 | 2016-02-24 16:34:38 -0800 | [diff] [blame] | 48 | argp.add_argument('build_files', nargs='+', default=[]) |
Craig Tiller | 0fe5ee7 | 2015-12-22 12:50:36 -0800 | [diff] [blame] | 49 | argp.add_argument('--templates', nargs='+', default=[]) |
Craig Tiller | 3ab2fe0 | 2016-04-11 20:11:18 -0700 | [diff] [blame] | 50 | argp.add_argument('--output_merged', default=None, type=str) |
Craig Tiller | 1297170 | 2016-01-21 13:38:35 -0800 | [diff] [blame] | 51 | argp.add_argument('--jobs', '-j', default=multiprocessing.cpu_count(), type=int) |
Craig Tiller | 0fe5ee7 | 2015-12-22 12:50:36 -0800 | [diff] [blame] | 52 | args = argp.parse_args() |
| 53 | |
Craig Tiller | 560c901 | 2016-02-24 16:34:38 -0800 | [diff] [blame] | 54 | json = args.build_files |
Craig Tiller | 1a71811 | 2015-07-03 12:07:06 -0700 | [diff] [blame] | 55 | |
| 56 | test = {} if 'TEST' in os.environ else None |
| 57 | |
| 58 | plugins = sorted(glob.glob('tools/buildgen/plugins/*.py')) |
| 59 | |
Craig Tiller | 0fe5ee7 | 2015-12-22 12:50:36 -0800 | [diff] [blame] | 60 | templates = args.templates |
| 61 | if not templates: |
| 62 | for root, dirs, files in os.walk('templates'): |
| 63 | for f in files: |
| 64 | templates.append(os.path.join(root, f)) |
| 65 | |
Craig Tiller | 560c901 | 2016-02-24 16:34:38 -0800 | [diff] [blame] | 66 | pre_jobs = [] |
| 67 | base_cmd = ['python2.7', 'tools/buildgen/mako_renderer.py'] |
| 68 | cmd = base_cmd[:] |
| 69 | for plugin in plugins: |
| 70 | cmd.append('-p') |
| 71 | cmd.append(plugin) |
| 72 | for js in json: |
| 73 | cmd.append('-d') |
| 74 | cmd.append(js) |
| 75 | cmd.append('-w') |
| 76 | preprocessed_build = '.preprocessed_build' |
| 77 | cmd.append(preprocessed_build) |
Craig Tiller | 3ab2fe0 | 2016-04-11 20:11:18 -0700 | [diff] [blame] | 78 | if args.output_merged is not None: |
| 79 | cmd.append('-M') |
| 80 | cmd.append(args.output_merged) |
Craig Tiller | 560c901 | 2016-02-24 16:34:38 -0800 | [diff] [blame] | 81 | pre_jobs.append(jobset.JobSpec(cmd, shortname='preprocess', timeout_seconds=None)) |
| 82 | |
Craig Tiller | 1a71811 | 2015-07-03 12:07:06 -0700 | [diff] [blame] | 83 | jobs = [] |
Craig Tiller | 560c901 | 2016-02-24 16:34:38 -0800 | [diff] [blame] | 84 | for template in reversed(sorted(templates)): |
Craig Tiller | 0fe5ee7 | 2015-12-22 12:50:36 -0800 | [diff] [blame] | 85 | root, f = os.path.split(template) |
| 86 | if os.path.splitext(f)[1] == '.template': |
| 87 | out_dir = '.' + root[len('templates'):] |
| 88 | out = out_dir + '/' + os.path.splitext(f)[0] |
| 89 | if not os.path.exists(out_dir): |
| 90 | os.makedirs(out_dir) |
Craig Tiller | 560c901 | 2016-02-24 16:34:38 -0800 | [diff] [blame] | 91 | cmd = base_cmd[:] |
| 92 | cmd.append('-P') |
| 93 | cmd.append(preprocessed_build) |
Craig Tiller | 0fe5ee7 | 2015-12-22 12:50:36 -0800 | [diff] [blame] | 94 | cmd.append('-o') |
| 95 | if test is None: |
| 96 | cmd.append(out) |
| 97 | else: |
| 98 | tf = tempfile.mkstemp() |
| 99 | test[out] = tf[1] |
| 100 | os.close(tf[0]) |
| 101 | cmd.append(test[out]) |
| 102 | cmd.append(root + '/' + f) |
Craig Tiller | 590105a | 2016-01-19 13:03:46 -0800 | [diff] [blame] | 103 | jobs.append(jobset.JobSpec(cmd, shortname=out, timeout_seconds=None)) |
Craig Tiller | 1a71811 | 2015-07-03 12:07:06 -0700 | [diff] [blame] | 104 | |
Craig Tiller | 560c901 | 2016-02-24 16:34:38 -0800 | [diff] [blame] | 105 | jobset.run(pre_jobs, maxjobs=args.jobs) |
Craig Tiller | 1297170 | 2016-01-21 13:38:35 -0800 | [diff] [blame] | 106 | jobset.run(jobs, maxjobs=args.jobs) |
Craig Tiller | 1a71811 | 2015-07-03 12:07:06 -0700 | [diff] [blame] | 107 | |
| 108 | if test is not None: |
Nicolas "Pixel" Noble | 22232ae | 2015-07-11 22:40:22 +0200 | [diff] [blame] | 109 | for s, g in test.iteritems(): |
Craig Tiller | 259e627 | 2015-08-31 16:58:18 -0700 | [diff] [blame] | 110 | if os.path.isfile(g): |
Nicolas "Pixel" Noble | 9baaead | 2015-10-03 02:25:02 +0200 | [diff] [blame] | 111 | assert 0 == os.system('diff %s %s' % (s, g)), s |
Craig Tiller | 259e627 | 2015-08-31 16:58:18 -0700 | [diff] [blame] | 112 | os.unlink(g) |
| 113 | else: |
Nicolas "Pixel" Noble | 9baaead | 2015-10-03 02:25:02 +0200 | [diff] [blame] | 114 | assert 0 == os.system('diff -r %s %s' % (s, g)), s |
Craig Tiller | 259e627 | 2015-08-31 16:58:18 -0700 | [diff] [blame] | 115 | shutil.rmtree(g, ignore_errors=True) |