Craig Tiller | 1a71811 | 2015-07-03 12:07:06 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python2.7 |
| 2 | |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 3 | # Copyright 2015 gRPC authors. |
Craig Tiller | 1a71811 | 2015-07-03 12:07:06 -0700 | [diff] [blame] | 4 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
Craig Tiller | 1a71811 | 2015-07-03 12:07:06 -0700 | [diff] [blame] | 8 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
Craig Tiller | 1a71811 | 2015-07-03 12:07:06 -0700 | [diff] [blame] | 10 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
Craig Tiller | 1a71811 | 2015-07-03 12:07:06 -0700 | [diff] [blame] | 16 | |
Craig Tiller | 0fe5ee7 | 2015-12-22 12:50:36 -0800 | [diff] [blame] | 17 | import argparse |
Craig Tiller | 1a71811 | 2015-07-03 12:07:06 -0700 | [diff] [blame] | 18 | import glob |
| 19 | import os |
Craig Tiller | 259e627 | 2015-08-31 16:58:18 -0700 | [diff] [blame] | 20 | import shutil |
Craig Tiller | 1a71811 | 2015-07-03 12:07:06 -0700 | [diff] [blame] | 21 | import sys |
| 22 | import tempfile |
David Garcia Quintas | 985f22e | 2015-10-04 23:14:37 -0700 | [diff] [blame] | 23 | import multiprocessing |
Jan Tattermusch | 5c79a31 | 2016-12-20 11:02:50 +0100 | [diff] [blame] | 24 | sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', 'run_tests', 'python_utils')) |
Craig Tiller | 1a71811 | 2015-07-03 12:07:06 -0700 | [diff] [blame] | 25 | |
| 26 | assert sys.argv[1:], 'run generate_projects.sh instead of this directly' |
| 27 | |
| 28 | import jobset |
| 29 | |
| 30 | os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '..', '..')) |
Craig Tiller | 0fe5ee7 | 2015-12-22 12:50:36 -0800 | [diff] [blame] | 31 | |
| 32 | argp = argparse.ArgumentParser() |
Craig Tiller | 560c901 | 2016-02-24 16:34:38 -0800 | [diff] [blame] | 33 | argp.add_argument('build_files', nargs='+', default=[]) |
Craig Tiller | 0fe5ee7 | 2015-12-22 12:50:36 -0800 | [diff] [blame] | 34 | argp.add_argument('--templates', nargs='+', default=[]) |
Craig Tiller | 3ab2fe0 | 2016-04-11 20:11:18 -0700 | [diff] [blame] | 35 | argp.add_argument('--output_merged', default=None, type=str) |
Craig Tiller | 1297170 | 2016-01-21 13:38:35 -0800 | [diff] [blame] | 36 | argp.add_argument('--jobs', '-j', default=multiprocessing.cpu_count(), type=int) |
Craig Tiller | 0fe5ee7 | 2015-12-22 12:50:36 -0800 | [diff] [blame] | 37 | args = argp.parse_args() |
| 38 | |
Craig Tiller | 560c901 | 2016-02-24 16:34:38 -0800 | [diff] [blame] | 39 | json = args.build_files |
Craig Tiller | 1a71811 | 2015-07-03 12:07:06 -0700 | [diff] [blame] | 40 | |
| 41 | test = {} if 'TEST' in os.environ else None |
| 42 | |
| 43 | plugins = sorted(glob.glob('tools/buildgen/plugins/*.py')) |
| 44 | |
Craig Tiller | 0fe5ee7 | 2015-12-22 12:50:36 -0800 | [diff] [blame] | 45 | templates = args.templates |
| 46 | if not templates: |
| 47 | for root, dirs, files in os.walk('templates'): |
| 48 | for f in files: |
| 49 | templates.append(os.path.join(root, f)) |
| 50 | |
Craig Tiller | 560c901 | 2016-02-24 16:34:38 -0800 | [diff] [blame] | 51 | pre_jobs = [] |
| 52 | base_cmd = ['python2.7', 'tools/buildgen/mako_renderer.py'] |
| 53 | cmd = base_cmd[:] |
| 54 | for plugin in plugins: |
| 55 | cmd.append('-p') |
| 56 | cmd.append(plugin) |
| 57 | for js in json: |
| 58 | cmd.append('-d') |
| 59 | cmd.append(js) |
| 60 | cmd.append('-w') |
| 61 | preprocessed_build = '.preprocessed_build' |
| 62 | cmd.append(preprocessed_build) |
Craig Tiller | 3ab2fe0 | 2016-04-11 20:11:18 -0700 | [diff] [blame] | 63 | if args.output_merged is not None: |
| 64 | cmd.append('-M') |
| 65 | cmd.append(args.output_merged) |
Craig Tiller | 560c901 | 2016-02-24 16:34:38 -0800 | [diff] [blame] | 66 | pre_jobs.append(jobset.JobSpec(cmd, shortname='preprocess', timeout_seconds=None)) |
| 67 | |
Craig Tiller | 1a71811 | 2015-07-03 12:07:06 -0700 | [diff] [blame] | 68 | jobs = [] |
Craig Tiller | 560c901 | 2016-02-24 16:34:38 -0800 | [diff] [blame] | 69 | for template in reversed(sorted(templates)): |
Craig Tiller | 0fe5ee7 | 2015-12-22 12:50:36 -0800 | [diff] [blame] | 70 | root, f = os.path.split(template) |
| 71 | if os.path.splitext(f)[1] == '.template': |
| 72 | out_dir = '.' + root[len('templates'):] |
| 73 | out = out_dir + '/' + os.path.splitext(f)[0] |
| 74 | if not os.path.exists(out_dir): |
| 75 | os.makedirs(out_dir) |
Craig Tiller | 560c901 | 2016-02-24 16:34:38 -0800 | [diff] [blame] | 76 | cmd = base_cmd[:] |
| 77 | cmd.append('-P') |
| 78 | cmd.append(preprocessed_build) |
Craig Tiller | 0fe5ee7 | 2015-12-22 12:50:36 -0800 | [diff] [blame] | 79 | cmd.append('-o') |
| 80 | if test is None: |
| 81 | cmd.append(out) |
| 82 | else: |
| 83 | tf = tempfile.mkstemp() |
| 84 | test[out] = tf[1] |
| 85 | os.close(tf[0]) |
| 86 | cmd.append(test[out]) |
| 87 | cmd.append(root + '/' + f) |
Craig Tiller | 590105a | 2016-01-19 13:03:46 -0800 | [diff] [blame] | 88 | jobs.append(jobset.JobSpec(cmd, shortname=out, timeout_seconds=None)) |
Craig Tiller | 1a71811 | 2015-07-03 12:07:06 -0700 | [diff] [blame] | 89 | |
Craig Tiller | 560c901 | 2016-02-24 16:34:38 -0800 | [diff] [blame] | 90 | jobset.run(pre_jobs, maxjobs=args.jobs) |
Craig Tiller | 1297170 | 2016-01-21 13:38:35 -0800 | [diff] [blame] | 91 | jobset.run(jobs, maxjobs=args.jobs) |
Craig Tiller | 1a71811 | 2015-07-03 12:07:06 -0700 | [diff] [blame] | 92 | |
| 93 | if test is not None: |
Nicolas "Pixel" Noble | 22232ae | 2015-07-11 22:40:22 +0200 | [diff] [blame] | 94 | for s, g in test.iteritems(): |
Craig Tiller | 259e627 | 2015-08-31 16:58:18 -0700 | [diff] [blame] | 95 | if os.path.isfile(g): |
Nicolas "Pixel" Noble | 9baaead | 2015-10-03 02:25:02 +0200 | [diff] [blame] | 96 | assert 0 == os.system('diff %s %s' % (s, g)), s |
Craig Tiller | 259e627 | 2015-08-31 16:58:18 -0700 | [diff] [blame] | 97 | os.unlink(g) |
| 98 | else: |
Nicolas "Pixel" Noble | 9baaead | 2015-10-03 02:25:02 +0200 | [diff] [blame] | 99 | assert 0 == os.system('diff -r %s %s' % (s, g)), s |
Craig Tiller | 259e627 | 2015-08-31 16:58:18 -0700 | [diff] [blame] | 100 | shutil.rmtree(g, ignore_errors=True) |