Ben Murdoch | 097c5b2 | 2016-05-18 11:27:45 +0100 | [diff] [blame] | 1 | # Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
| 5 | # This script computs the number of concurrent links we want to run in the build |
| 6 | # as a function of machine spec. It's based on GetDefaultConcurrentLinks in GYP. |
| 7 | |
| 8 | import optparse |
| 9 | import os |
| 10 | import re |
| 11 | import subprocess |
| 12 | import sys |
| 13 | |
| 14 | def _GetDefaultConcurrentLinks(is_lto): |
| 15 | # Inherit the legacy environment variable for people that have set it in GYP. |
| 16 | pool_size = int(os.getenv('GYP_LINK_CONCURRENCY', 0)) |
| 17 | if pool_size: |
| 18 | return pool_size |
| 19 | |
| 20 | if sys.platform in ('win32', 'cygwin'): |
| 21 | import ctypes |
| 22 | |
| 23 | class MEMORYSTATUSEX(ctypes.Structure): |
| 24 | _fields_ = [ |
| 25 | ("dwLength", ctypes.c_ulong), |
| 26 | ("dwMemoryLoad", ctypes.c_ulong), |
| 27 | ("ullTotalPhys", ctypes.c_ulonglong), |
| 28 | ("ullAvailPhys", ctypes.c_ulonglong), |
| 29 | ("ullTotalPageFile", ctypes.c_ulonglong), |
| 30 | ("ullAvailPageFile", ctypes.c_ulonglong), |
| 31 | ("ullTotalVirtual", ctypes.c_ulonglong), |
| 32 | ("ullAvailVirtual", ctypes.c_ulonglong), |
| 33 | ("sullAvailExtendedVirtual", ctypes.c_ulonglong), |
| 34 | ] |
| 35 | |
| 36 | stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX)) |
| 37 | ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)) |
| 38 | |
| 39 | # VS 2015 uses 20% more working set than VS 2013 and can consume all RAM |
| 40 | # on a 64 GB machine. |
| 41 | mem_limit = max(1, stat.ullTotalPhys / (5 * (2 ** 30))) # total / 5GB |
| 42 | hard_cap = max(1, int(os.getenv('GYP_LINK_CONCURRENCY_MAX', 2**32))) |
| 43 | return min(mem_limit, hard_cap) |
| 44 | elif sys.platform.startswith('linux'): |
| 45 | if os.path.exists("/proc/meminfo"): |
| 46 | with open("/proc/meminfo") as meminfo: |
| 47 | memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB') |
| 48 | for line in meminfo: |
| 49 | match = memtotal_re.match(line) |
| 50 | if not match: |
| 51 | continue |
| 52 | mem_total_gb = float(match.group(1)) / (2 ** 20) |
| 53 | # Allow 8Gb per link on Linux because Gold is quite memory hungry |
| 54 | mem_per_link_gb = 8 |
| 55 | if is_lto: |
| 56 | mem_total_gb -= 10 # Reserve |
| 57 | # For LTO builds the RAM requirements are even higher |
| 58 | mem_per_link_gb = 16 |
| 59 | return int(max(1, mem_total_gb / mem_per_link_gb)) |
| 60 | return 1 |
| 61 | elif sys.platform == 'darwin': |
| 62 | try: |
| 63 | avail_bytes = int(subprocess.check_output(['sysctl', '-n', 'hw.memsize'])) |
| 64 | # A static library debug build of Chromium's unit_tests takes ~2.7GB, so |
| 65 | # 4GB per ld process allows for some more bloat. |
| 66 | return max(1, avail_bytes / (4 * (2 ** 30))) # total / 4GB |
| 67 | except Exception: |
| 68 | return 1 |
| 69 | else: |
| 70 | # TODO(scottmg): Implement this for other platforms. |
| 71 | return 1 |
| 72 | |
| 73 | def main(): |
| 74 | parser = optparse.OptionParser() |
| 75 | parser.add_option('--lto', action="store_true", default=False, |
| 76 | help='This is an LTO build with higher memory requirements') |
| 77 | parser.disable_interspersed_args() |
| 78 | options, args = parser.parse_args() |
| 79 | |
| 80 | print _GetDefaultConcurrentLinks(is_lto=options.lto) |
| 81 | return 0 |
| 82 | |
| 83 | if __name__ == '__main__': |
| 84 | sys.exit(main()) |