blob: 23c956bb8ec3ad2928c6f4703c7584df8e9c9e72 [file] [log] [blame]
Masahiro Yamada21343422014-08-27 14:05:51 +09001#!/usr/bin/env python2
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +09002#
3# Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
4#
5# SPDX-License-Identifier: GPL-2.0+
6#
7
8"""
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +09009Converter from Kconfig and MAINTAINERS to a board database.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090010
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090011Run 'tools/genboardscfg.py' to create a board database.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090012
13Run 'tools/genboardscfg.py -h' for available options.
Masahiro Yamada21343422014-08-27 14:05:51 +090014
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090015Python 2.6 or later, but not Python 3.x is necessary to run this script.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090016"""
17
18import errno
19import fnmatch
20import glob
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090021import multiprocessing
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090022import optparse
23import os
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090024import subprocess
25import sys
26import tempfile
27import time
28
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090029sys.path.append(os.path.join(os.path.dirname(__file__), 'buildman'))
30import kconfiglib
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090031
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090032### constant variables ###
33OUTPUT_FILE = 'boards.cfg'
34CONFIG_DIR = 'configs'
35SLEEP_TIME = 0.03
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090036COMMENT_BLOCK = '''#
37# List of boards
38# Automatically generated by %s: don't edit
39#
Masahiro Yamadaca418dd2014-08-06 13:42:34 +090040# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090041
42''' % __file__
43
44### helper functions ###
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090045def try_remove(f):
46 """Remove a file ignoring 'No such file or directory' error."""
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090047 try:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090048 os.remove(f)
49 except OSError as exception:
50 # Ignore 'No such file or directory' error
51 if exception.errno != errno.ENOENT:
52 raise
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090053
54def check_top_directory():
55 """Exit if we are not at the top of source directory."""
56 for f in ('README', 'Licenses'):
57 if not os.path.exists(f):
Masahiro Yamada31e21412014-08-16 00:59:26 +090058 sys.exit('Please run at the top of source directory.')
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090059
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090060def output_is_new(output):
61 """Check if the output file is up to date.
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090062
63 Returns:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090064 True if the given output file exists and is newer than any of
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090065 *_defconfig, MAINTAINERS and Kconfig*. False otherwise.
66 """
67 try:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090068 ctime = os.path.getctime(output)
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090069 except OSError as exception:
70 if exception.errno == errno.ENOENT:
71 # return False on 'No such file or directory' error
72 return False
73 else:
74 raise
75
76 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
77 for filename in fnmatch.filter(filenames, '*_defconfig'):
78 if fnmatch.fnmatch(filename, '.*'):
79 continue
80 filepath = os.path.join(dirpath, filename)
81 if ctime < os.path.getctime(filepath):
82 return False
83
84 for (dirpath, dirnames, filenames) in os.walk('.'):
85 for filename in filenames:
86 if (fnmatch.fnmatch(filename, '*~') or
87 not fnmatch.fnmatch(filename, 'Kconfig*') and
88 not filename == 'MAINTAINERS'):
89 continue
90 filepath = os.path.join(dirpath, filename)
91 if ctime < os.path.getctime(filepath):
92 return False
93
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090094 # Detect a board that has been removed since the current board database
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090095 # was generated
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090096 with open(output) as f:
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090097 for line in f:
98 if line[0] == '#' or line == '\n':
99 continue
100 defconfig = line.split()[6] + '_defconfig'
101 if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
102 return False
103
104 return True
105
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900106### classes ###
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900107class KconfigScanner:
108
109 """Kconfig scanner."""
110
111 ### constant variable only used in this class ###
112 _SYMBOL_TABLE = {
113 'arch' : 'SYS_ARCH',
114 'cpu' : 'SYS_CPU',
115 'soc' : 'SYS_SOC',
116 'vendor' : 'SYS_VENDOR',
117 'board' : 'SYS_BOARD',
118 'config' : 'SYS_CONFIG_NAME',
119 'options' : 'SYS_EXTRA_OPTIONS'
120 }
121
122 def __init__(self):
123 """Scan all the Kconfig files and create a Config object."""
124 # Define environment variables referenced from Kconfig
125 os.environ['srctree'] = os.getcwd()
126 os.environ['UBOOTVERSION'] = 'dummy'
127 os.environ['KCONFIG_OBJDIR'] = ''
128 self._conf = kconfiglib.Config()
129
130 def __del__(self):
131 """Delete a leftover temporary file before exit.
132
133 The scan() method of this class creates a temporay file and deletes
134 it on success. If scan() method throws an exception on the way,
135 the temporary file might be left over. In that case, it should be
136 deleted in this destructor.
137 """
138 if hasattr(self, '_tmpfile') and self._tmpfile:
139 try_remove(self._tmpfile)
140
141 def scan(self, defconfig):
142 """Load a defconfig file to obtain board parameters.
143
144 Arguments:
145 defconfig: path to the defconfig file to be processed
146
147 Returns:
148 A dictionary of board parameters. It has a form of:
149 {
150 'arch': <arch_name>,
151 'cpu': <cpu_name>,
152 'soc': <soc_name>,
153 'vendor': <vendor_name>,
154 'board': <board_name>,
155 'target': <target_name>,
156 'config': <config_header_name>,
157 'options': <extra_options>
158 }
159 """
160 # strip special prefixes and save it in a temporary file
161 fd, self._tmpfile = tempfile.mkstemp()
162 with os.fdopen(fd, 'w') as f:
163 for line in open(defconfig):
164 colon = line.find(':CONFIG_')
165 if colon == -1:
166 f.write(line)
167 else:
168 f.write(line[colon + 1:])
169
170 self._conf.load_config(self._tmpfile)
171
172 try_remove(self._tmpfile)
173 self._tmpfile = None
174
175 params = {}
176
177 # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
178 # Set '-' if the value is empty.
179 for key, symbol in self._SYMBOL_TABLE.items():
180 value = self._conf.get_symbol(symbol).get_value()
181 if value:
182 params[key] = value
183 else:
184 params[key] = '-'
185
186 defconfig = os.path.basename(defconfig)
187 params['target'], match, rear = defconfig.partition('_defconfig')
188 assert match and not rear, '%s : invalid defconfig' % defconfig
189
190 # fix-up for aarch64
191 if params['arch'] == 'arm' and params['cpu'] == 'armv8':
192 params['arch'] = 'aarch64'
193
194 # fix-up options field. It should have the form:
195 # <config name>[:comma separated config options]
196 if params['options'] != '-':
197 params['options'] = params['config'] + ':' + \
198 params['options'].replace(r'\"', '"')
199 elif params['config'] != params['target']:
200 params['options'] = params['config']
201
202 return params
203
204def scan_defconfigs_for_multiprocess(queue, defconfigs):
205 """Scan defconfig files and queue their board parameters
206
207 This function is intended to be passed to
208 multiprocessing.Process() constructor.
209
210 Arguments:
211 queue: An instance of multiprocessing.Queue().
212 The resulting board parameters are written into it.
213 defconfigs: A sequence of defconfig files to be scanned.
214 """
215 kconf_scanner = KconfigScanner()
216 for defconfig in defconfigs:
217 queue.put(kconf_scanner.scan(defconfig))
218
219def read_queues(queues, params_list):
220 """Read the queues and append the data to the paramers list"""
221 for q in queues:
222 while not q.empty():
223 params_list.append(q.get())
224
225def scan_defconfigs(jobs=1):
226 """Collect board parameters for all defconfig files.
227
228 This function invokes multiple processes for faster processing.
229
230 Arguments:
231 jobs: The number of jobs to run simultaneously
232 """
233 all_defconfigs = []
234 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
235 for filename in fnmatch.filter(filenames, '*_defconfig'):
236 if fnmatch.fnmatch(filename, '.*'):
237 continue
238 all_defconfigs.append(os.path.join(dirpath, filename))
239
240 total_boards = len(all_defconfigs)
241 processes = []
242 queues = []
243 for i in range(jobs):
244 defconfigs = all_defconfigs[total_boards * i / jobs :
245 total_boards * (i + 1) / jobs]
246 q = multiprocessing.Queue(maxsize=-1)
247 p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess,
248 args=(q, defconfigs))
249 p.start()
250 processes.append(p)
251 queues.append(q)
252
253 # The resulting data should be accumulated to this list
254 params_list = []
255
256 # Data in the queues should be retrieved preriodically.
257 # Otherwise, the queues would become full and subprocesses would get stuck.
258 while any([p.is_alive() for p in processes]):
259 read_queues(queues, params_list)
260 # sleep for a while until the queues are filled
261 time.sleep(SLEEP_TIME)
262
263 # Joining subprocesses just in case
264 # (All subprocesses should already have been finished)
265 for p in processes:
266 p.join()
267
268 # retrieve leftover data
269 read_queues(queues, params_list)
270
271 return params_list
272
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900273class MaintainersDatabase:
274
275 """The database of board status and maintainers."""
276
277 def __init__(self):
278 """Create an empty database."""
279 self.database = {}
280
281 def get_status(self, target):
282 """Return the status of the given board.
283
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900284 The board status is generally either 'Active' or 'Orphan'.
285 Display a warning message and return '-' if status information
286 is not found.
287
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900288 Returns:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900289 'Active', 'Orphan' or '-'.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900290 """
Masahiro Yamadab8828e82014-08-25 12:39:43 +0900291 if not target in self.database:
292 print >> sys.stderr, "WARNING: no status info for '%s'" % target
293 return '-'
294
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900295 tmp = self.database[target][0]
296 if tmp.startswith('Maintained'):
297 return 'Active'
298 elif tmp.startswith('Orphan'):
299 return 'Orphan'
300 else:
Masahiro Yamadab8828e82014-08-25 12:39:43 +0900301 print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" %
302 (tmp, target))
303 return '-'
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900304
305 def get_maintainers(self, target):
306 """Return the maintainers of the given board.
307
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900308 Returns:
309 Maintainers of the board. If the board has two or more maintainers,
310 they are separated with colons.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900311 """
Masahiro Yamadab8828e82014-08-25 12:39:43 +0900312 if not target in self.database:
313 print >> sys.stderr, "WARNING: no maintainers for '%s'" % target
314 return ''
315
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900316 return ':'.join(self.database[target][1])
317
318 def parse_file(self, file):
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900319 """Parse a MAINTAINERS file.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900320
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900321 Parse a MAINTAINERS file and accumulates board status and
322 maintainers information.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900323
324 Arguments:
325 file: MAINTAINERS file to be parsed
326 """
327 targets = []
328 maintainers = []
329 status = '-'
330 for line in open(file):
Masahiro Yamada5dff8442014-09-16 14:11:49 +0900331 # Check also commented maintainers
332 if line[:3] == '#M:':
333 line = line[1:]
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900334 tag, rest = line[:2], line[2:].strip()
335 if tag == 'M:':
336 maintainers.append(rest)
337 elif tag == 'F:':
338 # expand wildcard and filter by 'configs/*_defconfig'
339 for f in glob.glob(rest):
340 front, match, rear = f.partition('configs/')
341 if not front and match:
342 front, match, rear = rear.rpartition('_defconfig')
343 if match and not rear:
344 targets.append(front)
345 elif tag == 'S:':
346 status = rest
Masahiro Yamada9c2d60c2014-08-22 14:10:43 +0900347 elif line == '\n':
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900348 for target in targets:
349 self.database[target] = (status, maintainers)
350 targets = []
351 maintainers = []
352 status = '-'
353 if targets:
354 for target in targets:
355 self.database[target] = (status, maintainers)
356
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900357def insert_maintainers_info(params_list):
358 """Add Status and Maintainers information to the board parameters list.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900359
360 Arguments:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900361 params_list: A list of the board parameters
362 """
363 database = MaintainersDatabase()
364 for (dirpath, dirnames, filenames) in os.walk('.'):
365 if 'MAINTAINERS' in filenames:
366 database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
367
368 for i, params in enumerate(params_list):
369 target = params['target']
370 params['status'] = database.get_status(target)
371 params['maintainers'] = database.get_maintainers(target)
372 params_list[i] = params
373
374def format_and_output(params_list, output):
375 """Write board parameters into a file.
376
377 Columnate the board parameters, sort lines alphabetically,
378 and then write them to a file.
379
380 Arguments:
381 params_list: The list of board parameters
382 output: The path to the output file
383 """
384 FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
385 'options', 'maintainers')
386
387 # First, decide the width of each column
388 max_length = dict([ (f, 0) for f in FIELDS])
389 for params in params_list:
390 for f in FIELDS:
391 max_length[f] = max(max_length[f], len(params[f]))
392
393 output_lines = []
394 for params in params_list:
395 line = ''
396 for f in FIELDS:
397 # insert two spaces between fields like column -t would
398 line += ' ' + params[f].ljust(max_length[f])
399 output_lines.append(line.strip())
400
401 # ignore case when sorting
402 output_lines.sort(key=str.lower)
403
404 with open(output, 'w') as f:
405 f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
406
407def gen_boards_cfg(output, jobs=1, force=False):
408 """Generate a board database file.
409
410 Arguments:
411 output: The name of the output file
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900412 jobs: The number of jobs to run simultaneously
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900413 force: Force to generate the output even if it is new
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900414 """
Masahiro Yamada79d45d32014-08-25 12:39:46 +0900415 check_top_directory()
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900416
417 if not force and output_is_new(output):
418 print "%s is up to date. Nothing to do." % output
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +0900419 sys.exit(0)
420
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900421 params_list = scan_defconfigs(jobs)
422 insert_maintainers_info(params_list)
423 format_and_output(params_list, output)
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900424
425def main():
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900426 try:
427 cpu_count = multiprocessing.cpu_count()
428 except NotImplementedError:
429 cpu_count = 1
430
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900431 parser = optparse.OptionParser()
432 # Add options here
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +0900433 parser.add_option('-f', '--force', action="store_true", default=False,
434 help='regenerate the output even if it is new')
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900435 parser.add_option('-j', '--jobs', type='int', default=cpu_count,
436 help='the number of jobs to run simultaneously')
437 parser.add_option('-o', '--output', default=OUTPUT_FILE,
438 help='output file [default=%s]' % OUTPUT_FILE)
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900439 (options, args) = parser.parse_args()
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +0900440
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900441 gen_boards_cfg(options.output, jobs=options.jobs, force=options.force)
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900442
443if __name__ == '__main__':
444 main()