Mike Frysinger | d03e6b5 | 2019-08-03 12:49:01 -0400 | [diff] [blame^] | 1 | #!/usr/bin/python2 |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 2 | |
| 3 | # Copyright 2017 The Chromium OS Authors. All rights reserved. |
| 4 | # Use of this source code is governed by a BSD-style license that can be |
| 5 | # found in the LICENSE file. |
| 6 | |
| 7 | """Load generator for devserver.""" |
| 8 | |
David Riley | f0897b9 | 2017-12-14 13:25:35 -0800 | [diff] [blame] | 9 | import argparse |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 10 | import itertools |
| 11 | import json |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 12 | import re |
| 13 | import sys |
| 14 | |
| 15 | import common |
| 16 | from chromite.lib import commandline |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 17 | |
| 18 | |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 19 | # Default keys to skip displaying. |
| 20 | DEFAULT_SKIP = [ |
| 21 | 'build_name', |
| 22 | 'devserver', |
| 23 | 'name', |
| 24 | 'parent', |
| 25 | 'quick_provision', |
David Riley | 7a2a7b9 | 2017-12-06 15:24:20 -0800 | [diff] [blame] | 26 | 'trigger_response', |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 27 | ] |
| 28 | |
| 29 | # List of commandline arguments for easy filtering. |
| 30 | FILTER_ARGS = [ |
| 31 | 'board', |
| 32 | 'build_name', |
| 33 | 'devserver', |
| 34 | 'name', |
| 35 | 'status', |
| 36 | ] |
| 37 | |
| 38 | |
| 39 | def get_parser(): |
David Riley | 48cac47 | 2017-12-18 12:23:28 -0800 | [diff] [blame] | 40 | """Creates the argparse parser.""" |
| 41 | parser = commandline.ArgumentParser(description=__doc__) |
| 42 | parser.add_argument('infile', nargs='*', type=argparse.FileType('r'), |
| 43 | help='Path to JSON file to read.', |
| 44 | default=[sys.stdin]) |
| 45 | parser.add_argument('--boards', type=str, action='store', |
| 46 | help='Boards to show.') |
| 47 | parser.add_argument('--group', type=str, action='store', |
| 48 | help='Comma-spearated list of keys to group by.') |
| 49 | parser.add_argument('--dump', action='store_true', |
| 50 | help='Dump all filtered entries.') |
| 51 | parser.add_argument('--skip', type=str, action='store', |
| 52 | help='Comma-separated list of keys to skip displaying.', |
| 53 | default=','.join(DEFAULT_SKIP)) |
| 54 | parser.add_argument('--filter', type=str, action='store', |
| 55 | help='Filter expression to apply to each node.') |
| 56 | for arg in FILTER_ARGS: |
| 57 | parser.add_argument('--%s' % arg, type=str, action='store', |
| 58 | help='Comma-separated list of %s to filter by.' % |
| 59 | arg) |
David Riley | e3e67db | 2018-01-16 08:53:03 -0800 | [diff] [blame] | 60 | parser.add_argument('--no-summary', action='store_false', dest='summary', |
| 61 | help='Disable summary.') |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 62 | |
David Riley | 48cac47 | 2017-12-18 12:23:28 -0800 | [diff] [blame] | 63 | return parser |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 64 | |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 65 | def summarize_entries(entries, skip=set()): |
David Riley | 48cac47 | 2017-12-18 12:23:28 -0800 | [diff] [blame] | 66 | """Summarize a list of entries.""" |
| 67 | TAG_KEYS = [ |
| 68 | 'board', 'build_name', 'devserver', 'name', |
| 69 | 'parent', 'quick_provision', 'status' |
| 70 | ] |
| 71 | VALUE_KEYS = [ |
| 72 | 'avg_active', 'elapsed', |
| 73 | ] |
| 74 | summary = { |
| 75 | 'COUNT': len(entries), |
| 76 | } |
| 77 | summary.update({key: summarize_tags(entries, key) for key in TAG_KEYS |
| 78 | if key not in skip}) |
| 79 | summary.update({key: summarize_values(entries, key) for key in VALUE_KEYS |
| 80 | if key not in skip}) |
| 81 | return summary |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 82 | |
| 83 | def summarize_tags(entries, key): |
David Riley | 48cac47 | 2017-12-18 12:23:28 -0800 | [diff] [blame] | 84 | """Summarize all the different string values for a given key.""" |
| 85 | tags = {str(entry[key]) for entry in entries} |
| 86 | return list(tags) |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 87 | |
| 88 | def summarize_values(entries, key): |
David Riley | 48cac47 | 2017-12-18 12:23:28 -0800 | [diff] [blame] | 89 | """Summarize the numeric values for a given key.""" |
| 90 | if entries is None or len(entries) == 0: |
| 91 | return None |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 92 | |
David Riley | 48cac47 | 2017-12-18 12:23:28 -0800 | [diff] [blame] | 93 | values = [entry[key] for entry in entries if key in entry] |
| 94 | summary = {} |
| 95 | num_values = len(values) |
| 96 | if num_values: |
| 97 | summary['min'] = min(values) |
| 98 | summary['max'] = max(values) |
| 99 | summary['avg'] = sum(values) / num_values |
| 100 | num_skipped = len(entries) - num_values |
| 101 | if num_skipped: |
| 102 | summary['num'] = num_values |
| 103 | summary['skipped'] = num_skipped |
| 104 | return summary |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 105 | |
| 106 | def group_entries(keys, entries): |
David Riley | 48cac47 | 2017-12-18 12:23:28 -0800 | [diff] [blame] | 107 | """Group entries based on different values of given keys. |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 108 | |
David Riley | 48cac47 | 2017-12-18 12:23:28 -0800 | [diff] [blame] | 109 | @param keys: A list of keys to group by. |
| 110 | @param entries: A list of entries to split into groups. |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 111 | |
David Riley | 48cac47 | 2017-12-18 12:23:28 -0800 | [diff] [blame] | 112 | @return A list of list of entries, where each list has a different key |
| 113 | value. |
| 114 | """ |
| 115 | if not keys: |
| 116 | return [entries] |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 117 | |
David Riley | 48cac47 | 2017-12-18 12:23:28 -0800 | [diff] [blame] | 118 | # Divide the group based on the first key. |
| 119 | indexed = {} |
| 120 | for entry in entries: |
| 121 | value = str(entry[keys[0]]) |
| 122 | indexed.setdefault(value, []).append(entry) |
| 123 | groups = [indexed[value] for value in sorted(indexed.keys())] |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 124 | |
David Riley | 48cac47 | 2017-12-18 12:23:28 -0800 | [diff] [blame] | 125 | # Recursively subdivide all the groups based on the rest of the keys. |
| 126 | subgroups = [] |
| 127 | for group in groups: |
| 128 | subgroups.extend(group_entries(keys[1:], group)) |
| 129 | return subgroups |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 130 | |
| 131 | def main(argv): |
David Riley | 48cac47 | 2017-12-18 12:23:28 -0800 | [diff] [blame] | 132 | """Load generator for a devserver.""" |
| 133 | parser = get_parser() |
| 134 | options = parser.parse_args(argv) |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 135 | |
David Riley | 48cac47 | 2017-12-18 12:23:28 -0800 | [diff] [blame] | 136 | # Read entries from the specified file. |
| 137 | all_entries = [] |
| 138 | for f in options.infile: |
| 139 | all_entries.extend([json.loads(line) for line in f]) |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 140 | |
David Riley | 48cac47 | 2017-12-18 12:23:28 -0800 | [diff] [blame] | 141 | # Filter entries: |
| 142 | # - Ignore non-provisions. |
| 143 | # - Filter via the specified FILTER_ARGS arguments. |
| 144 | # - Filter via explicit filter request. |
| 145 | entries = filter(lambda x: x['name'] != 'Runner', all_entries) |
| 146 | for arg in FILTER_ARGS: |
| 147 | if options.__dict__.get(arg): |
| 148 | entries = filter(lambda x: x[arg] in |
| 149 | options.__dict__[arg].split(','), |
| 150 | entries) |
| 151 | if options.filter: |
David Riley | e3e67db | 2018-01-16 08:53:03 -0800 | [diff] [blame] | 152 | entries = filter(lambda x: eval(options.filter, {'re': re}, x), entries) |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 153 | |
David Riley | 48cac47 | 2017-12-18 12:23:28 -0800 | [diff] [blame] | 154 | # Group the entries based on specified keys. |
| 155 | groups = group_entries(options.group.split(',') if options.group else None, |
| 156 | entries) |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 157 | |
David Riley | 48cac47 | 2017-12-18 12:23:28 -0800 | [diff] [blame] | 158 | # Dump all filtered entries as groups, including their parents. |
| 159 | if options.dump: |
| 160 | dump_entries = itertools.chain(*groups) |
| 161 | # Dump all entries, tracking needed parents. |
| 162 | parents = [] |
| 163 | for entry in dump_entries: |
| 164 | print(json.dumps(entry)) |
David Riley | e3e67db | 2018-01-16 08:53:03 -0800 | [diff] [blame] | 165 | if 'parent' in entry and entry['parent'] not in parents: |
David Riley | 48cac47 | 2017-12-18 12:23:28 -0800 | [diff] [blame] | 166 | parents.append(entry['parent']) |
| 167 | # Dump all parents. |
| 168 | for entry in all_entries: |
| 169 | if entry['id'] in parents: |
| 170 | print(json.dumps(entry)) |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 171 | |
David Riley | 48cac47 | 2017-12-18 12:23:28 -0800 | [diff] [blame] | 172 | # Summarize the entries, group by group. |
David Riley | e3e67db | 2018-01-16 08:53:03 -0800 | [diff] [blame] | 173 | if options.summary: |
| 174 | skip = options.skip.split(',') if options.skip else set() |
| 175 | summaries = [summarize_entries(group, skip) for group in groups] |
| 176 | print(json.dumps(summaries, indent=2)) |
David Riley | 9c38278 | 2017-11-23 16:41:34 -0800 | [diff] [blame] | 177 | |
| 178 | if __name__ == '__main__': |
David Riley | 48cac47 | 2017-12-18 12:23:28 -0800 | [diff] [blame] | 179 | sys.exit(main(sys.argv[1:])) |