lmr | 6f669ce | 2009-05-31 19:02:42 +0000 | [diff] [blame^] | 1 | import re, os, sys, StringIO |
| 2 | from autotest_lib.client.common_lib import error |
| 3 | |
| 4 | """ |
| 5 | KVM configuration file utility functions. |
| 6 | |
| 7 | @copyright: Red Hat 2008-2009 |
| 8 | """ |
| 9 | |
| 10 | class config: |
| 11 | """ |
| 12 | Parse an input file or string that follows the KVM Test Config File format |
| 13 | and generate a list of dicts that will be later used as configuration |
| 14 | parameters by the the KVM tests. |
| 15 | |
| 16 | @see: http://www.linux-kvm.org/page/KVM-Autotest/Test_Config_File |
| 17 | """ |
| 18 | |
| 19 | def __init__(self, filename=None, debug=False): |
| 20 | """ |
| 21 | Initialize the list and optionally parse filename. |
| 22 | |
| 23 | @param filename: Path of the file that will be taken |
| 24 | """ |
| 25 | self.list = [{"name": "", "shortname": "", "depend": []}] |
| 26 | self.debug = debug |
| 27 | self.filename = filename |
| 28 | if filename: |
| 29 | self.parse_file(filename) |
| 30 | |
| 31 | |
| 32 | def set_debug(self, debug=False): |
| 33 | """ |
| 34 | Enable or disable debugging output. |
| 35 | |
| 36 | @param debug: Whether debug is enabled (True) or disabled (False). |
| 37 | """ |
| 38 | self.debug = debug |
| 39 | |
| 40 | |
| 41 | def parse_file(self, filename): |
| 42 | """ |
| 43 | Parse filename, return the resulting list and store it in .list. If |
| 44 | filename does not exist, raise an exception. |
| 45 | |
| 46 | @param filename: Path of the configuration file. |
| 47 | """ |
| 48 | if not os.path.exists(filename): |
| 49 | raise Exception, "File %s not found" % filename |
| 50 | self.filename = filename |
| 51 | file = open(filename, "r") |
| 52 | self.list = self.parse(file, self.list) |
| 53 | file.close() |
| 54 | return self.list |
| 55 | |
| 56 | |
| 57 | def parse_string(self, str): |
| 58 | """ |
| 59 | Parse a string, return the resulting list and store it in .list. |
| 60 | |
| 61 | @param str: String that will be parsed. |
| 62 | """ |
| 63 | file = StringIO.StringIO(str) |
| 64 | self.list = self.parse(file, self.list) |
| 65 | file.close() |
| 66 | return self.list |
| 67 | |
| 68 | |
| 69 | def get_list(self): |
| 70 | """ |
| 71 | Return the list of dictionaries. This should probably be called after |
| 72 | parsing something. |
| 73 | """ |
| 74 | return self.list |
| 75 | |
| 76 | |
| 77 | def match(self, filter, dict): |
| 78 | """ |
| 79 | Return True if dict matches filter. |
| 80 | |
| 81 | @param filter: A regular expression that defines the filter. |
| 82 | @param dict: Dictionary that will be inspected. |
| 83 | """ |
| 84 | filter = re.compile("(\\.|^)" + filter + "(\\.|$)") |
| 85 | return filter.search(dict["name"]) != None |
| 86 | |
| 87 | |
| 88 | def filter(self, filter, list=None): |
| 89 | """ |
| 90 | Filter a list of dicts. |
| 91 | |
| 92 | @param filter: A regular expression that will be used as a filter. |
| 93 | @param list: A list of dictionaries that will be filtered. |
| 94 | """ |
| 95 | if list == None: |
| 96 | list = self.list |
| 97 | filtered_list = [] |
| 98 | for dict in list: |
| 99 | if self.match(filter, dict): |
| 100 | filtered_list.append(dict) |
| 101 | return filtered_list |
| 102 | |
| 103 | |
| 104 | # Currently unused, will be removed if it remains unused |
| 105 | def get_match_block_indices(self, filter, list=None): |
| 106 | """ |
| 107 | Get the indexes of a list that match a given filter. |
| 108 | |
| 109 | @param filter: A regular expression that will filter the list. |
| 110 | @param list: List which we want to know the indexes that match |
| 111 | a filter. |
| 112 | """ |
| 113 | if list == None: |
| 114 | list = self.list |
| 115 | block_list = [] |
| 116 | prev_match = False |
| 117 | for index in range(len(list)): |
| 118 | dict = list[index] |
| 119 | if self.match(filter, dict): |
| 120 | if not prev_match: |
| 121 | block_list.append([index]) |
| 122 | prev_match = True |
| 123 | else: |
| 124 | if prev_match: |
| 125 | block_list[-1].append(index) |
| 126 | prev_match = False |
| 127 | if prev_match: |
| 128 | block_list[-1].append(len(list)) |
| 129 | return block_list |
| 130 | |
| 131 | |
| 132 | def split_and_strip(self, str, sep="="): |
| 133 | """ |
| 134 | Split str and strip quotes from the resulting parts. |
| 135 | |
| 136 | @param str: String that will be processed |
| 137 | @param sep: Separator that will be used to split the string |
| 138 | """ |
| 139 | temp = str.split(sep) |
| 140 | for i in range(len(temp)): |
| 141 | temp[i] = temp[i].strip() |
| 142 | temp[i] = temp[i].strip("\"\'") |
| 143 | return temp |
| 144 | |
| 145 | |
| 146 | def get_next_line(self, file): |
| 147 | """ |
| 148 | Get the next non-empty, non-comment line in a file like object. |
| 149 | |
| 150 | @param file: File like object |
| 151 | @return: If no line is available, return None. |
| 152 | """ |
| 153 | while True: |
| 154 | line = file.readline() |
| 155 | if line == "": return None |
| 156 | stripped_line = line.strip() |
| 157 | if len(stripped_line) > 0 \ |
| 158 | and not stripped_line.startswith('#') \ |
| 159 | and not stripped_line.startswith('//'): |
| 160 | return line |
| 161 | |
| 162 | |
| 163 | def get_next_line_indent(self, file): |
| 164 | """ |
| 165 | Return the indent level of the next non-empty, non-comment line in file. |
| 166 | |
| 167 | @param file: File like object. |
| 168 | @return: If no line is available, return -1. |
| 169 | """ |
| 170 | pos = file.tell() |
| 171 | line = self.get_next_line(file) |
| 172 | if not line: |
| 173 | file.seek(pos) |
| 174 | return -1 |
| 175 | line = line.expandtabs() |
| 176 | indent = 0 |
| 177 | while line[indent] == ' ': |
| 178 | indent += 1 |
| 179 | file.seek(pos) |
| 180 | return indent |
| 181 | |
| 182 | |
| 183 | def add_name(self, str, name, append=False): |
| 184 | """ |
| 185 | Add name to str with a separator dot and return the result. |
| 186 | |
| 187 | @param str: String that will be processed |
| 188 | @param name: name that will be appended to the string. |
| 189 | @return: If append is True, append name to str. |
| 190 | Otherwise, pre-pend name to str. |
| 191 | """ |
| 192 | if str == "": |
| 193 | return name |
| 194 | # Append? |
| 195 | elif append: |
| 196 | return str + "." + name |
| 197 | # Prepend? |
| 198 | else: |
| 199 | return name + "." + str |
| 200 | |
| 201 | |
| 202 | def parse_variants(self, file, list, subvariants=False, prev_indent=-1): |
| 203 | """ |
| 204 | Read and parse lines from file like object until a line with an indent |
| 205 | level lower than or equal to prev_indent is encountered. |
| 206 | |
| 207 | @brief: Parse a 'variants' or 'subvariants' block from a file-like |
| 208 | object. |
| 209 | @param file: File-like object that will be parsed |
| 210 | @param list: List of dicts to operate on |
| 211 | @param subvariants: If True, parse in 'subvariants' mode; |
| 212 | otherwise parse in 'variants' mode |
| 213 | @param prev_indent: The indent level of the "parent" block |
| 214 | @return: The resulting list of dicts. |
| 215 | """ |
| 216 | new_list = [] |
| 217 | |
| 218 | while True: |
| 219 | indent = self.get_next_line_indent(file) |
| 220 | if indent <= prev_indent: |
| 221 | break |
| 222 | indented_line = self.get_next_line(file).rstrip() |
| 223 | line = indented_line.strip() |
| 224 | |
| 225 | # Get name and dependencies |
| 226 | temp = line.strip("- ").split(":") |
| 227 | name = temp[0] |
| 228 | if len(temp) == 1: |
| 229 | dep_list = [] |
| 230 | else: |
| 231 | dep_list = temp[1].split() |
| 232 | |
| 233 | # See if name should be added to the 'shortname' field |
| 234 | add_to_shortname = True |
| 235 | if name.startswith("@"): |
| 236 | name = name.strip("@") |
| 237 | add_to_shortname = False |
| 238 | |
| 239 | # Make a deep copy of list |
| 240 | temp_list = [] |
| 241 | for dict in list: |
| 242 | new_dict = dict.copy() |
| 243 | new_dict["depend"] = dict["depend"][:] |
| 244 | temp_list.append(new_dict) |
| 245 | |
| 246 | if subvariants: |
| 247 | # If we're parsing 'subvariants', we need to modify the list first |
| 248 | self.__modify_list_subvariants(temp_list, name, dep_list, add_to_shortname) |
| 249 | temp_list = self.parse(file, temp_list, |
| 250 | restricted=True, prev_indent=indent) |
| 251 | else: |
| 252 | # If we're parsing 'variants', we need to parse first and then modify the list |
| 253 | if self.debug: |
| 254 | self.__debug_print(indented_line, "Entering variant '%s' (variant inherits %d dicts)" % (name, len(list))) |
| 255 | temp_list = self.parse(file, temp_list, |
| 256 | restricted=False, prev_indent=indent) |
| 257 | self.__modify_list_variants(temp_list, name, dep_list, add_to_shortname) |
| 258 | |
| 259 | new_list += temp_list |
| 260 | |
| 261 | return new_list |
| 262 | |
| 263 | |
| 264 | def parse(self, file, list, restricted=False, prev_indent=-1): |
| 265 | """ |
| 266 | Read and parse lines from file until a line with an indent level lower |
| 267 | than or equal to prev_indent is encountered. |
| 268 | |
| 269 | @brief: Parse a file-like object. |
| 270 | @param file: A file-like object |
| 271 | @param list: A list of dicts to operate on (list is modified in |
| 272 | place and should not be used after the call) |
| 273 | @param restricted: if True, operate in restricted mode |
| 274 | (prohibit 'variants') |
| 275 | @param prev_indent: the indent level of the "parent" block |
| 276 | @return: Return the resulting list of dicts. |
| 277 | @note: List is destroyed and should not be used after the call. |
| 278 | Only the returned list should be used. |
| 279 | """ |
| 280 | while True: |
| 281 | indent = self.get_next_line_indent(file) |
| 282 | if indent <= prev_indent: |
| 283 | break |
| 284 | indented_line = self.get_next_line(file).rstrip() |
| 285 | line = indented_line.strip() |
| 286 | words = line.split() |
| 287 | |
| 288 | len_list = len(list) |
| 289 | |
| 290 | # Look for a known operator in the line |
| 291 | operators = ["?+=", "?<=", "?=", "+=", "<=", "="] |
| 292 | op_found = None |
| 293 | for op in operators: |
| 294 | if op in line: |
| 295 | op_found = op |
| 296 | break |
| 297 | |
| 298 | # Found an operator? |
| 299 | if op_found: |
| 300 | if self.debug and not restricted: |
| 301 | self.__debug_print(indented_line, |
| 302 | "Parsing operator (%d dicts in current" |
| 303 | "context)" % len_list) |
| 304 | (left, value) = self.split_and_strip(line, op_found) |
| 305 | filters_and_key = self.split_and_strip(left, ":") |
| 306 | filters = filters_and_key[:-1] |
| 307 | key = filters_and_key[-1] |
| 308 | filtered_list = list |
| 309 | for filter in filters: |
| 310 | filtered_list = self.filter(filter, filtered_list) |
| 311 | # Apply the operation to the filtered list |
| 312 | for dict in filtered_list: |
| 313 | if op_found == "=": |
| 314 | dict[key] = value |
| 315 | elif op_found == "+=": |
| 316 | dict[key] = dict.get(key, "") + value |
| 317 | elif op_found == "<=": |
| 318 | dict[key] = value + dict.get(key, "") |
| 319 | elif op_found.startswith("?") and dict.has_key(key): |
| 320 | if op_found == "?=": |
| 321 | dict[key] = value |
| 322 | elif op_found == "?+=": |
| 323 | dict[key] = dict.get(key, "") + value |
| 324 | elif op_found == "?<=": |
| 325 | dict[key] = value + dict.get(key, "") |
| 326 | |
| 327 | # Parse 'no' and 'only' statements |
| 328 | elif words[0] == "no" or words[0] == "only": |
| 329 | if len(words) <= 1: |
| 330 | continue |
| 331 | filters = words[1:] |
| 332 | filtered_list = [] |
| 333 | if words[0] == "no": |
| 334 | for dict in list: |
| 335 | for filter in filters: |
| 336 | if self.match(filter, dict): |
| 337 | break |
| 338 | else: |
| 339 | filtered_list.append(dict) |
| 340 | if words[0] == "only": |
| 341 | for dict in list: |
| 342 | for filter in filters: |
| 343 | if self.match(filter, dict): |
| 344 | filtered_list.append(dict) |
| 345 | break |
| 346 | list = filtered_list |
| 347 | if self.debug and not restricted: |
| 348 | self.__debug_print(indented_line, |
| 349 | "Parsing no/only (%d dicts in current" |
| 350 | "context, %d remain)" % |
| 351 | (len_list, len(list))) |
| 352 | |
| 353 | # Parse 'variants' |
| 354 | elif line == "variants:": |
| 355 | # 'variants' not allowed in restricted mode |
| 356 | # (inside an exception or inside subvariants) |
| 357 | if restricted: |
| 358 | e_msg = "Using variants in this context is not allowed" |
| 359 | raise error.AutotestError() |
| 360 | if self.debug and not restricted: |
| 361 | self.__debug_print(indented_line, |
| 362 | "Entering variants block (%d dicts in" |
| 363 | "current context)" % len_list) |
| 364 | list = self.parse_variants(file, list, subvariants=False, |
| 365 | prev_indent=indent) |
| 366 | |
| 367 | # Parse 'subvariants' (the block is parsed for each dict |
| 368 | # separately) |
| 369 | elif line == "subvariants:": |
| 370 | if self.debug and not restricted: |
| 371 | self.__debug_print(indented_line, |
| 372 | "Entering subvariants block (%d dicts in" |
| 373 | "current context)" % len_list) |
| 374 | new_list = [] |
| 375 | # Remember current file position |
| 376 | pos = file.tell() |
| 377 | # Read the lines in any case |
| 378 | self.parse_variants(file, [], subvariants=True, |
| 379 | prev_indent=indent) |
| 380 | # Iterate over the list... |
| 381 | for index in range(len(list)): |
| 382 | # Revert to initial file position in this 'subvariants' |
| 383 | # block |
| 384 | file.seek(pos) |
| 385 | # Everything inside 'subvariants' should be parsed in |
| 386 | # restricted mode |
| 387 | new_list += self.parse_variants(file, list[index:index+1], |
| 388 | subvariants=True, |
| 389 | prev_indent=indent) |
| 390 | list = new_list |
| 391 | |
| 392 | # Parse 'include' statements |
| 393 | elif words[0] == "include": |
| 394 | if len(words) <= 1: |
| 395 | continue |
| 396 | if self.debug and not restricted: |
| 397 | self.__debug_print(indented_line, |
| 398 | "Entering file %s" % words[1]) |
| 399 | if self.filename: |
| 400 | filename = os.path.join(os.path.dirname(self.filename), |
| 401 | words[1]) |
| 402 | if not os.path.exists(filename): |
| 403 | e_msg = "Cannot include %s -- file not found" % filename |
| 404 | raise error.AutotestError(e_msg) |
| 405 | new_file = open(filename, "r") |
| 406 | list = self.parse(new_file, list, restricted) |
| 407 | new_file.close() |
| 408 | if self.debug and not restricted: |
| 409 | self.__debug_print("", "Leaving file %s" % words[1]) |
| 410 | else: |
| 411 | e_msg = "Cannot include anything because no file is open" |
| 412 | raise error.AutotestError(e_msg) |
| 413 | |
| 414 | # Parse multi-line exceptions |
| 415 | # (the block is parsed for each dict separately) |
| 416 | elif line.endswith(":"): |
| 417 | if self.debug and not restricted: |
| 418 | self.__debug_print(indented_line, |
| 419 | "Entering multi-line exception block" |
| 420 | "(%d dicts in current context outside" |
| 421 | "exception)" % len_list) |
| 422 | line = line.strip(":") |
| 423 | new_list = [] |
| 424 | # Remember current file position |
| 425 | pos = file.tell() |
| 426 | # Read the lines in any case |
| 427 | self.parse(file, [], restricted=True, prev_indent=indent) |
| 428 | # Iterate over the list... |
| 429 | for index in range(len(list)): |
| 430 | if self.match(line, list[index]): |
| 431 | # Revert to initial file position in this |
| 432 | # exception block |
| 433 | file.seek(pos) |
| 434 | # Everything inside an exception should be parsed in |
| 435 | # restricted mode |
| 436 | new_list += self.parse(file, list[index:index+1], |
| 437 | restricted=True, prev_indent=indent) |
| 438 | else: |
| 439 | new_list += list[index:index+1] |
| 440 | list = new_list |
| 441 | |
| 442 | return list |
| 443 | |
| 444 | |
| 445 | def __debug_print(self, str1, str2=""): |
| 446 | """ |
| 447 | Nicely print two strings and an arrow. |
| 448 | |
| 449 | @param str1: First string |
| 450 | @param str2: Second string |
| 451 | """ |
| 452 | if str2: |
| 453 | str = "%-50s ---> %s" % (str1, str2) |
| 454 | else: |
| 455 | str = str1 |
| 456 | print str |
| 457 | |
| 458 | |
| 459 | def __modify_list_variants(self, list, name, dep_list, add_to_shortname): |
| 460 | """ |
| 461 | Make some modifications to list, as part of parsing a 'variants' block. |
| 462 | |
| 463 | @param list |
| 464 | """ |
| 465 | for dict in list: |
| 466 | # Prepend name to the dict's 'name' field |
| 467 | dict["name"] = self.add_name(dict["name"], name) |
| 468 | # Prepend name to the dict's 'shortname' field |
| 469 | if add_to_shortname: |
| 470 | dict["shortname"] = self.add_name(dict["shortname"], name) |
| 471 | # Prepend name to each of the dict's dependencies |
| 472 | for i in range(len(dict["depend"])): |
| 473 | dict["depend"][i] = self.add_name(dict["depend"][i], name) |
| 474 | # Add new dependencies |
| 475 | dict["depend"] += dep_list |
| 476 | |
| 477 | |
| 478 | def __modify_list_subvariants(self, list, name, dep_list, add_to_shortname): |
| 479 | """ |
| 480 | Make some modifications to list, as part of parsing a |
| 481 | 'subvariants' block. |
| 482 | |
| 483 | @param list: List that will be processed |
| 484 | @param name: Name that will be prepended to the dictionary name |
| 485 | @param dep_list: List of dependencies to be added to the list |
| 486 | dictionaries |
| 487 | @param add_to_shortname: Whether we'll add a shortname parameter to |
| 488 | the dictionaries. |
| 489 | """ |
| 490 | for dict in list: |
| 491 | # Add new dependencies |
| 492 | for dep in dep_list: |
| 493 | dep_name = self.add_name(dict["name"], dep, append=True) |
| 494 | dict["depend"].append(dep_name) |
| 495 | # Append name to the dict's 'name' field |
| 496 | dict["name"] = self.add_name(dict["name"], name, append=True) |
| 497 | # Append name to the dict's 'shortname' field |
| 498 | if add_to_shortname: |
| 499 | dict["shortname"] = self.add_name(dict["shortname"], name, |
| 500 | append=True) |
| 501 | |
| 502 | |
| 503 | if __name__ == "__main__": |
| 504 | if len(sys.argv) >= 2: |
| 505 | filename = sys.argv[1] |
| 506 | else: |
| 507 | filename = os.path.join(os.path.dirname(sys.argv[0]), "kvm_tests.cfg") |
| 508 | list = config(filename, debug=True).get_list() |
| 509 | i = 0 |
| 510 | for dict in list: |
| 511 | print "Dictionary #%d:" % i |
| 512 | keys = dict.keys() |
| 513 | keys.sort() |
| 514 | for key in keys: |
| 515 | print " %s = %s" % (key, dict[key]) |
| 516 | i += 1 |