user_activity: Created utility module for grouping parsing functions.
The modules contains the functions that do the parsing for the pprof top
and tree, CWP data and Chrome OS groups files.
BUG=None
TEST=utils_unittest passes
Change-Id: Ida94a2d456250afe4630a146349937eb0bab0767
Reviewed-on: https://chrome-internal-review.googlesource.com/291201
Reviewed-by: Luis Lozano <llozano@chromium.org>
Reviewed-by: Evelina Dumitrescu <evelinad@google.com>
Tested-by: Evelina Dumitrescu <evelinad@google.com>
Reviewed-on: https://chromium-review.googlesource.com/435915
Commit-Ready: Luis Lozano <llozano@chromium.org>
Tested-by: Luis Lozano <llozano@chromium.org>
diff --git a/user_activity_benchmarks/utils.py b/user_activity_benchmarks/utils.py
new file mode 100644
index 0000000..ead56df
--- /dev/null
+++ b/user_activity_benchmarks/utils.py
@@ -0,0 +1,327 @@
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Utility functions for parsing pprof, CWP data and Chrome OS groups files."""
+
+from collections import defaultdict
+
+import csv
+import os
+import re
+
+SEPARATOR_REGEX = re.compile(r'-+\+-+')
+FUNCTION_STATISTIC_REGEX = \
+ re.compile(r'(\S+)\s+(\S+)%\s+(\S+)%\s+(\S+)\s+(\S+)%')
+CHILD_FUNCTION_PERCENTAGE_REGEX = re.compile(r'([0-9.]+)%')
+FUNCTION_KEY_SEPARATOR_REGEX = re.compile(r'\|\s+')
+# Constants used to identify if a function is common in the pprof and CWP
+# files.
+COMMON_FUNCTION = 'common'
+EXTRA_FUNCTION = 'extra'
+PARENT_CHILD_FUNCTIONS_SEPARATOR = ';;'
+# List of pairs of strings used for make substitutions in file names to make
+# CWP and pprof data consistent.
+FILE_NAME_REPLACING_PAIR_STRINGS = [('/build/gnawty', '/build/BOARD'),
+ ('/build/amd64-generic', '/build/BOARD'),
+ (' ../sysdeps', ',sysdeps'),
+ (' ../nptl', ',nptl'),
+ (' aes-x86_64.s', ',aes-x86_64.s'),
+ (' (inline)', ''),
+ (' (partial-inline)', ''),
+ (' ../', ','),
+ ('../', '')]
+# Separator used to delimit the function from the file name.
+FUNCTION_FILE_SEPARATOR = ' /'
+
+
+def MakeCWPAndPprofFileNamesConsistent(file_name):
+ """Makes the CWP and pprof file names consistent.
+
+ For the same function, it may happen for some file paths to differ slightly
+ in the CWP data compared to the pprof output. In a file name, for each tuple
+ element of the list, we substitute the first element with the second one.
+
+ Args:
+ file_name: A string representing the name of the file.
+
+ Returns:
+ A string representing the modified name of tihe file.
+ """
+ file_name = file_name.replace(', ', '; ')
+ for replacing_pair_string in FILE_NAME_REPLACING_PAIR_STRINGS:
+ file_name = file_name.replace(replacing_pair_string[0],
+ replacing_pair_string[1])
+
+ return file_name
+
+def MakePprofFunctionKey(function_and_file_name):
+ """Creates the function key from the function and file name.
+
+ Parsing the the pprof --top and --tree outputs is difficult due to the fact
+ that it hard to extract the function and file name (i.e the function names
+ can have a lot of unexpected charachters such as spaces, operators etc).
+ For the moment, we used FUNCTION_FILE_SEPARATOR as delimiter between the
+ function and the file name. However, there are some cases where the file name
+ does not start with / and we treat this cases separately (i.e ../sysdeps,
+ ../nptl, aes-x86_64.s).
+
+ Args:
+ function_and_file_name: A string representing the function and the file name
+ as it appears in the pprof output.
+
+ Returns:
+ A string representing the function key, composed from the function and file
+ name, comma separated.
+ """
+ # TODO(evelinad): Use pprof --topproto instead of pprof --top to parse
+ # protobuffers instead of text output. Investigate if there is an equivalent
+ # for pprof --tree that gives protobuffer output.
+ #
+ # In the CWP output, we replace the , with ; as a workaround for parsing
+ # csv files. We do the same for the pprof output.
+ #
+ # TODO(evelinad): Use dremel --csv_dialect=excel-tab in the queries for
+ # replacing the , delimiter with tab.
+ function_and_file_name = function_and_file_name.replace(', ', '; ')
+ # If the function and file name sequence contains the FUNCTION_FILE_SEPARATOR,
+ # we normalize the path name of the file and make the string subtitutions
+ # to make the CWP and pprof data consistent. The returned key is composed
+ # from the function name and normalized file path name, separated by a comma.
+ # If the function and file name does not contain the FUNCTION_FILE_SEPARATOR,
+ # we just do the strings substitution.
+ if FUNCTION_FILE_SEPARATOR in function_and_file_name:
+ function_name, file_name = \
+ function_and_file_name.split(FUNCTION_FILE_SEPARATOR)
+ file_name = \
+ MakeCWPAndPprofFileNamesConsistent(os.path.normpath("/" + file_name))
+ return ','.join([function_name, file_name])
+
+ return MakeCWPAndPprofFileNamesConsistent(function_and_file_name)
+
+def ParseFunctionGroups(cwp_function_groups_lines):
+ """Parses the contents of the function groups file.
+
+ Args:
+ cwp_function_groups_lines: A list of the lines contained in the CWP
+ function groups file. A line contains the group name and the file path
+ that describes the group, separated by a space.
+
+ Returns:
+ A list of tuples containing the group name and the file path.
+ """
+ # The order of the groups mentioned in the cwp_function_groups file
+ # matters. A function declared in a file will belong to the first
+ # mentioned group that matches its path to the one of the file.
+ # It is possible to have multiple paths that belong to the same group.
+ return [tuple(line.split()) for line in cwp_function_groups_lines]
+
+
+def ParsePprofTopOutput(file_name):
+ """Parses a file that contains the output of the pprof --top command.
+
+ Args:
+ file_name: The name of the file containing the pprof --top output.
+
+ Returns:
+ A dict having as a key the name of the function and the file containing
+ the declaration of the function, separated by a comma, and as a value
+ a tuple containing the flat, flat percentage, sum percentage, cummulative
+ and cummulative percentage values.
+ """
+
+ pprof_top_statistics = {}
+
+ # In the pprof top output, the statistics of the functions start from the
+ # 6th line.
+ with open(file_name) as input_file:
+ pprof_top_content = input_file.readlines()[6:]
+
+ for line in pprof_top_content:
+ function_statistic_match = FUNCTION_STATISTIC_REGEX.search(line)
+ flat, flat_p, sum_p, cum, cum_p = function_statistic_match.groups()
+ flat_p = str(float(flat_p) / 100.0)
+ sum_p = str(float(sum_p) / 100.0)
+ cum_p = str(float(cum_p) / 100.0)
+ lookup_index = function_statistic_match.end()
+ function_and_file_name = line[lookup_index + 2 : -1]
+ key = MakePprofFunctionKey(function_and_file_name)
+ pprof_top_statistics[key] = (flat, flat_p, sum_p, cum, cum_p)
+ return pprof_top_statistics
+
+
+def ParsePprofTreeOutput(file_name):
+ """Parses a file that contains the output of the pprof --tree command.
+
+ Args:
+ file_name: The name of the file containing the pprof --tree output.
+
+ Returns:
+ A dict including the statistics for pairs of parent and child functions.
+ The key is the name of the parent function and the file where the
+ function is declared, separated by a comma. The value is a dict having as
+ a key the name of the child function and the file where the function is
+ delcared, comma separated and as a value the percentage of time the
+ parent function spends in the child function.
+ """
+
+ # In the pprof output, the statistics of the functions start from the 9th
+ # line.
+ with open(file_name) as input_file:
+ pprof_tree_content = input_file.readlines()[9:]
+
+ pprof_tree_statistics = defaultdict(lambda: defaultdict(float))
+ track_child_functions = False
+
+ # The statistics of a given function, its parent and child functions are
+ # included between two separator marks.
+ # All the parent function statistics are above the line containing the
+ # statistics of the given function.
+ # All the statistics of a child function are below the statistics of the
+ # given function.
+ # The statistics of a parent or a child function contain the calls, calls
+ # percentage, the function name and the file where the function is declared.
+ # The statistics of the given function contain the flat, flat percentage,
+ # sum percentage, cummulative, cummulative percentage, function name and the
+ # name of the file containing the declaration of the function.
+ for line in pprof_tree_content:
+ separator_match = SEPARATOR_REGEX.search(line)
+
+ if separator_match:
+ track_child_functions = False
+ continue
+
+ parent_function_statistic_match = FUNCTION_STATISTIC_REGEX.search(line)
+
+ if parent_function_statistic_match:
+ track_child_functions = True
+ lookup_index = parent_function_statistic_match.end()
+ parent_function_key_match = \
+ FUNCTION_KEY_SEPARATOR_REGEX.search(line, pos=lookup_index)
+ lookup_index = parent_function_key_match.end()
+ parent_function_key = MakePprofFunctionKey(line[lookup_index:-1])
+ continue
+
+ if not track_child_functions:
+ continue
+
+ child_function_statistic_match = \
+ CHILD_FUNCTION_PERCENTAGE_REGEX.search(line)
+ child_function_percentage = \
+ float(child_function_statistic_match.group(1))
+ lookup_index = child_function_statistic_match.end()
+ child_function_key_match = \
+ FUNCTION_KEY_SEPARATOR_REGEX.search(line, pos=lookup_index)
+ lookup_index = child_function_key_match.end()
+ child_function_key = MakePprofFunctionKey(line[lookup_index:-1])
+
+ pprof_tree_statistics[parent_function_key][child_function_key] += \
+ child_function_percentage / 100.0
+
+ return pprof_tree_statistics
+
+
+def ParseCWPInclusiveCountFile(file_name):
+ """Parses the CWP inclusive count files.
+
+ A line should contain the name of the function, the file name with the
+ declaration, the inclusive count and inclusive count fraction out of the
+ total extracted inclusive count values.
+
+ Args:
+ file_name: The file containing the inclusive count values of the CWP
+ functions.
+
+ Returns:
+ A dict containing the inclusive count statistics. The key is the name of
+ the function and the file name, comma separated. The value represents a
+ tuple with the object name containing the function declaration, the
+ inclusive count and inclusive count fraction values, and a marker to
+ identify if the function is present in one of the benchmark profiles.
+ """
+ cwp_inclusive_count_statistics = defaultdict(lambda: ('', 0, 0.0, 0))
+
+ with open(file_name) as input_file:
+ statistics_reader = csv.DictReader(input_file, delimiter=',')
+ for statistic in statistics_reader:
+ function_name = statistic['function']
+ file_name = MakeCWPAndPprofFileNamesConsistent(
+ os.path.normpath(statistic['file']))
+ dso_name = statistic['dso']
+ inclusive_count = statistic['inclusive_count']
+ inclusive_count_fraction = statistic['inclusive_count_fraction']
+
+ # We ignore the lines that have empty fields(i.e they specify only the
+ # addresses of the functions and the inclusive counts values).
+ if all([
+ function_name, file_name, dso_name, inclusive_count,
+ inclusive_count_fraction
+ ]):
+ key = '%s,%s' % (function_name, file_name)
+
+ # There might be situations where a function appears in multiple files
+ # or objects. Such situations can occur when in the Dremel queries there
+ # are not specified the Chrome OS version and the name of the board (i.e
+ # the files can belong to different kernel or library versions).
+ inclusive_count_sum = \
+ cwp_inclusive_count_statistics[key][1] + int(inclusive_count)
+ inclusive_count_fraction_sum = \
+ cwp_inclusive_count_statistics[key][2] + \
+ float(inclusive_count_fraction)
+
+ # All the functions are initially marked as EXTRA_FUNCTION.
+ value = \
+ (dso_name, inclusive_count_sum, inclusive_count_fraction_sum,
+ EXTRA_FUNCTION)
+ cwp_inclusive_count_statistics[key] = value
+
+ return cwp_inclusive_count_statistics
+
+
+def ParseCWPPairwiseInclusiveCountFile(file_name):
+ """Parses the CWP pairwise inclusive count files.
+
+ A line of the file should contain a pair of a parent and a child function,
+ concatenated by the PARENT_CHILD_FUNCTIONS_SEPARATOR, the name of the file
+ where the child function is declared and the inclusive count fractions of
+ the pair of functions out of the total amount of inclusive count values.
+
+ Args:
+ file_name: The file containing the pairwise inclusive_count statistics of
+ the
+ CWP functions.
+
+ Returns:
+ A dict containing the statistics of the parent functions and each of
+ their child functions. The key of the dict is the name of the parent
+ function. The value is a dict having as a key the name of the child
+ function with its file name separated by a ',' and as a value the
+ inclusive count value of the parent-child function pair.
+ """
+ pairwise_inclusive_count_statistics = defaultdict(lambda: defaultdict(float))
+
+ with open(file_name) as input_file:
+ statistics_reader = csv.DictReader(input_file, delimiter=',')
+
+ for statistic in statistics_reader:
+ parent_function_name, child_function_name = \
+ statistic['parent_child_functions'].split(
+ PARENT_CHILD_FUNCTIONS_SEPARATOR)
+ child_function_file_name = MakeCWPAndPprofFileNamesConsistent(
+ os.path.normpath(statistic['child_function_file']))
+ inclusive_count = statistic['inclusive_count']
+
+ # There might be situations where a child function appears in
+ # multiple files or objects. Such situations can occur when in the
+ # Dremel queries are not specified the Chrome OS version and the
+ # name of the board (i.e the files can belong to different kernel or
+ # library versions), when the child function is a template function
+ # that is declared in a header file or there are name collisions
+ # between multiple executable objects.
+ # If a pair of child and parent functions appears multiple times, we
+ # add their inclusive count values.
+ child_function_key = ','.join(
+ [child_function_name, child_function_file_name])
+ pairwise_inclusive_count_statistics[parent_function_name] \
+ [child_function_key] += float(inclusive_count)
+
+ return pairwise_inclusive_count_statistics