client/tools/crash_handler.py

#!/usr/bin/python
"""
Simple crash handling application for autotest

@copyright Red Hat Inc 2009
@author Lucas Meneghel Rodrigues <lmr@redhat.com>
"""
import sys, os, commands, glob, tempfile, shutil, syslog, re, time


def get_parent_pid(pid):
    """
    Returns the parent PID for a given PID, converted to an integer.

    @param pid: Process ID.
    """
    try:
        ppid = int(open('/proc/%s/stat' % pid).read().split()[3])
    except:
        # It is not possible to determine the parent because the process
        # already left the process table.
        ppid = 1

    return ppid


def write_to_file(filename, data, compress=False):
    """
    Write contents to a given file path specified. If not specified, the file
    will be created. Optionally, compress the destination file.

    @param file_path: Path to a given file.
    @param data: File contents.
    @param compress: Whether the file is going to be compressed at the end of
            the data write process.
    """
    f = open(filename, 'w')
    try:
        f.write(data)
    finally:
        f.close()

    if compress:
        s, o = commands.getstatusoutput('bzip2 %s' % filename)
        if s:
            syslog.syslog("File %s compression failed: %s" % (filename, o))
        else:
            filename += '.bz2'

    return filename


def get_results_dir_list(pid, core_dir_basename):
    """
    Get all valid output directories for the core file and the report. It works
    by inspecting files created by each test on /tmp and verifying if the
    PID of the process that crashed is a child or grandchild of the autotest
    test process. If it can't find any relationship (maybe a daemon that died
    during a test execution), it will write the core file to the debug dirs
    of all tests currently being executed. If there are no active autotest
    tests at a particular moment, it will return a list with ['/tmp'].

    @param pid: PID for the process that generated the core
    @param core_dir_basename: Basename for the directory that will hold both
            the core dump and the crash report.
    """
    pid_dir_dict = {}
    for debugdir_file in glob.glob("/tmp/autotest_results_dir.*"):
        a_pid = os.path.splitext(debugdir_file)[1]
        results_dir = open(debugdir_file).read().strip()
        pid_dir_dict[a_pid] = os.path.join(results_dir, core_dir_basename)

    results_dir_list = []
    # If a bug occurs and we can't grab the PID for the process that died, just
    # return all directories available and write to all of them.
    if pid is not None:
        while pid > 1:
            if pid in pid_dir_dict:
                results_dir_list.append(pid_dir_dict[pid])
            pid = get_parent_pid(pid)
    else:
        results_dir_list = pid_dir_dict.values()

    return (results_dir_list or
            pid_dir_dict.values() or
            [os.path.join("/tmp", core_dir_basename)])


def get_info_from_core(path):
    """
    Reads a core file and extracts a dictionary with useful core information.
    Right now, the only information extracted is the full executable name.

    @param path: Path to core file.
    """
    full_exe_path = None
    output = commands.getoutput('gdb -c %s batch' % path)
    path_pattern = re.compile("Core was generated by `([^\0]+)'", re.IGNORECASE)
    match = re.findall(path_pattern, output)
    for m in match:
        # Sometimes the command line args come with the core, so get rid of them
        m = m.split(" ")[0]
        if os.path.isfile(m):
            full_exe_path = m
            break

    if full_exe_path is None:
        syslog.syslog("Could not determine from which application core file %s "
                      "is from" % path)

    return {'full_exe_path': full_exe_path}


if __name__ == "__main__":
    syslog.openlog('AutotestCrashHandler', 0, syslog.LOG_DAEMON)
    try:
        try:
            full_functionality = False
            try:
                (crashed_pid, crash_time, uid, signal, hostname, exe) = sys.argv[1:]
                full_functionality = True
            except ValueError, e:
                # Probably due a kernel bug, we can't exactly map the parameters
                # passed to this script. So we have to reduce the functionality
                # of the script (just write the core at a fixed place).
                syslog.syslog("Unable to unpack parameters passed to the "
                              "script. Operating with limited functionality.")

            core_name = 'core'
            report_name = 'report'

            core_tmp_dir = tempfile.mkdtemp(prefix='core_', dir='/tmp')
            core_tmp_path = os.path.join(core_tmp_dir, core_name)
            gdb_command_path = os.path.join(core_tmp_dir, 'gdb_command')

            if full_functionality:
                core_dir_name = 'crash.%s.%s' % (exe, crashed_pid)
            else:
                crashed_pid = None
                core_dir_name = os.path.basename(core_tmp_dir)

            # Get the filtered results dir list
            current_results_dir_list = get_results_dir_list(crashed_pid,
                                                            core_dir_name)

            # Write the core file to the appropriate directory
            # (we are piping it to this script)
            core_file = sys.stdin.read()
            # Write the core file to its temporary location, let's keep it
            # there in case something goes wrong
            core_tmp_path = write_to_file(core_tmp_path, core_file)
            processing_succeed = False

            if not full_functionality:
                syslog.syslog("Writing core files to %s" %
                              current_results_dir_list)
                for result_dir in current_results_dir_list:
                    if not os.path.isdir(result_dir):
                        os.makedirs(result_dir)
                    core_path = os.path.join(result_dir, 'core')
                    core_path = write_to_file(core_path, core_file,
                                              compress=True)
                    processing_succeed = True
                raise ValueError("Incorrect params passed to handler "
                                 "script: %s." % sys.argv[1:])

            # Get full command path
            exe_path = get_info_from_core(core_tmp_path)['full_exe_path']

            if exe_path is not None:
                # Write a command file for GDB
                gdb_command = 'bt full\n'
                write_to_file(gdb_command_path, gdb_command)

                # Take a backtrace from the running program
                gdb_cmd = ('gdb -e %s -c %s -x %s -n -batch -quiet' %
                           (exe_path, core_tmp_path, gdb_command_path))
                backtrace = commands.getoutput(gdb_cmd)
                # Sanitize output before passing it to the report
                backtrace = backtrace.decode('utf-8', 'ignore')
            else:
                exe_path = "Unknown"
                backtrace = ("Could not determine backtrace for core file %s" %
                             core_tmp_path)

            # Composing the format_dict
            report = "Program: %s\n" % exe_path
            report += "PID: %s\n" % crashed_pid
            report += "Signal: %s\n" % signal
            report += "Hostname: %s\n" % hostname
            report += "Time of the crash: %s\n" % time.ctime(float(crash_time))
            report += "Program backtrace:\n%s\n" % backtrace

            syslog.syslog("Application %s, PID %s crashed" %
                          (exe_path, crashed_pid))

            # Now, for all results dir, let's create the directory if it doesn't
            # exist, and write the core file and the report to it.
            syslog.syslog("Writing core files and reports to %s" %
                          current_results_dir_list)
            for result_dir in current_results_dir_list:
                if not os.path.isdir(result_dir):
                    os.makedirs(result_dir)
                core_path = os.path.join(result_dir, 'core')
                core_path = write_to_file(core_path, core_file, compress=True)
                report_path = os.path.join(result_dir, 'report')
                write_to_file(report_path, report)
            processing_succeed = True

        except Exception, e:
            syslog.syslog("Crash handler had a problem: %s" % e)

    finally:
        if processing_succeed:
            if os.path.isdir(core_tmp_dir):
                shutil.rmtree(core_tmp_dir)
        else:
            syslog.syslog("Crash handler failed to process the core file. "
                          "A copy of the file was kept at %s" %
                          core_tmp_path)