| #!/usr/bin/python |
| """ |
| Simple crash handling application for autotest |
| |
| @copyright Red Hat Inc 2009 |
| @author Lucas Meneghel Rodrigues <lmr@redhat.com> |
| """ |
| import sys, os, commands, glob, tempfile, shutil, syslog, re, time |
| |
| |
| def get_parent_pid(pid): |
| """ |
| Returns the parent PID for a given PID, converted to an integer. |
| |
| @param pid: Process ID. |
| """ |
| try: |
| ppid = int(open('/proc/%s/stat' % pid).read().split()[3]) |
| except: |
| # It is not possible to determine the parent because the process |
| # already left the process table. |
| ppid = 1 |
| |
| return ppid |
| |
| |
| def write_to_file(filename, data, compress=False): |
| """ |
| Write contents to a given file path specified. If not specified, the file |
| will be created. Optionally, compress the destination file. |
| |
| @param file_path: Path to a given file. |
| @param data: File contents. |
| @param compress: Whether the file is going to be compressed at the end of |
| the data write process. |
| """ |
| f = open(filename, 'w') |
| try: |
| f.write(data) |
| finally: |
| f.close() |
| |
| if compress: |
| s, o = commands.getstatusoutput('bzip2 %s' % filename) |
| if s: |
| syslog.syslog("File %s compression failed: %s" % (filename, o)) |
| else: |
| filename += '.bz2' |
| |
| return filename |
| |
| |
| def get_results_dir_list(pid, core_dir_basename): |
| """ |
| Get all valid output directories for the core file and the report. It works |
| by inspecting files created by each test on /tmp and verifying if the |
| PID of the process that crashed is a child or grandchild of the autotest |
| test process. If it can't find any relationship (maybe a daemon that died |
| during a test execution), it will write the core file to the debug dirs |
| of all tests currently being executed. If there are no active autotest |
| tests at a particular moment, it will return a list with ['/tmp']. |
| |
| @param pid: PID for the process that generated the core |
| @param core_dir_basename: Basename for the directory that will hold both |
| the core dump and the crash report. |
| """ |
| pid_dir_dict = {} |
| for debugdir_file in glob.glob("/tmp/autotest_results_dir.*"): |
| a_pid = os.path.splitext(debugdir_file)[1] |
| results_dir = open(debugdir_file).read().strip() |
| pid_dir_dict[a_pid] = os.path.join(results_dir, core_dir_basename) |
| |
| results_dir_list = [] |
| # If a bug occurs and we can't grab the PID for the process that died, just |
| # return all directories available and write to all of them. |
| if pid is not None: |
| while pid > 1: |
| if pid in pid_dir_dict: |
| results_dir_list.append(pid_dir_dict[pid]) |
| pid = get_parent_pid(pid) |
| else: |
| results_dir_list = pid_dir_dict.values() |
| |
| return (results_dir_list or |
| pid_dir_dict.values() or |
| [os.path.join("/tmp", core_dir_basename)]) |
| |
| |
| def get_info_from_core(path): |
| """ |
| Reads a core file and extracts a dictionary with useful core information. |
| Right now, the only information extracted is the full executable name. |
| |
| @param path: Path to core file. |
| """ |
| full_exe_path = None |
| output = commands.getoutput('gdb -c %s batch' % path) |
| path_pattern = re.compile("Core was generated by `([^\0]+)'", re.IGNORECASE) |
| match = re.findall(path_pattern, output) |
| for m in match: |
| # Sometimes the command line args come with the core, so get rid of them |
| m = m.split(" ")[0] |
| if os.path.isfile(m): |
| full_exe_path = m |
| break |
| |
| if full_exe_path is None: |
| syslog.syslog("Could not determine from which application core file %s " |
| "is from" % path) |
| |
| return {'full_exe_path': full_exe_path} |
| |
| |
| if __name__ == "__main__": |
| syslog.openlog('AutotestCrashHandler', 0, syslog.LOG_DAEMON) |
| try: |
| try: |
| full_functionality = False |
| try: |
| (crashed_pid, crash_time, uid, signal, hostname, exe) = sys.argv[1:] |
| full_functionality = True |
| except ValueError, e: |
| # Probably due a kernel bug, we can't exactly map the parameters |
| # passed to this script. So we have to reduce the functionality |
| # of the script (just write the core at a fixed place). |
| syslog.syslog("Unable to unpack parameters passed to the " |
| "script. Operating with limited functionality.") |
| |
| core_name = 'core' |
| report_name = 'report' |
| |
| core_tmp_dir = tempfile.mkdtemp(prefix='core_', dir='/tmp') |
| core_tmp_path = os.path.join(core_tmp_dir, core_name) |
| gdb_command_path = os.path.join(core_tmp_dir, 'gdb_command') |
| |
| if full_functionality: |
| core_dir_name = 'crash.%s.%s' % (exe, crashed_pid) |
| else: |
| crashed_pid = None |
| core_dir_name = os.path.basename(core_tmp_dir) |
| |
| # Get the filtered results dir list |
| current_results_dir_list = get_results_dir_list(crashed_pid, |
| core_dir_name) |
| |
| # Write the core file to the appropriate directory |
| # (we are piping it to this script) |
| core_file = sys.stdin.read() |
| # Write the core file to its temporary location, let's keep it |
| # there in case something goes wrong |
| core_tmp_path = write_to_file(core_tmp_path, core_file) |
| processing_succeed = False |
| |
| if not full_functionality: |
| syslog.syslog("Writing core files to %s" % |
| current_results_dir_list) |
| for result_dir in current_results_dir_list: |
| if not os.path.isdir(result_dir): |
| os.makedirs(result_dir) |
| core_path = os.path.join(result_dir, 'core') |
| core_path = write_to_file(core_path, core_file, |
| compress=True) |
| processing_succeed = True |
| raise ValueError("Incorrect params passed to handler " |
| "script: %s." % sys.argv[1:]) |
| |
| # Get full command path |
| exe_path = get_info_from_core(core_tmp_path)['full_exe_path'] |
| |
| if exe_path is not None: |
| # Write a command file for GDB |
| gdb_command = 'bt full\n' |
| write_to_file(gdb_command_path, gdb_command) |
| |
| # Take a backtrace from the running program |
| gdb_cmd = ('gdb -e %s -c %s -x %s -n -batch -quiet' % |
| (exe_path, core_tmp_path, gdb_command_path)) |
| backtrace = commands.getoutput(gdb_cmd) |
| # Sanitize output before passing it to the report |
| backtrace = backtrace.decode('utf-8', 'ignore') |
| else: |
| exe_path = "Unknown" |
| backtrace = ("Could not determine backtrace for core file %s" % |
| core_tmp_path) |
| |
| # Composing the format_dict |
| report = "Program: %s\n" % exe_path |
| report += "PID: %s\n" % crashed_pid |
| report += "Signal: %s\n" % signal |
| report += "Hostname: %s\n" % hostname |
| report += "Time of the crash: %s\n" % time.ctime(float(crash_time)) |
| report += "Program backtrace:\n%s\n" % backtrace |
| |
| syslog.syslog("Application %s, PID %s crashed" % |
| (exe_path, crashed_pid)) |
| |
| # Now, for all results dir, let's create the directory if it doesn't |
| # exist, and write the core file and the report to it. |
| syslog.syslog("Writing core files and reports to %s" % |
| current_results_dir_list) |
| for result_dir in current_results_dir_list: |
| if not os.path.isdir(result_dir): |
| os.makedirs(result_dir) |
| core_path = os.path.join(result_dir, 'core') |
| core_path = write_to_file(core_path, core_file, compress=True) |
| report_path = os.path.join(result_dir, 'report') |
| write_to_file(report_path, report) |
| processing_succeed = True |
| |
| except Exception, e: |
| syslog.syslog("Crash handler had a problem: %s" % e) |
| |
| finally: |
| if processing_succeed: |
| if os.path.isdir(core_tmp_dir): |
| shutil.rmtree(core_tmp_dir) |
| else: |
| syslog.syslog("Crash handler failed to process the core file. " |
| "A copy of the file was kept at %s" % |
| core_tmp_path) |