[autotest] Re-land "Add logic to symbolicate dumps using dev server, stage symbols on-demand"
re-run /usr/local/autotest/utils/build_externals.py before trying to use this
outside the chroot.
When trying to symbolicate a crash dump using the dev server, we first
have to stage debug symbols on-demand. We can't stage them up front,
sadly. So, block and stage them on-demand, doing so iff we actually get
crashes during a test.
This reverts commit 77e95f2dd7a025966aa2cdf19cfb5f0bfcf2608f.
BUG=chromium-os:29730,chromium-os:30400
TEST=run_remote_tests a test that generates a crash dump; symbolication should
TEST=occur locally.
TEST=set up a dev server that does not have symbols staged, and run a test
TEST=that generates a crash dump. The symbols should get staged and the dump
TEST=symbolicated. Now run another. The symbols should be re-used.
STATUS=Fixed
Change-Id: I85c9f16ec054210ebacf22c4713d9ba65927551b
Reviewed-on: https://gerrit.chromium.org/gerrit/22866
Tested-by: Chris Masone <cmasone@chromium.org>
Reviewed-by: Scott Zawalski <scottz@chromium.org>
Commit-Ready: Chris Masone <cmasone@chromium.org>
diff --git a/client/common_lib/cros/dev_server.py b/client/common_lib/cros/dev_server.py
index a665c88..1c02d71 100644
--- a/client/common_lib/cros/dev_server.py
+++ b/client/common_lib/cros/dev_server.py
@@ -3,10 +3,12 @@
# found in the LICENSE file.
import httplib
+import logging
import urllib2
import HTMLParser
from autotest_lib.client.common_lib import error, global_config
+# TODO(cmasone): redo this class using requests module; http://crosbug.com/30107
CONFIG = global_config.global_config
@@ -172,7 +174,7 @@
def get_control_file(self, build, control_path):
"""Ask the dev server for the contents of a control file.
- Ask the dev server at |self._dev_server|for the contents of the
+ Ask the dev server at |self._dev_server| for the contents of the
control file at |control_path| for |build|.
@param build: The build (e.g. x86-mario-release/R18-1586.0.0-a1-b1514)
@@ -188,6 +190,50 @@
@remote_devserver_call
+ def symbolicate_dump(self, minidump_path, build):
+ """Ask the dev server to symbolicate the dump at minidump_path.
+
+ Stage the debug symbols for |build| and, if that works, ask the
+ dev server at |self._dev_server| to symbolicate the dump at
+ minidump_path.
+
+ @param minidump_path: the on-disk path of the minidump.
+ @param build: The build (e.g. x86-mario-release/R18-1586.0.0-a1-b1514)
+ whose debug symbols are needed for symbolication.
+ @return The contents of the stack trace
+ @raise DevServerException upon any return code that's not HTTP OK.
+ """
+ try:
+ import requests
+ except ImportError:
+ logging.warning("Can't 'import requests' to connect to dev server.")
+ return ''
+ # Stage debug symbols.
+ call = self._build_call(
+ 'stage_debug',
+ archive_url=_get_image_storage_server() + build)
+ request = requests.get(call)
+ if (request.status_code != requests.codes.ok or
+ request.text != 'Success'):
+ raise urllib2.HTTPError(call,
+ request.status_code,
+ request.text,
+ request.headers,
+ None)
+ # Symbolicate minidump.
+ call = self._build_call('symbolicate_dump')
+ request = requests.post(call,
+ files={'minidump': open(minidump_path, 'rb')})
+ if request.status_code == requests.codes.OK:
+ return request.text
+ raise urllib2.HTTPError(call,
+ request.status_code,
+ '%d' % request.status_code,
+ request.headers,
+ None)
+
+
+ @remote_devserver_call
def get_latest_build(self, target, milestone=''):
"""Ask the dev server for the latest build for a given target.
diff --git a/server/cros/dynamic_suite.py b/server/cros/dynamic_suite.py
index 63a6285..f80adb8 100644
--- a/server/cros/dynamic_suite.py
+++ b/server/cros/dynamic_suite.py
@@ -941,7 +941,8 @@
name='/'.join([self._build, self._tag, test.name]),
control_type=test.test_type.capitalize(),
meta_hosts=[meta_hosts],
- dependencies=job_deps)
+ dependencies=job_deps,
+ keyvals={'build': self._build, 'suite': self._tag})
setattr(test_obj, 'test_name', test.name)
diff --git a/server/cros/dynamic_suite_unittest.py b/server/cros/dynamic_suite_unittest.py
index 2986866..e9a600d 100755
--- a/server/cros/dynamic_suite_unittest.py
+++ b/server/cros/dynamic_suite_unittest.py
@@ -649,7 +649,9 @@
mox.StrContains(test.name)),
control_type=mox.IgnoreArg(),
meta_hosts=[dynamic_suite.VERSION_PREFIX + self._BUILD],
- dependencies=[]).AndReturn(FakeJob())
+ dependencies=[],
+ keyvals={'build': self._BUILD, 'suite': self._TAG}
+ ).AndReturn(FakeJob())
def testScheduleTests(self):
diff --git a/server/site_crashcollect.py b/server/site_crashcollect.py
index 89dd7a6..5450c6e 100644
--- a/server/site_crashcollect.py
+++ b/server/site_crashcollect.py
@@ -4,7 +4,9 @@
import logging
import os
+import urllib2
from autotest_lib.client.common_lib import utils as client_utils
+from autotest_lib.client.common_lib.cros import dev_server
from autotest_lib.client.cros import constants
from autotest_lib.server import utils
@@ -14,16 +16,39 @@
This function expects the debug symbols to reside under:
/build/<board>/usr/lib/debug
+
+ @param minidump_path: absolute path to minidump to by symbolicated.
+ @raise client_utils.error.CmdError if minidump_stackwalk return code != 0.
"""
symbol_dir = '%s/../../../lib/debug' % utils.get_server_dir()
logging.info('symbol_dir: %s' % symbol_dir)
- try:
- result = client_utils.run('minidump_stackwalk %s %s > %s.txt' %
- (minidump_path, symbol_dir, minidump_path))
- rc = result.exit_status
- except client_utils.error.CmdError, err:
- rc = err.result_obj.exit_status
- return rc
+ client_utils.run('minidump_stackwalk %s %s > %s.txt' %
+ (minidump_path, symbol_dir, minidump_path))
+
+
+def symbolicate_minidump_with_devserver(minidump_path, resultdir):
+ """
+ Generates a stack trace for the specified minidump by consulting devserver.
+
+ This function assumes the debug symbols have been staged on the devserver.
+
+ @param minidump_path: absolute path to minidump to by symbolicated.
+ @param resultdir: server job's result directory.
+ @raise DevServerException upon failure, HTTP or otherwise.
+ """
+ # First, look up what build we tested. If we can't find this, we can't
+ # get the right debug symbols, so we might as well give up right now.
+ keyvals = client_utils.read_keyval(resultdir)
+ if 'build' not in keyvals:
+ raise dev_server.DevServerException(
+ 'Cannot determine build being tested.')
+
+ devserver = dev_server.DevServer.create()
+ trace_text = devserver.symbolicate_dump(minidump_path, keyvals['build'])
+ if not trace_text:
+ raise dev_server.DevServerException('Unknown error!!')
+ with open(minidump_path + '.txt', 'w') as trace_file:
+ trace_file.write(trace_text)
def find_and_generate_minidump_stacktraces(host_resultdir):
@@ -40,13 +65,25 @@
if not file.endswith('.dmp'):
continue
minidump = os.path.join(dir, file)
- rc = generate_minidump_stacktrace(minidump)
- if rc == 0:
+
+ # First, try to symbolicate locally.
+ try:
+ generate_minidump_stacktrace(minidump)
logging.info('Generated stack trace for dump %s', minidump)
- return
- else:
- logging.warn('Failed to generate stack trace for ' \
- 'dump %s (rc=%d)' % (minidump, rc))
+ continue
+ except client_utils.error.CmdError as err:
+ logging.warn('Failed to generate stack trace locally for '
+ 'dump %s (rc=%d):\n%r',
+ minidump, err.result_obj.exit_status, err)
+
+ # If that did not succeed, try to symbolicate using the dev server.
+ try:
+ symbolicate_minidump_with_devserver(minidump, host_resultdir)
+ logging.info('Generated stack trace for dump %s', minidump)
+ continue
+ except dev_server.DevServerException as e:
+ logging.warn('Failed to generate stack trace on devserver for '
+ 'dump %s:\n%r', minidump, e)
def fetch_orphaned_crashdumps(host, host_resultdir):
diff --git a/utils/external_packages.py b/utils/external_packages.py
index 0b8cb56..18e50fe 100644
--- a/utils/external_packages.py
+++ b/utils/external_packages.py
@@ -654,6 +654,18 @@
return True
+class RequestsPackage(ExternalPackage):
+ version = '0.11.2'
+ local_filename = 'requests-%s.tar.gz' % version
+ urls = ('http://pypi.python.org/packages/source/r/requests/' +
+ local_filename,)
+ hex_sum = '00a49e8bd6dd8955acf6f6269d1b85f50c70b712'
+
+ _build_and_install = ExternalPackage._build_and_install_from_package
+ _build_and_install_current_dir = (
+ ExternalPackage._build_and_install_current_dir_setup_py)
+
+
class SimplejsonPackage(ExternalPackage):
version = '2.0.9'
local_filename = 'simplejson-%s.tar.gz' % version