Delete a bunch of legacy code that is no longer used.
High-level
1) Remove test_scheduler.py (scheduler before suite_scheduler)
2) Remove chromeos_test -- a bunch of related library code from
the Kirkland team that we've replaced.
3) Remove the old autotest dashboard code.
4) Old admin/deployment code.
BUG=None
TEST=Moblab test
Change-Id: I49213b98f33b2d04e026e9e532ef99f1d6ca65aa
Reviewed-on: https://chromium-review.googlesource.com/274426
Tested-by: Chris Sosa <sosa@chromium.org>
Reviewed-by: Dan Shi <dshi@chromium.org>
Commit-Queue: Chris Sosa <sosa@chromium.org>
diff --git a/site_utils/admin/cron/chrometest2.cron b/site_utils/admin/cron/chrometest2.cron
deleted file mode 100644
index 15b8a3b..0000000
--- a/site_utils/admin/cron/chrometest2.cron
+++ /dev/null
@@ -1,27 +0,0 @@
-USER=chromeos-test
-MAILTO=chromeos-test-cron@google.com
-PATH=/home/chromeos-test/bin:/usr/local/scripts:/usr/kerberos/bin:/usr/local/buildtools/java/jdk/bin:/home/chromeos-test/depot_tools:/home/chromeos-test/gsutil:/usr/local/symlinks:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-
-CHROMEOS_BASE=/usr/local/google/chromeos
-CRON_SCRIPT_BASE=/home/chromeos-test/autotest-tools/cron
-
-# Standard cron format:
-# m h dom mon dow command
-
-# Dev Server cleanup and auto-restart on failure.
-0 0 * * * cd /usr/local/google/images; ./clean_dir.sh
-*/5 * * * * cd $CRON_SCRIPT_BASE; ./start_dev_server.sh
-
-# Autotest pipeline spice. Pulls images and schedules new tests.
-*/5 * * * * cd $CRON_SCRIPT_BASE; . ./ssh_agent.sh; flock -n -x /tmp/downloader.lock -c 'runcron ./downloader.py --cros_checkout $CHROMEOS_BASE'
-*/5 * * * * cd $CRON_SCRIPT_BASE; . ./ssh_agent.sh; flock -n -x /tmp/test_scheduler.lock -c 'runcron ./test_scheduler.py'
-
-# Sync source repo used for building update.gz image. Every day at 7am so we
-# can fix any issues when we arrive in the morning.
-0 7 * * * cd $CHROMEOS_BASE; runcron repo sync -q
-
-# Sync autotest-tools repo.
-0 * * * * cd $CRON_SCRIPT_BASE; git pull -q
-
-# Schedule daily test runs at midnight.
-0 0 * * * cd $CRON_SCRIPT_BASE; . ./ssh_agent.sh; runcron ./test_scheduler.py --config daily_test_config.json
diff --git a/site_utils/chromeos_test/__init__.py b/site_utils/chromeos_test/__init__.py
deleted file mode 100644
index 8b13789..0000000
--- a/site_utils/chromeos_test/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/site_utils/chromeos_test/autotest_util.py b/site_utils/chromeos_test/autotest_util.py
deleted file mode 100644
index 67faa92..0000000
--- a/site_utils/chromeos_test/autotest_util.py
+++ /dev/null
@@ -1,294 +0,0 @@
-# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""A common Autotest utility library used by other Chrome OS scripts.
-
-Various helper functions for finding hosts, creating control files, and creating
-Autotest jobs.
-"""
-
-__author__ = 'dalecurtis@google.com (Dale Curtis)'
-
-import getpass
-import logging
-import optparse
-import os
-import posixpath
-import re
-
-import common_util
-
-
-# Pre-built Autotest location. Does not contain [client/server]/site_tests.
-if 'chromeos-test' == getpass.getuser():
- # The Autotest user should use the local install directly.
- AUTOTEST_BIN_DIR = os.path.abspath(os.path.join(
- os.path.dirname(__file__), '..', '..'))
-else:
- # All regular users need to use this for now.
- # Until we can drop SSO this is required.
- AUTOTEST_BIN_DIR = '/home/chromeos-test/autotest'
-
-# Path to atest executable.
-ATEST_PATH = os.path.join(AUTOTEST_BIN_DIR, 'cli/atest')
-
-# Maximum retries for test importer failures (rsync or site_test_importer.sh).
-MAX_IMPORT_RETRY = 3
-
-# Amount of time to sleep (in seconds) between import command retries.
-RETRY_SLEEP_SECS = 5
-
-# Directories to be rsync'd to remote server from extracted autotest tarball.
-SYNC_DIRS = ['autotest/server', 'autotest/client']
-
-
-def CreateJob(name, control, platforms, labels=None, server=True,
- sync=None, update_url=None, cli=ATEST_PATH, mail=None,
- priority='medium'):
- """Creates an Autotest job using the provided parameters.
-
- Uses the atest CLI tool to create a job for the given hosts with the given
- parameters.
-
- Args:
- name: Name of job to create.
- control: Path to Autotest control file to use.
- platforms: Platform labels to schedule job for.
- labels: Only use hosts with these labels. Can be a list or a comma delimited
- str.
- server: Run the job as a server job? Defaults to true.
- sync: Number of hosts to process synchronously.
- update_url: Dev Server update URL each host should pull update from.
- cli: Path to atest (Autotest CLI) to use for this job.
- mail: Comma separated list of email addresses to notify upon job completion.
- priority: The job priority (low, medium, high, urgent), default medium.
-
- Returns:
- Job id if successful.
-
- Raises:
- common_util.ChromeOSTestError: If Autotest job can't be created.
- """
- cmd_list = [cli, 'job create', '--machine ' + platforms,
- '--control-file ' + control]
-
- if server:
- cmd_list.append('--server')
- if sync:
- cmd_list.append('--synch_count %d' % sync)
- if update_url:
- cmd_list.append('--image ' + update_url)
- if mail:
- cmd_list.append('--email ' + mail)
- if priority:
- cmd_list.append('--priority %s' % priority.lower())
-
- if labels:
- if isinstance(labels, list):
- # Convert labels to comma separated list and escape labels w/ commas.
- # if we were given a list.
- labels = ','.join([label.replace(',', r'\\,') for label in labels])
- cmd_list.append('--dependencies ' + labels)
-
- cmd_list.append(name)
- msg = 'Failed to create Autotest job %s !' % name
- output = common_util.RunCommand(
- cmd=' '.join(cmd_list), error_msg=msg, output=True)
- return re.sub(r'[^\d-]+', '', (output.split('id')[1].strip()))
-
-
-def ImportTests(hosts, staging_dir):
- """Distributes tests to a list of hosts and runs site_test_importer.
-
- Given a list of hosts, rsync the contents of SYNC_DIRS from the Autotest
- tarball to the remote directory specified. The following script will be called
- on each host until one of them completes successfully.
-
- <dir>/utils>/site_test_importer.sh
-
- Method assumes password-less login has been setup for the hosts.
-
- Args:
- hosts: List of Autotest hosts containing host, user, and path; e.g.,
- [{'host': '127.0.0.1', 'user': 'user', 'path': '/usr/local/autotest'},
- {'host': '127.0.0.2', 'user': 'user', 'path': '/usr/local/autotest'}}
- staging_dir: Directory containing extracted autotest.tar
-
- Returns:
- True if all hosts synced successfully. False otherwise.
- """
- all_ok = True
- imported_tests = False
- for host in hosts:
- try:
- # Copy relevant files and directories, keep permissions on remote host.
- for sdir in SYNC_DIRS:
- cmd = 'rsync -a --safe-links --no-p %s %s@%s:%s' % (
- sdir, host['user'], host['host'], host['path'])
- msg = 'Failed to rsync %s to %s@%s:%s' % (
- sdir, host['user'], host['host'], host['path'])
- common_util.RunCommand(cmd, cwd=staging_dir, error_msg=msg,
- retries=MAX_IMPORT_RETRY,
- retry_sleep=RETRY_SLEEP_SECS)
-
- # Run test importer.
- if not imported_tests:
- cmd = 'ssh %s@%s %s' % (host['user'], host['host'],
- posixpath.join(host['path'],
- 'utils/site_test_importer.sh'))
- msg = 'Failed to run site_test_importer.sh on %s@%s!' % (
- host['user'], host['host'])
- common_util.RunCommand(cmd, error_msg=msg, retries=MAX_IMPORT_RETRY,
- retry_sleep=RETRY_SLEEP_SECS)
- imported_tests = True
- except common_util.ChromeOSTestError, e:
- logging.exception(e)
- all_ok = False
-
- return all_ok
-
-
-def GetPlatformDict(cli=ATEST_PATH):
- """Returns a dict of platforms + labels usable by the current user.
-
- @returns a dict with platform as a key and value of a set representing labels
- associated with a given platform.
- """
-
- cmd = '%s host list --parse --user $USER' % cli
- msg = 'Failed to retrieve host list from Autotest.'
- output = common_util.RunCommand(cmd=cmd, error_msg=msg, output=True)
-
- # atest host list will return a tabular data set with columns separated by |
- # characters. From each line we only want the platform label.
- platform_dict = {}
- if output:
- for line in output.splitlines():
- temp_dict = {}
- for entry in line.split('|'):
- key, values = entry.split('=', 1)
- temp_dict[key.strip()] = values.strip()
- platform = temp_dict.get('Platform', None)
- if not platform:
- continue
- labels = temp_dict.get('Labels', '').split()
- for label in labels:
- if label.startswith('rps') or label.startswith('cros'):
- continue
- platform_dict.setdefault(platform, set()).add(label)
-
- return platform_dict
-
-
-def GetHostList(cli=ATEST_PATH, acl=None, label=None, user=None, status=None):
- """Returns a list containing hosts retrieved from the atest CLI.
-
- Args:
- cli: path to atest.
- acl: acl to use in atest query.
- label: label to use in atest query.
- user: user to use in atest query.
- status: status to use in atest query.
-
- Returns:
- A list of host names.
-
- Raises:
- common_util.ChromeOSTestError: If host list can't be retrieved.
- """
- return [host for host in GetHostData(cli, acl, label, user, status)]
-
-
-def GetHostData(cli=ATEST_PATH, acl=None, label=None, user=None, status=None):
- """Returns a dict containing full host info retrieved from the atest CLI.
-
- Args:
- cli: path to atest.
- acl: acl to use in atest query.
- label: label to use in atest query.
- user: user to use in atest query.
- status: status to use in atest query.
-
- Returns:
- A dict of host data.
-
- Raises:
- common_util.ChromeOSTestError: If host data can't be retrieved.
- """
- afe_hosts = {}
-
- cmd = [cli, 'host list']
- if acl:
- cmd += ['-a', acl]
- if label:
- cmd += ['-b', label]
- if user:
- cmd += ['-u', user]
- if status:
- cmd += ['-s', status]
- cmd_str = ' '.join(cmd)
-
- msg = 'Failed to retrieve hosts data from autotest.'
- atest_out = common_util.RunCommand(cmd=cmd_str, error_msg=msg, output=True)
- if not atest_out:
- return afe_hosts
-
- lines = atest_out.splitlines()[1:]
- for line in lines:
- # The only two word status is 'Repair Failed'. In that case insert a '_'
- # to make a single word status 'Repair_Failed'.
- fields = line.replace('Repair Failed', 'Repair_Failed').split()
- if len(fields) > 3:
- afe_hosts[fields[0]] = {
- 'status': fields[1],
- 'locked': fields[2],
- 'platform': fields[3],
- 'labels': []}
- if len(fields) > 4:
- afe_hosts[fields[0]]['labels'] = fields[4:]
-
- return afe_hosts
-
-
-def AddOptions(parser, cli_only=False):
- """Add command line option group for atest usage.
-
- Optional method to add helpful command line options to calling programs.
- Adds Options:
- --cli: Location of atest
- --acl: acl to use in atest query
- --label: label to use in atest query
- --status: status to use in atest query
- --user: user to use in atest query
-
- Args:
- parser: OptionParser to add autotest flags to.
- cli_only: When true only add the --cli flag to argument list.
-
- Returns:
- OptionGroup: Users can add additional options to the returned group.
- """
- group = optparse.OptionGroup(parser,
- title='Autotest CLI Configuration',
- description=('Options specifying the location'
- ' and arguments to Atest.'))
- group.add_option('--cli',
- help='Autotest CLI executable location [default: %default]',
- default=ATEST_PATH)
- if not cli_only:
- group.add_option('--acl',
- help=('Autotest ACL Group to query for host machines. '
- 'http://cautotest/afe/server/admin/afe/aclgroup/'
- ' [default: %default]'),
- default='acl_cros_test')
- group.add_option('--label',
- help=('Only run on hosts with the specified label. '
- 'Examples: board_x86-generic, has_80211n, has_ssd. '
- 'See http://cautotest/afe/server/admin/afe/label/'))
- group.add_option('--status',
- help=('Only run on hosts with the specified status. '
- 'Examples: Running, Ready, Repair.'))
- group.add_option('--user',
- help='Only run on hosts with the specified user.')
- return parser.add_option_group(group)
diff --git a/site_utils/chromeos_test/build_util.py b/site_utils/chromeos_test/build_util.py
deleted file mode 100644
index dccbb66..0000000
--- a/site_utils/chromeos_test/build_util.py
+++ /dev/null
@@ -1,358 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""A common build utility library used by other Chrome OS scripts.
-
-Various helper functions for checking build versions, downloading builds,
-processing Chrome OS build components, and extracting tests.
-"""
-
-__author__ = 'dalecurtis@google.com (Dale Curtis)'
-
-import logging
-import os
-import re
-import tempfile
-import urlparse
-
-import common_util
-
-
-# Name of autotest tarball in downloaded archive.
-AUTOTEST = 'autotest.tar'
-
-# Directory to mount rootfs to during MountImage.
-ROOTFS_MOUNT_DIR = 'rootfs'
-
-# Relative path to scripts directory from Chrome OS source root.
-SCRIPTS_DIR = 'src/scripts'
-
-# Directory to mount stateful partition to during MountImage.
-STATEFUL_MOUNT_DIR = 'stateful_partition'
-
-# Name of test image in downloaded archive.
-TEST_IMAGE = 'chromiumos_test_image.bin'
-
-
-def GetLatestBuildbotBuildVersion(archive_server, board, boto=None,
- archive_path=None, build_pattern=None):
- """Retrieves the latest build version from Buildbot for the given board.
-
- Uses gsutil to build a list of builds matching the archive_path and
- build_pattern after wildcard expansion, sorts it by timestamp, and returns the
- latest build.
-
- Args:
- archive_server: Base Google Storage URL.
- board: Board name for this build; e.g., x86-generic-rel
- boto: If a Google Storage URL is provided, path to credentials file for use
- with gsutil; e.g., ~/.boto
- archive_path: Path to image file. Will be joined with archive_server.
- %(build)s values will be expanded with build_pattern if provided,
- otherwise *. All other format variables will be expanded as *.
- build_pattern: Wildcard expansion for %(build)s variable in archive_path.
-
- Returns:
- Latest build version; e.g., 0.8.61.0-r1cf43296-b269.
-
- Raises:
- common_util.ChromeOSTestError: if the latest build version can't be
- retrieved.
- """
- if not build_pattern:
- build_pattern = '*'
-
- # Create RegEx for extracting the build from the URL.
- regex_path = archive_path % {'board': '[\w-]+', 'build': '([\.\w-]+)',
- 'build_version': '[\.\w-]+'}
-
- # Wildcard expansion for the Google Storage URL...
- archive_path %= {'board': '*', 'build': build_pattern, 'build_version': '*'}
-
- archive_url = '/'.join([archive_server, archive_path])
- regex_url = '/'.join([archive_server, regex_path])
-
- env = {}
- if boto:
- env['BOTO_CONFIG'] = boto
-
- # Run gsutil, strip last line, sort by timestamp, extract URL.
- cmd = ("gsutil ls -l %s | sed '$d' | sort -k 2,2 | tail -1 |"
- " awk '{print $3}'" % archive_url)
- msg = 'Could not retrieve latest build version for board %s.' % board
- latest_url = common_util.RunCommand(
- cmd=cmd, env=env, error_msg=msg, output=True)
-
- if latest_url:
- # Fail loudly here by letting exception raise on unrecognized format.
- return re.match(regex_url, latest_url).group(1)
-
-
-def DownloadAndExtractBuild(archive_server, board, boto, build,
- archive_path=None):
- """Downloads the specified build and extracts it to a temporary folder.
-
- Looks for the file '<archive_server>/<board>/<build>/image.zip'. The archive
- is expected to contain chromiumos_test_image.bin and autotest.tar. Both
- Google Storage and http(s) URLs are okay. If a Google Storage URL is provided,
- gsutil is used to download the file, while for http(s) wget is used. wget and
- gsutil must be in the path. Downloaded archive is deleted if all steps
- complete successfully.
-
- Args:
- archive_server: Google Storage or http(s) archive URL.
- board: Board name for this build; e.g., x86-generic-rel
- boto: If a Google Storage URL is provided, path to credentials file for use
- with gsutil; e.g., ~/.boto
- build: Full build string to look for; e.g., R16-1000.0.0-a1-b269
- archive_path: Optional path to image on the archive server. Will be
- formatted against:
- {
- 'board': <e.g. 'x86-generic-rel'>,
- 'build': <e.g. 'R16-1000.0.0-a1-b269'>,
- 'build_version': <e.g. '1000.0.0'>,
- }
-
- Returns:
- A tuple of two paths (local_staging_dir, remote_arhive_path)
- local_staging_dir is the path to the staging directory where the build
- has been downloaded and relevant components extracted.
- remote_archive_url is the path where the image was downloaded from
- remote site.
-
- Raises:
- common_util.ChromeOSTestError: If any steps in the process fail to complete.
- """
- if archive_path is None:
- archive_path = '%(board)s/%(build)s/image.zip'
-
- # Format user specified archive path against parsed build variables.
- build_release = ''
- build_version, build_hash, build_num = build.rsplit('-', 2)
- if '-' in build_version:
- build_release, build_version = build_version.split('-')
- archive_path %= {'board': board, 'build': build,
- 'build_version': build_version}
- archive_url = '/'.join([archive_server, archive_path])
-
- # Create temporary directory for extraction and processing of build.
- staging_dir = tempfile.mkdtemp()
-
- # Standardize file name of archive to be downloaded.
- download_path = os.path.join(staging_dir, 'image.zip')
-
- env = {}
-
- # Choose download method based on URL protocol.
- scheme = urlparse.urlparse(archive_url).scheme
- if scheme == 'gs':
- if boto:
- env['BOTO_CONFIG'] = boto
- cmd = 'gsutil cp %s %s' % (archive_url, download_path)
- elif scheme in ['http', 'https']:
- cmd = 'wget -O %s --no-proxy %s' % (download_path, archive_url)
- else:
- raise common_util.ChromeOSTestError('Unknown archive URL protocol.')
-
- msg = 'Failed to download build! Tried "%s"' % archive_url
- common_util.RunCommand(cmd=cmd, env=env, error_msg=msg)
-
- # Extract test image and autotest tarball.
- cmd = 'unzip -u -o %s %s %s' % (download_path, TEST_IMAGE, AUTOTEST)
- msg = 'Failed to extract build!'
- common_util.RunCommand(cmd=cmd, cwd=staging_dir, error_msg=msg)
-
- # Extract autotest components. Use root to ensure when files are inserted into
- # the image later, that they have the proper permissions. Failure to do so
- # will break certain tests.
- cmd = 'sudo tar xf %s' % os.path.join(staging_dir, AUTOTEST)
- msg = 'Failed to extract autotest.tar !'
- common_util.RunCommand(cmd=cmd, cwd=staging_dir, error_msg=msg)
-
- # Everything went okay, so remove archive file.
- os.unlink(download_path)
- return staging_dir, archive_url
-
-
-def CreateUpdateZip(cros_checkout, staging_dir, image_file=TEST_IMAGE,
- output_dir=None, source_image=None):
- """Create update.gz from an image using cros_generate_update_payload.
-
- Args:
- cros_checkout: Location of a ChromeOS source code check out. A valid chroot
- is required to call the cros_generate_update_payload script.
- staging_dir: Work directory. Should contain a ChromeOS image.
- image_file: Name of the image to process.
- output_dir: Path relative to staging_dir to store update.gz in. Defaults to
- the root of the staging_dir.
- source_image: If specified, used to generate a delta update. Must be located
- in the chroot.
-
- Raises:
- common_util.ChromeOSTestError: If any steps in the process fail to complete.
- """
- # Create mount point for image temp in ChromeOS chroot.
- chroot_dir = os.path.join(cros_checkout, 'chroot')
- in_chroot_dir = os.sep + os.path.relpath(
- tempfile.mkdtemp(dir=os.path.join(chroot_dir, 'tmp')), chroot_dir)
- # Skip '/' in in_chroot_dir otherwise os.path.join will treat it as an
- # absolute path and reset the whole join.
- out_chroot_dir = os.path.join(chroot_dir, in_chroot_dir[1:])
-
- # Mount staging directory into the chroot.
- cmd = 'sudo mount --bind %s %s' % (staging_dir, out_chroot_dir)
- msg = 'Failed to mount image directory in chroot!'
- common_util.RunCommand(cmd=cmd, error_msg=msg)
-
- scripts_dir = os.path.join(cros_checkout, SCRIPTS_DIR)
-
- update_path = in_chroot_dir
- if output_dir:
- update_path = os.path.join(update_path, output_dir)
-
- # Use cros_generate_update_payload in the chroot to create update.gz.
- # TODO(dalecurtis): May need to add special failure case for lazy unmounts, no
- # sense in aborting if the only issue is we can't unmount the staging dir.
- cmd = ('cros_sdk -- cros_generate_update_payload --image %s --output %s' %
- (os.path.join(in_chroot_dir, image_file),
- os.path.join(update_path, 'update.gz')))
-
- if source_image:
- cmd += ' --src_image %s' % os.path.join(in_chroot_dir, source_image)
-
- msg = 'Failed to create update.gz!'
- # cros_generate_update_payload is a frequent source of errors. Which is why we
- # want to set error_file=True so the errors will appear in the logs.
- common_util.RunCommand(
- cmd=cmd, cwd=scripts_dir, error_msg=msg, error_file=True)
-
- # Unmount chroot temp directory. Exit chroot unmounts automatically only if
- # there are no other cros_sdk instances open.
- cmd = 'sudo umount ' + out_chroot_dir
- common_util.RunCommand(cmd=cmd, ignore_errors=True)
-
- # Remove mount point.
- os.rmdir(out_chroot_dir)
-
-
-def MountImage(cros_checkout, staging_dir, image_file=TEST_IMAGE,
- image_dir='.'):
- """Mount an image to ROOTFS_MOUNT_DIR and STATEFUL_MOUNT_DIR in staging_dir.
-
- Uses mount_gpt_image.sh from outside the chroot to setup the mounts. Image is
- mounted in safe mode (read only rootfs).
-
- Args:
- cros_checkout: Location of a ChromeOS source code check out. A valid chroot
- is required to call the cros_generate_update_payload script.
- staging_dir: Work directory. Should also contain a ChromeOS image. If the
- image is elsewhere, specify using image_dir.
- image_file: Name of the image to process.
- image_dir: The directory of the image, using staging_dir if not given.
- """
- scripts_dir = os.path.join(cros_checkout, SCRIPTS_DIR)
- # Mount rootfs and stateful partitions. Mount rootfs as read_only.
- common_util.MakedirsExisting(os.path.join(staging_dir, ROOTFS_MOUNT_DIR))
- common_util.MakedirsExisting(os.path.join(staging_dir,
- STATEFUL_MOUNT_DIR))
- cmd = ('sudo %s/mount_gpt_image.sh --image %s --from %s --rootfs_mountpt=%s'
- ' --stateful_mountpt=%s --safe' % (scripts_dir, image_file, image_dir,
- ROOTFS_MOUNT_DIR, STATEFUL_MOUNT_DIR))
- msg = 'Failed to mount partitions!'
- common_util.RunCommand(cmd=cmd, cwd=staging_dir, error_msg=msg)
-
-
-def UnmountImage(cros_checkout, staging_dir):
- """Unmount image in staging_dir from ROOTFS_MOUNT_DIR and STATEFUL_MOUNT_DIR.
-
- Uses mount_gpt_image.sh from outside the chroot to teardown the mounts.
-
- Args:
- cros_checkout: Location of a ChromeOS source code check out. A valid chroot
- is required to call the cros_generate_update_payload script.
- staging_dir: Work directory. Should also contain a ChromeOS image. If the
- image is elsewhere, specify using image_dir.
- """
- scripts_dir = os.path.join(cros_checkout, SCRIPTS_DIR)
-
- # Unmount partitions.
- cmd = ('sudo %s/mount_gpt_image.sh --unmount --rootfs_mountpt=%s'
- ' --stateful_mountpt=%s' %
- (scripts_dir, ROOTFS_MOUNT_DIR, STATEFUL_MOUNT_DIR))
- msg = 'Failed to unmount partitions!'
- common_util.RunCommand(cmd=cmd, cwd=staging_dir, error_msg=msg)
-
-
-def PrepareAutotestPkgs(autotest_dir, staging_dir, test_name='all'):
- """Create autotest client packages inside staging_dir.
-
- So they could be uploaded later, either to a mounted stateful partition or a
- remote dev server.
-
- Args:
- autotest_dir: Location of Autotest directory. Absolute or relative to the
- staging_dir.
- staging_dir: Work directory. Should also contain a ChromeOS image. If the
- image is elsewhere, specify using image_dir.
- test_name: Name of test to package. Defaults to all tests.
-
- Raises:
- common_util.ChromeOSTestError: If any steps in the process fail to complete.
- """
- common_util.MakedirsExisting(os.path.join(staging_dir, 'autotest-pkgs'))
-
- cmd_list = ['sudo', os.path.join(autotest_dir, 'utils/packager.py'),
- 'upload', '--repository autotest-pkgs']
- if test_name == 'all':
- cmd_list.append('--all')
- else:
- cmd_list.append('--client --test ' + test_name)
-
- # Upload autotest packages onto remote server.
- msg = 'Failed to create autotest packages!'
- common_util.RunCommand(cmd=' '.join(cmd_list), cwd=staging_dir,
- error_msg=msg)
-
-
-def CreateStatefulZip(cros_checkout, staging_dir, image_file=TEST_IMAGE):
- """Create stateful.tgz from using cros_generate_stateful_update_payload.
-
- Args:
- cros_checkout: Location of a ChromeOS source code check out. A valid chroot
- is required to call the cros_generate_update_payload script.
- staging_dir: Work directory. Should also contain a ChromeOS image. If the
- image is elsewhere, specify using image_dir.
- image_file: Name of the image to process.
-
- Raises:
- common_util.ChromeOSTestError: If any steps in the process fail to complete.
- """
- chroot_bin_dir = os.path.join(cros_checkout, 'chroot/usr/bin')
-
- # Generate stateful update.
- cmd = ('sudo %s/cros_generate_stateful_update_payload --image_path %s '
- '--output_dir .' % (chroot_bin_dir, image_file))
- msg = 'Failed to generate stateful.tgz!'
- common_util.RunCommand(cmd=cmd, cwd=staging_dir, error_msg=msg)
-
-
-def CreateBuildComponents(cros_checkout, staging_dir):
- """Creates various build components from chromiumos_test_image.bin.
-
- Given a staging directory containing a chromiumos_test_image.bin and autotest
- tarball, method creates update.gz and stateful.image.gz.
-
- Args:
- cros_checkout: Location of a ChromeOS source code check out. A valid chroot
- is required to call the cros_generate_update_payload script.
- staging_dir: Directory containing unzipped Buildbot image.
-
- Raises:
- common_util.ChromeOSTestError: If any steps in the process fail to complete.
- """
- CreateUpdateZip(cros_checkout, staging_dir)
- PrepareAutotestPkgs('autotest', staging_dir)
- CreateStatefulZip(cros_checkout, staging_dir)
diff --git a/site_utils/chromeos_test/colors.py b/site_utils/chromeos_test/colors.py
deleted file mode 100644
index 4414cfd..0000000
--- a/site_utils/chromeos_test/colors.py
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2010 Google Inc. All Rights Reserved.
-
-"""A class to help with colorizing console output."""
-
-__author__ = 'dalecurtis@google.com (Dale Curtis)'
-
-
-class Colors(object):
- """A class to help with colorizing console output."""
-
- _COLOR_BASE = '\033[3%dm'
-
- _BOLD_COLOR_BASE = '\033[1;3%dm'
-
- BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = [
- _COLOR_BASE % i for i in range(8)]
-
- (BOLD_BLACK, BOLD_RED, BOLD_GREEN, BOLD_YELLOW, BOLD_BLUE, BOLD_MAGENTA,
- BOLD_CYAN, BOLD_WHITE) = [_BOLD_COLOR_BASE % i for i in range(8)]
-
- OFF = '\033[m'
-
- @classmethod
- def Color(cls, color, string):
- return ''.join([color, string, cls.OFF])
diff --git a/site_utils/chromeos_test/common_util.py b/site_utils/chromeos_test/common_util.py
deleted file mode 100644
index d72d89d..0000000
--- a/site_utils/chromeos_test/common_util.py
+++ /dev/null
@@ -1,211 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""A common function library used by other Chrome OS scripts.
-
-Various helper functions for running commands and handling exceptions.
-"""
-
-__author__ = 'dalecurtis@google.com (Dale Curtis)'
-
-import errno
-import logging
-import os
-import subprocess
-import sys
-import tempfile
-import time
-
-
-_SSH_OPTIONS = ('-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'
- ' -o ConnectTimeout=30')
-
-
-class ChromeOSTestError(Exception):
- """Generic error for ChromeOS Test exceptions."""
-
-
-def RunCommand(cmd, cwd=None, env=None, error_msg=None, output=False,
- retries=0, retry_sleep=0, ignore_errors=False, error_file=False):
- """Executes a command with the given environment and working directory.
-
- Unless output is set to True, all output (stdout, stderr) is suppressed. The
- command success is determined by an exit code of zero.
-
- Args:
- cmd: Command to execute.
- cwd: Set current working directory.
- env: Dictionary of environment variables to pass to subprocess.Popen. Merged
- on top of os.environ.
- error_msg: Message used when raising an exception after command failure.
- output: Should output be kept? If used with error_file, output file name is
- returned on completion.
- retries: Number of times to retry a command before raising an exception.
- retry_sleep: Amount of time to sleep between retries.
- ignore_errors: Don't raise an exception on error.
- error_file: Store output to a file. On failure include file name in
- exception, otherwise the file is deleted if command is successful (
- unless ouput is True). Multiple retries are written to the same file.
-
- Returns:
- If output is True and error_file is False, the contents of stdout will be
- returned upon command success. Returns None if there is no output (after
- passing through strip()).
-
- If output is True and error_file is True, the output file name will be
- returned upon command success.
-
- Raises:
- ChromeOSTestError: If command fails. Message is set by the error_msg
- parameter.
- """
- logging.debug('Running command "%s"', cmd)
-
- # Import environment variables from os so we have proper PATH, etc.
- if env is not None:
- local_env = os.environ.copy()
- local_env.update(env)
- env = local_env
-
- # Setup output pipes depending on if output was requested. Use a /dev/null
- # file handle to save on having to store output.
- if error_file:
- pipe, temp_fn = tempfile.mkstemp(
- prefix=os.path.basename(sys.argv[0]).split('.')[0], suffix='.log')
- elif output:
- pipe = subprocess.PIPE
- else:
- pipe = os.open(os.devnull, os.O_WRONLY)
-
- for retry in xrange(0, retries + 1):
- # Use Popen instead of call so we don't deadlock on massive amounts of
- # output (like the autotest image import...).
- p = subprocess.Popen(cmd, env=env, cwd=cwd, shell=True, stdout=pipe,
- stderr=pipe)
-
- # Used instead of p.wait() to prevent deadlock. See subprocess man page.
- stdout, stderr = p.communicate()
-
- if p.returncode == 0:
- break
-
- if retry < retries:
- logging.warning('%s [Retrying; attempt #%d]', error_msg, retry + 1)
- time.sleep(retry_sleep)
-
- if pipe != subprocess.PIPE:
- os.close(pipe)
-
- if stdout:
- stdout = stdout.strip()
-
- if p.returncode != 0 and not ignore_errors:
- if error_file:
- raise ChromeOSTestError(error_msg,
- 'Command: %s' % cmd,
- 'Exit code: %s' % p.returncode,
- 'Output file: %s' % temp_fn)
- elif output:
- raise ChromeOSTestError(error_msg,
- 'Command: %s' % cmd,
- 'Exit code: %s' % p.returncode,
- 'Output: %s' % stdout, 'Error: %s' % stderr)
- else:
- raise ChromeOSTestError(error_msg, 'Command: %s' % cmd,
- 'Exit code: %s' % p.returncode)
- elif output and error_file:
- return temp_fn
- elif output:
- return stdout
- elif error_file:
- os.unlink(temp_fn)
-
-
-def MakedirsExisting(path, mode=None):
- """Wrapper method for os.makedirs to provide mkdir -p functionality.
-
- Args:
- path: Local directory to create.
- mode: Numeric mode to set path to, e.g., 0755 for world readable.
- """
- try:
- os.makedirs(path)
- except OSError, e:
- if e.errno == errno.EEXIST:
- pass
- else:
- raise
- if mode is not None:
- os.chmod(path, mode)
-
-
-def _MakeSSHCommand(private_key=None):
- """Helper function for building rsync command line.
-
- Args:
- private_key: Path to SSH private key for password less ssh login.
-
- Returns:
- Command line for using SSH.
- """
- ssh_cmd_list = ['ssh', _SSH_OPTIONS, '-o', 'Compression=no']
- if private_key:
- ssh_cmd_list.append('-i')
- ssh_cmd_list.append(private_key)
- return ' '.join(ssh_cmd_list)
-
-
-def _MakeRSyncCommand(private_key=None):
- """Helper function for building rsync command line.
-
- Args:
- private_key: Path to SSH private key for password less ssh login.
-
- Returns:
- Command line for using rsync.
- """
- return ' '.join(['rsync', '-az', '-e', '"%s"' % _MakeSSHCommand(private_key)])
-
-
-def RemoteCommand(remote_host, remote_user, cmd, private_key=None, **kwargs):
- """Wrapper function for RunCommand to execute commands via SSH.
-
- Takes the cmd argument and prepends "ssh <user>@<ip> ". See definition for
- RunCommand for complete argument definitions.
-
- Args:
- remote_host: Name of the remote host.
- remote_user: User name used to ssh into the remote host.
- cmd: Command to execute on remote host.
- private_key: Path to SSH private key for password less ssh login.
-
- Returns:
- Results from RunCommand()
- """
- return RunCommand(
- '%s %s@%s "%s"' % (
- _MakeSSHCommand(private_key), remote_user, remote_host, cmd),
- **kwargs)
-
-
-def RemoteCopy(remote_host, remote_user, src, dest, private_key=None, **kwargs):
- """Wrapper function for RunCommand to copy files via rsync.
-
- Takes a source and remote destination and uses rsync to copy the files. See
- definition for common_util.RunCommand for complete argument definitions.
-
- Args:
- remote_host: Name of the remote host.
- remote_user: User name used to ssh into the remote host.
- src: Local path/file to pass into rsync.
- dest: Remote destination on Dev Server.
- private_key: Path to SSH private key for password less ssh login.
-
- Returns:
- Results from RunCommand()
- """
- return RunCommand(
- '%s %s %s@%s:%s' % (
- _MakeRSyncCommand(private_key), src, remote_user, remote_host, dest),
- **kwargs)
diff --git a/site_utils/chromeos_test/common_util_test.py b/site_utils/chromeos_test/common_util_test.py
deleted file mode 100755
index 82ffc33..0000000
--- a/site_utils/chromeos_test/common_util_test.py
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-# pylint: disable-msg=C0111
-
-"""Unit tests for common utility class."""
-
-__author__ = 'dalecurtis@google.com (Dale Curtis)'
-
-import os
-import tempfile
-import unittest
-
-import common_util
-
-
-class CommonUtilityTest(unittest.TestCase):
-
- def testRunCommandFailedCommand(self):
- self.assertRaises(common_util.ChromeOSTestError,
- common_util.RunCommand, cmd='exit 1')
-
- def testRunCommandOutput(self):
- self.assertEqual(
- common_util.RunCommand(cmd='echo " Test "', output=True),
- 'Test')
-
- def testRunCommandEnvironment(self):
- old_env = os.environ.copy()
- # Ensure variables from local environment are present.
- try:
- user = os.environ['USER']
- except KeyError:
- raise unittest.SkipTest('USER environment variable is not set.')
-
- self.assertEqual(
- common_util.RunCommand(cmd='echo $test_var-$USER',
- env={'test_var': 'Test'}, output=True),
- 'Test-' + user)
- # Ensure local environment is untampered.
- self.assertEqual(old_env, os.environ)
-
- def testRunCommandCustomError(self):
- try:
- common_util.RunCommand(cmd='exit 1', error_msg='test')
- self.fail('No exception raised for invalid command.')
- except common_util.ChromeOSTestError, e:
- self.assertEqual(e.args[0], 'test')
-
- def testRunCommandRetry(self):
- tmp_fd, tmp_fn = tempfile.mkstemp()
- os.close(tmp_fd)
-
- cmd = 'if [ -f %s ]; then rm %s; exit 1; else exit 0; fi' % (tmp_fn, tmp_fn)
- try:
- common_util.RunCommand(cmd=cmd, error_msg='test', retries=2)
- except common_util.ChromeOSTestError:
- self.fail('Command failed after retry.')
-
- def testIgnoreErrors(self):
- common_util.RunCommand(cmd='exit 1', ignore_errors=True)
-
- def testErrorFile(self):
- err_str = ' 1 2 3 '
- try:
- common_util.RunCommand(cmd='echo "%s"; exit 1' % err_str, error_file=True)
- self.fail('No exception raised for invalid command.')
- except common_util.ChromeOSTestError, e:
- error_file = e[-1].split()[-1]
- with open(error_file, 'r') as f:
- self.assertEquals(err_str + '\n', f.read())
- os.unlink(error_file)
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/site_utils/chromeos_test/dash_util.py b/site_utils/chromeos_test/dash_util.py
deleted file mode 100644
index a3d8999..0000000
--- a/site_utils/chromeos_test/dash_util.py
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2011 Google Inc. All Rights Reserved.
-
-"""A utility library used by other Chrome OS scripts.
-
-Routines to upload chromeos build and autotest job data to appengine based
-dashboard.
-"""
-
-__author__ = 'ericli@google.com (Eric Li)'
-
-import getpass
-import common_util
-
-
-def UploadBuild(appengine_cfg, board, build, image_archive_url):
- """Upload chromeos build data onto appengine.
-
- Args:
- appengine_cfg: A dictionary of appengine configuration.
- board: Name of the board.
- build: A build version string.
- image_archive_url: A string of the url location of the image downloaded.
-
- Raises:
- ChromeOSTestError: If command fails. Message is set by the error_msg
- parameter.
- """
- cmd = ('/usr/local/autotest/cautotest-dashboard/build_poster.py '
- '--board %s --build %s --image_url %s --url %s' %
- (board, build, image_archive_url, appengine_cfg['dash_url']))
- msg = 'Failed to post build (%s-%s) onto appengine.' % (board, build)
- common_util.RemoteCommand(appengine_cfg['upload_from'], getpass.getuser(),
- cmd, private_key=None, error_msg=msg)
-
-
-def UploadJob(appengine_cfg, job_id):
- """Upload chromeos autotest job data onto appengine.
-
- Args:
- appengine_cfg: A dictionary of appengine configuration.
- job_id: afe_job_id from autotest database.
-
- Raises:
- ChromeOSTestError: If command fails. Message is set by the error_msg
- parameter.
- """
- cmd = ('/usr/local/autotest/cautotest-dashboard/job_poster.py '
- '--job_id %s --url %s' % (job_id, appengine_cfg['dash_url']))
- msg = 'Failed to post job (%s) onto appengine.' % job_id
- common_util.RemoteCommand(appengine_cfg['upload_from'], getpass.getuser(),
- cmd, private_key=None, error_msg=msg)
diff --git a/site_utils/chromeos_test/dev_server.py b/site_utils/chromeos_test/dev_server.py
deleted file mode 100644
index f2b8e92..0000000
--- a/site_utils/chromeos_test/dev_server.py
+++ /dev/null
@@ -1,445 +0,0 @@
-# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Helper class for interacting with the Dev Server.
-
-DevServer is the controlling class for all Dev Server interactions. All Dev
-Server centric methods are collected here.
-
-Using the locking methods provided in this class, multiple instances of various
-tools and methods can be run concurrently with guaranteed safety.
-"""
-
-__author__ = 'dalecurtis@google.com (Dale Curtis)'
-
-import logging
-import os
-import posixpath
-
-# Autotest imports
-import common
-
-import common_util
-
-
-class DevServer(object):
- """Helper class for interacting with the Dev Server.
-
- All methods assume password-less login for the Dev Server has been set up or
- methods are being run interactively.
- """
- AU_BASE = 'au'
- LATEST = 'LATEST'
- ROOT_UPDATE = 'update.gz'
- STATEFUL_UPDATE = 'stateful.tgz'
- TEST_IMAGE = 'chromiumos_test_image.bin'
-
- def __init__(self, dev_host, path, user, private_key=None, remote_host=None):
- """Initializes class variables and fixes private key permissions.
-
- Args:
- @param host: Address of the Dev Server.
- @param path: Images directory root on Dev Server.
- @param user: Dev Server SSH user name.
- @param private_key: Optional private key file for password-less login.
- If the key file has any group or world permissions
- they will be removed.
- @param remote_host: If a different hostname/ip should be needed for
- uploading images to the dev server.
- """
- self._dev_host = dev_host
- if remote_host:
- self._remote_host = remote_host
- else:
- self._remote_host = dev_host
- self._user = user
- self._images = path
- self._private_key = private_key
- if self._private_key:
- # Check that there are no group or world permissions.
- perms = os.stat(self._private_key).st_mode
- if perms & 0o77 != 0:
- # Remove group and world permissions, keep higher order bits.
- os.chmod(self._private_key, (perms >> 6) << 6)
- logging.warning(
- 'Removing group and world permissions from private key %s to make'
- ' SSH happy.', self._private_key)
-
-
- def UploadAutotestPackages(self, remote_dir, staging_dir):
- """Uploads Autotest packages from staging directory to Dev Server.
-
- Specifically, the autotest-pkgs directory is uploaded from the staging
- directory to the specified Dev Server.
-
- Args:
- @param remote_dir: Directory to upload build components into.
- @param staging_dir: Directory containing update.gz and stateful.tgz
-
- Raises:
- common_util.ChromeOSTestError: If any steps in the process fail to
- complete.
- """
- if os.path.isdir(os.path.join(staging_dir, 'autotest-pkgs')):
- # Upload autotest-pkgs to Dev Server.
- remote_pkgs_dir = posixpath.join(remote_dir, 'autotest')
- msg = 'Failed to upload autotest packages to Dev Server!'
- self.RemoteCopy(src='autotest-pkgs/*', dest=remote_pkgs_dir,
- cwd=staging_dir, error_msg=msg)
-
- def UploadBuildComponents(self, remote_dir, staging_dir, upload_image=False):
- """Uploads various build components from staging directory to Dev Server.
-
- Specifically, the following components are uploaded:
- - update.gz
- - stateful.tgz
- - chromiumos_test_image.bin
- - The entire contents of the au directory. Symlinks are generated for each
- au payload as well.
- - Contents of autotest-pkgs directory.
- - Control files from autotest/server/{tests, site_tests}
-
- Args:
- @param remote_dir: Directory to upload build components into.
- @param staging_dir: Directory containing update.gz and stateful.tgz
- @param upload_image: Should the chromiumos_test_image.bin be uploaded?
-
- Raises:
- common_util.ChromeOSTestError: If any steps in the process fail to
- complete.
- """
- upload_list = [self.ROOT_UPDATE, self.STATEFUL_UPDATE]
- if upload_image:
- upload_list.append(self.TEST_IMAGE)
- else:
- # Create blank chromiumos_test_image.bin. Otherwise the Dev Server will
- # try to rebuild it unnecessarily.
- cmd = 'touch ' + posixpath.join(remote_dir, self.TEST_IMAGE)
- msg = 'Failed to create %s on Dev Server!' % self.TEST_IMAGE
- self.RemoteCommand(cmd, error_msg=msg)
-
- # Upload AU payloads.
- au_path = os.path.join(staging_dir, self.AU_BASE)
- if os.path.isdir(au_path):
- upload_list.append(self.AU_BASE)
-
- # For each AU payload, setup symlinks to the main payloads.
- cwd = os.getcwd()
- for au in os.listdir(au_path):
- os.chdir(os.path.join(staging_dir, au_path, au))
- os.symlink('../../%s' % self.TEST_IMAGE, self.TEST_IMAGE)
- os.symlink('../../%s' % self.STATEFUL_UPDATE, self.STATEFUL_UPDATE)
- os.chdir(cwd)
-
- msg = 'Failed to upload build components to the Dev Server!'
- self.RemoteCopy(
- ' '.join(upload_list), dest=remote_dir, cwd=staging_dir, error_msg=msg)
-
- self.UploadAutotestPackages(remote_dir, staging_dir)
-
- if os.path.isdir(os.path.join(staging_dir, 'autotest')):
- remote_server_dir = posixpath.join(remote_dir, 'server')
- cmd = 'mkdir -p ' + remote_server_dir
- msg = 'Failed to create autotest server dir on Dev Server!'
- self.RemoteCommand(cmd, error_msg=msg)
-
- # Upload autotest/server/{tests,site_tests} onto Dev Server.
- msg = 'Failed to upload autotest/server/{tests,site_tests} to Dev Server!'
- self.RemoteCopy(src='autotest/server/{tests,site_tests}',
- dest=remote_server_dir, cwd=staging_dir, error_msg=msg)
-
- def AcquireLock(self, tag):
- """Acquires a Dev Server lock for a given tag.
-
- Creates a directory for the specified tag on the Dev Server, telling other
- components the resource/task represented by the tag is unavailable.
-
- Args:
- @param tag: Unique resource/task identifier. Use '/' for nested tags.
-
- Returns:
- Path to the created directory on Dev Server or None if creation failed.
-
- Raises:
- common_util.ChromeOSTestError: If Dev Server lock can't be acquired.
- """
- remote_dir = posixpath.join(self._images, tag)
-
- # Attempt to make the directory '<image dir>/<tag>' on the Dev Server. Doing
- # so tells other components that this build is being/was processed by
- # another instance. Directory creation is atomic and will return a non-zero
- # exit code if the directory already exists.
- cmd = 'mkdir ' + remote_dir
- self.RemoteCommand(cmd)
- return remote_dir
-
- def ReleaseLock(self, tag):
- """Releases Dev Server lock for a given tag. Removes lock directory content.
-
- Used to release an acquired Dev Server lock. If lock directory is not empty
- the lock will fail to release.
-
- Args:
- @param tag: Unique resource/task identifier. Use '/' for nested tags.
-
- Raises:
- common_util.ChromeOSTestError: If processing lock can't be released.
- """
- remote_dir = posixpath.join(self._images, tag)
-
- cmd = 'rmdir ' + remote_dir
- self.RemoteCommand(cmd)
-
- def UpdateLatestBuild(self, board, build):
- """Create and upload LATEST file to the Dev Server for the given build.
-
- If a LATEST file already exists, it's renamed to LATEST.n-1
-
- Args:
- @param board: Board name for this build; e.g., x86-generic-rel
- @param build: Full build string to look for; e.g., 0.8.61.0-r1cf43296-b269
- """
- try:
- latest_path = posixpath.join(self._images, board, self.LATEST)
-
- # Update the LATEST file and move any existing LATEST to LATEST.n-1. Use
- # cp instead of mv to prevent any race conditions elsewhere.
- cmd = '[ -f "%s" ] && cp "%s" "%s.n-1"; echo %s>"%s"' % (
- latest_path, latest_path, latest_path, build, latest_path)
- self.RemoteCommand(cmd=cmd)
- except common_util.ChromeOSTestError:
- # Log an error, but don't raise an exception. We don't want to blow away
- # all the work we did just because we can't update the LATEST file.
- logging.error('Could not update %s file for board %s, build %s.',
- self.LATEST, board, build)
-
- def GetUpdateUrl(self, board, build):
- """Returns Dev Server update URL for use with memento updater.
-
- Args:
- @param board: Board name for this build; e.g., x86-generic-rel
- @param build: Full build string to look for; e.g., 0.8.61.0-r1cf43296-b269
-
- Returns:
- Properly formatted Dev Server update URL.
- """
- return 'http://%s:8080/update/%s/%s' % (self._dev_host, board, build)
-
- def FindMatchingBoard(self, board):
- """Returns a list of boards given a partial board name.
-
- Args:
- @param board: Partial board name for this build; e.g., x86-generic
-
- Returns:
- Returns a list of boards given a partial board and build.
- """
- cmd = 'cd %s; ls -d %s*' % (self._images, board)
- output = self.RemoteCommand(cmd, ignore_errors=True, output=True)
- if output:
- return output.splitlines()
- else:
- return []
-
- def FindMatchingBuild(self, board, build):
- """Returns a list of matching builds given a board and partial build.
-
- Args:
- @param board: Partial board name for this build; e.g., x86-generic-rel
- @param build: Partial build string to look for; e.g., 0.8.61.0
-
- Returns:
- Returns a list of (board, build) tuples given a partial board and build.
- """
- cmd = 'cd %s; find \$(ls -d %s*) -maxdepth 1 -type d -name "%s*"' % (
- self._images, board, build)
- results = self.RemoteCommand(cmd, output=True)
- if results:
- return [tuple(line.split('/')) for line in results.splitlines()]
- else:
- return []
-
- def RemoteCommand(self, cmd, **kwargs):
- """Wrapper function for executing commands on the Dev Server.
-
- Args:
- @param cmd: Command to execute on Dev Server.
- @param kwargs: Dicionary of optional args.
-
- Returns:
- Results from common_util.RunCommand()
- """
- return common_util.RemoteCommand(self._remote_host, self._user, cmd,
- private_key=self._private_key, **kwargs)
-
- def RemoteCopy(self, src, dest, **kwargs):
- """Wrapper function for copying a file to the Dev Server.
-
- Copies from a local source to a remote destination (on the Dev Server). See
- definition for common_util.RunCommand for complete argument definitions.
-
- Args:
- @param src: Local path/file.
- @param dest: Remote destination on Dev Server.
- @param kwargs: Dictionary of optional args.
-
- Returns:
- Results from common_util.RemoteCopy()
- """
- return common_util.RemoteCopy(self._remote_host, self._user, src, dest,
- private_key=self._private_key, **kwargs)
-
- def PrepareDevServer(self, tag, force=False):
- """Prepare Dev Server file system to recieve build components.
-
- Checks if the component directory for the given build is available and if
- not creates it.
-
- Args:
- @param tag: Unique resource/task identifier. Use '/' for nested tags.
- @param force: Force re-creation of remote_build_dir even if it already
- exists.
-
- Returns:
- Tuple of (remote_build_dir, exists).
- remote_build_dir: The path on Dev Server to the remote build.
- exists: Indicates whether the directory was already present.
- """
- # Check Dev Server for the build before we begin processing.
- remote_build_dir = posixpath.join(self._images, tag)
-
- # If force is request, delete the existing remote build directory.
- if force:
- self.RemoteCommand('rm -rf ' + remote_build_dir)
-
- # Create remote directory. Will fail if it already exists.
- exists = self.RemoteCommand('[ -d %s ] && echo exists || mkdir -p %s' % (
- remote_build_dir, remote_build_dir), output=True) == 'exists'
-
- return remote_build_dir, exists
-
- def FindDevServerBuild(self, board, build):
- """Given partial build and board ids, figure out the appropriate build.
-
- Args:
- @param board: Partial board name for this build; e.g., x86-generic
- @param build: Partial build string to look for; e.g., 0.8.61.0
-
- Returns:
- Tuple of (board, build):
- board: Fully qualified board name; e.g., x86-generic-rel
- build: Fully qualified build string; e.g., 0.8.61.0-r1cf43296-b269
-
- Raises:
- common_util.ChromeOSTestError: If no boards, no builds, or too many builds
- are matched.
- """
- # Find matching updates on Dev Server.
- if build.lower().strip() == 'latest':
- raise NotImplementedException('FindDevServerBuild no longer supports '
- 'the value "latest". You must pass a '
- 'build string to look for.')
-
- builds = self.FindMatchingBuild(board, build)
- if not builds:
- raise common_util.ChromeOSTestError(
- 'No builds matching %s could be found for board %s.' % (
- build, board))
-
- if len(builds) > 1:
- raise common_util.ChromeOSTestError(
- 'The given build id is ambiguous. Disambiguate by using one of'
- ' these instead: %s' % ', '.join([b[1] for b in builds]))
-
- board, build = builds[0]
-
- return board, build
-
- def CloneDevServerBuild(self, board, build, tag, force=False):
- """Clone existing Dev Server build. Returns path to cloned build.
-
- Args:
- @param board: Fully qualified board name; e.g., x86-generic-rel
- @param build: Fully qualified build string; e.g., 0.8.61.0-r1cf43296-b269
- @param tag: Unique resource/task identifier. Use '/' for nested tags.
- @param force: Force re-creation of remote_build_dir even if it already
- exists.
-
- Returns:
- The path on Dev Server to the remote build.
- """
- # Prepare the Dev Server for this build.
- remote_build_dir, exists = self.PrepareDevServer(tag, force=force)
-
- if not exists:
- # Make a copy of the official build, only take necessary files.
- self.RemoteCommand('cp %s %s %s %s' % (
- os.path.join(self._images, board, build, self.TEST_IMAGE),
- os.path.join(self._images, board, build, self.ROOT_UPDATE),
- os.path.join(self._images, board, build, self.STATEFUL_UPDATE),
- remote_build_dir))
-
- return remote_build_dir
-
- def GetControlFile(self, board, build, control):
- """Attempts to pull the requested control file from the Dev Server.
-
- Args:
- @param board: Fully qualified board name; e.g., x86-generic-rel
- @param build: Fully qualified build string; e.g., 0.8.61.0-r1cf43296-b269
- @param control: Path to control file on remote host relative to Autotest
- root.
-
- Returns:
- Contents of the control file.
-
- Raises:
- common_util.ChromeOSTestError: If control file can't be retrieved
- """
- # Create temporary file to target via scp; close.
- return self.RemoteCommand(
- 'cat %s' % posixpath.join(self._images, board, build, control),
- output=True)
-
- def ListAutoupdateTargets(self, board, build):
- """Returns a list of autoupdate test targets for the given board, build.
-
- Args:
- @param board: Fully qualified board name; e.g., x86-generic-rel
- @param build: Fully qualified build string; e.g., 0.8.61.0-r1cf43296-b269
-
- Returns:
- List of autoupdate test targets; e.g., ['0.14.747.0-r2bf8859c-b2927_nton']
-
- Raises:
- common_util.ChromeOSTestError: If control file can't be retrieved
- """
- msg = 'Unable to retrieve list of autoupdate targets!'
- return [os.path.basename(t) for t in self.RemoteCommand(
- 'ls -d %s/*' % posixpath.join(self._images, board, build, self.AU_BASE),
- output=True, error_msg=msg).split()]
-
- def GetImage(self, board, build, staging_dir):
- """Retrieve the TEST_IMAGE for the specified board and build.
-
- Downloads the image using wget via the Dev Server HTTP interface. The image
- is given a new random file name in the staging directory.
-
- Args:
- @param board: Fully qualified board name; e.g., x86-generic-rel
- @param build: Fully qualified build string; e.g., 0.8.61.0-r1cf43296-b269
- @param staging_dir: Directory to store downloaded image in.
-
- Returns:
- File name of the image in the staging directory.
- """
- image_url = '%s/%s' % (
- self.GetUpdateUrl(board, build).replace('update', 'static/archive'),
- self.TEST_IMAGE)
- image_file = self.TEST_IMAGE + '.n-1'
- msg = 'Failed to retrieve the specified image from the Dev Server!'
- common_util.RunCommand(
- 'wget -O %s --timeout=30 --tries=1 --no-proxy %s' % (
- image_file, image_url), cwd=staging_dir, error_msg=msg)
- return image_file
diff --git a/site_utils/chromeos_test/dev_server_test.py b/site_utils/chromeos_test/dev_server_test.py
deleted file mode 100755
index 6f6194c..0000000
--- a/site_utils/chromeos_test/dev_server_test.py
+++ /dev/null
@@ -1,258 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# pylint: disable-msg=C0111
-
-"""Unit tests for DevServer class."""
-
-__author__ = 'dalecurtis@google.com (Dale Curtis)'
-
-import os
-import shutil
-import socket
-import tempfile
-import unittest
-
-import common_util
-import dev_server
-
-
-# Testing board name.
-TEST_BOARD_NAME = 'board-test'
-
-# Test lock name.
-TEST_LOCK = 'test-lock'
-
-# Test version string.
-TEST_BUILD = '0.99.99.99-r0ABCDEFG-b9999'
-
-# Fake Dev Server Layout:
-TEST_LAYOUT = {
- 'test-board-1': ['0.1.2.3-r12345678-b12', '0.1.2.4-rAba45678-b126']
-}
-
-
-class DevServerTest(unittest.TestCase):
-
- def setUp(self):
- self._test_path = tempfile.mkdtemp()
- self._board_path = os.path.join(self._test_path, TEST_BOARD_NAME)
-
- self._dev = dev_server.DevServer(socket.gethostname(), self._test_path,
- os.environ['USER'])
-
- # Create a fully functional Dev Server layout mimicing the lab deployment.
- os.mkdir(self._board_path)
- for board, builds in TEST_LAYOUT.iteritems():
- board_path = os.path.join(self._test_path, board)
- os.mkdir(board_path)
-
- with open(os.path.join(board_path, self._dev.LATEST), 'w') as f:
- f.write(builds[-1])
-
- for build in builds:
- build_path = os.path.join(board_path, build)
- os.mkdir(build_path)
- with open(os.path.join(build_path, self._dev.TEST_IMAGE), 'w') as f:
- f.write(TEST_BUILD)
- with open(os.path.join(build_path,
- self._dev.STATEFUL_UPDATE), 'w') as f:
- f.write(TEST_BUILD)
- with open(os.path.join(build_path, self._dev.ROOT_UPDATE), 'w') as f:
- f.write(TEST_BUILD)
-
- def tearDown(self):
- shutil.rmtree(self._test_path)
-
- def testUploadBuildComponents(self):
- # Write text to file so we can verify later, any text will do.
- with open(os.path.join(self._test_path, self._dev.ROOT_UPDATE), 'w') as f:
- f.write(TEST_BUILD)
-
- with open(os.path.join(self._test_path,
- self._dev.STATEFUL_UPDATE), 'w') as f:
- f.write(TEST_BUILD)
-
- with open(os.path.join(self._test_path, self._dev.TEST_IMAGE), 'w') as f:
- f.write(TEST_BUILD)
-
- au_test_path = os.path.join(self._test_path, self._dev.AU_BASE, 'au_test')
- os.makedirs(au_test_path)
- with open(os.path.join(au_test_path, self._dev.ROOT_UPDATE), 'w') as f:
- f.write(TEST_BUILD)
-
- self._dev.UploadBuildComponents(remote_dir=self._board_path,
- staging_dir=self._test_path,
- upload_image=True)
-
- self.assertTrue(os.path.isfile(os.path.join(self._board_path,
- self._dev.TEST_IMAGE)))
- self.assertTrue(os.path.isfile(os.path.join(self._board_path,
- self._dev.ROOT_UPDATE)))
- self.assertTrue(os.path.isfile(os.path.join(self._board_path,
- self._dev.STATEFUL_UPDATE)))
- # Verify AU symlink and files exist...
- self.assertTrue(os.path.islink(os.path.join(self._board_path,
- self._dev.AU_BASE, 'au_test',
- self._dev.TEST_IMAGE)))
- self.assertTrue(os.path.isfile(os.path.join(self._board_path,
- self._dev.AU_BASE, 'au_test',
- self._dev.ROOT_UPDATE)))
- self.assertTrue(os.path.islink(os.path.join(self._board_path,
- self._dev.AU_BASE, 'au_test',
- self._dev.STATEFUL_UPDATE)))
-
- with open(os.path.join(self._board_path, self._dev.ROOT_UPDATE), 'r') as f:
- self.assertEquals(f.readlines(), [TEST_BUILD])
-
- with open(os.path.join(self._board_path,
- self._dev.STATEFUL_UPDATE), 'r') as f:
- self.assertEquals(f.readlines(), [TEST_BUILD])
-
- with open(os.path.join(self._board_path, self._dev.TEST_IMAGE), 'r') as f:
- self.assertEquals(f.readlines(), [TEST_BUILD])
-
- with open(os.path.join(self._board_path, self._dev.AU_BASE, 'au_test',
- self._dev.ROOT_UPDATE), 'r') as f:
- self.assertEquals(f.readlines(), [TEST_BUILD])
-
- def testAcquireReleaseLockSuccess(self):
- self.assertTrue(os.path.exists(self._dev.AcquireLock(TEST_LOCK)))
- self._dev.ReleaseLock(TEST_LOCK)
-
- def testAcquireLockFailure(self):
- self._dev.AcquireLock(TEST_LOCK)
- self.assertRaises(common_util.ChromeOSTestError, self._dev.AcquireLock,
- TEST_LOCK)
- self._dev.ReleaseLock(TEST_LOCK)
-
- def testReleaseLockFailure(self):
- self.assertRaises(common_util.ChromeOSTestError,
- self._dev.ReleaseLock, TEST_LOCK)
-
- def testUpdateLatestBuild(self):
- self._dev.UpdateLatestBuild(board=TEST_BOARD_NAME, build=TEST_BUILD)
-
- self.assertTrue(os.path.isfile(os.path.join(self._board_path,
- self._dev.LATEST)))
-
- with open(os.path.join(self._board_path, self._dev.LATEST), 'r') as f:
- self.assertEquals(f.readlines(), [TEST_BUILD + '\n'])
-
- # Update a second time to ensure n-1 file is created.
- self._dev.UpdateLatestBuild(board=TEST_BOARD_NAME, build=TEST_BUILD + 'n-1')
-
- self.assertTrue(os.path.isfile(os.path.join(self._board_path,
- self._dev.LATEST)))
-
- self.assertTrue(os.path.isfile(os.path.join(self._board_path,
- self._dev.LATEST + '.n-1')))
-
- def testFindMatchingBoard(self):
- # Try a partial match with a single board.
- self.assertEqual(
- self._dev.FindMatchingBoard(TEST_BOARD_NAME[:-5]),
- [TEST_BOARD_NAME])
-
- for key in TEST_LAYOUT:
- # Partial match with multiple boards.
- self.assertEqual(
- set(self._dev.FindMatchingBoard(key[:-5])),
- set(TEST_LAYOUT.keys()))
-
- # Absolute match.
- self.assertEqual(self._dev.FindMatchingBoard(key), [key])
-
- # Invalid partial match.
- self.assertEqual(self._dev.FindMatchingBoard('asdfsadf'), [])
-
- def testFindMatchingBuild(self):
- for board, builds in TEST_LAYOUT.iteritems():
- build = builds[0]
-
- # Try a partial board and build match with single match.
- self.assertEqual(
- self._dev.FindMatchingBuild(board[:-5], build[:-5]),
- [(board, build)])
-
- # Try a partial board and build match with multiple match.
- self.assertEqual(
- set(self._dev.FindMatchingBuild(board[:-5], build[:5])),
- set([(board, build), (board, builds[1])]))
-
- # Try very partial board with build match.
- self.assertEqual(
- self._dev.FindMatchingBuild(board[:5], build[:-5]),
- [(board, build)])
-
- def testPrepareDevServer(self):
- test_prefix = 'abc'
- test_tag = test_prefix + '/123'
- abc_path = os.path.join(self._test_path, test_tag)
-
- os.mkdir(os.path.join(self._test_path, test_prefix))
-
- # Verify leaf path is created and proper values returned.
- remote_dir, exists = self._dev.PrepareDevServer(test_tag)
- self.assertEquals(remote_dir, abc_path)
- self.assertFalse(exists)
- self.assertTrue(os.path.exists(abc_path))
-
- # Test existing remote dir.
- remote_dir, exists = self._dev.PrepareDevServer(test_tag)
- self.assertEquals(remote_dir, abc_path)
- self.assertTrue(exists)
- self.assertTrue(os.path.exists(abc_path))
-
- # Verify force properly removes the old directory.
- junk_path = os.path.join(remote_dir, 'junk')
- with open(junk_path, 'w') as f:
- f.write('hello!')
-
- remote_dir, exists = self._dev.PrepareDevServer(test_tag, force=True)
- self.assertEquals(remote_dir, abc_path)
- self.assertFalse(exists)
- self.assertTrue(os.path.exists(abc_path))
- self.assertFalse(os.path.exists(junk_path))
-
-
- def testCloneDevServerBuild(self):
- test_prefix = 'abc'
- test_tag = test_prefix + '/123'
- abc_path = os.path.join(self._test_path, test_tag)
-
- os.mkdir(os.path.join(self._test_path, test_prefix))
-
- # Verify leaf path is created and proper values returned.
- board, builds = TEST_LAYOUT.items()[0]
- remote_dir = self._dev.CloneDevServerBuild(board, builds[0], test_tag)
- self.assertEquals(remote_dir, abc_path)
- self.assertTrue(os.path.exists(abc_path))
- self.assertTrue(os.path.isfile(os.path.join(
- abc_path, self._dev.TEST_IMAGE)))
- self.assertTrue(os.path.isfile(os.path.join(
- abc_path, self._dev.ROOT_UPDATE)))
- self.assertTrue(os.path.isfile(os.path.join(
- abc_path, self._dev.STATEFUL_UPDATE)))
-
- # Verify force properly removes the old directory.
- junk_path = os.path.join(remote_dir, 'junk')
- with open(junk_path, 'w') as f:
- f.write('hello!')
- remote_dir = self._dev.CloneDevServerBuild(
- board, builds[0], test_tag, force=True)
- self.assertEquals(remote_dir, abc_path)
- self.assertTrue(os.path.exists(abc_path))
- self.assertTrue(os.path.isfile(os.path.join(
- abc_path, self._dev.TEST_IMAGE)))
- self.assertTrue(os.path.isfile(os.path.join(
- abc_path, self._dev.ROOT_UPDATE)))
- self.assertTrue(os.path.isfile(os.path.join(
- abc_path, self._dev.STATEFUL_UPDATE)))
- self.assertFalse(os.path.exists(junk_path))
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/site_utils/chromeos_test/log_util.py b/site_utils/chromeos_test/log_util.py
deleted file mode 100644
index b83d8aa..0000000
--- a/site_utils/chromeos_test/log_util.py
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2010 Google Inc. All Rights Reserved.
-
-"""Helper class for setting up the logging subsystem."""
-
-__author__ = 'dalecurtis@google.com (Dale Curtis)'
-
-import logging
-import optparse
-
-
-def InitializeLogging(verbose=False):
- """Configure the global logger for time/date stamping console output."""
- logging.basicConfig(format='%(asctime)s - %(levelname)s: %(message)s')
-
- # Enable verbose output if specified.
- if verbose:
- logging.getLogger().setLevel(logging.DEBUG)
-
-
-def AddOptions(parser):
- """Add command line option group for Logging.
-
- Optional method to add helpful command line options to calling programs. Adds
- the option value "verbose".
-
- Args:
- parser: OptionParser instance.
- """
- group = optparse.OptionGroup(parser, 'Logging Options')
- group.add_option('--verbose', dest='verbose', action='store_true',
- default=False,
- help='Enable verbose output. Script is quiet by default.')
-
- parser.add_option_group(group)
diff --git a/site_utils/chromeos_test/log_util_test.py b/site_utils/chromeos_test/log_util_test.py
deleted file mode 100755
index 837cf90..0000000
--- a/site_utils/chromeos_test/log_util_test.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2010 Google Inc. All Rights Reserved.
-
-"""Unit tests for log utility class."""
-
-__author__ = 'dalecurtis@google.com (Dale Curtis)'
-
-import logging
-import optparse
-import unittest
-
-import log_util
-
-
-class LogUtilityTest(unittest.TestCase):
-
- def testVerbose(self):
- log_util.InitializeLogging(verbose=True)
- self.assertEqual(logging.getLogger().getEffectiveLevel(), logging.DEBUG)
-
- def testNoVerbose(self):
- log_util.InitializeLogging(verbose=False)
- self.assertEqual(logging.getLogger().getEffectiveLevel(), logging.WARNING)
-
- def testParseOptionsVerbose(self):
- parser = optparse.OptionParser()
- log_util.AddOptions(parser)
-
- self.assertEqual(parser.parse_args(['--verbose'])[0].verbose, True)
-
- def testParseOptionsDefaults(self):
- parser = optparse.OptionParser()
- log_util.AddOptions(parser)
-
- self.assertEqual(parser.parse_args([])[0].verbose, False)
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/site_utils/chromeos_test/mp_log_util.py b/site_utils/chromeos_test/mp_log_util.py
deleted file mode 100644
index b32f234..0000000
--- a/site_utils/chromeos_test/mp_log_util.py
+++ /dev/null
@@ -1,133 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Helper class for setting up the logging with mp_thread_pool."""
-
-__author__ = 'pauldean@google.com (Paul Pendlebury)'
-
-import logging.handlers
-import optparse
-import os
-
-# Max size to grow log files, in bytes.
-MAX_FILE_SIZE = 1024000
-
-# Number of backups RotatingFileHandler should keep.
-BACKUP_FILE_COUNT = 10
-
-
-def InitializeLogging(logger=None, log_file=None, skip_console=False,
- verbose=False, **kwargs):
- """Configure a logger instance.
-
- Args:
- logger: logger instance to setup, if non configures global logger.
- log_file: log file name.
- skip_console: if true do not write log message to console.
- verbose: set logger level to DEBUG.
- kwargs: ignored extra args.
- """
- if logger is None:
- logger = logging.getLogger()
-
- fmt = '%(levelname)s: %(message)s'
- dbg_fmt = ''
- log_level = logging.INFO
- if verbose:
- # For debug level logging add process and thread info to messages.
- dbg_fmt = '%(process)d:%(threadName)s '
- log_level = logging.DEBUG
-
- if log_file:
- # Setup file logging. Parent PID is useful in log files when messages
- # overlap from different instances of a script. So add it to the log
- # messages.
- log_fmt = '%(asctime)s ' + repr(os.getppid()) + ': '
- hf = logging.Formatter(log_fmt + dbg_fmt + fmt)
- h = logging.handlers.RotatingFileHandler(filename=log_file,
- maxBytes=MAX_FILE_SIZE,
- backupCount=BACKUP_FILE_COUNT)
- h.setFormatter(hf)
- logger.addHandler(h)
-
- # Setup console logging.
- if not skip_console:
- log_fmt = '%(asctime)s '
- cf = logging.Formatter(log_fmt + dbg_fmt + fmt, '%H:%M:%S')
- c = logging.StreamHandler()
- c.setFormatter(cf)
- logger.addHandler(c)
-
- logger.setLevel(log_level)
-
-
-def LogWithHeader(message, logger=None, width=60, symbol='='):
- """Display a message surrounded by solid lines to make it stand out.
-
- Print a leading and trailing line of width=count of character=symbol.
- Print a centered string=message, and if there is space insert symbol into
- front and end of string.
-
- PrettyPrintHeader('Run Start: Graph HTML', 50) would display:
- =================================================
- ============= Run Start: Graph HTML =============
- =================================================
-
- If message is longer than width, it is printed as is between lines of symbol.
-
- If message is shorter than width but contains newline characters the output
- will not be padded with symbols on the left/right.
-
- Args:
- message: text string to print.
- logger: logger to use
- width: number of characters per line.
- symbol: character to print as decoration.
- """
- if logger is None:
- logger = logging.getLogger()
-
- msg = message
- msg_header = width * symbol
-
- # +2 for space on either side of message.
- if width > len(message) + 2 and not message.count('\n'):
- spaced_msg = ' %s ' % message
- fill_space = width - len(spaced_msg)
- if fill_space % 2 != 0: spaced_msg += ' ' # Put uneven space on right.
- fill_space /= 2
- symbol_fill = symbol * fill_space
- msg = symbol_fill + spaced_msg + symbol_fill
-
- log_msg = '\n'.join(['', # empty string to start output on next line.
- msg_header,
- msg,
- msg_header])
- logger.info(log_msg)
-
-
-def AddOptions(parser):
- """Add command line option group for Logging.
-
- Optional method to add helpful command line options to calling programs. Adds
- the option value "verbose".
-
- Args:
- parser: OptionParser instance.
-
- Returns:
- OptionGroup: Users can add additional options to the returned group.
- """
- group = optparse.OptionGroup(parser, title='Logging',
- description='Logging Configuration Options')
- group.add_option('--log_file',
- help='Write log messages to specified log file.')
- group.add_option('--skip_console', action='store_true', default=False,
- help='Do not write log messages to the console.')
- group.add_option('--verbose', dest='verbose', action='store_true',
- default=False,
- help='Enable verbose output. Script is quiet by default.')
- return parser.add_option_group(group)
\ No newline at end of file
diff --git a/site_utils/chromeos_test/mp_thread_pool.py b/site_utils/chromeos_test/mp_thread_pool.py
deleted file mode 100755
index 82c20af..0000000
--- a/site_utils/chromeos_test/mp_thread_pool.py
+++ /dev/null
@@ -1,409 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""A thread pool library that spreads work across processors.
-
-By default python restricts itself to a single processor. So even when
-additional threads are created they still execute sequentially on one processor
-due to python's Global Interpreter Lock. To work around this limitation and
-take advantage of multi-core and multi-processor computers this library spawns
-additional python processes which execute on different processors. Then each
-of these processes create additional threads to attempt to fully parallelize the
-work submitted to the pool.
-
-To use this library instantiate an instance of MultiProcWorkPool and call
-ExecuteWorkItems(work_items).
- Args:
- work_items: A list of objects with an Execute() method to call.
-
- Examples:
-
- class WorkClass(object):
- def __init__(self, val):
- self.val = val
-
- def Execute(self):
- self.val = self.val * self.val
-
- def ExecuteWithLogger(self, logger):
- self.val = self.val * self.val
- logger.info('val=%s', self.val)
-
- def SomeOtherFunction()
- # Build some objects to submit to work pool
- work_items = [WorkClass(i) for i in range(100)]
-
- # Submit objects to pool and get results
- mp_tp = tp.MultiProcWorkPool()
- work_items = mp_tp.ExecuteWorkItems(work_items)
-
- # Call again with a logger provided to the method
- work_items = mp_tp.ExecuteWorkItems(work_items, 'ExecuteWithLogger',
- provide_logger=True)
-
-
-Callback responsibilities
- 1 - Do not hang. The thread executing the callback cannot detect or deal
- with a hung callback itself. (The pool cannot use timers to deal with
- this situation because timers only fire on the main thread in a process
- and the hung work item is on a spawned thread.)
- 2 - Only throw a CallbackError exception. This is exception is handled. If
- another exception is thrown it will go unhandled and terminate the thread
- executing this callback and prevent it from picking up more work from the
- queue. A watchdog thread will eventually notice this and add more
- threads to the pool, but this is generally bad.
- 3 - When CallbackError is thrown, expect that as the return from the callback.
- If you normally return a single int as a return code and sometimes throw
- this exception, your post processing may go wonky if you don't expect to
- see the exception message and stack trace as the return.
-
-
-Help! It is not working for me. How do I troubleshoot?
- If things don't work narrow down the issue by constraining the pool in this
- order to help identify where the problem is.
- 1 - Set proc count to 0: This will prevent any processes from being
- created and the work will execute in the main calling process. This
- mode will help distinguish between errors dealing with any work being
- submitted to the pool and errors from work completing on different
- processes. Different processes do not share resources that your
- callback may expect them to.
- 2 - One Proc One Thread: Make sure you work works when executed in the pool.
- Creating a single process will force the work to be done in a different
- process than the requesting process.
- 2 - One Proc Multi Thread: Make sure threads don't cause issues.
- 3 - Multi Proc One Thread: Make sure processes don't cause issues.
-"""
-
-
-__author__ = 'pauldean@google.com (Paul Pendlebury)'
-
-
-import inspect
-import logging
-import multiprocessing as mp
-import os
-import Queue
-import threading
-import types
-
-
-class Error(Exception):
- """Base exception for this module."""
- pass
-
-
-class CallbackError(Error):
- """Only exception caught by the thread executing a user callback."""
- pass
-
-
-# Time in seconds the stall check timer waits to check for missing threads.
-DEFAULT_STALL_CHECK_INTERVAL = 5
-
-
-# Default number of threads each worker processes should create.
-DEFAULT_THREADS_PER_PROC = 8
-
-
-# Time in seconds to wait for a process to complete it's portion of the work.
-DEFAULT_PROCESS_TIMEOUT = 120
-
-
-class ExecuteWorkItemsWorkerThread(threading.Thread):
- """Thread that runs work method on user items."""
-
- def __init__(self, input_queue, input_queue_lock, output_queue,
- output_queue_lock, method_name, logger=None):
- threading.Thread.__init__(self)
- self._input_queue = input_queue
- self._input_queue_lock = input_queue_lock
- self._output_queue = output_queue
- self._output_queue_lock = output_queue_lock
- self._method_name = method_name
- self._logger = logger
-
- def run(self):
- """Execute work in input_queue."""
- while True:
- try:
- self._input_queue_lock.acquire()
- work_obj = self._input_queue.get_nowait()
- except Queue.Empty:
- break
- finally:
- self._input_queue_lock.release()
-
- try:
- func = getattr(work_obj, self._method_name, None)
- if self._logger:
- func(self._logger)
- else:
- func()
-
- if self._output_queue:
- self._output_queue_lock.acquire()
- self._output_queue.put(work_obj)
- self._output_queue_lock.release()
- except CallbackError:
- pass
- finally:
- self._input_queue_lock.acquire()
- self._input_queue.task_done()
- self._input_queue_lock.release()
-
-
-def ExecuteWorkItemsThreadInjectionCallback(input_queue, input_queue_lock,
- output_queue, output_queue_lock,
- method_name, max_threads, logger):
- """Timer callback. Watchdog function that makes sure threads are working.
-
- This callback checks the number of active threads in the process and makes
- sure there are threads working on the queue. Threads die when the callback
- throws an exception other than CallbackError. So if 100 work items were
- submitted to this process, this process started 10 threads, and half way
- through the work 10 exceptions were thrown by callbacks, work would stall and
- the process would deadlock. The 10 spawned threads would be terminated by
- unhandled exceptions and the work queue would still have half the work in it.
-
- This callback avoids this situation by checking every
- DEFAULT_STALL_CHECK_INTERVAL seconds that the number of active threads is
- still the target number, and if it is less it spawns new threads to continue
- working.
-
- @param input_queue: multiprocessing.manager.Queue() holding submitted work
- items.
- @param input_queue_lock: lock for the input queue.
- @param output_queue: multiprocessing.manager.Queue() holding finished work
- items.
- @param output_queue_lock: lock for the output queue.
- @param method_name: name of method to execute on each item.
- @param max_threads: maximum threads to start per worker process.
- @param logger: multiprocessing logger.
- """
- # We only care about the case when this timer fires and there is still work
- # in the queue.
- input_queue_lock.acquire()
- work_remaining = input_queue.qsize()
- input_queue_lock.release()
-
- if work_remaining > 0:
- # Enumerated threads include the thread this callback is running on as well
- # as the main thread of the process. So to get the number of active worker
- # threads subtract 2 from the enumeration.
- active_worker_threads = len(threading.enumerate()) - 2
-
- # If we have fewer threads than we want, either some threads have died from
- # a problem in execution, or they've completed work. But since the
- # work_queue is not empty no threads should have terminated from a lack of
- # work. So lets restart some threads to avoid a stall in the work pool.
- if active_worker_threads < max_threads:
- required_threads = min(work_remaining,
- max_threads - active_worker_threads)
- for i in range(required_threads):
- ExecuteWorkItemsWorkerThread(input_queue, input_queue_lock,
- output_queue, output_queue_lock,
- method_name, logger).start()
-
- # Setup a new timer now that this one has fired.
- threading.Timer(DEFAULT_STALL_CHECK_INTERVAL,
- ExecuteWorkItemsThreadInjectionCallback,
- (input_queue, input_queue_lock,
- output_queue, output_queue_lock, method_name, max_threads,
- logger)).start()
-
-
-def ExecuteWorkItemsProcessCallback(input_queue, output_queue, method_name,
- max_threads, provide_logger=False,
- logger_init_callback=None,
- **logger_init_args):
- """Starts threads in a new process to perform requested work.
-
- @param input_queue: multiprocessing.manager.Queue() holding submitted work
- items.
- @param output_queue: multiprocessing.manager.Queue() holding finished work
- items.
- @param method_name: name of method to execute on each item.
- @param max_threads: maximum threads to start per worker process.
- @param provide_logger: if true provide a multiprocessing logger to the
- callback.
- @param logger_init_callback: optional callback where user can setup logging.
- @param logger_init_args: optional arguments to logger_init_callback.
- """
- # Setup logging.
- logger = None
- if provide_logger:
- if logger_init_callback:
- logger = mp.get_logger()
- logger_init_callback(logger, **logger_init_args)
- else:
- logger = mp.log_to_stderr()
- logger.setLevel(logging.INFO)
-
- # The queue proxy objects from the multiprocessing manager are safe for
- # multiple processes to use simultaneously, but not safe for multiple threads
- # in those processes to access simultaneously. So we need a lock per process
- # to synchronize access to the work queues.
- input_queue_lock = threading.Lock()
- output_queue_lock = None
- if output_queue:
- output_queue_lock = threading.Lock()
-
- assert max_threads > 0, 'Must request at least 1 thread per process.'
- thread_count = min(max_threads, input_queue.qsize())
- for i in range(thread_count):
- ExecuteWorkItemsWorkerThread(input_queue, input_queue_lock, output_queue,
- output_queue_lock, method_name, logger).start()
-
- # Start this processes watchdog thread.
- t = threading.Timer(DEFAULT_STALL_CHECK_INTERVAL,
- ExecuteWorkItemsThreadInjectionCallback,
- (input_queue, input_queue_lock,
- output_queue, output_queue_lock, method_name,
- max_threads, logger))
- t.start()
-
- # Wait for queue to drain.
- try:
- input_queue.join()
- except (KeyboardInterrupt, SystemExit):
- raise
- finally:
- t.cancel()
-
-
-class MultiProcWorkPool(object):
- """Multi Processor Thread Pool implementation."""
-
- # TODO: Fix crosbug.com/38902 regarding procs=0 not being correctly
- # handled. Unit test for this module is disabled for now as a result
- # of bug.
- def __init__(self, procs=None, threads_per_proc=DEFAULT_THREADS_PER_PROC,
- max_threads=None):
- """Create an instance of MultiProcWorkPool.
-
-
- @param procs: Number of worker processes to spawn. Default CPU count. If
- procs is 0 no processes will be created and the work will be
- performed on the main process.
- @param threads_per_proc: Number of threads per processor to run.
- @param max_threads: Limit on total threads across all processors.
- """
- if procs is not None:
- assert procs >= 0, 'procs cannot be negative.'
- self._proc_count = procs
- else:
- self._proc_count = mp.cpu_count()
-
- assert threads_per_proc > 0, 'Must run at least 1 thread per process.'
- self._threads_per_proc = threads_per_proc
-
- # If max_threads does not divide evenly into proc_count the remainder will
- # be ignored and the work will be run on fewer threads rather than go over
- # the user supplied max_threads. UNLESS the user asks for fewer threads
- # than proc_count, then we will use proc_count threads as each proc always
- # gets one worker thread.
- if max_threads:
- self._threads_per_proc = max(max_threads / max(self._proc_count, 1), 1)
-
- self._pool = mp.Pool(processes=self._proc_count)
- self._manager = mp.Manager()
-
- def ExecuteWorkItems(self, object_list, method_name='Execute',
- return_objects=True, provide_logger=False,
- logger_init_callback=None, **logger_init_args):
- """Distrubutes work on a list of objects across processes/threads.
-
-
- @param object_list: List of objects to call work method on.
- @param method_name: Name of method to execute on objects.
- @param return_objects: When true return a list of the objects after the work
- has been executed.
- @param provide_logger: Pass a mp logger object to the execute method.
- @param logger_init_callback: Callback to be called once per process to allow
- user to configure logging. A bare logger will be
- passed into the callback. If logging is requested
- and not callback is provided the default logging
- will go to stderr.
- @param logger_init_args: Arguments to pass into logger_init_callback.
-
- @return: Either None or a list of objects when return_objects is True.
- """
- input_queue = self._manager.Queue()
- output_queue = None
- if return_objects:
- output_queue = self._manager.Queue()
-
- if logger_init_callback:
- assert callable(logger_init_callback), ('logger_init_callback is not a '
- 'callable method.')
- argspec = inspect.getargspec(logger_init_callback)
- if logger_init_args:
- assert len(argspec.args) >= 2
- else:
- assert len(argspec.args) == 1
-
- assert object_list, 'Must supply work items.'
- assert type(object_list) is types.ListType, ('object_list parameter \"%s\" '
- 'is not a List.'
- % repr(object_list))
- first_obj = object_list[0]
- assert hasattr(first_obj, method_name), ('%s method missing from work '
- 'items.' % method_name)
- func = getattr(first_obj, method_name)
- assert callable(func), '%s is not a callable method.' % method_name
- argspec = inspect.getargspec(func)
- if provide_logger:
- assert len(argspec.args) == 2, ('Logging was requested. The parameters '
- 'to %s should be [\'self\', \'logger\'] '
- 'and not %s' % (method_name,
- argspec.args))
- else:
- assert len(argspec.args) == 1, ('Logging was not requested. The '
- 'parameter to %s should be [\'self\'] '
- 'and not %s' % (method_name,
- argspec.args))
- obj_type = type(first_obj)
- for obj in object_list:
- assert obj_type == type(obj), 'Different types submitted.'
- input_queue.put(obj)
-
- # ZeroProc debug mode. Don't spawn subprocesses to see if the problem
- # is related to interprocess communication/resource sharing.
- if self._proc_count == 0:
- ExecuteWorkItemsProcessCallback(input_queue, output_queue, method_name,
- self._threads_per_proc, provide_logger,
- logger_init_callback, logger_init_args)
- else:
- for i in range(self._proc_count):
- self._pool.apply_async(ExecuteWorkItemsProcessCallback,
- (input_queue, output_queue, method_name,
- self._threads_per_proc, provide_logger,
- logger_init_callback), logger_init_args)
- # Wait for work to finish
- try:
- input_queue.join()
- except (KeyboardInterrupt, SystemExit):
- self._pool.terminate()
- raise
-
- # If the caller requested results take the mutated objects from the queue
- # and put them in a simple list to return.
- if output_queue:
- result_list = []
- while True:
- try:
- result_obj = output_queue.get_nowait()
- result_list.append(result_obj)
- except Queue.Empty:
- break
- return result_list
-
-
-def ThreadDebugInfo():
- """Debug helper returning a string of the current process and thread."""
-
- return '[(pid:%s) %s : %s ]' % (os.getpid(), mp.current_process().name,
- threading.current_thread().name)
diff --git a/site_utils/chromeos_test/mp_thread_pool_test.py.disabled b/site_utils/chromeos_test/mp_thread_pool_test.py.disabled
deleted file mode 100755
index 89ed928..0000000
--- a/site_utils/chromeos_test/mp_thread_pool_test.py.disabled
+++ /dev/null
@@ -1,130 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Unit tests for mp_thread_pool module."""
-
-__author__ = 'pauldean@google.com (Paul Pendlebury)'
-
-
-import logging
-import unittest
-import mp_thread_pool as tp
-
-
-class WorkItemClass(object):
- """Class used for ExecuteWorkItems* tests.
-
- The test methods do easy to verify manipulations on the initial value of val.
- """
-
- def __init__(self, val):
- self.val = val
- self.new_val = None
-
- def Execute(self):
- """Set new_val to the square of val."""
- self.new_val = self.val * self.val
-
- def ExecuteWithLogger(self, logger=None):
- """Set val to new_val, and set new_val to twice new_val."""
- assert logger is not None, 'Logger missing in ExecuteWithLogger'
- self.val = self.new_val
- self.new_val = self.val + self.val
-
-
-def SetLoggerFormat(logger):
- """Default logger formatting method."""
- logger.setLevel(logging.WARNING)
- stream_handler = logging.StreamHandler()
- stream_handler.setLevel(logging.WARNING)
- format_str = '%(asctime)s - %(levelname)s - %(message)s'
- stream_handler.setFormatter(logging.Formatter(format_str))
- logger.addHandler(stream_handler)
-
-
-class MultiProcWorkPoolTest(unittest.TestCase):
-
- def ExecuteWorkItems(self, work_pool=None, iterations=100, use_logger=False):
- """Verify Execute and ExecuteWithLogger methods."""
-
- mp_tp = work_pool
- if not mp_tp:
- mp_tp = tp.MultiProcWorkPool()
-
- work_items = []
- for i in range(iterations):
- work_items.append(WorkItemClass(i))
-
- work_items = mp_tp.ExecuteWorkItems(work_items)
- for i in range(iterations):
- self.assertTrue(work_items[i].val * work_items[i].val ==
- work_items[i].new_val)
-
- if use_logger:
- work_items = mp_tp.ExecuteWorkItems(work_items, 'ExecuteWithLogger',
- provide_logger=True,
- logger_init_callback=SetLoggerFormat)
- for i in range(iterations):
- self.assertTrue(work_items[i].val + work_items[i].val ==
- work_items[i].new_val)
-
- def test01SingleExecution(self):
- """01 - Verify a single item can be submitted to the pool."""
- self.ExecuteWorkItems(iterations=1)
-
- def test02SingleExecution(self):
- """02 - Verify a single item can be submitted to the pool with a logger."""
- self.ExecuteWorkItems(iterations=1, use_logger=True)
-
- def test03SingleProcMultiThread(self):
- """03 - Verify work completes when using only 1 process."""
- mp_tp = tp.MultiProcWorkPool(procs=0)
- self.ExecuteWorkItems(mp_tp)
-
- def test04SingleProcMultiThread(self):
- """04 - Verify work completes when using only 1 process with a logger."""
- mp_tp = tp.MultiProcWorkPool(procs=0)
- self.ExecuteWorkItems(mp_tp, use_logger=True)
-
- def test05MultiProcSingleThread(self):
- """05 - Verify work completes using only 1 thread per proc."""
- mp_tp = tp.MultiProcWorkPool(threads_per_proc=1)
- self.ExecuteWorkItems(mp_tp)
-
- def test06MultiProcSingleThread(self):
- """06 - Verify work completes using only 1 thread per proc with a logger."""
- mp_tp = tp.MultiProcWorkPool(threads_per_proc=1)
- self.ExecuteWorkItems(mp_tp)
-
- def test07SingleProcSingleThread(self):
- """07 - Verify using only 1 process and 1 thread."""
- mp_tp = tp.MultiProcWorkPool(procs=0, threads_per_proc=1)
- self.ExecuteWorkItems(mp_tp)
-
- def test08SingleProcSingleThread(self):
- """08 - Verify using only 1 process and 1 thread with a logger."""
- mp_tp = tp.MultiProcWorkPool(procs=0, threads_per_proc=1)
- self.ExecuteWorkItems(mp_tp)
-
- def test09MultipleExecuteSamePool(self):
- """09 - Verify the same mp_tp object can perform repeated executions."""
- for i in range(10):
- self.ExecuteWorkItems(iterations=1000)
-
- def test10MultipleExecuteSamePoolWithLogger(self):
- """10 - Verify logger is provided to callbacks on all processes/threads."""
- for i in range(10):
- self.ExecuteWorkItems(iterations=100, use_logger=True)
-
-
-def main():
- suite = unittest.TestLoader().loadTestsFromTestCase(MultiProcWorkPoolTest)
- alltests = unittest.TestSuite([suite])
- unittest.TextTestRunner(verbosity=2).run(alltests)
-
-
-if __name__ == '__main__':
- main()
diff --git a/site_utils/chromeos_test/test_config.py b/site_utils/chromeos_test/test_config.py
deleted file mode 100644
index a3f619f..0000000
--- a/site_utils/chromeos_test/test_config.py
+++ /dev/null
@@ -1,93 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Helper class for interacting and loading the JSON ChromeOS Test Config."""
-
-__author__ = 'dalecurtis@google.com (Dale Curtis)'
-
-import json
-import optparse
-import os
-import re
-
-
-# Test configuration file.
-DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__), '..',
- 'chromeos_test_config.json')
-
-
-class TestConfig(object):
- """Utility class for interacting with the JSON ChromeOS Test Config."""
-
- def __init__(self, config_file=DEFAULT_CONFIG_FILE):
- """Initializes class variables and parses JSON configuration file.
-
- @param config_file: Path to Chrome OS test configuration file.
- """
- self._config = json.load(open(config_file))
-
- # Is the config file based off another config?
- if '__base__' in self._config:
- # Rebase the config based on the specified config. Prevent usage of paths.
- base_config = json.load(open(os.path.basename(self._config['__base__'])))
- base_config.update(self._config)
- self._config = base_config
-
- # Cleanup the base tag.
- del self._config['__base__']
-
- def GetConfig(self):
- """Returns test configuration object."""
- return self._config
-
- def ParseConfigGroups(self, board_re=None):
- """Returns 3-tuple of valid boards, groups, and platforms from config.
-
- @param board_re: If specified, only return platforms for boards
- matching this regular expression.
-
- @return: Tuple of (boards, groups, platforms)
- """
- boards = sorted(self._config['boards'].keys())
- groups = sorted(self._config['groups'].keys())
-
- platforms = []
- for board in boards:
- if board_re and not re.search(board_re, board):
- continue
- for platform in self._config['boards'][board]['platforms']:
- platforms.append(platform['platform'])
-
- platforms = sorted(set(platforms))
-
- return boards, groups, platforms
-
- def GetBoardPlatformPairs(self):
- """Returns a generator for (board, platform) defined in the config file.
-
- Example use:
- for board, platform in testconfig.GetBoardPlatformPairs():
- do_something_neat(board, platform)
-
- Yields:
- 2-tuple of valid (board, platform) defined in the config file.
- """
- for board in self._config['boards']:
- for platform in self._config['boards'][board]['platforms']:
- yield (board, platform['platform'])
-
-
-def AddOptions(parser):
- """Add command line option group for Test Config.
-
- Optional method to add helpful command line options to calling programs. Adds
- the option value "config".
-
- @param parser: OptionParser instance.
- """
- group = optparse.OptionGroup(parser, 'Test Config Options')
- group.add_option('--config', dest='config', default=DEFAULT_CONFIG_FILE,
- help=('Specify an alternate test configuration file. '
- 'Defaults to "%default".'))
- parser.add_option_group(group)
diff --git a/site_utils/chromeos_test/test_config_test.py b/site_utils/chromeos_test/test_config_test.py
deleted file mode 100755
index 4f8e1e3..0000000
--- a/site_utils/chromeos_test/test_config_test.py
+++ /dev/null
@@ -1,164 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-# pylint: disable-msg=C0111
-"""Unit tests for TestConfig class."""
-
-__author__ = 'dalecurtis@google.com (Dale Curtis)'
-
-import json
-import optparse
-import os
-import shutil
-import tempfile
-import unittest
-
-import test_config
-
-
-# Fake platform name.
-_SAMPLE_PLATFORM = 'netbook_TEST'
-
-# Fake test config layout.
-_SAMPLE_TEST_CONFIG_NAME = 'test_config.json'
-_SAMPLE_TEST_CONFIG = {
- "boards": {
- "test-board-1": {
- "archive_server": "http://awesometown/dev-channel",
- "archive_path": "test-board-1/%(build_version)s/test.zip",
- "build_pattern": "0\\.1\\..*",
- "platforms": [
- {"platform": _SAMPLE_PLATFORM, "extra_groups": ["abc"]}
- ]
- }
- },
-
- "default_groups": [
- "tests_a", "tests_b"
- ],
-
- "groups": {
- "test_a": [
- {"name": "a",
- "control": "tests/suites/control.a"}
- ],
-
- "test_b": [
- {"name": "b",
- "control": "tests/suites/control.b"}
- ]
- },
-
- "import_hosts": [
- {
- "host": "test.corp.google.com",
- "path": "/usr/local/autotest",
- "user": "chromeos-test"
- }
- ],
-
- "dev_server": {
- "host": "127.0.0.1",
- "path": "/usr/local/google/images",
- "user": "testing"
- },
-
- "appengine": {
- "dash_url": "https://localhost",
- "upload_from": "test.corp.google.com"
- }
-}
-
-_SAMPLE_OVERRIDE_CONFIG_NAME = 'test_override.json'
-_SAMPLE_OVERRIDE_CONFIG = {
- "__base__": _SAMPLE_TEST_CONFIG_NAME,
-
- "boards": {
- "test-board-2": {
- "platforms": [
- {"platform": _SAMPLE_PLATFORM}
- ]
- }
- },
-
- "default_groups": ["test_c"],
-
- "groups": {
- "test_c": [
- {"name": "c",
- "control": "tests/suites/control.c"}
- ]
- }
-}
-
-
-class TestConfigTest(unittest.TestCase):
-
- def setUp(self):
- self._test_path = tempfile.mkdtemp()
- self._test_config_path = os.path.join(
- self._test_path, _SAMPLE_TEST_CONFIG_NAME)
- self._test_override_config_path = os.path.join(
- self._test_path, _SAMPLE_OVERRIDE_CONFIG_NAME)
-
- with open(self._test_config_path, 'w') as f:
- json.dump(_SAMPLE_TEST_CONFIG, f)
-
- with open(self._test_override_config_path, 'w') as f:
- json.dump(_SAMPLE_OVERRIDE_CONFIG, f)
-
- self._config = test_config.TestConfig()
-
- self._test_config = test_config.TestConfig(self._test_config_path)
-
- shutil.copy(test_config.DEFAULT_CONFIG_FILE, self._test_path)
- cwd = os.getcwd()
- os.chdir(self._test_path)
- self._override_config = test_config.TestConfig(
- self._test_override_config_path)
- os.chdir(cwd)
-
- def tearDown(self):
- shutil.rmtree(self._test_path)
-
- def testValidConfig(self):
- self.assertEquals(sorted(self._config.GetConfig().keys()),
- sorted(['appengine', 'boards', 'default_groups',
- 'dev_server', 'groups', 'import_hosts',
- 'default_tot_groups']))
-
- def testParseConfigGroups(self):
- boards, groups, platforms = self._test_config.ParseConfigGroups()
-
- self.assertEqual(set(boards), set(_SAMPLE_TEST_CONFIG['boards'].keys()))
- self.assertEqual(set(groups), set(_SAMPLE_TEST_CONFIG['groups'].keys()))
- self.assertEqual(platforms, [_SAMPLE_PLATFORM])
-
- boards, groups, platforms = self._override_config.ParseConfigGroups()
- self.assertEqual(set(boards), set(_SAMPLE_OVERRIDE_CONFIG['boards'].keys()))
- self.assertEqual(set(groups), set(_SAMPLE_OVERRIDE_CONFIG['groups'].keys()))
- self.assertEqual(platforms, [_SAMPLE_PLATFORM])
-
- _, _, platforms = self._test_config.ParseConfigGroups(board_re='wont match')
- self.assertEqual([], platforms)
-
- _, _, platforms = self._test_config.ParseConfigGroups(board_re='test')
- self.assertEqual(platforms, [_SAMPLE_PLATFORM])
-
- def testParseOptionsConfig(self):
- parser = optparse.OptionParser()
- test_config.AddOptions(parser)
-
- self.assertEqual(
- parser.parse_args(['--config', 'tests.json'])[0].config, 'tests.json')
-
- def testOverrideConfig(self):
- self.assertTrue(not '__base__' in self._override_config.GetConfig())
- for key in self._test_config.GetConfig().keys():
- self.assertTrue(key in self._override_config.GetConfig())
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/site_utils/chromeos_test_common.py b/site_utils/chromeos_test_common.py
deleted file mode 100644
index 4b72d8f..0000000
--- a/site_utils/chromeos_test_common.py
+++ /dev/null
@@ -1,21 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""A common helper that adds chromeos_test libraries to the path.
-
-Also defines:
- chromeos_test_common.CURRENT_DIR as the current directory.
- chromeos_test_common.CRON_DIR as the autotest-tools/cron directory.
- chromeos_test_common.CROS_DIR as path to the ChromeOS enlistment.
-"""
-
-import os
-import sys
-
-# Figure out our absolute path so we can simplify configuration.
-CURRENT_DIR = os.path.realpath(os.path.abspath(os.path.join(
- os.getcwd(), os.path.dirname(__file__))))
-CROS_DIR = os.path.abspath(os.path.join(CURRENT_DIR, '../../../../..'))
-CRON_DIR = CURRENT_DIR
-sys.path.append(CRON_DIR)
diff --git a/site_utils/dashboard/__init__.py b/site_utils/dashboard/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/site_utils/dashboard/__init__.py
+++ /dev/null
diff --git a/site_utils/dashboard/alert_email.py b/site_utils/dashboard/alert_email.py
deleted file mode 100755
index cb93ba8..0000000
--- a/site_utils/dashboard/alert_email.py
+++ /dev/null
@@ -1,250 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Analyze perf keyvals and email regressions."""
-
-import logging
-import os
-
-from django.shortcuts import render_to_response
-
-import dash_util
-
-from common_email import EmailNotifier
-from preprocess_functions import PreprocessFunctions
-from stats_functions import StatsFunctions
-
-# String resources.
-from dash_strings import ALERT_CHECKED_PREFIX
-from dash_strings import BUILD_PERFORMANCE_FILE
-from dash_strings import DASHBOARD_MAIN
-from dash_strings import PERFORMANCE_DIR
-from dash_strings import PERFORMANCE_REGRESSED_EMAIL
-
-
-class AlertEmailNotifier(EmailNotifier):
- """Class to check for keyval alerts (perf regressions) and send emails."""
-
- def __init__(self, base_dir, netbook, board_type,
- use_sheriffs, extra_emails, plot_options):
- super(AlertEmailNotifier, self).__init__(
- base_dir, netbook, board_type, use_sheriffs, extra_emails,
- ALERT_CHECKED_PREFIX, PERFORMANCE_DIR)
- self._regressed_tests = {}
- self._preprocessors = PreprocessFunctions()
- self._stats = StatsFunctions()
- self._plot_options = plot_options
- self._preprocessed = False
- self._float_converter = dash_util.HumanReadableFloat()
-
- def ExpandWildcardKeys(self, test_name, perf_checks):
- """Add keys to the check list using wildcard prefixes."""
- keyvals = self._dash_view.GetTestKeyVals(
- self._netbook, self._board_type, test_name)
- if not keyvals:
- return []
- expanded_checks = perf_checks.copy()
- key_list = sorted(expanded_checks.keys())
- for key_match in key_list:
- if key_match[-1] == '*':
- key_len = len(key_match) - 1
- check_dict = expanded_checks[key_match]
- for perf_key in keyvals:
- if perf_key[:key_len] == key_match[:key_len]:
- if not perf_key in expanded_checks:
- expanded_checks[perf_key] = check_dict
- del expanded_checks[key_match]
- return expanded_checks
-
- def InvokePreprocessor(self, function_name, params, keyvals, checks):
- self._preprocessed = True
- return self._preprocessors.Invoke(function_name, params, keyvals, checks)
-
- def PreProcess(self, items, preprocess_functions):
- """Hook to manipulate keyvals before entering emailer pipeline."""
- categories, test_name, perf_checks = items
- for fn_name, params in preprocess_functions:
- keyvals = self._dash_view.GetTestKeyVals(
- self._netbook, self._board_type, test_name)
- if keyvals:
- self.InvokePreprocessor(fn_name, params, keyvals, perf_checks)
-
- def PreprocessorHTML(self, test_name, regressed_key_details):
- results = ''
- if self._preprocessed:
- results = self._preprocessors.PreprocessorHTML(
- test_name, regressed_key_details)
- return results
-
- def IsPreprocessedKey(self, key):
- return self._preprocessors.IsPreprocessedKey(key)
-
- def InvokeStats(self, function_name, params, vals, build):
- return self._stats.Invoke(function_name, params, vals, build)
-
- def GetTestPath(self, test_name):
- _, test_path = self._dash_view.GetAutotestInfo(test_name)
- return test_path
-
- def GetPlotLink(self, test_name, key_name):
- """Dig through the plot config json to get plot filename for linking."""
- plot_file = None
- for plot_id, plot_definition in self._plot_options.iteritems():
- if test_name == plot_definition['test']:
- if key_name in plot_definition['keys']:
- plot_file = '%s%s' % (test_name, plot_id)
- return plot_file
-
- def CheckItems(self, items):
- """Iterate through test categories and send email for failed tests."""
- categories, test_name, perf_checks = items
- for category in categories:
- for build in self.GetBuilds(category):
- if not self.Checked(category, build):
- regressed_tests = self._regressed_tests.setdefault(build, {})
- for alert_key, alert_checks in perf_checks.iteritems():
- vals = self._dash_view.GetTestPerfVals(
- self._netbook, self._board_type, test_name, alert_key)
- if not vals:
- logging.debug(
- 'No keyvals found for configuration requested: '
- '%s, %s, %s, %s.',
- self._board_type, self._netbook, test_name, alert_key)
- continue
- if not build in vals:
- logging.debug(
- 'No keyval found for configuration requested and build: '
- '%s, %s, %s, %s, %s.',
- self._board_type, self._netbook, test_name, alert_key,
- build)
- continue
- for fn_name, fn_params in alert_checks.iteritems():
- stats_result, stats_data = self.InvokeStats(
- fn_name, fn_params, vals, build)
- if stats_result:
- regressed_keys = regressed_tests.setdefault(test_name, {})
- regressed_stats = regressed_keys.setdefault(alert_key, {})
- regressed_stats.update(stats_data)
- # Write the sentinel file
- self.SetChecked(category, build)
-
- def GenerateEmail(self):
- """Send email to aid troubleshooting performance regressions.
-
- Emails are broken into 3 or 4 sections:
- 1. Intro with summary of failing build and netbook combination.
- 2. An optional section of ui if preprocessing.
- 3 A list of regressed keys, details and related plots inline.
- 4. Inline build log for blame.
- """
-
- for tpl_build, regressed_tests in self._regressed_tests.iteritems():
- if not regressed_tests:
- continue
- logging.debug(
- 'Build %s has %s regressed test names to email.',
- tpl_build, len(regressed_tests))
- preprocessed_html = []
- # Move work to Django templates. Django prefers lists of dicts.
- tpl_regressed_tests = []
- for test_name, regressed_keys in regressed_tests.iteritems():
- test_name_keys = []
- tpl_regressed_tests.append({
- 'test_name': test_name,
- 'test_path': self.GetTestPath(test_name),
- 'test_keys': test_name_keys})
- preprocessed_html.extend(
- self.PreprocessorHTML(test_name, regressed_keys))
- # Organize the keys with their corresponding plots.
- for test_key, regressed_stats in regressed_keys.iteritems():
- if self.IsPreprocessedKey(test_key):
- continue
- test_key_headers = set()
- test_key_stats = []
- stat_keys = regressed_stats.keys()
- stat_keys.sort()
- sort_key = regressed_stats[stat_keys[0]]
- for stat_key in stat_keys:
- stat_data = regressed_stats[stat_key]
- test_key_headers.add(stat_key.replace('_', ' '))
- if type(stat_data) == float:
- stat_data = self._float_converter.Convert(stat_data)
- test_key_stats.append({
- 'stat_name': stat_key, 'stat_val': stat_data})
-
- test_name_keys.append({
- 'test_key': test_key,
- 'key_plot': self.GetPlotLink(test_name, test_key),
- 'key_headers': sorted(test_key_headers),
- 'key_stats': test_key_stats,
- 'sort_key': sort_key})
-
- # Inline build log.
- use_json = False
- tpl_build_log = ''
-
- # Assemble the final email.
- tpl_board = self._board_type
- tpl_netbook = self._netbook
- tpl_preprocessed_html = ''.join(preprocessed_html)
- body = render_to_response(
- os.path.join('emails', PERFORMANCE_REGRESSED_EMAIL), locals()).content
-
- # Assemble a build performance page.
- tpl_last_updated = self._dash_view.GetFormattedLastUpdated()
- performance_file = '%s_%s' % (tpl_build, BUILD_PERFORMANCE_FILE)
- dash_util.SaveHTML(
- os.path.join(self.GetPerformanceDir(), performance_file),
- render_to_response(
- os.path.join('tables/performance', BUILD_PERFORMANCE_FILE),
- locals()).content)
-
- # Send it.
- subject = 'Performance keyvals for %s(%s) on %s' % (
- tpl_board, tpl_build, tpl_netbook[8:])
- self.SendEmail(subject, body)
-
-
-def AlertAll(dash_base_dir, dash_view, dash_options):
- """All the work of checking and sending email.
-
- Args:
- dash_base_dir: Base dir of the output files.
- dash_view: Reference to our data model.
- dash_options: From alert_config.json.
- """
-
- plot_options = dash_options['plots']
- for alert in dash_options['alerts']:
- categories = alert['categories']
- use_sheriffs = alert['sheriffs']
- cc = alert['cc']
- test_name = alert['test']
- preprocess_functions = None
- if 'preprocess' in alert:
- preprocess_functions = alert['preprocess']
- perf_checks = alert['checks']
-
- if not use_sheriffs and not cc:
- logging.warning('Email requires sheriffs or cc.')
- continue
-
- if not categories or not test_name or not perf_checks:
- logging.warning('Alerts require categories, test and checks.')
- continue
-
- for platform in alert['platforms']:
- for board, netbook in platform.iteritems():
- notifier = AlertEmailNotifier(
- dash_base_dir, netbook, board, use_sheriffs, cc, plot_options)
- expanded_checks = notifier.ExpandWildcardKeys(test_name, perf_checks)
- if preprocess_functions:
- notifier.PreProcess(
- (categories, test_name, expanded_checks), preprocess_functions)
- notifier.CheckItems((categories, test_name, expanded_checks))
- notifier.GenerateEmail()
-
-
-if __name__ == '__main__':
- print 'Run %s with --alert-generate.' % DASHBOARD_MAIN
diff --git a/site_utils/dashboard/build_info.py b/site_utils/dashboard/build_info.py
deleted file mode 100644
index b79f396..0000000
--- a/site_utils/dashboard/build_info.py
+++ /dev/null
@@ -1,197 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Wrap everything we do with builds in a dash class."""
-
-import commands
-import json
-import logging
-import os
-import re
-import time
-import urllib
-
-import dash_util
-
-# String resources.
-from dash_strings import BUILDTIME_PREFIX
-from dash_strings import LOCAL_TMP_DIR
-from dash_strings import WGET_CMD
-
-
-class BuildInfo(object):
- """Data and functions from build log."""
-
- class __impl:
- """Nested class implements code wrapped by singleton."""
-
- def __init__(self):
- # Store build entries as {started_time, finished_time, chrome_version}
- # indexed by board and build#.
- self._build_time_cache = {}
- self._formatted_time_cache = {}
- self._chrome_version_cache = {}
- self._chrome_parse = re.compile(
- "^Started chromeos-base/chromeos-chrome-([\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3})_rc.*", re.M)
-
- def GetChromeVersion(self, board, build):
- # Get the string w.x.y.z Chrome version and a zzzz svn revision.
- self.FetchBuildInfo(board, build)
- return self._build_time_cache[board][build]['chrome_version']
-
- def GetStartedTime(self, board, build):
- # This is a float - seconds since the epoch.
- self.FetchBuildInfo(board, build)
- return self._build_time_cache[board][build]['started_time']
-
- def GetFinishedTime(self, board, build):
- # This is a float - seconds since the epoch.
- self.FetchBuildInfo(board, build)
- return self._build_time_cache[board][build]['finished_time']
-
- def GetElapsedTime(self, board, build):
- # This is a float.
- return (
- self.GetFinishedTime(board, build) -
- self.GetStartedTime(board, build))
-
- def GetFormattedTime(self, time_seconds, short=None):
- if short:
- # Formatted as: Wed 09/08 12:37.
- result = time.strftime('%a %m/%d %H:%M', time.localtime(time_seconds))
- else:
- # Formatted as: Wed Sep 8 12:37:56 2010.
- result = time.ctime(time_seconds)
- return result
-
- def GetFormattedStartedTime(self, board, build, short=None):
- return self.GetFormattedTime(
- self.GetStartedTime(board, build), short)
-
- def GetFormattedFinishedTime(self, board, build, short=None):
- return self.GetFormattedTime(
- self.GetFinishedTime(board, build), short)
-
- def GetFormattedElapsedTime(self, board, build, short=None):
- time_seconds = self.GetElapsedTime(board, build)
- if short:
- # Formatted as: 06:16:12.
- result = time.strftime('%H:%M:%S', time.gmtime(time_seconds))
- else:
- # Formatted as: 04 hrs, 27 mins, 03 secs.
- result = time.strftime(
- '%H hrs, %M mins, %S secs', time.gmtime(time_seconds))
- return result
-
- def GetFormattedBuildTimes(self, board, build):
- """Perf optimize on the pattern on repeat/retrieval of all."""
- time_key = (board, build)
- if time_key in self._formatted_time_cache:
- return self._formatted_time_cache[time_key]
- result = (self.GetFormattedStartedTime(board, build),
- self.GetFormattedFinishedTime(board, build),
- self.GetFormattedElapsedTime(board, build),
- self.GetFormattedFinishedTime(board, build, True))
- self._formatted_time_cache[time_key] = result
- return result
-
- def FetchChromeVersion(self, full_build):
- """Grab the Chrome version from the chromeos-images version map."""
- chromeos_build = full_build.split('_')[-1].split('-')[0]
- if chromeos_build in self._chrome_version_cache:
- return self._chrome_version_cache[chromeos_build]
- map_file = os.path.join(os.path.abspath(os.path.dirname(__file__)),
- 'chromeos-chrome-version.json')
- if not os.path.exists(map_file):
- return (None, None)
- chrome_versions = json.load(open(map_file))
- if not chrome_versions or not chromeos_build in chrome_versions:
- return (None, None)
- dot_version = chrome_versions[chromeos_build]
- omaha_url = 'http://omahaproxy.appspot.com/revision?version=%s' % (
- dot_version)
- omaha_wget = '%s "%s"' % (WGET_CMD, omaha_url)
- svn_revision = commands.getoutput(omaha_wget)
- results = (dot_version, svn_revision)
- self._chrome_version_cache[chromeos_build] = results
- return results
-
- def GetCacheFilename(self, board, build, dir=True):
- filename = '%s%s_%s' % (BUILDTIME_PREFIX, board, build)
- if dir:
- return '%s/%s' % (LOCAL_TMP_DIR, filename)
- else:
- return filename
-
- def FetchBuildInfo(self, board, build):
- """Load start_time, end_time into memory from file cache or web lookup."""
- # Use an in-memory cache (dictionary) for repeats.
- board_builds = self._build_time_cache.setdefault(board, {})
- build_keys = board_builds.setdefault(build, {})
- if not build_keys:
- build_keys['started_time'] = 0.0
- build_keys['finished_time'] = 0.0
- build_keys['chrome_version'] = [None, None]
-
- build_log_json = None
- cache_filename = self.GetCacheFilename(board, build)
- if os.path.isfile(cache_filename):
- f = open(cache_filename, 'r')
- build_log_text = f.read()
- if build_log_text:
- build_log_json = json.loads(build_log_text)
- f.close()
- else:
- if not os.path.exists(LOCAL_TMP_DIR):
- os.makedirs(LOCAL_TMP_DIR, 0755)
- dash_util.SaveHTML(cache_filename, '')
-
- def PruneTmpFiles(self, dash_view):
- """Remove cached build_time data that is no longer useful."""
-
- build_set = set()
- for board in dash_view.GetBoardTypes():
- for build in dash_view.GetBoardtypeBuilds(board):
- build_set.add(unicode(self.GetCacheFilename(board, build, False)))
- files_set = set(os.listdir(LOCAL_TMP_DIR))
- for f in files_set - build_set:
- if f.find(BUILDTIME_PREFIX) > -1:
- logging.info('Pruning %s from %s.', f, LOCAL_TMP_DIR)
- os.remove('%s/%s' % (LOCAL_TMP_DIR, f))
-
- def ShowCache(self):
- logging.debug("*")
- logging.debug("*BUILDINFO CACHE***************")
- logging.debug("*")
- for board, build_times in self._build_time_cache.iteritems():
- for build, time_info in build_times.iteritems():
- logging.debug(" %s: %s: %s ~ %s (%s).",
- board, build,
- time_info['started_time'],
- time_info['finished_time'],
- time_info['chrome_version'])
-
- # Instance reference for singleton behavior.
- __instance = None
- __refs = 0
-
- def __init__(self):
- if BuildInfo.__instance is None:
- BuildInfo.__instance = BuildInfo.__impl()
-
- self.__dict__["_BuildInfo__instance"] = BuildInfo.__instance
- BuildInfo.__refs += 1
-
- def __del__(self):
- BuildInfo.__refs -= 1
- if not BuildInfo.__instance is None and BuildInfo.__refs == 0:
- BuildInfo.__instance.ShowCache()
- del BuildInfo.__instance
- BuildInfo.__instance = None
-
- def __getattr__(self, attr):
- return getattr(BuildInfo.__instance, attr)
-
- def __setattr__(self, attr, value):
- return setattr(BuildInfo.__instance, attr, value)
diff --git a/site_utils/dashboard/common_email.py b/site_utils/dashboard/common_email.py
deleted file mode 100644
index f8f46d1..0000000
--- a/site_utils/dashboard/common_email.py
+++ /dev/null
@@ -1,159 +0,0 @@
-# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Common email routines."""
-
-import base64
-import commands
-import getpass
-import hashlib
-import logging
-import os
-import shutil
-
-import dash_util
-
-from build_info import BuildInfo
-from dash_view import AutotestDashView
-
-# String resources.
-from dash_strings import AUTOTEST_USER
-from dash_strings import EMAIL_BUILDS_TO_CHECK
-from dash_strings import EMAIL_DIR
-from dash_strings import LOCAL_TMP_DIR
-from dash_strings import PERFORMANCE_DIR
-from dash_strings import WGET_CMD
-
-
-class EmailNotifier(object):
- """Base class to send emails based on some condition."""
-
- def __init__(self, base_dir, netbook, board_type, use_sheriffs,
- extra_emails, email_prefix, email_type):
- self._dash_view = AutotestDashView()
- self._netbook = netbook
- self._board_type = board_type
- self._use_sheriffs = use_sheriffs
- self._extra_emails = extra_emails
- self._email_prefix = email_prefix
-
- self._build_info = BuildInfo()
-
- self._base_dir = base_dir
- self._cache_dir = os.path.join(base_dir, LOCAL_TMP_DIR, netbook, board_type)
- dash_util.MakeChmodDirs(self._cache_dir)
- self._email_dir = os.path.join(base_dir, EMAIL_DIR, email_type, board_type)
- dash_util.MakeChmodDirs(self._email_dir)
- self._performance_dir = os.path.join(
- base_dir, netbook, board_type, PERFORMANCE_DIR)
- dash_util.MakeChmodDirs(self._performance_dir)
-
- def GetEmailDir(self):
- return self._email_dir
-
- def GetPerformanceDir(self):
- return self._performance_dir
-
- def GetBuilds(self, category, build_count=EMAIL_BUILDS_TO_CHECK):
- return self._dash_view.GetBuilds(
- self._netbook, self._board_type, category)[:build_count]
-
- def GetTestNamesInBuild(self, category, build, regex):
- return self._dash_view.GetTestNamesInBuild(
- self._netbook, self._board_type, category, build, regex)
-
- def GetTestDetails(self, category, test_name, build):
- return self._dash_view.GetTestDetails(
- self._netbook, self._board_type, category, test_name, build)
-
- def GetTestErrorLog(self, log_url):
- diag = dash_util.DebugTiming()
- command = '%s %s' % (WGET_CMD, log_url)
- logging.debug(command)
- error_log = commands.getoutput(command)
- del diag
- return error_log
-
- def GetEmailFilename(self, category, build):
- return '%s/%s_%s_%s' % (
- self._cache_dir, self._email_prefix, category, build)
-
- def Checked(self, category, build):
- return os.path.exists(self.GetEmailFilename(category, build))
-
- def SetChecked(self, category, build):
- dash_util.PruneOldDirsFiles(path=self._cache_dir, dirs=False,
- older_than_days=20)
- dash_util.SaveHTML(self.GetEmailFilename(category, build), '')
-
- def FindCurrentSheriffs(self):
- # sheriff.js normally returns a comma separated list of sheriffs,
- # but it may return the following on a weekend:
- # document.write('None (channel is sheriff)')
- sheriff_from_js = (
- '%s '
- 'http://build.chromium.org/buildbot/chromiumos/sheriff.js'
- ' 2>/dev/null | sed "s/.*\'\\(.*\\)\'.*/\\1/"' % WGET_CMD)
- logging.debug(sheriff_from_js)
- out = commands.getoutput(sheriff_from_js)
- if out[:4] == 'None':
- return None
- else:
- return [name.strip() for name in out.split(',') if name.strip()]
-
- def SendEmail(self, subject, body):
- """Utility sendmail function.
-
- Wraps the command line call to send email.
- """
- email_to = []
- email_cc = []
- me = getpass.getuser()
- if me == AUTOTEST_USER:
- if self._use_sheriffs:
- sheriffs = self.FindCurrentSheriffs()
- # Sometimes no sheriffs assigned.
- if sheriffs:
- email_to.extend(sheriffs)
- email_cc.append('chromeos-bvt@google.com')
- if self._extra_emails:
- email_to.extend(self._extra_emails)
- else:
- # For debugging email.
- email_to.append(me)
- to = []
- for user in email_to:
- if user.find('@') > -1:
- to.append(user)
- else:
- to.append('%s@chromium.org' % user)
- to = ';'.join(to)
- email_cc = ';'.join(email_cc)
-
- logging.info(
- "Sending email to: '%s' cc: '%s' with subject: '%s'.",
- to, email_cc, subject)
- p = os.popen('/usr/sbin/sendmail -t', 'w')
- p.write('To: %s\n' % to)
- if email_cc:
- p.write('Cc: %s\n' % email_cc)
- p.write('From: chromeos-automated-test-failures@google.com\n')
- p.write('Subject: %s\n' % subject)
- p.write('Content-Type: text/html')
- p.write('\n') # blank line separating headers from body
- p.write(body)
- p.write('\n')
- return_code = p.close()
- if return_code is not None:
- logging.error('Sendmail exit status %s', return_code)
-
- def CheckItems(self, items):
- # Implemented by derived class.
- # Sets state in the class.
- pass
-
- def GenerateEmail(self):
- # Implemented by derived class.
- # Uses state set by CheckItems.
- pass
diff --git a/site_utils/dashboard/dash_common.py b/site_utils/dashboard/dash_common.py
deleted file mode 100644
index 1a3fa69..0000000
--- a/site_utils/dashboard/dash_common.py
+++ /dev/null
@@ -1,8 +0,0 @@
-import os, sys
-dirname = os.path.dirname(sys.modules[__name__].__file__)
-autotest_dir = os.path.abspath(os.path.join(dirname, "../.."))
-client_dir = os.path.join(autotest_dir, "client")
-sys.path.insert(0, client_dir)
-import setup_modules
-sys.path.pop(0)
-setup_modules.setup(base_path=autotest_dir, root_module_name="autotest_lib")
diff --git a/site_utils/dashboard/dash_config.json b/site_utils/dashboard/dash_config.json
deleted file mode 100644
index 210dded..0000000
--- a/site_utils/dashboard/dash_config.json
+++ /dev/null
@@ -1,230 +0,0 @@
-{
- "keylist": {
- "alerts": "Unordered list of the performance checks which send email.",
- "alternatelandings": "Dictionary of exceptions to default landing (bvt).",
- "blacklistboards": "Boards to ignore.",
- "customboardfilter": "Board filter for parsing boards from jobnames.",
- "resultmail": "Unordered list of test groups to check for failures.",
- "plots": "Ordered list of plots to generate on dash details page.",
- "priorityboards": "Ordered list of boards at top of summary dash."},
- "alerts": [
- {
- "platforms": [
- {"x86-alex-r20": "netbook_alex"},
- {"x86-alex-r19": "netbook_alex"},
- {"x86-alex-r18": "netbook_alex"},
- {"x86-mario-r20": "netbook_mario_mp"},
- {"x86-mario-r19": "netbook_mario_mp"},
- {"x86-mario-r18": "netbook_mario_mp"},
- {"x86-zgb-r20": "netbook_acer_zgb"},
- {"x86-zgb-r19": "netbook_acer_zgb"},
- {"x86-zgb-r18": "netbook_acer_zgb"}],
- "categories": ["perfalerts"],
- "sheriffs": false,
- "cc": [
- "chromeos-autotest@google.com",
- "chromeos-perf-test@google.com"],
- "test": "platform_BootPerfServer",
- "preprocess": [
- ["GroupDeltas", ""]],
- "checks": {
- "seconds_kernel_to_login": {
- "PrintAverages": "",
- "PrintStats": "",
- "PrintHistogram": {
- "numbins": 16, "limits": [5, 9],
- "width": 1024, "height": 768},
- "PrintIterations": {
- "width": 1024, "height": 768}},
- "seconds_kernel_to_*": {
- "PrintAverages": "",
- "PrintStats": ""},
- "seconds_power_on_to_login": {
- "PrintAverages": ""},
- "rdbytes_kernel_to_*": {
- "PrintAverages": "",
- "PrintStats": ""}}}],
- "resultmail_doc": [
- "---------------------------------------------------------------------",
- "The following keys are available:",
- " platforms: ",
- " <board>: <cautotest_platform>",
- " filters: ",
- " categories: regex_string [default: .*]",
- " tests: regex_string [default: .*]",
- " sheriffs: boolean true or false [default: false]",
- " cc: list of email aliases [default: None]",
- " [NOTE: sheriffs: or cc: must be set for every filter.]",
- " trigger: failed|result_changed|completed [default: 'failed']",
- "--------------------------------------------------------------------"],
- "resultmail": [
- {"platforms": {"butterfly": ["butterfly"],
- "daisy": ["snow"],
- "kiev": ["kiev"],
- "link": ["link"],
- "lumpy": ["lumpy", "netbook_lumpy"],
- "parrot": ["parrot"],
- "stout": ["stout"],
- "stumpy": ["stumpy", "desktop_stumpy"],
- "x86-alex": ["alex", "netbook_alex"],
- "x86-mario": ["mario", "netbook_mario_mp"],
- "x86-zgb": ["zgb", "netbook_acer_zgb"]},
- "filters": [
- {"categories": "bvt",
- "sheriffs": true,
- "include_experimental": false,
- "cc": ["chromeos-test@google.com",
- "chromeos-tpms+automail@google.com"]},
- {"categories": "audio",
- "cc": ["rohitbm@google.com"]},
- {"categories": "audiovideo|graphics",
- "cc": ["josephv@google.com"]},
- {"categories": "browsertests",
- "cc": ["chromeos-ui-sheriffs@google.com"]},
- {"categories": "build",
- "cc": ["krisr@google.com"]},
- {"categories": "desktopui|login|realtimecomm",
- "cc": ["tturchetto@google.com"]},
- {"categories": "enterprise",
- "cc": ["craigdh@google.com",
- "scunningham@google.com"]},
- {"categories": "kernel", "tests": ".*Trackpad",
- "trigger": "completed", "cc": ["josephsih@google.com"]},
- {"categories": "logging|platform",
- "cc": ["scunningham@google.com"]},
- {"categories": "mton_au|nton_au|regression",
- "cc": ["chromeos-test@google.com"]},
- {"categories": "perfalerts",
- "cc": ["jrbarnette@google.com",
- "puneetster@google.com"]},
- {"categories": "pyauto|pyauto_basic"},
- {"categories": "pyauto_perf",
- "cc": ["dennisjeffrey@google.com",
- "puneetster@google.com"]},
- {"categories": "video",
- "cc": ["rohitbm@google.com"]},
- {"categories": "faft*",
- "cc": ["vbendeb@google.com",
- "waihong@google.com",
- "victoryang@google.com",
- "jrbarnette@google.com"]}]},
- {"platforms": {"x86-generic-full": ["netbook_sandy_bridge"]},
- "filters": [
- {"categories": "bvt|browsertests|pyauto|sync",
- "cc": ["marcheu@google.com"]},
- {"categories": "security",
- "cc": ["keescook@google.com"]}]},
- {"platforms": {"x86-mario": ["netbook_rf_enclosure"]},
- "filters": [
- {"categories": "network_wifi",
- "cc": ["stanleyw@google.com"]}]},
- {"platforms": {"x86-mario": ["netbook_mario_mp_enterprise"],
- "x86-alex": ["netbook_alex_enterprise"],
- "x86-zgb": ["netbook_acer_zgb_enterprise"]},
- "filters": [
- {"categories": "enterprise",
- "cc": ["craigdh@google.com",
- "scunningham@google.com"]}]}],
- "plots": {
- "10": {
- "test": "platform_BootPerfServer",
- "keys": [
- "seconds_kernel_to_startup",
- "seconds_kernel_to_startup_done",
- "seconds_kernel_to_x_started",
- "seconds_kernel_to_chrome_exec",
- "seconds_kernel_to_chrome_main",
- "seconds_kernel_to_login",
- "seconds_kernel_to_network"],
- "ylabel": "Seconds"},
- "11": {
- "test": "platform_BootPerfServer",
- "keys": [
- "seconds_power_on_to_login"],
- "ylabel": "Seconds"},
- "12": {
- "test": "platform_BootPerfServer",
- "keys": [
- "seconds_reboot_time",
- "seconds_shutdown_time"],
- "ylabel": "Seconds"},
- "15": {
- "test": "platform_BootPerfServer",
- "keys": [
- "rdbytes_kernel_to_startup",
- "rdbytes_kernel_to_startup_done",
- "rdbytes_kernel_to_chrome_exec",
- "rdbytes_kernel_to_chrome_main",
- "rdbytes_kernel_to_login"],
- "ylabel": "bytes"},
- "17": {
- "test": "platform_BootPerfServer",
- "keys": [
- "seconds_firmware_boot"],
- "ylabel": "seconds"},
- "30": {
- "test": "build_RootFilesystemSize",
- "keys": ["bytes_rootfs_prod"],
- "ylabel": "100M Bytes"},
- "35": {
- "test": "build_RootFilesystemSize",
- "keys": ["bytes_rootfs_test"],
- "ylabel": "100M Bytes"},
- "40": {
- "test": "power_LoadTest.WIFI",
- "keys": ["w_energy_rate"]},
- "41": {
- "test": "power_LoadTest.WIFI",
- "keys": ["minutes_battery_life"]},
- "45": {
- "test": "power_LoadTest.WIRED",
- "keys": ["w_energy_rate"]},
- "46": {
- "test": "power_LoadTest.WIRED",
- "keys": ["minutes_battery_life"]},
- "50": {
- "test": "power_Idle",
- "keys": [
- "w_energy_rate"]},
- "60": {
- "test": "power_Resume",
- "keys": [
- "seconds_system_resume",
- "seconds_system_suspend",
- "seconds_system_resume_firmware",
- "seconds_system_resume_kernel"]},
- "70": {
- "test": "hardware_MemoryThroughput",
- "keys": [
- "mb_per_sec_memory_cp_256k_seq",
- "mb_per_sec_memory_r_256k_ran",
- "mb_per_sec_memory_rw_256k_seq",
- "mb_per_sec_memory_set_256k_seq",
- "mb_per_sec_memory_w_256k_seq"]}},
- "alternatelandings": {
- "x86-alex-r18": {
- "netbook_ALEX_CDMA_GOBI2K": "network",
- "netbook_ALEX_CDMA_GOBI3K": "network"},
- "x86-alex-r17": {
- "netbook_ALEX_CDMA_GOBI2K": "network",
- "netbook_ALEX_CDMA_GOBI3K": "network"},
- "x86-alex-r16": {
- "netbook_ALEX_CDMA_GOBI2K": "network",
- "netbook_ALEX_CDMA_GOBI3K": "network"},
- "x86-mario-r18": {
- "netbook_MARIO_MP_CDMA_GOBI2K": "network",
- "netbook_RF_ENCLOSURE": "network"},
- "x86-mario-r17": {
- "netbook_MARIO_MP_CDMA_GOBI2K": "network",
- "netbook_RF_ENCLOSURE": "network"},
- "x86-mario-r16": {
- "netbook_MARIO_MP_CDMA_GOBI2K": "network",
- "netbook_RF_ENCLOSURE": "network"}},
- "priorityboards_tot": ["x86-alex-r20", "x86-mario-r20", "x86-zgb-r20"],
- "priorityboards": [
- "x86-alex-r20", "x86-alex-r19", "x86-alex-r18",
- "x86-mario-r20", "x86-mario-r19", "x86-mario-r18",
- "x86-zgb-r20", "x86-zgb-r19", "x86-zgb-r18"],
- "blacklistboards": ["x86-alex-test"],
- "customboardfilter": "x86|tegra2|stumpy|lumpy"
-}
diff --git a/site_utils/dashboard/dash_data_analyze.py b/site_utils/dashboard/dash_data_analyze.py
deleted file mode 100755
index 08f71f2..0000000
--- a/site_utils/dashboard/dash_data_analyze.py
+++ /dev/null
@@ -1,141 +0,0 @@
-#!/usr/bin/python
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Load dash data model and analyze data."""
-
-import logging
-import optparse
-import os
-
-import dash_common
-import dash_util
-
-settings = 'autotest_lib.frontend.settings'
-os.environ['DJANGO_SETTINGS_MODULE'] = settings
-
-from dash_view import AutotestDashView
-
-
-# String resources.
-from dash_strings import LAST_N_JOBS_LIMIT
-
-
-def test_first_two_keyval_data_points(dash_view, dash_options):
- """Show analysis of the effect of dropping first two points."""
- float_converter = dash_util.HumanReadableFloat()
- boards = dash_view.GetBoardTypes()
-
- summary = {}
-
- # Print stats for each build/key that has an 'out of range' first or second
- # data point.
- for board in boards:
- netbooks = dash_view.GetNetbooksWithBoardTypeCategory(board, "perfalerts")
- board_summary = summary.setdefault(board, {})
- for netbook in netbooks:
- keyvals = dash_view.GetTestKeyVals(
- netbook, board, "platform_BootPerfServer")
- if not keyvals:
- continue
- netbook_summary = board_summary.setdefault(netbook, {})
- for key, build_dict in keyvals.iteritems():
- key_summary = netbook_summary.setdefault(key, {
- 'sum1': 0.0, 'sum2': 0.0, 'summinus2': 0.0,
- 'count1': 0, 'countminus2': 0, 'build_averages': {}})
- key_printed = False
- for seq, data_tuple in build_dict.iteritems():
- data = data_tuple[0]
- list_len = len(data)
- key_summary['build_averages'][seq] = sum(data, 0.0) / list_len
- if list_len > 2:
- key_summary['sum1'] += data[0]
- key_summary['sum2'] += data[1]
- sum_minus2 = sum(data[2:], 0.0)
- len_minus2 = list_len - 2
- key_summary['summinus2'] += sum_minus2
- key_summary['count1'] += 1
- key_summary['countminus2'] += len_minus2
- list_avg = sum_minus2 / len_minus2
- d = list_avg * 0.1
- if not dash_options.datapoints:
- continue
- if (abs(data[0]-list_avg) > d or
- abs(data[1]-list_avg) > d):
- if not key_printed:
- logging.debug('%s-%s-%s:', board, netbook, key)
- key_printed = True
- logging.debug('%s, %s, %s, %s', board, netbook, key, seq)
- logging.debug(' %s', [float_converter.Convert(n) for n in data])
-
- # Now print a summary.
- if dash_options.summaryaverages:
- logging.debug('-----------------------------')
- logging.debug('SUMMARY:')
- logging.debug(
- ' AVG1 AVG2 AVGALL-2 AVG-ALL '
- 'BOARD NETBOOK KEY')
- for board, netbooks in summary.iteritems():
- for netbook, keys in netbooks.iteritems():
- for key, key_stats in keys.iteritems():
- logging.debug(
- '%8s %8s %8s %8s %s %s %s',
- float_converter.Convert(
- key_stats['sum1'] / key_stats['count1']),
- float_converter.Convert(
- key_stats['sum2'] / key_stats['count1']),
- float_converter.Convert(
- key_stats['summinus2'] / key_stats['countminus2']),
- float_converter.Convert(
- (key_stats['sum1'] + key_stats['sum2'] +
- key_stats['summinus2']) / (
- key_stats['count1'] * 2 + key_stats['countminus2'])),
- board, netbook, key)
- if dash_options.buildaverages:
- logging.debug('-----------------------------')
- for board, netbooks in summary.iteritems():
- for netbook, keys in netbooks.iteritems():
- for key, key_stats in keys.iteritems():
- for seq, build_average in key_stats['build_averages'].iteritems():
- logging.debug(
- '%s, %s, %s, %s: %s',
- board, netbook, key, seq,
- float_converter.Convert(build_average))
-
-
-def parse_args():
- """Support verbose flag."""
- parser = optparse.OptionParser()
- parser.add_option('-b', '--print-build-averages', help='show build averages',
- dest='buildaverages', action='store_true', default=False)
- parser.add_option('-f', '--file-name', help='output filename',
- dest='filename', default=None)
- parser.add_option('-j', '--job-limit', help='limit to last n jobs',
- dest='joblimit', default=LAST_N_JOBS_LIMIT)
- parser.add_option('-d', '--print-data-points', help='show data point values',
- dest='datapoints', action='store_true', default=False)
- parser.add_option('-s', '--print-summary', help='show summary of averages',
- dest='summaryaverages', action='store_true', default=False)
- options, args = parser.parse_args()
- logging_level = logging.DEBUG
- if options.filename:
- logging.basicConfig(
- level=logging.DEBUG, filename=options.filename, filemode='w')
- else:
- logging.basicConfig(level=logging_level)
- return options, args
-
-
-def main():
- options, args = parse_args()
-
- dash_view = AutotestDashView()
- dash_view.LoadPerfFromDB(int(options.joblimit))
- test_first_two_keyval_data_points(dash_view, options)
-
- if options.filename:
- os.chmod(options.filename, 0644)
-
-if __name__ == '__main__':
- main()
diff --git a/site_utils/dashboard/dash_email.py b/site_utils/dashboard/dash_email.py
deleted file mode 100755
index ba777c2..0000000
--- a/site_utils/dashboard/dash_email.py
+++ /dev/null
@@ -1,379 +0,0 @@
-# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-# pylint: disable-msg=C0111
-
-"""Handle dashboard analysis and email of test results.
-
-This code specializes the EmailNotifier defined in common_email.py.
-DashEmailNotifier is responsible for sending email with test results and
-links to aid in troubleshooting test failures. The parameters of these
-emails are generally described in dash_config.json under 'filters'.
-"""
-
-__author__ = ['truty@google.com (Mike Truty)']
-
-import logging
-import os
-
-import dash_util
-
-from django.shortcuts import render_to_response
-
-from build_info import BuildInfo
-from common_email import EmailNotifier
-
-# String resources.
-from dash_strings import CHANGELOG_URL
-from dash_strings import CHROME_CHANGELOG_URL
-from dash_strings import DASHBOARD_MAIN
-from dash_strings import EMAIL_TESTS_PER_ROW
-from dash_strings import EMAIL_TESTS_SUBJECT
-from dash_strings import EMAIL_TRIGGER_CHANGED
-from dash_strings import EMAIL_TRIGGER_COMPLETED
-from dash_strings import EMAIL_TRIGGER_FAILED
-from dash_strings import IMAGE_URLS
-from dash_strings import STATUS_FAILED
-from dash_strings import STATUS_PASSED
-from dash_strings import TEST_CHECKED_PREFIX
-from dash_strings import TEST_EMAIL_DIR
-from dash_strings import TESTS_STATUS_FILE
-
-
-def _ParseVersion(build):
- """Extract version from build string. Parses x.x.x.x* and Ryy-x.x.x* forms."""
- build_version = build.rsplit('-', 2)[0]
- if '-' in build_version:
- build_version = build_version.split('-')[1]
- return build_version
-
-
-class DashEmailNotifier(EmailNotifier):
- """Class to check for failed tests and send emails."""
-
- def __init__(self, base_dir, netbook, board_type, categories,
- use_sheriffs, extra_emails, trigger, prefix,
- include_experimental=True):
- """Specialize EmailNotifier and set a few of our own variables.
-
- Args:
- base_dir: Is use to check for previous emails sent.
- netbook: Autotest platform for filtering results: netbook_xxx
- board_type: Chrome OS board to be checked: x86-mario-rc, ...
- categories: Test grouping known by the dashboard code
- use_sheriffs: Indicates send email to the sheriffs
- extra_emails: Add others to receive the email
- trigger: Send email on test finished, failed or result state changed
- prefix: Custom prefix for cache tracking.
- include_experimental: Include failure results for tests marked
- experimental (Default: True)
- """
- super(DashEmailNotifier, self).__init__(
- base_dir, netbook, board_type, use_sheriffs, extra_emails,
- prefix, TEST_EMAIL_DIR)
- self._categories = categories
- self._trigger = trigger
- self._state_changed = {}
- self._failed_tests = {}
- self._failed_categories = set()
- self._crash_summaries = self._dash_view.GetCrashes().GetTestSummaries()
- self._crashes = {}
- self._previous_build = {}
- self._include_experimental = include_experimental
-
- def _FindTestFailures(self, build, category, test_name):
- """Scans test details for failures, retrieves artifacts, and finds crashes.
-
- Helper method for CheckItems which scans the test details for the given test
- in the given category on the given build. Pre-processed test artifacts are
- retrieved from the Autotest server if available.
-
- Failed tests are saved to self._failed_tests and crashes to self._crashes.
-
- Logs are added to the test details dictionary under the key 'test_logs'
-
- Args:
- build: a full build string: 0.8.73.0-r3ed8d12f-b719.
- category: a test group: bvt, regression, desktopui, graphics, ...
- test_name: test_name of Autotest test.
-
- Returns:
- True if no test failures are found, False if failures were found, and None
- if no details could be loaded for the given test_name.
- """
- # Check test details to see if this test failed.
- test_status = None
- crashes_dict = self._crashes.setdefault(build, {})
- failed_test_dict = self._failed_tests.setdefault(build, {})
- test_details = self.GetTestDetails(category, test_name, build)
- for t in test_details:
- # Skip experimental tests if flag says to do so
- if not self._include_experimental:
- if t.get('experimental', False):
- continue
-
- # Attempt to load pre-processed test artifacts from server.
- summary = self._crash_summaries.RetrieveTestSummary(t['tag'], test_name)
- if summary and summary.get('crashes'):
- self._failed_categories.add(category)
- # Keep track of crashes indexed by the crashed process and signal.
- for crash in summary['crashes']:
- crashes_dict.setdefault(crash, []).append((test_name, t))
-
- if t['status'] == 'GOOD':
- if test_status is None:
- test_status = True
- continue
-
- failed_test_dict.setdefault(test_name, []).append(t)
- self._failed_categories.add(category)
- test_status = False
-
- # Populate source path to test for processing by Django template later.
- t['test_path'] = self._dash_view.GetAutotestInfo(test_name)[1]
-
- # Add error logs if they exist.
- if summary:
- t['test_logs'] = summary['log'].strip()
-
- return test_status
-
- def _CheckStateChange(self, build, category, test_name, current_test_status):
- """Compares current test status and test status for a previous build.
-
- Helper method for CheckItems which scans the test details for the given test
- in the given category on the given build and compares that status against
- the status for the current build.
-
- Args:
- build: a full build string: 0.8.73.0-r3ed8d12f-b719.
- category: a test group: bvt, regression, desktopui, graphics, ...
- test_name: test_name of Autotest test.
- current_test_status: True for success, False for failure, None for none.
-
- Returns:
- True if the state changed, False otherwise.
- """
- state_changed = True
- # Tests for a board can be run for a build but
- # possibly not for this category.
- previous_details = self.GetTestDetails(category, test_name, build)
- if not current_test_status is None and previous_details:
- # Handle tricky state of multiple test results.
- # Any nongood is considered a failure even among a good.
- previous_test_status = True
- for t in previous_details:
- if not t['status'] == 'GOOD':
- previous_test_status = False
- break
- if current_test_status == previous_test_status:
- state_changed = False
- return state_changed
-
- def CheckItems(self, items):
- """Finds failed tests and crashes in the specified categories.
-
- CheckItems checks the latest build for a given category for any crashes or
- test failures. Failing tests are stored in self._failed_tests and crashes
- in self._crashes.
-
- When the trigger is EMAIL_TRIGGER_CHANGED, the last two builds are compared
- and any differences are recorded in self._state_changed.
-
- Args:
- items: Tuple of (category list, test_regex) to use for checks.
- """
- categories, test_regex = items
- for category in categories:
- # Retrieve the last two builds for this category.
- builds = self.GetBuilds(category, build_count=2)
- if not builds:
- continue
- build = builds[0]
- if len(builds) > 1:
- self._previous_build[build] = builds[1]
-
- # Check sentinel file to see if we've already processed this build.
- if self.Checked(category, build):
- continue
-
- for test_name in self.GetTestNamesInBuild(category, build, test_regex):
- # Scan the test details for failures. Fills out self._failed_tests and
- # self._crashes.
- test_status = self._FindTestFailures(build, category, test_name)
-
- # For efficiency, only check previous builds when needed.
- if not self._trigger == EMAIL_TRIGGER_CHANGED:
- continue
-
- # Once state-change has been discovered for any test in the build
- # there is no need to look for more changes.
- state_changed = self._state_changed.setdefault(build, False)
- if state_changed:
- continue
-
- # It is considered a state-change if no previous build is found.
- if len(builds) < 2:
- self._state_changed[build] = True
- else:
- self._state_changed[build] = self._CheckStateChange(
- builds[1], category, test_name, test_status)
-
- # Write the sentinel file
- self.SetChecked(category, build)
-
- def GenerateEmail(self):
- """Send email to aid troubleshooting failed tests.
-
- Emails are broken into 4 sections:
- 1. Intro with summary of failing build and netbook combination.
- 2. Table of failing tests.
- 3. Inline error logs for perusing.
- 4. Inline build log for blame.
- """
- buildinfo = BuildInfo()
- for tpl_build in set(self._failed_tests.keys() + self._crashes.keys()):
- # Sort crashes and failed tests.
- tpl_crashes = sorted(self._crashes.get(tpl_build, None).items())
- tpl_failed_tests = sorted(self._failed_tests.get(tpl_build, None).items())
- if ((self._trigger == EMAIL_TRIGGER_COMPLETED) or
- (self._trigger == EMAIL_TRIGGER_FAILED and
- (tpl_failed_tests or tpl_crashes)) or
- (self._trigger == EMAIL_TRIGGER_CHANGED and
- tpl_build in self._state_changed and
- self._state_changed[tpl_build])):
- tpl_netbook = self._netbook
- tpl_board = self._board_type
- categories = ', '.join(sorted(self._categories))
- if tpl_board in IMAGE_URLS:
- tpl_images_link = dash_util.UrlFix('%s/%s' % (
- IMAGE_URLS[tpl_board].rstrip('/'), _ParseVersion(tpl_build)))
- else:
- tpl_images_link = IMAGE_URLS['default']
- if tpl_build in self._previous_build:
- tpl_changelog_link = dash_util.UrlFix(CHANGELOG_URL % (
- _ParseVersion(self._previous_build[tpl_build]),
- _ParseVersion(tpl_build)))
- old_chrome_version = str(buildinfo.GetChromeVersion(
- tpl_board, self._previous_build[tpl_build])[0])
- new_chrome_version = str(buildinfo.GetChromeVersion(
- tpl_board, tpl_build)[0])
- if old_chrome_version and new_chrome_version:
- tpl_chrome_changelog_link = dash_util.UrlFix(
- CHROME_CHANGELOG_URL % (old_chrome_version, new_chrome_version))
-
- status = STATUS_PASSED
- if tpl_failed_tests:
- logging.debug(
- 'Build %s has %s failed test names to email.',
- tpl_build, len(tpl_failed_tests))
- # Django can't do basic math, so preprocess our failed tests into an
- # array of EMAIL_TESTS_PER_ROW-length arrays containing the failed
- # test data.
- tpl_index = 0
- tpl_split_failed_tests = [[]]
- for name, details in tpl_failed_tests:
- tpl_split_failed_tests[tpl_index].append((name, details[0]))
- if len(tpl_split_failed_tests[tpl_index]) == EMAIL_TESTS_PER_ROW:
- tpl_index += 1
- tpl_split_failed_tests.append([])
-
- if tpl_failed_tests or tpl_crashes:
- categories = ', '.join(sorted(list(self._failed_categories)))
- status = STATUS_FAILED
- template_file = TESTS_STATUS_FILE % status
-
- tpl_subject = EMAIL_TESTS_SUBJECT % {
- 'board': tpl_board,
- 'build': tpl_build,
- 'categories': categories,
- 'netbook': tpl_netbook[8:].lower(),
- 'status': status.lower()}
- body = render_to_response(
- os.path.join('emails', template_file), locals()).content
- self.SendEmail(tpl_subject, body)
-
-
-def EmailFromConfig(dash_base_dir, dash_view, email_options,
- prefix=TEST_CHECKED_PREFIX):
- """All the work of checking and sending email.
-
- Emails test failure summary based on entries in the dash config:
- -For boards/platforms specified
- -For only certain releases if desired
- -For groups (suites)/tests specified
- -To users/sheriffs specified
-
- Args:
- dash_base_dir: Base dir of the output files.
- dash_view: Reference to our data model.
- email_options: From email_config.json.
- prefix: String to uniquely identify sent emails in the cache.
- """
- triggers = [
- EMAIL_TRIGGER_COMPLETED, EMAIL_TRIGGER_FAILED, EMAIL_TRIGGER_CHANGED]
-
- for mailer in email_options['resultmail']:
- if not 'platforms' in mailer or not 'filters' in mailer:
- logging.warning('Emailer requires platforms and filters.')
- continue
- for board_prefix, netbooks in mailer['platforms'].iteritems():
- for netbook in netbooks:
- for board in [b for b in dash_view.GetNetbookBoardTypes(netbook)
- if b.startswith(board_prefix)]:
- for filter_ in mailer['filters']:
- use_sheriffs = filter_.get('sheriffs', False)
- include_experimental = filter_.get('include_experimental', True)
- cc = filter_.get('cc', None)
- if not use_sheriffs and not cc:
- logging.warning('Email requires sheriffs or cc.')
- continue
- trigger = filter_.get('trigger', EMAIL_TRIGGER_FAILED)
- if not trigger in triggers:
- logging.warning('Unknown trigger encountered, using default %s.',
- EMAIL_TRIGGER_FAILED)
- trigger = EMAIL_TRIGGER_FAILED
- categories = dash_view.GetCategories(netbook, board,
- filter_.get('categories'))
- notifier = DashEmailNotifier(dash_base_dir, netbook, board,
- categories, use_sheriffs, cc, trigger,
- prefix + trigger,
- include_experimental)
- notifier.CheckItems((categories, filter_.get('tests')))
- notifier.GenerateEmail()
-
-
-def EmailAllFailures(dash_base_dir, dash_view):
- """All the work of checking and sending email.
-
- Emails test failure summary for any test failure on any board to a common
- Google group. Subscribers may then choose to receive and filter email
- of their own accord instead of maintaining a config file.
-
- Re-uses existing emailer code by hand-building a config file surrogate.
-
- Args:
- dash_base_dir: Base dir of the output files.
- dash_view: Reference to our data model.
- """
- platform_configs = {}
- categories = set()
- for netbook in dash_view.netbooks:
- if not netbook.strip() or not dash_view.GetNetbookBoardTypes(netbook):
- continue
- for board in dash_view.GetNetbookBoardTypes(netbook):
- platform_configs.setdefault(board, []).append(netbook)
- categories |= set(dash_view.GetCategories(netbook, board))
-
- filter_configs = [{'categories': category,
- 'cc': ['chromeos-automated-test-failures@google.com']}
- for category in sorted(categories)]
-
- surrogate_config = {'resultmail': [{'platforms': platform_configs,
- 'filters': filter_configs}]}
- EmailFromConfig(dash_base_dir, dash_view, surrogate_config,
- '.tmp_allemailed_')
-
-
-if __name__ == '__main__':
- print 'Run %s with --mail-generate.' % DASHBOARD_MAIN
diff --git a/site_utils/dashboard/dash_strings.py b/site_utils/dashboard/dash_strings.py
deleted file mode 100644
index 61ca5e9..0000000
--- a/site_utils/dashboard/dash_strings.py
+++ /dev/null
@@ -1,114 +0,0 @@
-# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# Relocate the large string templates to one place.
-
-# Common constants.
-AUTOTEST_ARCHIVE = 'gs://chromeos-autotest-results'
-AUTOTEST_PATH = '/usr/local/autotest'
-AUTOTEST_SERVER = 'http://cautotest'
-AUTOTEST_USER = 'chromeos-test'
-
-BVT_TAG = 'bvt'
-KERNELTEST_TAG = 'kerneltest'
-
-EMAIL_TRIGGER_COMPLETED = 'completed'
-EMAIL_TRIGGER_FAILED = 'failed'
-EMAIL_TRIGGER_CHANGED = 'result_changed'
-
-DASHBOARD_MAIN = 'run_generate.py'
-
-EMAILS_SUMMARY_FILE = 'emails.html'
-TEST_LANDING_FILE = 'index.html'
-TEST_TABLE_FILE = 'table_index.html'
-TEST_WATERFALL_FILE = 'waterfall_index.html'
-KERNEL_TABLE_FILE = 'kernel_index.html'
-KERNEL_WATERFALL_FILE = 'waterfall_kernel.html'
-TEST_DETAILS_FILE = 'details.html'
-TESTS_STATUS_FILE = 'tests_%s.html'
-PERFORMANCE_REGRESSED_EMAIL = 'performance_regressed.html'
-BUILD_PERFORMANCE_FILE = 'build_performance.html'
-PLOT_FILE = 'index.html'
-PERF_INDEX_FILE = 'perf_index.html'
-PERF_BUILDS_FILE = 'perf_builds.html'
-PLOT_MONITORING_FILE = 'monitoring.html'
-
-LOCAL_TMP_DIR = './dashcache'
-EMAIL_DIR = 'emails'
-JOB_RESULT_DIR = 'job_results'
-PERFORMANCE_DIR = 'performance'
-TEST_EMAIL_DIR = 'test'
-
-BUILDTIME_PREFIX = '.tmp_buildtime_'
-TEST_CHECKED_PREFIX = '.tmp_emailed_'
-ALERT_CHECKED_PREFIX = '.tmp_alerted_'
-
-PREPROCESSED_TAG = '__pp__'
-
-LAST_N_JOBS_LIMIT = 500 # Scan only the last N jobs for relevant results.
-SUMMARY_TABLE_ROW_LIMIT = 50 # display max of the latest n test builds.
-
-# Email constants.
-EMAIL_BUILDS_TO_CHECK = 2
-EMAIL_TESTS_PER_ROW = 4
-
-# Image URLs.
-IMAGE_URLS = {
- 'default':
- 'https://sandbox.google.com/storage/chromeos-releases/',
- 'x86-generic-full':
- ('https://sandbox.google.com/storage/chromeos-image-archive/'
- 'x86-generic-full/')}
-
-CGI_RETRIEVE_LOGS_CMD = 'tko/retrieve_logs.cgi?job=/results'
-GSUTIL_GET_CMD = 'gsutil cat '
-WGET_CMD = 'wget --timeout=30 --tries=1 --no-proxy -qO- '
-
-# SUMMARY PAGE templates
-
-UNKNOWN_TIME_STR = 'None'
-
-# EMAIL templates.
-
-STATUS_PASSED = 'success'
-STATUS_FAILED = 'failure'
-
-EMAIL_TESTS_SUBJECT = (
- 'Autotest %(status)s in %(categories)s on %(board)s (%(build)s)')
-
-EMAIL_ALERT_DELTA_TABLE_SKELETON = """
-<table bgcolor="#e5e5c0" cellspacing="1"
-cellpadding="2" style="margin-right:200px;">
-<tr>
- <td colspan=5><center><h3>DELTA Summary for %(test_name)s<h3></center></td>
-</tr>
-<tr>
- <td><center>Key</center></td>
- <td><center>Delta Latest<br>Build</center></td>
- <td><center>Delta Average<br>Prev Builds</center></td>
- <td><center>Latest<br>Build</center></td>
- <td><center>Average<br>Prev Builds</center></td>
-</tr>
-%(body)s
-</table>
-<br>
-"""
-
-EMAIL_ALERT_DELTA_TABLE_ROW = """
-<tr>
- <td><center><b><tt>%(key)s</tt></b></center></td>
- <td><center><b><tt>%(pp_latest)s</tt></b></center></td>
- <td><center><b><tt>%(pp_average)s</tt></b></center></td>
- <td><center><b><tt>%(latest)s</tt></b></center></td>
- <td><center><b><tt>%(average)s</tt></b></center></td>
-</tr>
-"""
-
-# PLOT PAGE templates
-PLOT_ANCHOR = """
-<hr><center><a name="%(test_name)s">%(test_name)s</a><center><br>"""
-
-CHANGELOG_URL = 'http://chromeos-images/diff/report?from=%s&to=%s'
-CHROME_CHANGELOG_URL = (
- 'http://omahaproxy.appspot.com/changelog?old_version=%s&new_version=%s')
diff --git a/site_utils/dashboard/dash_util.py b/site_utils/dashboard/dash_util.py
deleted file mode 100644
index 6b312f5..0000000
--- a/site_utils/dashboard/dash_util.py
+++ /dev/null
@@ -1,292 +0,0 @@
-# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Common utility functions shared by multiple dashboard modules.
-
- Alphabetical where no dependency on another function.
-
- Functions: BuildNumberCmp
- MakeChmodDirs
- PruneOldDirsFiles
- SaveHTML
- ShowList
- ShowDict
- ShowStructure
- UrlFix
-
- Classes: DebugFunctionTiming
- DebugTiming
- HumanReadableFloat
- SimpleCounter
-"""
-
-import datetime
-import decimal
-import inspect
-import logging
-import os
-import re
-import shutil
-import urllib
-import urlparse
-
-from time import time
-
-BUILD_PARSE = re.compile('((?:[\d]+\.)?[\d]+\.[\d]+\.[\d]+)')
-
-
-def SplitBuild(build):
- """Find and split pure version number.
-
- From R18-xx.yy.zz => ['xx', 'yy', 'zz]
- From R18-xx.yy.zz-a1-b2 => ['xx', 'yy', 'zz]
- From 0.xx.yy.zz-a1-b2 => ['xx', 'yy', 'zz]
-
- Ignores old '-a1-bXXX' fields.
- """
- m = re.search(BUILD_PARSE, build)
- if m:
- return m.group(1).split('.')
- else:
- logging.debug('Unexpected build: %s.', build)
- return build
-
-
-def BuildNumberCmp(build1, build2):
- """Compare build numbers and return in descending order."""
- major1 = SplitBuild(build1)
- major2 = SplitBuild(build2)
-
- major_len = min([len(major1), len(major2)])
- for i in xrange(major_len):
- if major1[i] != major2[i]:
- return -cmp(int(major1[i]), int(major2[i]))
- return -cmp(build1, build2)
-
-
-def PruneOldDirsFiles(path, dirs=True, older_than_days=60):
- """Helper to prune dirs that are older.
-
- Job cache directory can easily exceed 32k limit.
- Prune older jobs.
-
- The waterfall/test displays of crash data do not
- generally exceed 3 weeks of results so choosing
- 60 days is reasonable with a buffer.
-
- Args:
- path: parent container directory to scan/prune.
- dirs: True to prune dirs, False to prune files.
- older_than: prune dirs older than this many days.
- """
- target_timedelta = datetime.timedelta(days=older_than_days)
- now_seconds = time()
- for entry in os.listdir(path):
- entry = os.path.join(path, entry)
- alive_seconds = now_seconds - os.path.getmtime(entry)
- if datetime.timedelta(seconds=alive_seconds) > target_timedelta:
- if dirs and os.path.isdir(entry):
- shutil.rmtree(entry)
- if not dirs and os.path.isfile(entry):
- os.remove(entry)
-
-def MakeChmodDirs(path):
- """Helper to make and chmod dirs."""
-
- return_code = os.system('mkdir -p %s' % path)
- if return_code:
- logging.error('mkdir (%s) exit status %s', path, return_code)
- return_code = os.system('chmod -R 0755 %s' % path)
- if return_code:
- logging.error('chmod (%s) exit status %s', path, return_code)
-
-
-def SaveHTML(html_file, html_content, style_section=None):
- """Helper to write our HTML files."""
- f = open(html_file, 'w')
- if style_section:
- f.write(style_section)
- f.write(html_content)
- f.close()
- os.chmod(html_file, 0644)
-
-
-def ShowList(current_list, indent=2):
- str_list = [str(s) for s in current_list]
- logging.debug("%s%s.", " " * indent, ", ".join(str_list))
-
-
-def ShowDict(current_dict, indent=2):
- for k, v in current_dict.iteritems():
- if type(v) == list:
- logging.debug("%s%s.", " " * indent, k)
- ShowList(v, indent+2)
- elif type(v) == set:
- logging.debug("%s%s.", " " * indent, k)
- ShowList(list(v), indent+2)
- elif type(v) == dict:
- logging.debug("%s%s.", " " * indent, k)
- ShowDict(v, indent+2)
- else:
- logging.debug("%s%s: %s.", " " * indent, k, unicode(v))
-
-
-def ShowStructure(title, var, doc=None):
- logging.debug("*")
- logging.debug("*%s***************", title)
- logging.debug("*")
- if doc:
- logging.debug(doc)
- logging.debug("*")
- if not var:
- logging.debug("None")
- elif type(var) == list:
- ShowList(var)
- elif type(var) == set:
- ShowList(list(var))
- elif type(var) == dict:
- ShowDict(var)
- else:
- logging.debug(str(var))
-
-
-def UrlFix(url):
- """Escapes a URL according to RFC2616."""
- scheme, netloc, path, qs, anchor = urlparse.urlsplit(url)
- path = urllib.quote(path, '/%')
- qs = urllib.quote_plus(qs, ':&=')
- return urlparse.urlunsplit((scheme, netloc, path, qs, anchor))
-
-
-class DebugFunctionTiming(object):
- def __init__(self):
- # Use the name of the parent frame record.
- self._function_name = 'function: ' + inspect.stack()[2][3]
- self._start_time = datetime.datetime.now()
-
- def GetElapsed(self):
- return self._function_name, (datetime.datetime.now() - self._start_time)
-
-
-class DebugTiming(object):
- """Class for simple timing of arbitrary blocks of code.
-
- USE:
- diag = dash_util.DebugTiming()
- ...block of code
- del diag
- """
-
- class __impl:
- """Nested class implements code wrapped by singleton."""
-
- def __init__(self):
- self._functions = {}
-
- def ShowFunctionTiming(self):
- logging.debug("*")
- logging.debug("*DEBUG FUNCTION TIMING***************")
- logging.debug("*")
- functions = self._functions.keys()
- functions.sort()
- for name in functions:
- logging.debug(" %s: %s.",
- name, str(self._functions[name]))
-
- def __del__(self):
- self.ShowFunctionTiming()
-
- def UpdateFunctionTime(self, name, elapsed):
- name_time = self._functions.setdefault(name, datetime.timedelta(0))
- self._functions[name] = name_time + elapsed
-
-
- # Instance reference for singleton behavior.
- __instance = None
- __refs = 0
- __functiontiming = []
-
- def __init__(self):
- if DebugTiming.__instance is None:
- DebugTiming.__instance = DebugTiming.__impl()
-
- self.__dict__["_DebugTiming__instance"] = DebugTiming.__instance
- DebugTiming.__refs += 1
-
- DebugTiming.__functiontiming.insert(0, DebugFunctionTiming())
-
- def __del__(self):
- timing = DebugTiming.__functiontiming.pop(0)
- name, elapsed = timing.GetElapsed()
- DebugTiming.__instance.UpdateFunctionTime(name, elapsed)
- del timing
-
- DebugTiming.__refs -= 1
- if not DebugTiming.__instance is None and DebugTiming.__refs == 0:
- del DebugTiming.__instance
- DebugTiming.__instance = None
-
- def __getattr__(self, attr):
- return getattr(DebugTiming.__instance, attr)
-
- def __setattr__(self, attr, value):
- return setattr(DebugTiming.__instance, attr, value)
-
-
-class HumanReadableFloat(object):
- """Class for converting floats to be human readable."""
-
- def __init__(self, use_short=True, use_suffix=None):
- if use_suffix: # Byte, Second...
- self._suffix = use_suffix
- else:
- self._suffix = ''
- self._prefixes = ' ,kilo,Mega,Giga,Tera,Peta,Exa,Zetta,Yotta'.split(',')
- if use_short:
- self._prefixes = [pre[0] for pre in self._prefixes]
- if self._suffix:
- self._suffix = self._suffix[0]
-
- def Convert(self, number, precision=3):
- if not type(number) == 'string':
- number = str(number)
- decimal_location = number.find('.')
- suffix_index = (decimal_location - 1) / 3
- if decimal_location > 3:
- # Put the decimal in the right spot
- new_decimal_location = decimal_location - suffix_index * 3
- number = list(number)
- number.remove('.')
- number.insert(new_decimal_location, '.')
- return '%s%s%s' % (
- round(float(''.join(number)), precision),
- self._prefixes[suffix_index],
- self._suffix)
-
- def BackToFloat(self, number_string):
- return float(number_string[:-1])
-
-
-class SimpleCounter(object):
- """Simple: a flavor of the Python Counter."""
- def __init__(self):
- """Track the 'counter' and the largest entry only."""
- self._counter = {}
- self._max_key = None
- self._max_val = -1
-
- def Push(self, k):
- """Enter new value and update largest tracking."""
- if k:
- self._counter[k] = self._counter.get(k, 0) + 1
- if self._counter[k] > self._max_val:
- self._max_key = k
- self._max_val = self._counter[k]
-
- def MaxKey(self):
- """Retrieve the largest."""
- if self._max_key and len(self._counter) > 1:
- return '%s (multiple)' % self._max_key
- else:
- return self._max_key
diff --git a/site_utils/dashboard/dash_view.py b/site_utils/dashboard/dash_view.py
deleted file mode 100755
index 1ae79b1..0000000
--- a/site_utils/dashboard/dash_view.py
+++ /dev/null
@@ -1,1513 +0,0 @@
-# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Classes for efficient data retrieval for dash utilities.
-
-To see live data for these data structures best, run test_dash_view.py and
-review its output. Output is produced by ShowDataModel() and ShowKeyVals().
-
-Includes: class CrashDashView(object)
- class AutotestDashView(object)
- class SummaryRanges(object)
-"""
-
-import datetime
-import itertools
-import logging
-import os
-import re
-
-import dash_util
-import test_summary
-
-settings = "autotest_lib.frontend.settings"
-os.environ["DJANGO_SETTINGS_MODULE"] = settings
-
-# For db access.
-from autotest_lib.frontend.afe import readonly_connection
-
-# String resources.
-from dash_strings import AUTOTEST_USER
-from dash_strings import JOB_RESULT_DIR
-from dash_strings import KERNELTEST_TAG
-from dash_strings import LAST_N_JOBS_LIMIT
-from dash_strings import LOCAL_TMP_DIR
-from dash_strings import UNKNOWN_TIME_STR
-
-
-GTEST_SUFFIXES = ["audio", "browsertests", "enterprise", "pagecycler", "pyauto",
- "pyauto_basic", "pyauto_perf", "sync", "video"]
-SUFFIXES_TO_SHOW = ["bvt", "flaky", "hwqual", "regression",
- KERNELTEST_TAG] + GTEST_SUFFIXES
-SERVER_JOB = "SERVER_JOB"
-LEGACY_PLATFORM_PREFIXES = ("netbook_", "desktop_")
-
-
-class CrashDashView(object):
- """View used to show crash information in summary and details views.
-
- An important reason we separate this class from AutotestDashView is that
- individual test details in AutotestDashView are frequently aliased into
- multiple categories. To dedup crash results we use this separate structure.
- """
-
- def __init__(self, dash_base_dir):
- """Initialize grouping containers used to retrieve crashes.
-
- Crashes are summarized per build for entire platforms, categories and
- test names. Data structures need to support retrieval of both detailed
- crash strings and counts of crashes discovered.
-
- The _crashes data structure is indexed as follows:
- +netbooks (netbook_HP_INDIANA, netbook_DELL_L13, ...)
- |+boards (x86-generic-bin, arm-generic-rel, ...)
- |+build
- |+test_name
- |-categories
- |-crash strings
-
- Args:
- dash_base_dir: root of the cache directory for crash results.
- """
- job_cache_dir = os.path.join(dash_base_dir, LOCAL_TMP_DIR, JOB_RESULT_DIR)
- dash_util.MakeChmodDirs(job_cache_dir)
- dash_util.PruneOldDirsFiles(job_cache_dir)
- self._test_summaries = test_summary.TestSummaryInfo(job_cache_dir)
- self._crashes = {}
-
- def _LookupCrashDict(self, netbook, board, build, test_name=None):
- """Retrieve a leaf level (test_name) or one-up (build) of the crash tree.
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
- @param board: one of our boards: x86-generic-full,
- x86-mario-full-chromeos, ...
- @param build: a full build string: 0.8.73.0-r3ed8d12f-b719.
- @param test_name: test_name of Autotest test.
-
- @return Leaf level tuple of the category list and a dictionary for crash
- string details indexed on result instance idx. If no test_name is
- supplied then the dict will contain crash results for all tests
- executed in that build (and the category list will be None).
-
- """
- netbook_dict = self._crashes.setdefault(netbook, {})
- board_dict = netbook_dict.setdefault(board, {})
- build_dict = board_dict.setdefault(build, {})
- if test_name:
- return build_dict.setdefault(test_name, (set(), {}))
- return None, build_dict
-
- def AddToCrashTree(self, netbook, board, build, test_name, result_idx,
- job_tag):
- """Update the crash strings container from the results summary file.
-
- This crash strings container is optimized to support the two views
- that consume it: the waterfall summary view (platform x build) and the
- details view (platform x build x test_name).
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
- @param board: one of our boards: x86-generic-full, x86-mario-full-chromeos,
- ...
- @param build: a full build string: 0.8.73.0-r3ed8d12f-b719.
- @param test_name: test_name of Autotest test.
- @param result_idx: unique identifier for a test result instance.
- @param job_tag: path base for finding test result file under CAutotest
- results.
-
- """
- crash_strings = self._LookupCrashDict(netbook, board, build, test_name)[1]
- if result_idx in crash_strings:
- # The same test result can be attempted for entry because we alias
- # some test results under multiple categories.
- return
- job_crashes = self._test_summaries.RetrieveTestSummary(job_tag, test_name)
- if job_crashes and job_crashes.get('crashes'):
- crash_strings[result_idx] = job_crashes['crashes']
-
- def AddCrashCategory(self, netbook, board, build, test_name, category):
- """Keep a list of the categories assigned to this test_name.
-
- Used to hyperlink from a crash summary to it's related details page.
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
- @param board: one of our boards: x86-generic-full, x86-mario-full-chromeos,
- ...
- @param build: a full build string: 0.8.73.0-r3ed8d12f-b719.
- @param test_name: test_name of Autotest test.
- @param category: test category (test prefix or job suffix usually).
-
- """
- categories = self._LookupCrashDict(netbook, board, build, test_name)[0]
- categories.add(category)
-
- def _CollapseCrashes(self, crash_strings):
- """Helper to change 'chrome sig 11' 'chrome sig 11' to '(2) chrome sig 11'
-
- This is needed to tighten up the popups when large crash quantities
- are encountered.
- """
- counted = {}
- for c in crash_strings:
- counted[c] = counted.setdefault(c, 0) + 1
- collapsed = []
- for c, v in counted.iteritems():
- collapsed.append('(%s) %s' % (v, c))
- return sorted(collapsed)
-
- def GetBuildCrashSummary(self, netbook, board, build, category=None):
- """Used to populate the waterfall summary page with a crash count.
-
- The cells on the waterfall summary page reflect all crashes found from
- all tests in all categories for a given platform/build combination. The
- cells on a category summary (kernel) page reflect crashes found only in a
- specific category for a given platform/build combination.
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
- @param board: one of our boards: x86-generic-full, x86-mario-full-chromeos,
- ...
- @param build: a full build string: 0.8.73.0-r3ed8d12f-b719.
- @param category: test category (test prefix or job suffix usually), None
- for all.
-
- @return Tuple used in watefall summary views of: crash_details list, a
- count of crashes and the first category for a hyperlink. The result
- tuple is ([], 0, None) if no crashes were discovered.
-
- """
- crashes = []
- n = 0
- all_categories = set()
- build_crashes = self._LookupCrashDict(netbook, board, build)[1]
- if build_crashes:
- for test_name in sorted(build_crashes):
- categories, crash_dict = build_crashes[test_name]
- if (not category) or (category and category in categories):
- new_crashes = sorted(list(itertools.chain(*crash_dict.values())))
- if new_crashes:
- crashes.append((test_name, self._CollapseCrashes(new_crashes)))
- n += len(new_crashes)
- all_categories |= categories
- if not crashes:
- return crashes, n, None
- return crashes, n, sorted(all_categories)[0]
-
- def GetBuildTestCrashSummary(self, netbook, board, build, test_name):
- """Used to populate the test details pages with crash counts per test.
-
- The cells on each category details page reflect crashes found only in a
- specific test for a given platform/build combination.
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
- @param board: one of our boards: x86-generic-full, x86-mario-full-chromeos,
- ...
- @param build: a full build string: 0.8.73.0-r3ed8d12f-b719.
- @param test_name: name of a specific test.
-
- @return Tuple used in details views: list of crash details and a count of
- crashes.
-
- """
- test_crashes = self._LookupCrashDict(netbook, board, build, test_name)[1]
- if not test_crashes:
- return [], 0
- new_crashes = sorted(list(itertools.chain(*test_crashes.values())))
- return self._CollapseCrashes(new_crashes), len(new_crashes)
-
- def GetTestSummaries(self):
- """Test Summaries are used to probe the crash cache for crashes in a job.
-
- Used by the test result summary emailer to include crashes and a link to
- each job with a crash for research.
-
- @return The TestSummaryInfo object that is shared.
-
- """
- return self._test_summaries
-
-
-class AutotestDashView(object):
- """View used by table_gen, plot_gen and dash_email."""
-
- class __impl:
- """Nested class implements code wrapped by singleton."""
-
- def __init__(self):
- """Setup common data structures for the models.
-
- Uses dashboard cache files for some (crash/timing) data.
- """
- self._dash_config = None
- self._cursor = readonly_connection.cursor()
- self._common_where = (
- "WHERE job_owner = %s"
- " AND NOT ISNULL(test_finished_time)"
- " AND NOT ISNULL(job_finished_time)"
- " AND NOT test_name LIKE 'CLIENT_JOB%%'"
- " AND NOT test_name LIKE 'boot.%%'"
- " AND NOT test_name IN ('Autotest.install', 'cleanup_test', "
- " 'lmbench', 'logfile.monitor', 'repair', "
- " 'sleeptest', 'tsc')")
-
- # Used in expression parsing - have slightly different captures.
-
- # New test suite job regex.
- # (x86-zgb)-release/((R19)-1913.0.0-a1-b1539) \
- # /(bvt)/(network_DisableInterface)
- self._jobname_testsuite_parse = re.compile(
- '(.*?)-release/((R\d+)-.*?)/(.*?)/(.*)')
-
- # (x86-alex-r18)-(R18-1660.71.0-a1-b75)_(bvt)
- # (x86-generic-full)-(R20-2112.0.0-a1-b2686)_(browsertests)
- self._jobname_parse = re.compile(
- '([\w-]+-[fr][\w]+)-(.*[.-][\d]+\.[\d]+\.[\d]+)(?:-[ar][\w]+-b[\d]+)?'
- '_([\w_]*)')
-
- # (R18-1660.71.0)-a1-b75
- # r18-1660.122.0_to_(R18-1660.122.0)-a1-b146
- self._subjob_parse = re.compile(
- '.*(R[\d]+-[\d]+\.[\d]+\.[\d]+)')
-
- self._board_parse = re.compile(
- '(x86|tegra2)-(.+)-(r[\d]+)')
-
- # (R19-1913.0.0)
- # (R19-1913.0.0)-a1-b1539
- # (0.8.73.0)-r3ed8d12f-b719.
- self._shortbuild1_parse = re.compile('(R[\d]+-[\d]+\.[\d]+\.[\d]+)')
- self._shortbuild2_parse = re.compile('([\d]+\.[\d]+\.[\d]+\.[\d]+)')
-
- self._release_parse = re.compile('r[\d]')
-
- # Test creation info (author, path).
- # Populated by QueryAutotests().
- self._autotests = {}
-
- self.TEST_TREE_DOC = """
- The test_tree is a dictionary of:
- +netbooks (netbook_HP_INDIANA, netbook_DELL_L13, ...)
- |+boards (x86-generic-bin, arm-generic-rel, ...)
- |+categories (platform, desktopui, bvt, regression, ...)
- |+test_name (platform_BootPerfServer, ...)
- |+builds (0.8.67.0-re7c459dc-b1135)
- |+indices [test_idx]
- This is our lookup index into tests.
- Populate netbooks by QueryNetbooks() and the rest by QueryTests().
- """
- self._test_tree = {}
-
- self.UI_CATEGORIES_DOC = """
- Many categories will not show on the dash but are available
- for use by emailer so must remain in the data model.
- This will be a subset of upper levels of the test_tree.
- +netbooks (netbook_HP_INDIANA, netbook_DELL_L13, ...)
- |+boards (x86-generic-bin, arm-generic-rel, ...)
- |+categories (platform, desktopui, bvt, regression, ...)
- Populated by QueryTests().
- """
- self._ui_categories = {}
-
- # Short subset of job_id's.
- # Populated by QueryBuilds().
- self._job_ids = set()
-
- self.BUILDS_DOC = """
- A little tree to track the builds for each of the boards.
- +board
- |-dictionary mapping short to long for lookups
- Populated by QueryBuilds().
- """
- self._builds = {}
-
- self.BUILD_TREE_DOC = """
- Need a tree of builds to show which builds were actually
- run for each netbook, board.
- +netbooks (netbook_HP_INDIANA, netbook_DELL_L13, ...)
- |+boards (x86-generic-bin, arm-generic-rel, ...)
- |+categories
- |+build
- |+aggregate build info
- |-latest job_id "job_id"
- |-earliest job started time "start"
- |-last job finished time "finish"
- |-number of 'GOOD' test names "ngood"
- |-number of total test names "ntotal"
- Used in netbook->board->category views.
- Populate netbooks by QueryNetbooks() and the rest by QueryTests().
- """
- self._build_tree = {}
-
- self.TESTS_DOC = """
- The test list is a dictionary of:
- +test_idx
- |+-test_name.
- -tag.
- -hostname.
- -status.
- -start (job_started_time)
- -finish (job_finished_time)
- -attr
- -experimental (optional boolean)
- The actual test data. Used to fill popups and test status.
- Populated by QueryTests().
- """
- self._tests = {}
-
- self.CRASHES_DOC = """
- The crashes object is a container of crashes that may be
- filtered by build, category and test_name.
- """
- self._crashes = None
-
- self.PERF_KEYVALS_DOC = """
- For performance counters.
- +netbooks
- |+boards
- |+test_name
- |+key
- |+build
- |+(value_list, test_idx_list, iteration_list)
- Used in plotting.
- Populated by QueryKeyVals().
- """
- self._perf_keyvals = {}
-
- # Constant for date comparisons
- self._null_datetime = datetime.datetime(2010, 1, 1)
- self._null_timedelta = datetime.timedelta(0)
-
- # Performance optimization
- self._formatted_time_cache = {}
- self._last_updated = datetime.datetime.ctime(datetime.datetime.now())
-
- def CrashSetup(self, dash_base_dir):
- """Set up the crash view.
-
- @param dash_base_dir: root of the cache directory for crash results.
-
- """
- self._crashes = CrashDashView(dash_base_dir)
-
- def GetCrashes(self):
- """Accessor for crash data and functions."""
- return self._crashes
-
- def SetDashConfig(self, dash_config):
- """Some preprocessing of dash_config.
-
- @param dash_config: dictionary of dash config entries.
-
- """
- self._dash_config = dash_config
- if 'customboardfilter' in dash_config:
- self._board_parse = re.compile(
- '(%s)-(.+)-(r[\d]+)' % dash_config['customboardfilter'])
-
- def GetAutotestInfo(self, name):
- """Return author and path of an autotest test.
-
- @param name: Autotest test_name.
-
- @return 2-Tuple of (author_name, test_path) used to locate test code.
-
- """
- name = name.split(".")[0]
- author = ""
- test_path = ""
- server_test_name = name + "Server"
- if name in self._autotests:
- author, test_path = self._autotests[name]
- elif server_test_name in self._autotests:
- author, test_path = self._autotests[server_test_name]
- if test_path:
- # convert server/tests/netpipe/control.srv --> server/tests/netpipe
- test_path = os.path.dirname(test_path)
- return author, test_path
-
- @property
- def netbooks(self):
- """Return a list of known netbooks - some may have not run tests.
-
- @return Unsorted List of all known netbooks (with netbook_ prefix). Some
- of these may have no tests run against them.
-
- """
- return self._test_tree.keys()
-
- def GetNetbooksWithBoardType(self, board):
- """Return list of netbooks with tests run under board.
-
- @param board: one of our boards: x86-generic-full,
- x86-mario-full-chromeos, ...
-
- @return Sorted List of netbooks (with netbook_ prefix) that have
- completed tests associated with the given board.
-
- """
- netbooks = self._test_tree.keys()
- netbooks.sort()
- return [n for n in netbooks if board in self._test_tree[n]]
-
- def GetNetbooksWithBoardTypeCategory(self, board, category):
- """Return list of netbooks with tests under board and category.
-
- @param board: one of our boards: x86-generic-full,
- x86-mario-full-chromeos, ...
- @param category: a test group: bvt, regression, desktopui, graphics, ...
-
- @return Sorted List of netbooks (with netbook_ prefix) that have
- completed tests of given category with the given board.
-
- """
- netbooks = self._build_tree.keys()
- netbooks.sort()
- return [n for n in netbooks if (
- board in self._build_tree[n] and
- category in self._build_tree[n][board])]
-
- def GetBoardTypes(self):
- """Return list of boards found.
-
- @return Unsorted List of all known boards: x86-generic-full,
- x86-mario-full-chromeos, ...
-
- """
- return self._builds.keys()
-
- def GetNetbookBoardTypes(self, netbook):
- """Return list of boards used in the given netbook.
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
-
- @return Unsorted List of boards which have completed tests on the
- given netbook (with netbook_ prefix).
-
- """
- if netbook in self._build_tree:
- return self._build_tree[netbook].keys()
- return []
-
- def GetAllBuilds(self):
- """Return list of all known builds that we used.
-
- @return Unsorted Set of unique builds across all boards.
-
- """
- results = set()
- for build_dict in self._builds.itervalues():
- for b in build_dict.itervalues():
- results.add(b)
- return results
-
- def GetBoardtypeBuilds(
- self, board, limit=LAST_N_JOBS_LIMIT, asc=False):
- """Return list of builds with tests run in the given board.
-
- @param board: one of our boards: x86-generic-full,
- x86-mario-full-chromeos, ...
- @param limit: common to truncate the build list for display.
- @param asc: if False, sort descending (tables) else ascending (plots).
-
- @return Sorted List of builds from with attempted jobs. These builds may
- NOT have associated test results if no tests completed on a netbook.
-
- """
- results = sorted(
- self._builds[board].values(),
- cmp=dash_util.BuildNumberCmp,
- reverse=asc)
- build_count = min(len(results), limit)
- if asc:
- return results[len(results)-build_count:]
- else:
- return results[:build_count]
-
- def GetBuilds(self, netbook, board, category):
- """Return list of builds with tests run in the given netbook.
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
- @param board: one of our boards: x86-generic-full,
- x86-mario-full-chromeos, ...
- @param category: a test group: bvt, regression, desktopui, graphics, ...
-
- @return Sorted List of builds with jobs attempted on the given netbook,
- board combination with tests attempted in the given category.
- Again, tests may not have been completed thus there may be no
- corresponding test results.
-
- """
- results = []
- if not netbook in self._build_tree:
- return results
- if (board in self._build_tree[netbook] and
- category in self._build_tree[netbook][board]):
- for b in self._build_tree[netbook][board][category].iterkeys():
- if not b in self._builds[board]:
- logging.warning(
- "***DATA WARNING: %s not in build list for %s, %s, %s!",
- b, netbook, board, category)
- else:
- results.append(self._builds[board][b])
- results.sort(dash_util.BuildNumberCmp)
- return results
-
- def GetUICategories(self, netbook, board):
- """Return categories for DASH UI of tests run in netbook - board.
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
- @param board: one of our boards: x86-generic-full,
- x86-mario-full-chromeos, ...
-
- @return Unsorted List of the UI categories (bvt, desktopui, ...) of tests
- with completed results run against the given netbook and board.
-
- """
- return list(self._ui_categories[netbook][board])
-
- def GetCategories(self, netbook, board, regex=None):
- """Return categories of tests run in netbook - board.
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
- @param board: one of our boards: x86-generic-full,
- x86-mario-full-chromeos, ...
- @param regex: optional match filter for categories.
-
- @return Unsorted List of the categories (bvt, desktopui, ...) of tests
- with completed results run against the given netbook and board.
-
- """
- if netbook in self._test_tree and board in self._test_tree[netbook]:
- if not regex:
- return self._test_tree[netbook][board].keys()
- return [c for c in self._test_tree[netbook][board].keys()
- if re.match(regex, c)]
- else:
- return []
-
- def GetTestNames(self, netbook, board, category):
- """Return unique test names run in netbook - board - category.
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
- @param board: one of our boards: x86-generic-full,
- x86-mario-full-chromeos, ...
- @param category: a test group: bvt, regression, desktopui, graphics, ...
-
- @return Unsorted or empty List of test names for building a table listing
- all tests in the given category with completed results on the given
- netbook and board.
-
- """
- if category not in self._test_tree[netbook][board]:
- return []
- return self._test_tree[netbook][board][category].keys()
-
- def GetTestNamesInBuild(self, netbook, board, category, build, regex=None):
- """Return the unique test names like GetTestNames() but for 1 build.
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
- @param board: one of our boards: x86-generic-full,
- x86-mario-full-chromeos, ...
- @param category: a test group: bvt, regression, desktopui, graphics, ...
- @param build: a full build string: 0.8.73.0.
- @param regex: optional match filter for test name.
-
- @return Sorted or empty List of the test names in the given category and
- given build with completed test results against the given netbook
- and board.
-
- """
- results = []
- try:
- for t, b in self._test_tree[netbook][board][category].iteritems():
- if build in b:
- if regex and not re.match(regex, t):
- continue
- results.append(t)
- results.sort()
- except KeyError:
- logging.debug("***KeyError: %s, %s, %s.", netbook, board, category)
- return results
-
- def GetCategorySummary(self, netbook, board, category, build):
- """Return ngood and ntotal for the given job.
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
- @param board: one of our boards: x86-generic-full,
- x86-mario-full-chromeos, ...
- @param category: a test group: bvt, regression, desktopui, graphics, ...
- @param build: a full build string: 0.8.73.0.
-
- @return 4-Tuple:
- -Boolean: True if the job was attempted
- -Boolean: True if all server jobs GOOD
- -Integer: number of tests completed GOOD
- -Integer: number of tests completed
-
- """
- ngood = 0
- ntotal = 0
- xngood = 0
- xntotal = 0
- job_attempted = False
- job_good = False
- if build in self._build_tree[netbook][board][category]:
- job = self._build_tree[netbook][board][category][build]
- ngood = job["ngood"]
- ntotal = job["ntotal"]
- xngood = job["xngood"]
- xntotal = job["xntotal"]
- job_good = job["server_good"]
- job_attempted = True
- return job_attempted, job_good, ngood, ntotal, xngood, xntotal
-
- def TestDetailIterator(self, netbook, board, category, build):
- """Common iterator for looking through test details.
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
- @param board: one of our boards: x86-generic-full,
- x86-mario-full-chromeos, ...
- @param category: a test group: bvt, regression, desktopui, graphics, ...
- @param build: a full build string: 0.8.73.0-r3ed8d12f-b719.
-
- @return Iterative test details using the Python generator (yield)
- mechanism.
-
- """
- tests = self.GetTestNamesInBuild(netbook, board, category, build)
- if not tests:
- return
- for test in tests:
- test_details = self.GetTestDetails(netbook, board, category, test,
- build)
- if not test_details:
- continue
- for t in test_details:
- yield t
-
- def GetCategoryKernel(self, netbook, board, category, build):
- """Return string name of the kernel version like: 2.6.38.3+.
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
- @param board: one of our boards: x86-generic-full,
- x86-mario-full-chromeos, ...
- @param category: a test group: bvt, regression, desktopui, graphics, ...
- @param build: a full build string: 0.8.73.0-r3ed8d12f-b719.
-
- @return String name of the kernel tested. If multiple kernels tested emit
- the one most used with a marker string.
-
- """
- kernel_votes = dash_util.SimpleCounter()
- for t in self.TestDetailIterator(netbook, board, category, build):
- kernel_votes.Push(t['attr'].get('sysinfo-uname', None))
- return kernel_votes.MaxKey()
-
- def GetCategoryFailedTests(self, netbook, board, category, build):
- """Return list of failed tests for easy popup display.
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
- @param board: one of our boards: x86-generic-full,
- x86-mario-full-chromeos, ...
- @param category: a test group: bvt, regression, desktopui, graphics, ...
- @param build: a full build string: 0.8.73.0-r3ed8d12f-b719.
-
- @return Tuple including a List of unique test names of failed tests,
- and a List of unique test names of experimental failed tests.
-
- """
- failed_tests = set()
- xfailed_tests = set()
- for t in self.TestDetailIterator(netbook, board, category, build):
- if t['status'] != 'GOOD':
- if t.get('experimental'):
- xfailed_tests.add(t['test_name'])
- else:
- failed_tests.add(t['test_name'])
- return (', '.join(sorted(failed_tests)),
- ', '.join(sorted(xfailed_tests)))
-
- def GetJobTimes(self, netbook, board, category, build):
- """Return job_start_time, job_end_time and elapsed for the given job.
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
- @param board: one of our boards: x86-generic-full,
- x86-mario-full-chromeos, ...
- @param category: a test group: bvt, regression, desktopui, graphics, ...
- @param build: a build on this netbook and board.
-
- @return 3-Tuple of datetime.datetime,datetime.datetime,datetime.timedelta
- for started_datetime, finished_datetime, elapsed_datetime. All are
- calculated across multiple jobs by looking at completed test results
- and choosing the earliest start time and the latest finish time.
-
- """
- job_started = self._null_datetime
- job_finished = self._null_datetime
- job_elapsed = self._null_timedelta
- if build in self._build_tree[netbook][board][category]:
- job = self._build_tree[netbook][board][category][build]
- job_started = job["start"]
- job_finished = job["finish"]
- job_elapsed = job_finished - job_started
- return job_started, job_finished, job_elapsed
-
- def GetJobTimesNone(self, netbook, board, category, build):
- """Translate null_datetime from GetJobTimes() to None.
-
- @param netbook: same as for GetJobTimes() above.
- @param board: same as for GetJobTimes() above.
- @param category: same as for GetJobTimes() above.
- @param build: same as for GetJobTimes() above.
-
- @return Same as for GetJobTimes() above, except with null datetimes
- translated to None.
-
- """
- job_started, job_finished, job_elapsed = self.GetJobTimes(
- netbook, board, category, build)
- if job_started == self._null_datetime:
- job_started = None
- if job_finished == self._null_datetime:
- job_finished = None
- if job_elapsed == self._null_datetime:
- job_elapsed = None
- return job_started, job_finished, job_elapsed
-
- def GetFormattedJobTimes(self, netbook, board, category, build):
- """Return job_start_time, job_end_time and elapsed in datetime format.
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
- @param board: one of our boards: x86-generic-full,
- x86-mario-full-chromeos, ...
- @param category: a test group: bvt, regression, desktopui, graphics, ...
- @param build: a build on this netbook and board.
-
- @return 3-Tuple of stringified started_datetime, finished_datetime, and
- elapsed_datetime. Returns a common string when invalid or no datetime
- was found.
-
- """
- time_key = (netbook, board, category, build)
- if time_key in self._formatted_time_cache:
- return self._formatted_time_cache[time_key]
- job_started_str = UNKNOWN_TIME_STR
- job_finished_str = UNKNOWN_TIME_STR
- job_elapsed_str = UNKNOWN_TIME_STR
- job_started, job_finished, job_elapsed = self.GetJobTimes(*time_key)
- if job_started != self._null_datetime:
- job_started_str = datetime.datetime.ctime(job_started)
- if job_finished != self._null_datetime:
- job_finished_str = datetime.datetime.ctime(job_finished)
- if job_elapsed != self._null_timedelta:
- job_elapsed_str = str(job_elapsed)
- result = (job_started_str, job_finished_str, job_elapsed_str)
- self._formatted_time_cache[time_key] = result
- return result
-
- def GetFormattedLastUpdated(self):
- """Return a string used to date-time stamp our reports."""
- return self._last_updated
-
- def GetTestDetails(self, netbook, board, category, test_name, build):
- """Return tests details for a given test_name x build cell.
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
- @param board: one of our boards: x86-generic-full,
- x86-mario-full-chromeos, ...
- @param category: a test group: bvt, regression, desktopui, graphics, ...
- @param test_name: test_name of Autotest test.
- @param build: a build on this netbook and board.
-
- @return Sorted or empty List of multiple test dictionaries for test
- instances in the given category that completed on the given netbook
- and board in the given build. The test dictionaries include common
- fields 'test_name', 'tag', 'hostname', 'status', experimental
- (optional) and an embedded dictionary of varying attributes under
- 'attr'.
-
- """
- test_details = []
- if build in self._test_tree[netbook][board][category][test_name]:
- test_index_list = (list(
- self._test_tree[netbook][board][category][test_name][build][0]))
- test_index_list.sort(reverse=True)
- for i in test_index_list:
- test_details.append(self._tests[i])
- return test_details
-
- def GetTestFromIdx(self, idx):
- """Returns all details about 1 specific instance of 1 test result.
-
- @param idx: unique index of the test result.
-
- @return A Dictionary with attributes for a test result instance including
- tag.
-
- """
- return self._tests[str(idx)]
-
- def GetPlatformKeyValTests(self, netbook, board):
- """Return list of tests that have keyvals for a given netbook and board.
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
- @param board: one of our boards: x86-generic-full,
- x86-mario-full-chromeos, ...
-
- @return None or a sorted list of the test names with keyvals.
-
- """
- if (not netbook in self._perf_keyvals or
- not board in self._perf_keyvals[netbook]):
- return []
- return sorted(self._perf_keyvals[netbook][board].keys())
-
- def GetTestKeys(self, netbook, board, test_name):
- """Return list of test keys with values for a given netbook and board.
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
- @param board: one of our boards: x86-generic-full,
- x86-mario-full-chromeos, ...
- @param test_name: test_name of Autotest test.
-
- @return None or a sorted list of the test keys with keyvals.
-
- """
- if (not netbook in self._perf_keyvals or
- not board in self._perf_keyvals[netbook] or
- not test_name in self._perf_keyvals[netbook][board]):
- return None
- return sorted(self._perf_keyvals[netbook][board][test_name].keys())
-
- def GetTestKeyVals(self, netbook, board, test_name):
- """Return keyvals for one test over our queried jobs/builds.
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
- @param board: one of our boards: x86-generic-full,
- x86-mario-full-chromeos, ...
- @param test_name: test_name of Autotest test.
-
- @return None or a dictionary of the performance key-values recorded during
- a completed test with the given netbook, board, test_name. The
- keyvals are from the overall set of jobs/builds that were discovered
- when querying the last n jobs/builds. The dictionary has keys of
- each performance key recorded and build dictionary. The build
- dictionary has keys of each build with the named performance key
- recorded and a value of the value list. The value list is a 2-Tuple
- of Lists. One is a list of the perf values and the other is a list
- of corresponding test_idx that may be used to look up job/test
- details from the point in a graphed plot.
-
- """
- if (not netbook in self._perf_keyvals or
- not board in self._perf_keyvals[netbook] or
- not test_name in self._perf_keyvals[netbook][board]):
- return None
- return self._perf_keyvals[netbook][board][test_name]
-
- def GetTestPerfVals(self, netbook, board, test_name, key):
- """Return values for one test/key over our queried jobs/builds.
-
- @param netbook: one of our netbooks with the netbook_ prefix:
- netbook_DELL_L13, netbook_ANDRETTI, ...
- @param board: one of our boards: x86-generic-full,
- x86-mario-full-chromeos, ...
- @param test_name: test_name of Autotest test.
- @param key: autotest perf key.
-
- @return None or a dictionary of the performance values recorded during
- a completed test with the given netbook, board, test_name, key. The
- vals are from the overall set of jobs/builds that were discovered
- when querying the last n jobs/builds. The dictionary has keys of
- each build with the named performance key recorded and a value of
- the value list. The value list is a 2-Tuple of Lists. One is a
- list of the perf values and the other is a list of corresponding
- test_idx that may be used to look up job/test details from the
- point in a graphed plot.
-
- """
- keyvals = self.GetTestKeyVals(netbook, board, test_name)
- if keyvals and key in keyvals:
- return keyvals[key]
- return None
-
- def ParseBoard(self, board):
- """Return simple board without release identifier: e.g. x86-mario.
-
- Examples:
- stumpy-r16
- tegra2-kaen-r16
- tegra2-seaboard
- tegra2-seaboard-rc
- x86-alex-r16
- x86-generic-full
- x86-mario-r15
-
- @param board: one of our boards: x86-generic-full,
- x86-mario-full-chromeos, ...
-
- @return (simple_board, release_if_found).
-
- """
- m = re.match(self._board_parse, board)
- if m and m.lastindex == 3:
- parsed_board = '%s-%s' % (m.group(1), m.group(2))
- release = m.group(3)
- else:
- split_board = board.split('-')
- found = False
- parsed_board = []
- for i in xrange(len(split_board)):
- if re.match(self._release_parse, split_board[i]):
- found = True
- break
- parsed_board.append(split_board[i])
- parsed_board = '-'.join(parsed_board)
- if found:
- release = split_board[i]
- else:
- release = None
- return parsed_board, release
-
- def ParseTestName(self, test_name):
- """Return category of test_name or a general category.
-
- A test_name defines a category if it has a prefix.
-
- @param test_name: test_name from autotest db.
-
- @return Single token test category.
-
- """
- if test_name.find(".") > 0:
- test_name = test_name.split(".")[0]
- if test_name.find("_") > 0:
- category = test_name.split("_")[0]
- else:
- category = "autotest"
-
- return category
-
- def ParseJobName(self, job_name):
- """Return board - build# and job_suffix from the job_name.
-
- @param job_name: complex string created by test_scheduler from a build
- image.
-
- @return Tuple of: board, a build#, a job group, and a bool True if
- experimental.
-
- """
- # (x86-zgb)-release/((R19)-1913.0.0-a1-b1539) \
- # /(bvt)/(network_DisableInterface)
- match = self._jobname_testsuite_parse.match(job_name)
- if match:
- board, build, milestone, suite, suffix = match.groups()
- # Put board in the format expected, e.g. x86-mario-r19.
- board = '%s-%s' % (board, milestone.lower())
- # Remove possible sequence artifacts, e.g.'-a1-b1539'
- build = self.ParseSimpleBuild(build)
- return (board, build, self.TranslateSuffix(suite),
- suffix.startswith('experimental_'))
-
- # Old suite style parsing.
- m = re.match(self._jobname_parse, job_name)
- if not m or not len(m.groups()) == 3:
- return None, None, None, None
-
- board, subjob, suffix = m.group(1, 2, 3)
-
- # Subjob handles multi-build au test job names.
- n = re.match(self._subjob_parse, subjob)
-
- # Old full_build is: build#-token-build_sequence#.
- # Trim token-buildsequence since it's not used anymore.
- if not n or not len(n.groups()) == 1:
- full_build = None
- else:
- full_build = n.group(1)
-
- # Translate new -release naming into old x86-mario-r19 style.
- # x86-alex-release-R19-1979.0.0-a1-b1798_security =>
- # x86-alex-r19-R19-1979.0.0-a1-b1798_security.
- # Also change x86-generic-full-R19... to x86-generic-full-r19.
- for terminator, replacement in [('-release', '-r'), ('-full', '-full-r')]:
- if board.endswith(terminator):
- board = board.replace(terminator,
- replacement + full_build.split('-')[0][1:])
-
- return board, full_build, self.TranslateSuffix(suffix), False
-
- def ParseSimpleBuild(self, build):
- """Strip out the 0.x.y.z portion of the build.
-
- Strip the core build string (1913.0.0 or 0.8.73.0) from a full build
- string:
- -(R19-1913.0.0)
- -(R19-1913.0.0)-a1-b1539
- -(0.8.73.0)-r3ed8d12f-b719.
-
- @param build: long/full build string.
-
- @return The simple numeric build number.
-
- """
- for p in [self._shortbuild1_parse, self._shortbuild2_parse]:
- m = re.match(p, build)
- if m:
- break
- if m:
- parsed_build = m.group(1)
- else:
- parsed_build = build
- return parsed_build
-
- def LoadFromDB(self, job_limit=None):
- """Initial queries from the db for test tables.
-
- @param job_limit: Limit query to last n jobs.
-
- """
- diag = dash_util.DebugTiming()
- if not self._autotests:
- self.QueryAutotests()
- if not self._test_tree:
- self.QueryNetbooks()
- if not self._builds:
- self.QueryBuilds(job_limit)
- if not self._tests:
- self.QueryTests()
- del diag
-
- def LoadPerfFromDB(self, job_limit=None):
- """Initial queries from db for perf checking.
-
- @param job_limit: Limit query to last n jobs.
-
- """
- diag = dash_util.DebugTiming()
- self.LoadFromDB(job_limit)
- if not self._perf_keyvals:
- self.QueryKeyVals()
- del diag
-
- def QueryDjangoSession(self):
- """Get current row count from django_session table."""
- query = [
- "SELECT COUNT(*)",
- "FROM django_session"]
- self._cursor.execute(" ".join(query))
- return self._cursor.fetchone()[0]
-
- def QueryAutotests(self):
- """Get test attributes like author and path."""
- query = [
- "SELECT name, path, author",
- "FROM afe_autotests",
- "ORDER BY name"]
- self._cursor.execute(" ".join(query))
- for (name, path, author) in self._cursor.fetchall():
- self._autotests[name] = [author, path]
-
- def ScrubNetbook(self, netbook):
- """Remove deprecated platform prefixes.
-
- If present, older prefixes are removed
- and the string is lower-cased for one
- common platform convention.
-
- @param netbook: platform from Autotest (e.g. ALEX).
-
- @return String with a 'scrubbed' netbook value.
-
- """
- for prefix in LEGACY_PLATFORM_PREFIXES:
- if netbook.startswith(prefix):
- netbook = netbook[len(prefix):]
- return netbook.lower()
-
- def QueryNetbooks(self):
- """Get the netbooks know the to database."""
- query = [
- "SELECT name",
- "FROM afe_labels",
- "WHERE platform AND NOT invalid",
- "UNION",
- "SELECT distinct machine_group as name",
- "FROM tko_machines",
- "ORDER BY name"]
- self._cursor.execute(" ".join(query))
- for (netbook,) in self._cursor.fetchall():
- netbook = self.ScrubNetbook(netbook)
- self._test_tree[netbook] = {}
- self._ui_categories[netbook] = {}
- self._build_tree[netbook] = {}
-
- def TranslateSuffix(self, suffix):
- """Allow processing of suffixes for aligning test suites.
-
- @param suffix: The suffix to process.
-
- @return The translated suffix.
-
- """
- if not suffix:
- return suffix
- if suffix.startswith('kernel_'):
- return KERNELTEST_TAG
- elif suffix.startswith('enroll_'):
- return 'enterprise'
- elif suffix.endswith('_bvt'):
- return 'bvt'
- return suffix
-
- def QueryBuilds(self, job_limit=None):
- """Get the boards and builds (jobs) to use.
-
- @param job_limit: Limit query to last n jobs.
-
- """
- query = [
- "SELECT j.id, j.name, complete",
- "FROM afe_jobs AS j",
- "INNER JOIN afe_host_queue_entries AS q ON j.id = q.job_id",
- "WHERE owner = %s",
- " AND NOT name LIKE '%%-try'"
- " AND NOT name LIKE '%%-test_suites/%%'"
- "ORDER BY created_on DESC",
- "LIMIT %s"]
- if not job_limit:
- job_limit = LAST_N_JOBS_LIMIT
- params = [AUTOTEST_USER, job_limit]
- self._cursor.execute(" ".join(query), params)
-
- incomplete_jobnames = set()
- jobname_to_jobid = {}
-
- for job_id, name, complete in self._cursor.fetchall():
- board, full_build, suffix, _ = self.ParseJobName(name)
- if not board or not full_build or not suffix:
- logging.debug("Ignoring invalid: %s (%s, %s, %s).", name, board,
- full_build, suffix)
- continue
- if (self._dash_config and
- 'blacklistboards' in self._dash_config and
- board in self._dash_config['blacklistboards']):
- continue
- str_job_id = str(job_id)
- self._job_ids.add(str_job_id)
- build_list_dict = self._builds.setdefault(board, {})
- build_list_dict.setdefault(full_build, full_build)
- # Track job_id's to later prune incomplete jobs.
- # Use a name common to all the jobs.
- tracking_name = "%s-%s" % (board, full_build)
- suffixes = jobname_to_jobid.setdefault(tracking_name, {})
- ids = suffixes.setdefault(suffix, [])
- ids.append(str_job_id)
- if not complete:
- incomplete_jobnames.add(name)
-
- # Now go prune out incomplete jobs.
- for name in incomplete_jobnames:
- logging.debug("Ignoring incomplete: %s.", name)
- board, full_build, suffix, _ = self.ParseJobName(name)
- tracking_name = "%s-%s" % (board, full_build)
- if suffix in jobname_to_jobid[tracking_name]:
- for str_job_id in jobname_to_jobid[tracking_name][suffix]:
- if str_job_id in self._job_ids:
- self._job_ids.remove(str_job_id)
- del jobname_to_jobid[tracking_name][suffix]
- if not jobname_to_jobid[tracking_name]:
- if full_build in self._builds[board]:
- del self._builds[board][full_build]
-
- def QueryTests(self):
- """Get and stash the test data and attributes."""
- if not self._job_ids:
- return
- query = [
- "SELECT test_idx, test_name, job_name, job_tag, afe_job_id,",
- " platform, hostname, status, job_started_time,"
- " job_finished_time, reason",
- "FROM tko_test_view_2",
- self._common_where,
- " AND afe_job_id IN (%s)" % ",".join(self._job_ids),
- "ORDER BY job_idx DESC"]
- params = [AUTOTEST_USER]
- self._cursor.execute(" ".join(query), params)
- results = self._cursor.fetchall()
- for (idx, test_name, job_name, job_tag, job_id, netbook,
- hostname, status, start_time, finish_time, reason) in results:
- netbook = self.ScrubNetbook(netbook)
- if not netbook in self.netbooks:
- continue
- board, full_build, suffix, experimental = self.ParseJobName(job_name)
- if not board or not full_build or not suffix:
- continue
- category = self.ParseTestName(test_name)
- ui_categories = self._ui_categories[netbook].setdefault(board, set())
- if suffix in SUFFIXES_TO_SHOW:
- ui_categories.add(suffix)
- if suffix in GTEST_SUFFIXES:
- category = suffix
- category_dict = self._test_tree[netbook].setdefault(board, {})
-
- if not test_name == SERVER_JOB:
- attribute_dict = {}
- attribute_dict["test_name"] = test_name
- attribute_dict["hostname"] = hostname
- attribute_dict["tag"] = job_tag
- attribute_dict["status"] = status
- attribute_dict["experimental"] = experimental
- attribute_dict["attr"] = {}
- if not status == 'GOOD':
- attribute_dict["attr"]["reason"] = reason[:min(len(reason), 120)]
- self._tests[str(idx)] = attribute_dict
- ui_categories.add(category)
- categories_to_load = [category, suffix]
- # Add crash string summary details.
- self._crashes.AddToCrashTree(netbook, board, full_build, test_name,
- idx, job_tag)
- else:
- categories_to_load = [suffix]
-
- for c in categories_to_load:
- self._crashes.AddCrashCategory(
- netbook, board, full_build, test_name, c)
- # Add earliest job started time and latest job_finished_time.
- build_board_dict = self._build_tree[netbook].setdefault(
- board, {})
- build_category_dict = build_board_dict.setdefault(c, {})
- build_info = build_category_dict.setdefault(full_build, {
- "start": datetime.datetime.now(),
- "finish": datetime.datetime(2010, 1, 1),
- "ngood": 0, # number of good test results excluding experimental
- "ntotal": 0, # number of tests run excluding experimental
- "xngood": 0, # number of good experimental test results
- "xntotal": 0, # number of experimental tests run
- "server_good": True})
-
- if start_time < build_info["start"]:
- build_info["start"] = start_time
- if finish_time > build_info["finish"]:
- build_info["finish"] = finish_time
-
- if test_name == SERVER_JOB:
- if not status == "GOOD":
- build_info["server_good"] = False
- continue
-
- test_dict = category_dict.setdefault(c, {})
- build_dict = test_dict.setdefault(test_name, {})
- test_index_list = build_dict.setdefault(full_build, [set(), None])
-
- test_index_list[0].add(str(idx))
- if not test_index_list[1]:
- test_index_list[1] = status
- if not experimental:
- build_info["ntotal"] += 1
- if status == "GOOD":
- build_info["ngood"] += 1
- else:
- build_info["xntotal"] += 1
- if status == "GOOD":
- build_info["xngood"] += 1
- elif not status == "GOOD" and test_index_list[1] == "GOOD":
- test_index_list[1] = status
- if not experimental:
- build_info["ngood"] -= 1
- else:
- build_info["xngood"] -= 1
-
- query = [
- "SELECT test_idx, attribute, value",
- "FROM tko_test_attributes",
- "WHERE test_idx in ('",
- "','".join(self._tests.keys()),
- "')",
- "ORDER BY test_idx, attribute"]
- self._cursor.execute(" ".join(query))
- for i, a, v in self._cursor.fetchall():
- self._tests[str(i)]["attr"][a] = v
-
- def QueryKeyVals(self):
- """Get the performance keyvals."""
- if not self._job_ids:
- return
- query = [
- "SELECT platform, job_name, hostname, test_idx, test_name, ",
- " iteration_key, iteration, iteration_value",
- "FROM tko_perf_view_2 as p",
- "INNER JOIN tko_jobs as j USING (job_idx)",
- self._common_where,
- " AND afe_job_id IN (%s)" % ",".join(self._job_ids),
- "AND NOT ISNULL(iteration_value)",
- "ORDER BY platform, job_name, test_name, iteration_key, ",
- "test_idx, iteration"]
- params = [AUTOTEST_USER]
- self._cursor.execute(" ".join(query), params)
- results = self._cursor.fetchall()
- for (netbook, job_name, hostname, test_idx, test_name,
- iteration_key, iteration, iteration_value) in results:
- if iteration_value < 0:
- continue
- board, full_build, _, _ = self.ParseJobName(job_name)
- if not board or not full_build:
- continue
- netbook = self.ScrubNetbook(netbook)
- board_dict = self._perf_keyvals.setdefault(netbook, {})
- test_dict = board_dict.setdefault(board, {})
- key_dict = test_dict.setdefault(test_name, {})
- build_dict = key_dict.setdefault(iteration_key, {})
- value_list = build_dict.setdefault(full_build, ([], [], [], []))
- value_list[0].append(iteration_value)
- # Save test_idx to retrieve job details from data point.
- value_list[1].append(test_idx)
- # Save iteration for plotting.
- value_list[2].append(iteration)
- # Save hostname for plotting.
- value_list[3].append(hostname)
-
- def ShowDataModel(self):
- """Dump the data model for inspection."""
- dash_util.ShowStructure("AUTOTESTS", self._autotests)
- dash_util.ShowStructure("NETBOOKS", self.netbooks)
- dash_util.ShowStructure("BOARDS", self.GetBoardTypes())
- dash_util.ShowStructure("JOB IDS", self._job_ids)
- dash_util.ShowStructure(
- "BUILDS", self._builds, self.BUILDS_DOC)
- dash_util.ShowStructure(
- "BUILD TREE", self._build_tree, self.BUILD_TREE_DOC)
- dash_util.ShowStructure(
- "UI CATEGORIES", self._ui_categories, self.UI_CATEGORIES_DOC)
- dash_util.ShowStructure(
- "TEST TREE", self._test_tree, self.TEST_TREE_DOC)
- dash_util.ShowStructure(
- "TESTS WITH ATTRIBUTES", self._tests, self.TESTS_DOC)
- dash_util.ShowStructure(
- "CRASHES WITH TESTS AND CATEGORIES", self._crashes, self.CRASHES_DOC)
-
- def ShowKeyVals(self):
- """Dump the perf keyvals for inspection."""
- dash_util.ShowStructure(
- "PERF KEYVALS", self._perf_keyvals, self.PERF_KEYVALS_DOC)
-
-
- # Instance reference for singleton behavior.
- __instance = None
- __refs = 0
-
- def __init__(self):
- if AutotestDashView.__instance is None:
- AutotestDashView.__instance = AutotestDashView.__impl()
-
- self.__dict__["_AutotestDashView__instance"] = AutotestDashView.__instance
- AutotestDashView.__refs += 1
-
- def __del__(self):
- AutotestDashView.__refs -= 1
- if not AutotestDashView.__instance is None and AutotestDashView.__refs == 0:
- del AutotestDashView.__instance
- AutotestDashView.__instance = None
-
- def __getattr__(self, attr):
- return getattr(AutotestDashView.__instance, attr)
-
- def __setattr__(self, attr, value):
- return setattr(AutotestDashView.__instance, attr, value)
-
-
-class SummaryRanges(object):
- """Each summary page needs list of each: boards, netbooks, builds."""
-
- def __init__(self, dash_view, category, summary_limit):
- self._summary_ranges = {}
- self._summary_kernels = {}
- boards = dash_view.GetBoardTypes() # Some may not have tests.
- for board in boards:
- netbooks = dash_view.GetNetbooksWithBoardTypeCategory(
- board, category)
- netbooks.sort()
-
- # If all jobs were filtered by the summary, do not show that netbook
- # in the summary (it will show in the details view).
- build_numbers = dash_view.GetBoardtypeBuilds(board, summary_limit)
- build_number_set = set(build_numbers)
- netbooks_copy = netbooks[:]
- for netbook in netbooks_copy:
- netbook_set = set(dash_view.GetBuilds(
- netbook, board, category))
- if (build_number_set - netbook_set) == build_number_set:
- netbooks.remove(netbook)
-
- if netbooks:
- self._summary_ranges[board] = (netbooks, build_numbers)
- # Populate kernels
- self._summary_kernels[board] = {}
- for n in netbooks:
- self._summary_kernels[board][n] = {}
- for b in build_numbers:
- self._summary_kernels[board][n][b] = dash_view.GetCategoryKernel(
- n, board, category, b)
-
- def GetBoards(self):
- """Gets all boards."""
- boards = self._summary_ranges.keys()
- boards.sort()
- return boards
-
- def GetNetbooks(self, board):
- """Gets all netbooks associated with a board.
-
- @param board: The associated board.
-
- @return A list of netbooks associated with the specified board.
-
- """
- return self._summary_ranges[board][0]
-
- def GetBuildNumbers(self, board):
- """Gets all build numbers associated with a board.
-
- @param board: The associated board.
-
- @return A list of build numbers associated with the specified board.
-
- """
- return self._summary_ranges[board][1]
-
- def GetKernel(self, board, netbook, build):
- """Gets the kernel assocaited with a board/netbook/build combination.
-
- @param board: The associated board.
- @param netbook: The associated netbook.
- @param build: The associated build.
-
- @return The kernel associated with the specified board/netbook/build
- combination.
-
- """
- try:
- return self._summary_kernels[board][netbook][build]
- except KeyError:
- return None
diff --git a/site_utils/dashboard/external/README.gprof2dot.py b/site_utils/dashboard/external/README.gprof2dot.py
deleted file mode 100644
index 2c2a8b8..0000000
--- a/site_utils/dashboard/external/README.gprof2dot.py
+++ /dev/null
@@ -1 +0,0 @@
-Retrieved from: http://code.google.com/p/jrfonseca/wiki/Gprof2Dot
diff --git a/site_utils/dashboard/external/__init__.py b/site_utils/dashboard/external/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/site_utils/dashboard/external/__init__.py
+++ /dev/null
diff --git a/site_utils/dashboard/external/gprof2dot.py b/site_utils/dashboard/external/gprof2dot.py
deleted file mode 100644
index 62b2276..0000000
--- a/site_utils/dashboard/external/gprof2dot.py
+++ /dev/null
@@ -1,2780 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright 2008-2009 Jose Fonseca
-#
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU Lesser General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-
-"""Generate a dot graph from the output of several profilers."""
-
-__author__ = "Jose Fonseca"
-
-__version__ = "1.0"
-
-
-import sys
-import math
-import os.path
-import re
-import textwrap
-import optparse
-import xml.parsers.expat
-
-
-try:
- # Debugging helper module
- import debug
-except ImportError:
- pass
-
-
-def times(x):
- return u"%u\xd7" % (x,)
-
-def percentage(p):
- return "%.02f%%" % (p*100.0,)
-
-def add(a, b):
- return a + b
-
-def equal(a, b):
- if a == b:
- return a
- else:
- return None
-
-def fail(a, b):
- assert False
-
-
-tol = 2 ** -23
-
-def ratio(numerator, denominator):
- try:
- ratio = float(numerator)/float(denominator)
- except ZeroDivisionError:
- # 0/0 is undefined, but 1.0 yields more useful results
- return 1.0
- if ratio < 0.0:
- if ratio < -tol:
- sys.stderr.write('warning: negative ratio (%s/%s)\n' % (numerator, denominator))
- return 0.0
- if ratio > 1.0:
- if ratio > 1.0 + tol:
- sys.stderr.write('warning: ratio greater than one (%s/%s)\n' % (numerator, denominator))
- return 1.0
- return ratio
-
-
-class UndefinedEvent(Exception):
- """Raised when attempting to get an event which is undefined."""
-
- def __init__(self, event):
- Exception.__init__(self)
- self.event = event
-
- def __str__(self):
- return 'unspecified event %s' % self.event.name
-
-
-class Event(object):
- """Describe a kind of event, and its basic operations."""
-
- def __init__(self, name, null, aggregator, formatter = str):
- self.name = name
- self._null = null
- self._aggregator = aggregator
- self._formatter = formatter
-
- def __eq__(self, other):
- return self is other
-
- def __hash__(self):
- return id(self)
-
- def null(self):
- return self._null
-
- def aggregate(self, val1, val2):
- """Aggregate two event values."""
- assert val1 is not None
- assert val2 is not None
- return self._aggregator(val1, val2)
-
- def format(self, val):
- """Format an event value."""
- assert val is not None
- return self._formatter(val)
-
-
-CALLS = Event("Calls", 0, add, times)
-SAMPLES = Event("Samples", 0, add)
-SAMPLES2 = Event("Samples", 0, add)
-
-TIME = Event("Time", 0.0, add, lambda x: '(' + str(x) + ')')
-TIME_RATIO = Event("Time ratio", 0.0, add, lambda x: '(' + percentage(x) + ')')
-TOTAL_TIME = Event("Total time", 0.0, fail)
-TOTAL_TIME_RATIO = Event("Total time ratio", 0.0, fail, percentage)
-
-
-class Object(object):
- """Base class for all objects in profile which can store events."""
-
- def __init__(self, events=None):
- if events is None:
- self.events = {}
- else:
- self.events = events
-
- def __hash__(self):
- return id(self)
-
- def __eq__(self, other):
- return self is other
-
- def __contains__(self, event):
- return event in self.events
-
- def __getitem__(self, event):
- try:
- return self.events[event]
- except KeyError:
- raise UndefinedEvent(event)
-
- def __setitem__(self, event, value):
- if value is None:
- if event in self.events:
- del self.events[event]
- else:
- self.events[event] = value
-
-
-class Call(Object):
- """A call between functions.
-
- There should be at most one call object for every pair of functions.
- """
-
- def __init__(self, callee_id):
- Object.__init__(self)
- self.callee_id = callee_id
- self.ratio = None
- self.weight = None
-
-
-class Function(Object):
- """A function."""
-
- def __init__(self, id, name):
- Object.__init__(self)
- self.id = id
- self.name = name
- self.module = None
- self.process = None
- self.calls = {}
- self.called = None
- self.weight = None
- self.cycle = None
-
- def add_call(self, call):
- if call.callee_id in self.calls:
- sys.stderr.write('warning: overwriting call from function %s to %s\n' % (str(self.id), str(call.callee_id)))
- self.calls[call.callee_id] = call
-
- def get_call(self, callee_id):
- if not callee_id in self.calls:
- call = Call(callee_id)
- call[SAMPLES] = 0
- call[SAMPLES2] = 0
- call[CALLS] = 0
- self.calls[callee_id] = call
- return self.calls[callee_id]
-
- # TODO: write utility functions
-
- def __repr__(self):
- return self.name
-
-
-class Cycle(Object):
- """A cycle made from recursive function calls."""
-
- def __init__(self):
- Object.__init__(self)
- # XXX: Do cycles need an id?
- self.functions = set()
-
- def add_function(self, function):
- assert function not in self.functions
- self.functions.add(function)
- # XXX: Aggregate events?
- if function.cycle is not None:
- for other in function.cycle.functions:
- if function not in self.functions:
- self.add_function(other)
- function.cycle = self
-
-
-class Profile(Object):
- """The whole profile."""
-
- def __init__(self):
- Object.__init__(self)
- self.functions = {}
- self.cycles = []
-
- def add_function(self, function):
- if function.id in self.functions:
- sys.stderr.write('warning: overwriting function %s (id %s)\n' % (function.name, str(function.id)))
- self.functions[function.id] = function
-
- def add_cycle(self, cycle):
- self.cycles.append(cycle)
-
- def validate(self):
- """Validate the edges."""
-
- for function in self.functions.itervalues():
- for callee_id in function.calls.keys():
- assert function.calls[callee_id].callee_id == callee_id
- if callee_id not in self.functions:
- sys.stderr.write('warning: call to undefined function %s from function %s\n' % (str(callee_id), function.name))
- del function.calls[callee_id]
-
- def find_cycles(self):
- """Find cycles using Tarjan's strongly connected components algorithm."""
-
- # Apply the Tarjan's algorithm successively until all functions are visited
- visited = set()
- for function in self.functions.itervalues():
- if function not in visited:
- self._tarjan(function, 0, [], {}, {}, visited)
- cycles = []
- for function in self.functions.itervalues():
- if function.cycle is not None and function.cycle not in cycles:
- cycles.append(function.cycle)
- self.cycles = cycles
- if 0:
- for cycle in cycles:
- sys.stderr.write("Cycle:\n")
- for member in cycle.functions:
- sys.stderr.write("\tFunction %s\n" % member.name)
-
- def _tarjan(self, function, order, stack, orders, lowlinks, visited):
- """Tarjan's strongly connected components algorithm.
-
- See also:
- - http://en.wikipedia.org/wiki/Tarjan's_strongly_connected_components_algorithm
- """
-
- visited.add(function)
- orders[function] = order
- lowlinks[function] = order
- order += 1
- pos = len(stack)
- stack.append(function)
- for call in function.calls.itervalues():
- callee = self.functions[call.callee_id]
- # TODO: use a set to optimize lookup
- if callee not in orders:
- order = self._tarjan(callee, order, stack, orders, lowlinks, visited)
- lowlinks[function] = min(lowlinks[function], lowlinks[callee])
- elif callee in stack:
- lowlinks[function] = min(lowlinks[function], orders[callee])
- if lowlinks[function] == orders[function]:
- # Strongly connected component found
- members = stack[pos:]
- del stack[pos:]
- if len(members) > 1:
- cycle = Cycle()
- for member in members:
- cycle.add_function(member)
- return order
-
- def call_ratios(self, event):
- # Aggregate for incoming calls
- cycle_totals = {}
- for cycle in self.cycles:
- cycle_totals[cycle] = 0.0
- function_totals = {}
- for function in self.functions.itervalues():
- function_totals[function] = 0.0
- for function in self.functions.itervalues():
- for call in function.calls.itervalues():
- if call.callee_id != function.id:
- callee = self.functions[call.callee_id]
- function_totals[callee] += call[event]
- if callee.cycle is not None and callee.cycle is not function.cycle:
- cycle_totals[callee.cycle] += call[event]
-
- # Compute the ratios
- for function in self.functions.itervalues():
- for call in function.calls.itervalues():
- assert call.ratio is None
- if call.callee_id != function.id:
- callee = self.functions[call.callee_id]
- if callee.cycle is not None and callee.cycle is not function.cycle:
- total = cycle_totals[callee.cycle]
- else:
- total = function_totals[callee]
- call.ratio = ratio(call[event], total)
-
- def integrate(self, outevent, inevent):
- """Propagate function time ratio allong the function calls.
-
- Must be called after finding the cycles.
-
- See also:
- - http://citeseer.ist.psu.edu/graham82gprof.html
- """
-
- # Sanity checking
- assert outevent not in self
- for function in self.functions.itervalues():
- assert outevent not in function
- assert inevent in function
- for call in function.calls.itervalues():
- assert outevent not in call
- if call.callee_id != function.id:
- assert call.ratio is not None
-
- # Aggregate the input for each cycle
- for cycle in self.cycles:
- total = inevent.null()
- for function in self.functions.itervalues():
- total = inevent.aggregate(total, function[inevent])
- self[inevent] = total
-
- # Integrate along the edges
- total = inevent.null()
- for function in self.functions.itervalues():
- total = inevent.aggregate(total, function[inevent])
- self._integrate_function(function, outevent, inevent)
- self[outevent] = total
-
- def _integrate_function(self, function, outevent, inevent):
- if function.cycle is not None:
- return self._integrate_cycle(function.cycle, outevent, inevent)
- else:
- if outevent not in function:
- total = function[inevent]
- for call in function.calls.itervalues():
- if call.callee_id != function.id:
- total += self._integrate_call(call, outevent, inevent)
- function[outevent] = total
- return function[outevent]
-
- def _integrate_call(self, call, outevent, inevent):
- assert outevent not in call
- assert call.ratio is not None
- callee = self.functions[call.callee_id]
- subtotal = call.ratio *self._integrate_function(callee, outevent, inevent)
- call[outevent] = subtotal
- return subtotal
-
- def _integrate_cycle(self, cycle, outevent, inevent):
- if outevent not in cycle:
-
- # Compute the outevent for the whole cycle
- total = inevent.null()
- for member in cycle.functions:
- subtotal = member[inevent]
- for call in member.calls.itervalues():
- callee = self.functions[call.callee_id]
- if callee.cycle is not cycle:
- subtotal += self._integrate_call(call, outevent, inevent)
- total += subtotal
- cycle[outevent] = total
-
- # Compute the time propagated to callers of this cycle
- callees = {}
- for function in self.functions.itervalues():
- if function.cycle is not cycle:
- for call in function.calls.itervalues():
- callee = self.functions[call.callee_id]
- if callee.cycle is cycle:
- try:
- callees[callee] += call.ratio
- except KeyError:
- callees[callee] = call.ratio
-
- for member in cycle.functions:
- member[outevent] = outevent.null()
-
- for callee, call_ratio in callees.iteritems():
- ranks = {}
- call_ratios = {}
- partials = {}
- self._rank_cycle_function(cycle, callee, 0, ranks)
- self._call_ratios_cycle(cycle, callee, ranks, call_ratios, set())
- partial = self._integrate_cycle_function(cycle, callee, call_ratio, partials, ranks, call_ratios, outevent, inevent)
- assert partial == max(partials.values())
- assert not total or abs(1.0 - partial/(call_ratio*total)) <= 0.001
-
- return cycle[outevent]
-
- def _rank_cycle_function(self, cycle, function, rank, ranks):
- if function not in ranks or ranks[function] > rank:
- ranks[function] = rank
- for call in function.calls.itervalues():
- if call.callee_id != function.id:
- callee = self.functions[call.callee_id]
- if callee.cycle is cycle:
- self._rank_cycle_function(cycle, callee, rank + 1, ranks)
-
- def _call_ratios_cycle(self, cycle, function, ranks, call_ratios, visited):
- if function not in visited:
- visited.add(function)
- for call in function.calls.itervalues():
- if call.callee_id != function.id:
- callee = self.functions[call.callee_id]
- if callee.cycle is cycle:
- if ranks[callee] > ranks[function]:
- call_ratios[callee] = call_ratios.get(callee, 0.0) + call.ratio
- self._call_ratios_cycle(cycle, callee, ranks, call_ratios, visited)
-
- def _integrate_cycle_function(self, cycle, function, partial_ratio, partials, ranks, call_ratios, outevent, inevent):
- if function not in partials:
- partial = partial_ratio*function[inevent]
- for call in function.calls.itervalues():
- if call.callee_id != function.id:
- callee = self.functions[call.callee_id]
- if callee.cycle is not cycle:
- assert outevent in call
- partial += partial_ratio*call[outevent]
- else:
- if ranks[callee] > ranks[function]:
- callee_partial = self._integrate_cycle_function(cycle, callee, partial_ratio, partials, ranks, call_ratios, outevent, inevent)
- call_ratio = ratio(call.ratio, call_ratios[callee])
- call_partial = call_ratio*callee_partial
- try:
- call[outevent] += call_partial
- except UndefinedEvent:
- call[outevent] = call_partial
- partial += call_partial
- partials[function] = partial
- try:
- function[outevent] += partial
- except UndefinedEvent:
- function[outevent] = partial
- return partials[function]
-
- def aggregate(self, event):
- """Aggregate an event for the whole profile."""
-
- total = event.null()
- for function in self.functions.itervalues():
- try:
- total = event.aggregate(total, function[event])
- except UndefinedEvent:
- return
- self[event] = total
-
- def ratio(self, outevent, inevent):
- assert outevent not in self
- assert inevent in self
- for function in self.functions.itervalues():
- assert outevent not in function
- assert inevent in function
- function[outevent] = ratio(function[inevent], self[inevent])
- for call in function.calls.itervalues():
- assert outevent not in call
- if inevent in call:
- call[outevent] = ratio(call[inevent], self[inevent])
- self[outevent] = 1.0
-
- def prune(self, node_thres, edge_thres):
- """Prune the profile"""
-
- # compute the prune ratios
- for function in self.functions.itervalues():
- try:
- function.weight = function[TOTAL_TIME_RATIO]
- except UndefinedEvent:
- pass
-
- for call in function.calls.itervalues():
- callee = self.functions[call.callee_id]
-
- if TOTAL_TIME_RATIO in call:
- # handle exact cases first
- call.weight = call[TOTAL_TIME_RATIO]
- else:
- try:
- # make a safe estimate
- call.weight = min(function[TOTAL_TIME_RATIO], callee[TOTAL_TIME_RATIO])
- except UndefinedEvent:
- pass
-
- # prune the nodes
- for function_id in self.functions.keys():
- function = self.functions[function_id]
- if function.weight is not None:
- if function.weight < node_thres:
- del self.functions[function_id]
-
- # prune the egdes
- for function in self.functions.itervalues():
- for callee_id in function.calls.keys():
- call = function.calls[callee_id]
- if callee_id not in self.functions or call.weight is not None and call.weight < edge_thres:
- del function.calls[callee_id]
-
- def dump(self):
- for function in self.functions.itervalues():
- sys.stderr.write('Function %s:\n' % (function.name,))
- self._dump_events(function.events)
- for call in function.calls.itervalues():
- callee = self.functions[call.callee_id]
- sys.stderr.write(' Call %s:\n' % (callee.name,))
- self._dump_events(call.events)
- for cycle in self.cycles:
- sys.stderr.write('Cycle:\n')
- self._dump_events(cycle.events)
- for function in cycle.functions:
- sys.stderr.write(' Function %s\n' % (function.name,))
-
- def _dump_events(self, events):
- for event, value in events.iteritems():
- sys.stderr.write(' %s: %s\n' % (event.name, event.format(value)))
-
-
-class Struct:
- """Masquerade a dictionary with a structure-like behavior."""
-
- def __init__(self, attrs = None):
- if attrs is None:
- attrs = {}
- self.__dict__['_attrs'] = attrs
-
- def __getattr__(self, name):
- try:
- return self._attrs[name]
- except KeyError:
- raise AttributeError(name)
-
- def __setattr__(self, name, value):
- self._attrs[name] = value
-
- def __str__(self):
- return str(self._attrs)
-
- def __repr__(self):
- return repr(self._attrs)
-
-
-class ParseError(Exception):
- """Raised when parsing to signal mismatches."""
-
- def __init__(self, msg, line):
- self.msg = msg
- # TODO: store more source line information
- self.line = line
-
- def __str__(self):
- return '%s: %r' % (self.msg, self.line)
-
-
-class Parser:
- """Parser interface."""
-
- def __init__(self):
- pass
-
- def parse(self):
- raise NotImplementedError
-
-
-class LineParser(Parser):
- """Base class for parsers that read line-based formats."""
-
- def __init__(self, file):
- Parser.__init__(self)
- self._file = file
- self.__line = None
- self.__eof = False
- self.line_no = 0
-
- def readline(self):
- line = self._file.readline()
- if not line:
- self.__line = ''
- self.__eof = True
- else:
- self.line_no += 1
- self.__line = line.rstrip('\r\n')
-
- def lookahead(self):
- assert self.__line is not None
- return self.__line
-
- def consume(self):
- assert self.__line is not None
- line = self.__line
- self.readline()
- return line
-
- def eof(self):
- assert self.__line is not None
- return self.__eof
-
-
-XML_ELEMENT_START, XML_ELEMENT_END, XML_CHARACTER_DATA, XML_EOF = range(4)
-
-
-class XmlToken:
-
- def __init__(self, type, name_or_data, attrs = None, line = None, column = None):
- assert type in (XML_ELEMENT_START, XML_ELEMENT_END, XML_CHARACTER_DATA, XML_EOF)
- self.type = type
- self.name_or_data = name_or_data
- self.attrs = attrs
- self.line = line
- self.column = column
-
- def __str__(self):
- if self.type == XML_ELEMENT_START:
- return '<' + self.name_or_data + ' ...>'
- if self.type == XML_ELEMENT_END:
- return '</' + self.name_or_data + '>'
- if self.type == XML_CHARACTER_DATA:
- return self.name_or_data
- if self.type == XML_EOF:
- return 'end of file'
- assert 0
-
-
-class XmlTokenizer:
- """Expat based XML tokenizer."""
-
- def __init__(self, fp, skip_ws = True):
- self.fp = fp
- self.tokens = []
- self.index = 0
- self.final = False
- self.skip_ws = skip_ws
-
- self.character_pos = 0, 0
- self.character_data = ''
-
- self.parser = xml.parsers.expat.ParserCreate()
- self.parser.StartElementHandler = self.handle_element_start
- self.parser.EndElementHandler = self.handle_element_end
- self.parser.CharacterDataHandler = self.handle_character_data
-
- def handle_element_start(self, name, attributes):
- self.finish_character_data()
- line, column = self.pos()
- token = XmlToken(XML_ELEMENT_START, name, attributes, line, column)
- self.tokens.append(token)
-
- def handle_element_end(self, name):
- self.finish_character_data()
- line, column = self.pos()
- token = XmlToken(XML_ELEMENT_END, name, None, line, column)
- self.tokens.append(token)
-
- def handle_character_data(self, data):
- if not self.character_data:
- self.character_pos = self.pos()
- self.character_data += data
-
- def finish_character_data(self):
- if self.character_data:
- if not self.skip_ws or not self.character_data.isspace():
- line, column = self.character_pos
- token = XmlToken(XML_CHARACTER_DATA, self.character_data, None, line, column)
- self.tokens.append(token)
- self.character_data = ''
-
- def next(self):
- size = 16*1024
- while self.index >= len(self.tokens) and not self.final:
- self.tokens = []
- self.index = 0
- data = self.fp.read(size)
- self.final = len(data) < size
- try:
- self.parser.Parse(data, self.final)
- except xml.parsers.expat.ExpatError, e:
- #if e.code == xml.parsers.expat.errors.XML_ERROR_NO_ELEMENTS:
- if e.code == 3:
- pass
- else:
- raise e
- if self.index >= len(self.tokens):
- line, column = self.pos()
- token = XmlToken(XML_EOF, None, None, line, column)
- else:
- token = self.tokens[self.index]
- self.index += 1
- return token
-
- def pos(self):
- return self.parser.CurrentLineNumber, self.parser.CurrentColumnNumber
-
-
-class XmlTokenMismatch(Exception):
-
- def __init__(self, expected, found):
- self.expected = expected
- self.found = found
-
- def __str__(self):
- return '%u:%u: %s expected, %s found' % (self.found.line, self.found.column, str(self.expected), str(self.found))
-
-
-class XmlParser(Parser):
- """Base XML document parser."""
-
- def __init__(self, fp):
- Parser.__init__(self)
- self.tokenizer = XmlTokenizer(fp)
- self.consume()
-
- def consume(self):
- self.token = self.tokenizer.next()
-
- def match_element_start(self, name):
- return self.token.type == XML_ELEMENT_START and self.token.name_or_data == name
-
- def match_element_end(self, name):
- return self.token.type == XML_ELEMENT_END and self.token.name_or_data == name
-
- def element_start(self, name):
- while self.token.type == XML_CHARACTER_DATA:
- self.consume()
- if self.token.type != XML_ELEMENT_START:
- raise XmlTokenMismatch(XmlToken(XML_ELEMENT_START, name), self.token)
- if self.token.name_or_data != name:
- raise XmlTokenMismatch(XmlToken(XML_ELEMENT_START, name), self.token)
- attrs = self.token.attrs
- self.consume()
- return attrs
-
- def element_end(self, name):
- while self.token.type == XML_CHARACTER_DATA:
- self.consume()
- if self.token.type != XML_ELEMENT_END:
- raise XmlTokenMismatch(XmlToken(XML_ELEMENT_END, name), self.token)
- if self.token.name_or_data != name:
- raise XmlTokenMismatch(XmlToken(XML_ELEMENT_END, name), self.token)
- self.consume()
-
- def character_data(self, strip = True):
- data = ''
- while self.token.type == XML_CHARACTER_DATA:
- data += self.token.name_or_data
- self.consume()
- if strip:
- data = data.strip()
- return data
-
-
-class GprofParser(Parser):
- """Parser for GNU gprof output.
-
- See also:
- - Chapter "Interpreting gprof's Output" from the GNU gprof manual
- http://sourceware.org/binutils/docs-2.18/gprof/Call-Graph.html#Call-Graph
- - File "cg_print.c" from the GNU gprof source code
- http://sourceware.org/cgi-bin/cvsweb.cgi/~checkout~/src/gprof/cg_print.c?rev=1.12&cvsroot=src
- """
-
- def __init__(self, fp):
- Parser.__init__(self)
- self.fp = fp
- self.functions = {}
- self.cycles = {}
-
- def readline(self):
- line = self.fp.readline()
- if not line:
- sys.stderr.write('error: unexpected end of file\n')
- sys.exit(1)
- line = line.rstrip('\r\n')
- return line
-
- _int_re = re.compile(r'^\d+$')
- _float_re = re.compile(r'^\d+\.\d+$')
-
- def translate(self, mo):
- """Extract a structure from a match object, while translating the types in the process."""
- attrs = {}
- groupdict = mo.groupdict()
- for name, value in groupdict.iteritems():
- if value is None:
- value = None
- elif self._int_re.match(value):
- value = int(value)
- elif self._float_re.match(value):
- value = float(value)
- attrs[name] = (value)
- return Struct(attrs)
-
- _cg_header_re = re.compile(
- # original gprof header
- r'^\s+called/total\s+parents\s*$|' +
- r'^index\s+%time\s+self\s+descendents\s+called\+self\s+name\s+index\s*$|' +
- r'^\s+called/total\s+children\s*$|' +
- # GNU gprof header
- r'^index\s+%\s+time\s+self\s+children\s+called\s+name\s*$'
- )
-
- _cg_ignore_re = re.compile(
- # spontaneous
- r'^\s+<spontaneous>\s*$|'
- # internal calls (such as "mcount")
- r'^.*\((\d+)\)$'
- )
-
- _cg_primary_re = re.compile(
- r'^\[(?P<index>\d+)\]?' +
- r'\s+(?P<percentage_time>\d+\.\d+)' +
- r'\s+(?P<self>\d+\.\d+)' +
- r'\s+(?P<descendants>\d+\.\d+)' +
- r'\s+(?:(?P<called>\d+)(?:\+(?P<called_self>\d+))?)?' +
- r'\s+(?P<name>\S.*?)' +
- r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +
- r'\s\[(\d+)\]$'
- )
-
- _cg_parent_re = re.compile(
- r'^\s+(?P<self>\d+\.\d+)?' +
- r'\s+(?P<descendants>\d+\.\d+)?' +
- r'\s+(?P<called>\d+)(?:/(?P<called_total>\d+))?' +
- r'\s+(?P<name>\S.*?)' +
- r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +
- r'\s\[(?P<index>\d+)\]$'
- )
-
- _cg_child_re = _cg_parent_re
-
- _cg_cycle_header_re = re.compile(
- r'^\[(?P<index>\d+)\]?' +
- r'\s+(?P<percentage_time>\d+\.\d+)' +
- r'\s+(?P<self>\d+\.\d+)' +
- r'\s+(?P<descendants>\d+\.\d+)' +
- r'\s+(?:(?P<called>\d+)(?:\+(?P<called_self>\d+))?)?' +
- r'\s+<cycle\s(?P<cycle>\d+)\sas\sa\swhole>' +
- r'\s\[(\d+)\]$'
- )
-
- _cg_cycle_member_re = re.compile(
- r'^\s+(?P<self>\d+\.\d+)?' +
- r'\s+(?P<descendants>\d+\.\d+)?' +
- r'\s+(?P<called>\d+)(?:\+(?P<called_self>\d+))?' +
- r'\s+(?P<name>\S.*?)' +
- r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +
- r'\s\[(?P<index>\d+)\]$'
- )
-
- _cg_sep_re = re.compile(r'^--+$')
-
- def parse_function_entry(self, lines):
- parents = []
- children = []
-
- while True:
- if not lines:
- sys.stderr.write('warning: unexpected end of entry\n')
- line = lines.pop(0)
- if line.startswith('['):
- break
-
- # read function parent line
- mo = self._cg_parent_re.match(line)
- if not mo:
- if self._cg_ignore_re.match(line):
- continue
- sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)
- else:
- parent = self.translate(mo)
- parents.append(parent)
-
- # read primary line
- mo = self._cg_primary_re.match(line)
- if not mo:
- sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)
- return
- else:
- function = self.translate(mo)
-
- while lines:
- line = lines.pop(0)
-
- # read function subroutine line
- mo = self._cg_child_re.match(line)
- if not mo:
- if self._cg_ignore_re.match(line):
- continue
- sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)
- else:
- child = self.translate(mo)
- children.append(child)
-
- function.parents = parents
- function.children = children
-
- self.functions[function.index] = function
-
- def parse_cycle_entry(self, lines):
-
- # read cycle header line
- line = lines[0]
- mo = self._cg_cycle_header_re.match(line)
- if not mo:
- sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)
- return
- cycle = self.translate(mo)
-
- # read cycle member lines
- cycle.functions = []
- for line in lines[1:]:
- mo = self._cg_cycle_member_re.match(line)
- if not mo:
- sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)
- continue
- call = self.translate(mo)
- cycle.functions.append(call)
-
- self.cycles[cycle.cycle] = cycle
-
- def parse_cg_entry(self, lines):
- if lines[0].startswith("["):
- self.parse_cycle_entry(lines)
- else:
- self.parse_function_entry(lines)
-
- def parse_cg(self):
- """Parse the call graph."""
-
- # skip call graph header
- while not self._cg_header_re.match(self.readline()):
- pass
- line = self.readline()
- while self._cg_header_re.match(line):
- line = self.readline()
-
- # process call graph entries
- entry_lines = []
- while line != '\014': # form feed
- if line and not line.isspace():
- if self._cg_sep_re.match(line):
- self.parse_cg_entry(entry_lines)
- entry_lines = []
- else:
- entry_lines.append(line)
- line = self.readline()
-
- def parse(self):
- self.parse_cg()
- self.fp.close()
-
- profile = Profile()
- profile[TIME] = 0.0
-
- cycles = {}
- for index in self.cycles.iterkeys():
- cycles[index] = Cycle()
-
- for entry in self.functions.itervalues():
- # populate the function
- function = Function(entry.index, entry.name)
- function[TIME] = entry.self
- if entry.called is not None:
- function.called = entry.called
- if entry.called_self is not None:
- call = Call(entry.index)
- call[CALLS] = entry.called_self
- function.called += entry.called_self
-
- # populate the function calls
- for child in entry.children:
- call = Call(child.index)
-
- assert child.called is not None
- call[CALLS] = child.called
-
- if child.index not in self.functions:
- # NOTE: functions that were never called but were discovered by gprof's
- # static call graph analysis dont have a call graph entry so we need
- # to add them here
- missing = Function(child.index, child.name)
- function[TIME] = 0.0
- function.called = 0
- profile.add_function(missing)
-
- function.add_call(call)
-
- profile.add_function(function)
-
- if entry.cycle is not None:
- try:
- cycle = cycles[entry.cycle]
- except KeyError:
- sys.stderr.write('warning: <cycle %u as a whole> entry missing\n' % entry.cycle)
- cycle = Cycle()
- cycles[entry.cycle] = cycle
- cycle.add_function(function)
-
- profile[TIME] = profile[TIME] + function[TIME]
-
- for cycle in cycles.itervalues():
- profile.add_cycle(cycle)
-
- # Compute derived events
- profile.validate()
- profile.ratio(TIME_RATIO, TIME)
- profile.call_ratios(CALLS)
- profile.integrate(TOTAL_TIME, TIME)
- profile.ratio(TOTAL_TIME_RATIO, TOTAL_TIME)
-
- return profile
-
-
-class CallgrindParser(LineParser):
- """Parser for valgrind's callgrind tool.
-
- See also:
- - http://valgrind.org/docs/manual/cl-format.html
- """
-
- _call_re = re.compile('^calls=\s*(\d+)\s+((\d+|\+\d+|-\d+|\*)\s+)+$')
-
- def __init__(self, infile):
- LineParser.__init__(self, infile)
-
- # Textual positions
- self.position_ids = {}
- self.positions = {}
-
- # Numeric positions
- self.num_positions = 1
- self.cost_positions = ['line']
- self.last_positions = [0]
-
- # Events
- self.num_events = 0
- self.cost_events = []
-
- self.profile = Profile()
- self.profile[SAMPLES] = 0
-
- def parse(self):
- # read lookahead
- self.readline()
-
- self.parse_key('version')
- self.parse_key('creator')
- while self.parse_part():
- pass
- if not self.eof():
- sys.stderr.write('warning: line %u: unexpected line\n' % self.line_no)
- sys.stderr.write('%s\n' % self.lookahead())
-
- # compute derived data
- self.profile.validate()
- self.profile.find_cycles()
- self.profile.ratio(TIME_RATIO, SAMPLES)
- self.profile.call_ratios(CALLS)
- self.profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
-
- return self.profile
-
- def parse_part(self):
- if not self.parse_header_line():
- return False
- while self.parse_header_line():
- pass
- if not self.parse_body_line():
- return False
- while self.parse_body_line():
- pass
- return True
-
- def parse_header_line(self):
- return \
- self.parse_empty() or \
- self.parse_comment() or \
- self.parse_part_detail() or \
- self.parse_description() or \
- self.parse_event_specification() or \
- self.parse_cost_line_def() or \
- self.parse_cost_summary()
-
- _detail_keys = set(('cmd', 'pid', 'thread', 'part'))
-
- def parse_part_detail(self):
- return self.parse_keys(self._detail_keys)
-
- def parse_description(self):
- return self.parse_key('desc') is not None
-
- def parse_event_specification(self):
- event = self.parse_key('event')
- if event is None:
- return False
- return True
-
- def parse_cost_line_def(self):
- pair = self.parse_keys(('events', 'positions'))
- if pair is None:
- return False
- key, value = pair
- items = value.split()
- if key == 'events':
- self.num_events = len(items)
- self.cost_events = items
- if key == 'positions':
- self.num_positions = len(items)
- self.cost_positions = items
- self.last_positions = [0]*self.num_positions
- return True
-
- def parse_cost_summary(self):
- pair = self.parse_keys(('summary', 'totals'))
- if pair is None:
- return False
- return True
-
- def parse_body_line(self):
- return \
- self.parse_empty() or \
- self.parse_comment() or \
- self.parse_cost_line() or \
- self.parse_position_spec() or \
- self.parse_association_spec()
-
- __subpos_re = r'(0x[0-9a-fA-F]+|\d+|\+\d+|-\d+|\*)'
- _cost_re = re.compile(r'^' +
- __subpos_re + r'( +' + __subpos_re + r')*' +
- r'( +\d+)*' +
- '$')
-
- def parse_cost_line(self, calls=None):
- line = self.lookahead().rstrip()
- mo = self._cost_re.match(line)
- if not mo:
- return False
-
- function = self.get_function()
-
- values = line.split()
- assert len(values) <= self.num_positions + self.num_events
-
- positions = values[0 : self.num_positions]
- events = values[self.num_positions : ]
- events += ['0']*(self.num_events - len(events))
-
- for i in range(self.num_positions):
- position = positions[i]
- if position == '*':
- position = self.last_positions[i]
- elif position[0] in '-+':
- position = self.last_positions[i] + int(position)
- elif position.startswith('0x'):
- position = int(position, 16)
- else:
- position = int(position)
- self.last_positions[i] = position
-
- events = map(float, events)
-
- if calls is None:
- function[SAMPLES] += events[0]
- self.profile[SAMPLES] += events[0]
- else:
- callee = self.get_callee()
- callee.called += calls
-
- try:
- call = function.calls[callee.id]
- except KeyError:
- call = Call(callee.id)
- call[CALLS] = calls
- call[SAMPLES] = events[0]
- function.add_call(call)
- else:
- call[CALLS] += calls
- call[SAMPLES] += events[0]
-
- self.consume()
- return True
-
- def parse_association_spec(self):
- line = self.lookahead()
- if not line.startswith('calls='):
- return False
-
- _, values = line.split('=', 1)
- values = values.strip().split()
- calls = int(values[0])
- call_position = values[1:]
- self.consume()
-
- self.parse_cost_line(calls)
-
- return True
-
- _position_re = re.compile('^(?P<position>[cj]?(?:ob|fl|fi|fe|fn))=\s*(?:\((?P<id>\d+)\))?(?:\s*(?P<name>.+))?')
-
- _position_table_map = {
- 'ob': 'ob',
- 'fl': 'fl',
- 'fi': 'fl',
- 'fe': 'fl',
- 'fn': 'fn',
- 'cob': 'ob',
- 'cfl': 'fl',
- 'cfi': 'fl',
- 'cfe': 'fl',
- 'cfn': 'fn',
- 'jfi': 'fl',
- }
-
- _position_map = {
- 'ob': 'ob',
- 'fl': 'fl',
- 'fi': 'fl',
- 'fe': 'fl',
- 'fn': 'fn',
- 'cob': 'cob',
- 'cfl': 'cfl',
- 'cfi': 'cfl',
- 'cfe': 'cfl',
- 'cfn': 'cfn',
- 'jfi': 'jfi',
- }
-
- def parse_position_spec(self):
- line = self.lookahead()
-
- if line.startswith('jump=') or line.startswith('jcnd='):
- self.consume()
- return True
-
- mo = self._position_re.match(line)
- if not mo:
- return False
-
- position, id, name = mo.groups()
- if id:
- table = self._position_table_map[position]
- if name:
- self.position_ids[(table, id)] = name
- else:
- name = self.position_ids.get((table, id), '')
- self.positions[self._position_map[position]] = name
-
- self.consume()
- return True
-
- def parse_empty(self):
- if self.eof():
- return False
- line = self.lookahead()
- if line.strip():
- return False
- self.consume()
- return True
-
- def parse_comment(self):
- line = self.lookahead()
- if not line.startswith('#'):
- return False
- self.consume()
- return True
-
- _key_re = re.compile(r'^(\w+):')
-
- def parse_key(self, key):
- pair = self.parse_keys((key,))
- if not pair:
- return None
- key, value = pair
- return value
- line = self.lookahead()
- mo = self._key_re.match(line)
- if not mo:
- return None
- key, value = line.split(':', 1)
- if key not in keys:
- return None
- value = value.strip()
- self.consume()
- return key, value
-
- def parse_keys(self, keys):
- line = self.lookahead()
- mo = self._key_re.match(line)
- if not mo:
- return None
- key, value = line.split(':', 1)
- if key not in keys:
- return None
- value = value.strip()
- self.consume()
- return key, value
-
- def make_function(self, module, filename, name):
- # FIXME: module and filename are not being tracked reliably
- #id = '|'.join((module, filename, name))
- id = name
- try:
- function = self.profile.functions[id]
- except KeyError:
- function = Function(id, name)
- if module:
- function.module = os.path.basename(module)
- function[SAMPLES] = 0
- function.called = 0
- self.profile.add_function(function)
- return function
-
- def get_function(self):
- module = self.positions.get('ob', '')
- filename = self.positions.get('fl', '')
- function = self.positions.get('fn', '')
- return self.make_function(module, filename, function)
-
- def get_callee(self):
- module = self.positions.get('cob', '')
- filename = self.positions.get('cfi', '')
- function = self.positions.get('cfn', '')
- return self.make_function(module, filename, function)
-
-
-class OprofileParser(LineParser):
- """Parser for oprofile callgraph output.
-
- See also:
- - http://oprofile.sourceforge.net/doc/opreport.html#opreport-callgraph
- """
-
- _fields_re = {
- 'samples': r'(\d+)',
- '%': r'(\S+)',
- 'linenr info': r'(?P<source>\(no location information\)|\S+:\d+)',
- 'image name': r'(?P<image>\S+(?:\s\(tgid:[^)]*\))?)',
- 'app name': r'(?P<application>\S+)',
- 'symbol name': r'(?P<symbol>\(no symbols\)|.+?)',
- }
-
- def __init__(self, infile):
- LineParser.__init__(self, infile)
- self.entries = {}
- self.entry_re = None
-
- def add_entry(self, callers, function, callees):
- try:
- entry = self.entries[function.id]
- except KeyError:
- self.entries[function.id] = (callers, function, callees)
- else:
- callers_total, function_total, callees_total = entry
- self.update_subentries_dict(callers_total, callers)
- function_total.samples += function.samples
- self.update_subentries_dict(callees_total, callees)
-
- def update_subentries_dict(self, totals, partials):
- for partial in partials.itervalues():
- try:
- total = totals[partial.id]
- except KeyError:
- totals[partial.id] = partial
- else:
- total.samples += partial.samples
-
- def parse(self):
- # read lookahead
- self.readline()
-
- self.parse_header()
- while self.lookahead():
- self.parse_entry()
-
- profile = Profile()
-
- reverse_call_samples = {}
-
- # populate the profile
- profile[SAMPLES] = 0
- for _callers, _function, _callees in self.entries.itervalues():
- function = Function(_function.id, _function.name)
- function[SAMPLES] = _function.samples
- profile.add_function(function)
- profile[SAMPLES] += _function.samples
-
- if _function.application:
- function.process = os.path.basename(_function.application)
- if _function.image:
- function.module = os.path.basename(_function.image)
-
- total_callee_samples = 0
- for _callee in _callees.itervalues():
- total_callee_samples += _callee.samples
-
- for _callee in _callees.itervalues():
- if not _callee.self:
- call = Call(_callee.id)
- call[SAMPLES2] = _callee.samples
- function.add_call(call)
-
- # compute derived data
- profile.validate()
- profile.find_cycles()
- profile.ratio(TIME_RATIO, SAMPLES)
- profile.call_ratios(SAMPLES2)
- profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
-
- return profile
-
- def parse_header(self):
- while not self.match_header():
- self.consume()
- line = self.lookahead()
- fields = re.split(r'\s\s+', line)
- entry_re = r'^\s*' + r'\s+'.join([self._fields_re[field] for field in fields]) + r'(?P<self>\s+\[self\])?$'
- self.entry_re = re.compile(entry_re)
- self.skip_separator()
-
- def parse_entry(self):
- callers = self.parse_subentries()
- if self.match_primary():
- function = self.parse_subentry()
- if function is not None:
- callees = self.parse_subentries()
- self.add_entry(callers, function, callees)
- self.skip_separator()
-
- def parse_subentries(self):
- subentries = {}
- while self.match_secondary():
- subentry = self.parse_subentry()
- subentries[subentry.id] = subentry
- return subentries
-
- def parse_subentry(self):
- entry = Struct()
- line = self.consume()
- mo = self.entry_re.match(line)
- if not mo:
- raise ParseError('failed to parse', line)
- fields = mo.groupdict()
- entry.samples = int(mo.group(1))
- if 'source' in fields and fields['source'] != '(no location information)':
- source = fields['source']
- filename, lineno = source.split(':')
- entry.filename = filename
- entry.lineno = int(lineno)
- else:
- source = ''
- entry.filename = None
- entry.lineno = None
- entry.image = fields.get('image', '')
- entry.application = fields.get('application', '')
- if 'symbol' in fields and fields['symbol'] != '(no symbols)':
- entry.symbol = fields['symbol']
- else:
- entry.symbol = ''
- if entry.symbol.startswith('"') and entry.symbol.endswith('"'):
- entry.symbol = entry.symbol[1:-1]
- entry.id = ':'.join((entry.application, entry.image, source, entry.symbol))
- entry.self = fields.get('self', None) != None
- if entry.self:
- entry.id += ':self'
- if entry.symbol:
- entry.name = entry.symbol
- else:
- entry.name = entry.image
- return entry
-
- def skip_separator(self):
- while not self.match_separator():
- self.consume()
- self.consume()
-
- def match_header(self):
- line = self.lookahead()
- return line.startswith('samples')
-
- def match_separator(self):
- line = self.lookahead()
- return line == '-'*len(line)
-
- def match_primary(self):
- line = self.lookahead()
- return not line[:1].isspace()
-
- def match_secondary(self):
- line = self.lookahead()
- return line[:1].isspace()
-
-
-class HProfParser(LineParser):
- """Parser for java hprof output
-
- See also:
- - http://java.sun.com/developer/technicalArticles/Programming/HPROF.html
- """
-
- trace_re = re.compile(r'\t(.*)\((.*):(.*)\)')
- trace_id_re = re.compile(r'^TRACE (\d+):$')
-
- def __init__(self, infile):
- LineParser.__init__(self, infile)
- self.traces = {}
- self.samples = {}
-
- def parse(self):
- # read lookahead
- self.readline()
-
- while not self.lookahead().startswith('------'): self.consume()
- while not self.lookahead().startswith('TRACE '): self.consume()
-
- self.parse_traces()
-
- while not self.lookahead().startswith('CPU'):
- self.consume()
-
- self.parse_samples()
-
- # populate the profile
- profile = Profile()
- profile[SAMPLES] = 0
-
- functions = {}
-
- # build up callgraph
- for id, trace in self.traces.iteritems():
- if not id in self.samples: continue
- mtime = self.samples[id][0]
- last = None
-
- for func, file, line in trace:
- if not func in functions:
- function = Function(func, func)
- function[SAMPLES] = 0
- profile.add_function(function)
- functions[func] = function
-
- function = functions[func]
- # allocate time to the deepest method in the trace
- if not last:
- function[SAMPLES] += mtime
- profile[SAMPLES] += mtime
- else:
- c = function.get_call(last)
- c[SAMPLES2] += mtime
-
- last = func
-
- # compute derived data
- profile.validate()
- profile.find_cycles()
- profile.ratio(TIME_RATIO, SAMPLES)
- profile.call_ratios(SAMPLES2)
- profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
-
- return profile
-
- def parse_traces(self):
- while self.lookahead().startswith('TRACE '):
- self.parse_trace()
-
- def parse_trace(self):
- l = self.consume()
- mo = self.trace_id_re.match(l)
- tid = mo.group(1)
- last = None
- trace = []
-
- while self.lookahead().startswith('\t'):
- l = self.consume()
- match = self.trace_re.search(l)
- if not match:
- #sys.stderr.write('Invalid line: %s\n' % l)
- break
- else:
- function_name, file, line = match.groups()
- trace += [(function_name, file, line)]
-
- self.traces[int(tid)] = trace
-
- def parse_samples(self):
- self.consume()
- self.consume()
-
- while not self.lookahead().startswith('CPU'):
- rank, percent_self, percent_accum, count, traceid, method = self.lookahead().split()
- self.samples[int(traceid)] = (int(count), method)
- self.consume()
-
-
-class SysprofParser(XmlParser):
-
- def __init__(self, stream):
- XmlParser.__init__(self, stream)
-
- def parse(self):
- objects = {}
- nodes = {}
-
- self.element_start('profile')
- while self.token.type == XML_ELEMENT_START:
- if self.token.name_or_data == 'objects':
- assert not objects
- objects = self.parse_items('objects')
- elif self.token.name_or_data == 'nodes':
- assert not nodes
- nodes = self.parse_items('nodes')
- else:
- self.parse_value(self.token.name_or_data)
- self.element_end('profile')
-
- return self.build_profile(objects, nodes)
-
- def parse_items(self, name):
- assert name[-1] == 's'
- items = {}
- self.element_start(name)
- while self.token.type == XML_ELEMENT_START:
- id, values = self.parse_item(name[:-1])
- assert id not in items
- items[id] = values
- self.element_end(name)
- return items
-
- def parse_item(self, name):
- attrs = self.element_start(name)
- id = int(attrs['id'])
- values = self.parse_values()
- self.element_end(name)
- return id, values
-
- def parse_values(self):
- values = {}
- while self.token.type == XML_ELEMENT_START:
- name = self.token.name_or_data
- value = self.parse_value(name)
- assert name not in values
- values[name] = value
- return values
-
- def parse_value(self, tag):
- self.element_start(tag)
- value = self.character_data()
- self.element_end(tag)
- if value.isdigit():
- return int(value)
- if value.startswith('"') and value.endswith('"'):
- return value[1:-1]
- return value
-
- def build_profile(self, objects, nodes):
- profile = Profile()
-
- profile[SAMPLES] = 0
- for id, object in objects.iteritems():
- # Ignore fake objects (process names, modules, "Everything", "kernel", etc.)
- if object['self'] == 0:
- continue
-
- function = Function(id, object['name'])
- function[SAMPLES] = object['self']
- profile.add_function(function)
- profile[SAMPLES] += function[SAMPLES]
-
- for id, node in nodes.iteritems():
- # Ignore fake calls
- if node['self'] == 0:
- continue
-
- # Find a non-ignored parent
- parent_id = node['parent']
- while parent_id != 0:
- parent = nodes[parent_id]
- caller_id = parent['object']
- if objects[caller_id]['self'] != 0:
- break
- parent_id = parent['parent']
- if parent_id == 0:
- continue
-
- callee_id = node['object']
-
- assert objects[caller_id]['self']
- assert objects[callee_id]['self']
-
- function = profile.functions[caller_id]
-
- samples = node['self']
- try:
- call = function.calls[callee_id]
- except KeyError:
- call = Call(callee_id)
- call[SAMPLES2] = samples
- function.add_call(call)
- else:
- call[SAMPLES2] += samples
-
- # Compute derived events
- profile.validate()
- profile.find_cycles()
- profile.ratio(TIME_RATIO, SAMPLES)
- profile.call_ratios(SAMPLES2)
- profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
-
- return profile
-
-
-class SharkParser(LineParser):
- """Parser for MacOSX Shark output.
-
- Author: tom@dbservice.com
- """
-
- def __init__(self, infile):
- LineParser.__init__(self, infile)
- self.stack = []
- self.entries = {}
-
- def add_entry(self, function):
- try:
- entry = self.entries[function.id]
- except KeyError:
- self.entries[function.id] = (function, { })
- else:
- function_total, callees_total = entry
- function_total.samples += function.samples
-
- def add_callee(self, function, callee):
- func, callees = self.entries[function.id]
- try:
- entry = callees[callee.id]
- except KeyError:
- callees[callee.id] = callee
- else:
- entry.samples += callee.samples
-
- def parse(self):
- self.readline()
- self.readline()
- self.readline()
- self.readline()
-
- match = re.compile(r'(?P<prefix>[|+ ]*)(?P<samples>\d+), (?P<symbol>[^,]+), (?P<image>.*)')
-
- while self.lookahead():
- line = self.consume()
- mo = match.match(line)
- if not mo:
- raise ParseError('failed to parse', line)
-
- fields = mo.groupdict()
- prefix = len(fields.get('prefix', 0)) / 2 - 1
-
- symbol = str(fields.get('symbol', 0))
- image = str(fields.get('image', 0))
-
- entry = Struct()
- entry.id = ':'.join([symbol, image])
- entry.samples = int(fields.get('samples', 0))
-
- entry.name = symbol
- entry.image = image
-
- # adjust the callstack
- if prefix < len(self.stack):
- del self.stack[prefix:]
-
- if prefix == len(self.stack):
- self.stack.append(entry)
-
- # if the callstack has had an entry, it's this functions caller
- if prefix > 0:
- self.add_callee(self.stack[prefix - 1], entry)
-
- self.add_entry(entry)
-
- profile = Profile()
- profile[SAMPLES] = 0
- for _function, _callees in self.entries.itervalues():
- function = Function(_function.id, _function.name)
- function[SAMPLES] = _function.samples
- profile.add_function(function)
- profile[SAMPLES] += _function.samples
-
- if _function.image:
- function.module = os.path.basename(_function.image)
-
- for _callee in _callees.itervalues():
- call = Call(_callee.id)
- call[SAMPLES] = _callee.samples
- function.add_call(call)
-
- # compute derived data
- profile.validate()
- profile.find_cycles()
- profile.ratio(TIME_RATIO, SAMPLES)
- profile.call_ratios(SAMPLES)
- profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
-
- return profile
-
-
-class XPerfParser(Parser):
- """Parser for CSVs generted by XPerf, from Microsoft Windows Performance Tools.
- """
-
- def __init__(self, stream):
- Parser.__init__(self)
- self.stream = stream
- self.profile = Profile()
- self.profile[SAMPLES] = 0
- self.column = {}
-
- def parse(self):
- import csv
- reader = csv.reader(
- self.stream,
- delimiter = ',',
- quotechar = None,
- escapechar = None,
- doublequote = False,
- skipinitialspace = True,
- lineterminator = '\r\n',
- quoting = csv.QUOTE_NONE)
- it = iter(reader)
- row = reader.next()
- self.parse_header(row)
- for row in it:
- self.parse_row(row)
-
- # compute derived data
- self.profile.validate()
- self.profile.find_cycles()
- self.profile.ratio(TIME_RATIO, SAMPLES)
- self.profile.call_ratios(SAMPLES2)
- self.profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
-
- return self.profile
-
- def parse_header(self, row):
- for column in range(len(row)):
- name = row[column]
- assert name not in self.column
- self.column[name] = column
-
- def parse_row(self, row):
- fields = {}
- for name, column in self.column.iteritems():
- value = row[column]
- for factory in int, float:
- try:
- value = factory(value)
- except ValueError:
- pass
- else:
- break
- fields[name] = value
-
- process = fields['Process Name']
- symbol = fields['Module'] + '!' + fields['Function']
- weight = fields['Weight']
- count = fields['Count']
-
- function = self.get_function(process, symbol)
- function[SAMPLES] += weight * count
- self.profile[SAMPLES] += weight * count
-
- stack = fields['Stack']
- if stack != '?':
- stack = stack.split('/')
- assert stack[0] == '[Root]'
- if stack[-1] != symbol:
- # XXX: some cases the sampled function does not appear in the stack
- stack.append(symbol)
- caller = None
- for symbol in stack[1:]:
- callee = self.get_function(process, symbol)
- if caller is not None:
- try:
- call = caller.calls[callee.id]
- except KeyError:
- call = Call(callee.id)
- call[SAMPLES2] = count
- caller.add_call(call)
- else:
- call[SAMPLES2] += count
- caller = callee
-
- def get_function(self, process, symbol):
- function_id = process + '!' + symbol
-
- try:
- function = self.profile.functions[function_id]
- except KeyError:
- module, name = symbol.split('!', 1)
- function = Function(function_id, name)
- function.process = process
- function.module = module
- function[SAMPLES] = 0
- self.profile.add_function(function)
-
- return function
-
-
-class SleepyParser(Parser):
- """Parser for GNU gprof output.
-
- See also:
- - http://www.codersnotes.com/sleepy/
- - http://sleepygraph.sourceforge.net/
- """
-
- def __init__(self, filename):
- Parser.__init__(self)
-
- from zipfile import ZipFile
-
- self.database = ZipFile(filename)
-
- self.version_0_7 = 'Version 0.7 required' in self.database.namelist()
-
- self.symbols = {}
- self.calls = {}
-
- self.profile = Profile()
-
- _symbol_re = re.compile(
- r'^(?P<id>\w+)' +
- r'\s+"(?P<module>[^"]*)"' +
- r'\s+"(?P<procname>[^"]*)"' +
- r'\s+"(?P<sourcefile>[^"]*)"' +
- r'\s+(?P<sourceline>\d+)$'
- )
-
- def parse_symbols(self):
- if self.version_0_7:
- symbols_txt = 'Symbols.txt'
- else:
- symbols_txt = 'symbols.txt'
- lines = self.database.read(symbols_txt).splitlines()
- for line in lines:
- mo = self._symbol_re.match(line)
- if mo:
- symbol_id, module, procname, sourcefile, sourceline = mo.groups()
-
- function_id = ':'.join([module, procname])
-
- try:
- function = self.profile.functions[function_id]
- except KeyError:
- function = Function(function_id, procname)
- function.module = module
- function[SAMPLES] = 0
- self.profile.add_function(function)
-
- self.symbols[symbol_id] = function
-
- def parse_callstacks(self):
- if self.version_0_7:
- callstacks_txt = 'Callstacks.txt'
- else:
- callstacks_txt = 'callstacks.txt'
- lines = self.database.read(callstacks_txt).splitlines()
- for line in lines:
- fields = line.split()
- samples = float(fields[0])
- callstack = fields[1:]
-
- callstack = [self.symbols[symbol_id] for symbol_id in callstack]
-
- callee = callstack[0]
-
- callee[SAMPLES] += samples
- self.profile[SAMPLES] += samples
-
- for caller in callstack[1:]:
- try:
- call = caller.calls[callee.id]
- except KeyError:
- call = Call(callee.id)
- call[SAMPLES2] = samples
- caller.add_call(call)
- else:
- call[SAMPLES2] += samples
-
- callee = caller
-
- def parse(self):
- profile = self.profile
- profile[SAMPLES] = 0
-
- self.parse_symbols()
- self.parse_callstacks()
-
- # Compute derived events
- profile.validate()
- profile.find_cycles()
- profile.ratio(TIME_RATIO, SAMPLES)
- profile.call_ratios(SAMPLES2)
- profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
-
- return profile
-
-
-class AQtimeTable:
-
- def __init__(self, name, fields):
- self.name = name
-
- self.fields = fields
- self.field_column = {}
- for column in range(len(fields)):
- self.field_column[fields[column]] = column
- self.rows = []
-
- def __len__(self):
- return len(self.rows)
-
- def __iter__(self):
- for values, children in self.rows:
- fields = {}
- for name, value in zip(self.fields, values):
- fields[name] = value
- children = dict([(child.name, child) for child in children])
- yield fields, children
- raise StopIteration
-
- def add_row(self, values, children=()):
- self.rows.append((values, children))
-
-
-class AQtimeParser(XmlParser):
-
- def __init__(self, stream):
- XmlParser.__init__(self, stream)
- self.tables = {}
-
- def parse(self):
- self.element_start('AQtime_Results')
- self.parse_headers()
- results = self.parse_results()
- self.element_end('AQtime_Results')
- return self.build_profile(results)
-
- def parse_headers(self):
- self.element_start('HEADERS')
- while self.token.type == XML_ELEMENT_START:
- self.parse_table_header()
- self.element_end('HEADERS')
-
- def parse_table_header(self):
- attrs = self.element_start('TABLE_HEADER')
- name = attrs['NAME']
- id = int(attrs['ID'])
- field_types = []
- field_names = []
- while self.token.type == XML_ELEMENT_START:
- field_type, field_name = self.parse_table_field()
- field_types.append(field_type)
- field_names.append(field_name)
- self.element_end('TABLE_HEADER')
- self.tables[id] = name, field_types, field_names
-
- def parse_table_field(self):
- attrs = self.element_start('TABLE_FIELD')
- type = attrs['TYPE']
- name = self.character_data()
- self.element_end('TABLE_FIELD')
- return type, name
-
- def parse_results(self):
- self.element_start('RESULTS')
- table = self.parse_data()
- self.element_end('RESULTS')
- return table
-
- def parse_data(self):
- rows = []
- attrs = self.element_start('DATA')
- table_id = int(attrs['TABLE_ID'])
- table_name, field_types, field_names = self.tables[table_id]
- table = AQtimeTable(table_name, field_names)
- while self.token.type == XML_ELEMENT_START:
- row, children = self.parse_row(field_types)
- table.add_row(row, children)
- self.element_end('DATA')
- return table
-
- def parse_row(self, field_types):
- row = [None]*len(field_types)
- children = []
- self.element_start('ROW')
- while self.token.type == XML_ELEMENT_START:
- if self.token.name_or_data == 'FIELD':
- field_id, field_value = self.parse_field(field_types)
- row[field_id] = field_value
- elif self.token.name_or_data == 'CHILDREN':
- children = self.parse_children()
- else:
- raise XmlTokenMismatch("<FIELD ...> or <CHILDREN ...>", self.token)
- self.element_end('ROW')
- return row, children
-
- def parse_field(self, field_types):
- attrs = self.element_start('FIELD')
- id = int(attrs['ID'])
- type = field_types[id]
- value = self.character_data()
- if type == 'Integer':
- value = int(value)
- elif type == 'Float':
- value = float(value)
- elif type == 'Address':
- value = int(value)
- elif type == 'String':
- pass
- else:
- assert False
- self.element_end('FIELD')
- return id, value
-
- def parse_children(self):
- children = []
- self.element_start('CHILDREN')
- while self.token.type == XML_ELEMENT_START:
- table = self.parse_data()
- assert table.name not in children
- children.append(table)
- self.element_end('CHILDREN')
- return children
-
- def build_profile(self, results):
- assert results.name == 'Routines'
- profile = Profile()
- profile[TIME] = 0.0
- for fields, tables in results:
- function = self.build_function(fields)
- children = tables['Children']
- for fields, _ in children:
- call = self.build_call(fields)
- function.add_call(call)
- profile.add_function(function)
- profile[TIME] = profile[TIME] + function[TIME]
- profile[TOTAL_TIME] = profile[TIME]
- profile.ratio(TOTAL_TIME_RATIO, TOTAL_TIME)
- return profile
-
- def build_function(self, fields):
- function = Function(self.build_id(fields), self.build_name(fields))
- function[TIME] = fields['Time']
- function[TOTAL_TIME] = fields['Time with Children']
- #function[TIME_RATIO] = fields['% Time']/100.0
- #function[TOTAL_TIME_RATIO] = fields['% with Children']/100.0
- return function
-
- def build_call(self, fields):
- call = Call(self.build_id(fields))
- call[TIME] = fields['Time']
- call[TOTAL_TIME] = fields['Time with Children']
- #call[TIME_RATIO] = fields['% Time']/100.0
- #call[TOTAL_TIME_RATIO] = fields['% with Children']/100.0
- return call
-
- def build_id(self, fields):
- return ':'.join([fields['Module Name'], fields['Unit Name'], fields['Routine Name']])
-
- def build_name(self, fields):
- # TODO: use more fields
- return fields['Routine Name']
-
-
-class PstatsParser:
- """Parser python profiling statistics saved with te pstats module."""
-
- def __init__(self, *filename):
- import pstats
- try:
- self.stats = pstats.Stats(*filename)
- except ValueError:
- import hotshot.stats
- self.stats = hotshot.stats.load(filename[0])
- self.profile = Profile()
- self.function_ids = {}
-
- def get_function_name(self, (filename, line, name)):
- module = os.path.splitext(filename)[0]
- module = os.path.basename(module)
- return "%s:%d:%s" % (module, line, name)
-
- def get_function(self, key):
- try:
- id = self.function_ids[key]
- except KeyError:
- id = len(self.function_ids)
- name = self.get_function_name(key)
- function = Function(id, name)
- self.profile.functions[id] = function
- self.function_ids[key] = id
- else:
- function = self.profile.functions[id]
- return function
-
- def parse(self):
- self.profile[TIME] = 0.0
- self.profile[TOTAL_TIME] = self.stats.total_tt
- for fn, (cc, nc, tt, ct, callers) in self.stats.stats.iteritems():
- callee = self.get_function(fn)
- callee.called = nc
- callee[TOTAL_TIME] = ct
- callee[TIME] = tt
- self.profile[TIME] += tt
- self.profile[TOTAL_TIME] = max(self.profile[TOTAL_TIME], ct)
- for fn, value in callers.iteritems():
- caller = self.get_function(fn)
- call = Call(callee.id)
- if isinstance(value, tuple):
- for i in xrange(0, len(value), 4):
- nc, cc, tt, ct = value[i:i+4]
- if CALLS in call:
- call[CALLS] += cc
- else:
- call[CALLS] = cc
-
- if TOTAL_TIME in call:
- call[TOTAL_TIME] += ct
- else:
- call[TOTAL_TIME] = ct
-
- else:
- call[CALLS] = value
- call[TOTAL_TIME] = ratio(value, nc)*ct
-
- caller.add_call(call)
- #self.stats.print_stats()
- #self.stats.print_callees()
-
- # Compute derived events
- self.profile.validate()
- self.profile.ratio(TIME_RATIO, TIME)
- self.profile.ratio(TOTAL_TIME_RATIO, TOTAL_TIME)
-
- return self.profile
-
-
-class Theme:
-
- def __init__(self,
- bgcolor = (0.0, 0.0, 1.0),
- mincolor = (0.0, 0.0, 0.0),
- maxcolor = (0.0, 0.0, 1.0),
- fontname = "Arial",
- minfontsize = 10.0,
- maxfontsize = 10.0,
- minpenwidth = 0.5,
- maxpenwidth = 4.0,
- gamma = 2.2,
- skew = 1.0):
- self.bgcolor = bgcolor
- self.mincolor = mincolor
- self.maxcolor = maxcolor
- self.fontname = fontname
- self.minfontsize = minfontsize
- self.maxfontsize = maxfontsize
- self.minpenwidth = minpenwidth
- self.maxpenwidth = maxpenwidth
- self.gamma = gamma
- self.skew = skew
-
- def graph_bgcolor(self):
- return self.hsl_to_rgb(*self.bgcolor)
-
- def graph_fontname(self):
- return self.fontname
-
- def graph_fontsize(self):
- return self.minfontsize
-
- def node_bgcolor(self, weight):
- return self.color(weight)
-
- def node_fgcolor(self, weight):
- return self.graph_bgcolor()
-
- def node_fontsize(self, weight):
- return self.fontsize(weight)
-
- def edge_color(self, weight):
- return self.color(weight)
-
- def edge_fontsize(self, weight):
- return self.fontsize(weight)
-
- def edge_penwidth(self, weight):
- return max(weight*self.maxpenwidth, self.minpenwidth)
-
- def edge_arrowsize(self, weight):
- return 0.5 * math.sqrt(self.edge_penwidth(weight))
-
- def fontsize(self, weight):
- return max(weight**2 * self.maxfontsize, self.minfontsize)
-
- def color(self, weight):
- weight = min(max(weight, 0.0), 1.0)
-
- hmin, smin, lmin = self.mincolor
- hmax, smax, lmax = self.maxcolor
-
- if self.skew < 0:
- raise ValueError("Skew must be greater than 0")
- elif self.skew == 1.0:
- h = hmin + weight*(hmax - hmin)
- s = smin + weight*(smax - smin)
- l = lmin + weight*(lmax - lmin)
- else:
- base = self.skew
- h = hmin + ((hmax-hmin)*(-1.0 + (base ** weight)) / (base - 1.0))
- s = smin + ((smax-smin)*(-1.0 + (base ** weight)) / (base - 1.0))
- l = lmin + ((lmax-lmin)*(-1.0 + (base ** weight)) / (base - 1.0))
-
- return self.hsl_to_rgb(h, s, l)
-
- def hsl_to_rgb(self, h, s, l):
- """Convert a color from HSL color-model to RGB.
-
- See also:
- - http://www.w3.org/TR/css3-color/#hsl-color
- """
-
- h = h % 1.0
- s = min(max(s, 0.0), 1.0)
- l = min(max(l, 0.0), 1.0)
-
- if l <= 0.5:
- m2 = l*(s + 1.0)
- else:
- m2 = l + s - l*s
- m1 = l*2.0 - m2
- r = self._hue_to_rgb(m1, m2, h + 1.0/3.0)
- g = self._hue_to_rgb(m1, m2, h)
- b = self._hue_to_rgb(m1, m2, h - 1.0/3.0)
-
- # Apply gamma correction
- r **= self.gamma
- g **= self.gamma
- b **= self.gamma
-
- return (r, g, b)
-
- def _hue_to_rgb(self, m1, m2, h):
- if h < 0.0:
- h += 1.0
- elif h > 1.0:
- h -= 1.0
- if h*6 < 1.0:
- return m1 + (m2 - m1)*h*6.0
- elif h*2 < 1.0:
- return m2
- elif h*3 < 2.0:
- return m1 + (m2 - m1)*(2.0/3.0 - h)*6.0
- else:
- return m1
-
-
-TEMPERATURE_COLORMAP = Theme(
- mincolor = (2.0/3.0, 0.80, 0.25), # dark blue
- maxcolor = (0.0, 1.0, 0.5), # satured red
- gamma = 1.0
-)
-
-PINK_COLORMAP = Theme(
- mincolor = (0.0, 1.0, 0.90), # pink
- maxcolor = (0.0, 1.0, 0.5), # satured red
-)
-
-GRAY_COLORMAP = Theme(
- mincolor = (0.0, 0.0, 0.85), # light gray
- maxcolor = (0.0, 0.0, 0.0), # black
-)
-
-BW_COLORMAP = Theme(
- minfontsize = 8.0,
- maxfontsize = 24.0,
- mincolor = (0.0, 0.0, 0.0), # black
- maxcolor = (0.0, 0.0, 0.0), # black
- minpenwidth = 0.1,
- maxpenwidth = 8.0,
-)
-
-
-class DotWriter:
- """Writer for the DOT language.
-
- See also:
- - "The DOT Language" specification
- http://www.graphviz.org/doc/info/lang.html
- """
-
- def __init__(self, fp):
- self.fp = fp
-
- def graph(self, profile, theme):
- self.begin_graph()
-
- fontname = theme.graph_fontname()
-
- self.attr('graph', fontname=fontname, ranksep=0.25, nodesep=0.125)
- self.attr('node', fontname=fontname, shape="box", style="filled", fontcolor="white", width=0, height=0)
- self.attr('edge', fontname=fontname)
-
- for function in profile.functions.itervalues():
- labels = []
- if function.process is not None:
- labels.append(function.process)
- if function.module is not None:
- labels.append(function.module)
- labels.append(function.name)
- for event in TOTAL_TIME_RATIO, TIME_RATIO:
- if event in function.events:
- label = event.format(function[event])
- labels.append(label)
- if function.called is not None:
- labels.append(u"%u\xd7" % (function.called,))
-
- if function.weight is not None:
- weight = function.weight
- else:
- weight = 0.0
-
- label = '\n'.join(labels)
- self.node(function.id,
- label = label,
- color = self.color(theme.node_bgcolor(weight)),
- fontcolor = self.color(theme.node_fgcolor(weight)),
- fontsize = "%.2f" % theme.node_fontsize(weight),
- )
-
- for call in function.calls.itervalues():
- callee = profile.functions[call.callee_id]
-
- labels = []
- for event in TOTAL_TIME_RATIO, CALLS:
- if event in call.events:
- label = event.format(call[event])
- labels.append(label)
-
- if call.weight is not None:
- weight = call.weight
- elif callee.weight is not None:
- weight = callee.weight
- else:
- weight = 0.0
-
- label = '\n'.join(labels)
-
- self.edge(function.id, call.callee_id,
- label = label,
- color = self.color(theme.edge_color(weight)),
- fontcolor = self.color(theme.edge_color(weight)),
- fontsize = "%.2f" % theme.edge_fontsize(weight),
- penwidth = "%.2f" % theme.edge_penwidth(weight),
- labeldistance = "%.2f" % theme.edge_penwidth(weight),
- arrowsize = "%.2f" % theme.edge_arrowsize(weight),
- )
-
- self.end_graph()
-
- def begin_graph(self):
- self.write('digraph {\n')
-
- def end_graph(self):
- self.write('}\n')
-
- def attr(self, what, **attrs):
- self.write("\t")
- self.write(what)
- self.attr_list(attrs)
- self.write(";\n")
-
- def node(self, node, **attrs):
- self.write("\t")
- self.id(node)
- self.attr_list(attrs)
- self.write(";\n")
-
- def edge(self, src, dst, **attrs):
- self.write("\t")
- self.id(src)
- self.write(" -> ")
- self.id(dst)
- self.attr_list(attrs)
- self.write(";\n")
-
- def attr_list(self, attrs):
- if not attrs:
- return
- self.write(' [')
- first = True
- for name, value in attrs.iteritems():
- if first:
- first = False
- else:
- self.write(", ")
- self.id(name)
- self.write('=')
- self.id(value)
- self.write(']')
-
- def id(self, id):
- if isinstance(id, (int, float)):
- s = str(id)
- elif isinstance(id, basestring):
- if id.isalnum() and not id.startswith('0x'):
- s = id
- else:
- s = self.escape(id)
- else:
- raise TypeError
- self.write(s)
-
- def color(self, (r, g, b)):
-
- def float2int(f):
- if f <= 0.0:
- return 0
- if f >= 1.0:
- return 255
- return int(255.0*f + 0.5)
-
- return "#" + "".join(["%02x" % float2int(c) for c in (r, g, b)])
-
- def escape(self, s):
- s = s.encode('utf-8')
- s = s.replace('\\', r'\\')
- s = s.replace('\n', r'\n')
- s = s.replace('\t', r'\t')
- s = s.replace('"', r'\"')
- return '"' + s + '"'
-
- def write(self, s):
- self.fp.write(s)
-
-
-class Main:
- """Main program."""
-
- themes = {
- "color": TEMPERATURE_COLORMAP,
- "pink": PINK_COLORMAP,
- "gray": GRAY_COLORMAP,
- "bw": BW_COLORMAP,
- }
-
- def main(self):
- """Main program."""
-
- parser = optparse.OptionParser(
- usage="\n\t%prog [options] [file] ...",
- version="%%prog %s" % __version__)
- parser.add_option(
- '-o', '--output', metavar='FILE',
- type="string", dest="output",
- help="output filename [stdout]")
- parser.add_option(
- '-n', '--node-thres', metavar='PERCENTAGE',
- type="float", dest="node_thres", default=0.5,
- help="eliminate nodes below this threshold [default: %default]")
- parser.add_option(
- '-e', '--edge-thres', metavar='PERCENTAGE',
- type="float", dest="edge_thres", default=0.1,
- help="eliminate edges below this threshold [default: %default]")
- parser.add_option(
- '-f', '--format',
- type="choice", choices=('prof', 'callgrind', 'oprofile', 'hprof', 'sysprof', 'pstats', 'shark', 'sleepy', 'aqtime', 'xperf'),
- dest="format", default="prof",
- help="profile format: prof, callgrind, oprofile, hprof, sysprof, shark, sleepy, aqtime, pstats, or xperf [default: %default]")
- parser.add_option(
- '-c', '--colormap',
- type="choice", choices=('color', 'pink', 'gray', 'bw'),
- dest="theme", default="color",
- help="color map: color, pink, gray, or bw [default: %default]")
- parser.add_option(
- '-s', '--strip',
- action="store_true",
- dest="strip", default=False,
- help="strip function parameters, template parameters, and const modifiers from demangled C++ function names")
- parser.add_option(
- '-w', '--wrap',
- action="store_true",
- dest="wrap", default=False,
- help="wrap function names")
- # add a new option to control skew of the colorization curve
- parser.add_option(
- '--skew',
- type="float", dest="theme_skew", default=1.0,
- help="skew the colorization curve. Values < 1.0 give more variety to lower percentages. Value > 1.0 give less variety to lower percentages")
- (self.options, self.args) = parser.parse_args(sys.argv[1:])
-
- if len(self.args) > 1 and self.options.format != 'pstats':
- parser.error('incorrect number of arguments')
-
- try:
- self.theme = self.themes[self.options.theme]
- except KeyError:
- parser.error('invalid colormap \'%s\'' % self.options.theme)
-
- # set skew on the theme now that it has been picked.
- if self.options.theme_skew:
- self.theme.skew = self.options.theme_skew
-
- if self.options.format == 'prof':
- if not self.args:
- fp = sys.stdin
- else:
- fp = open(self.args[0], 'rt')
- parser = GprofParser(fp)
- elif self.options.format == 'callgrind':
- if not self.args:
- fp = sys.stdin
- else:
- fp = open(self.args[0], 'rt')
- parser = CallgrindParser(fp)
- elif self.options.format == 'oprofile':
- if not self.args:
- fp = sys.stdin
- else:
- fp = open(self.args[0], 'rt')
- parser = OprofileParser(fp)
- elif self.options.format == 'sysprof':
- if not self.args:
- fp = sys.stdin
- else:
- fp = open(self.args[0], 'rt')
- parser = SysprofParser(fp)
- elif self.options.format == 'hprof':
- if not self.args:
- fp = sys.stdin
- else:
- fp = open(self.args[0], 'rt')
- parser = HProfParser(fp)
- elif self.options.format == 'pstats':
- if not self.args:
- parser.error('at least a file must be specified for pstats input')
- parser = PstatsParser(*self.args)
- elif self.options.format == 'xperf':
- if not self.args:
- fp = sys.stdin
- else:
- fp = open(self.args[0], 'rt')
- parser = XPerfParser(fp)
- elif self.options.format == 'shark':
- if not self.args:
- fp = sys.stdin
- else:
- fp = open(self.args[0], 'rt')
- parser = SharkParser(fp)
- elif self.options.format == 'sleepy':
- if len(self.args) != 1:
- parser.error('exactly one file must be specified for sleepy input')
- parser = SleepyParser(self.args[0])
- elif self.options.format == 'aqtime':
- if not self.args:
- fp = sys.stdin
- else:
- fp = open(self.args[0], 'rt')
- parser = AQtimeParser(fp)
- else:
- parser.error('invalid format \'%s\'' % self.options.format)
-
- self.profile = parser.parse()
-
- if self.options.output is None:
- self.output = sys.stdout
- else:
- self.output = open(self.options.output, 'wt')
-
- self.write_graph()
-
- _parenthesis_re = re.compile(r'\([^()]*\)')
- _angles_re = re.compile(r'<[^<>]*>')
- _const_re = re.compile(r'\s+const$')
-
- def strip_function_name(self, name):
- """Remove extraneous information from C++ demangled function names."""
-
- # Strip function parameters from name by recursively removing paired parenthesis
- while True:
- name, n = self._parenthesis_re.subn('', name)
- if not n:
- break
-
- # Strip const qualifier
- name = self._const_re.sub('', name)
-
- # Strip template parameters from name by recursively removing paired angles
- while True:
- name, n = self._angles_re.subn('', name)
- if not n:
- break
-
- return name
-
- def wrap_function_name(self, name):
- """Split the function name on multiple lines."""
-
- if len(name) > 32:
- ratio = 2.0/3.0
- height = max(int(len(name)/(1.0 - ratio) + 0.5), 1)
- width = max(len(name)/height, 32)
- # TODO: break lines in symbols
- name = textwrap.fill(name, width, break_long_words=False)
-
- # Take away spaces
- name = name.replace(", ", ",")
- name = name.replace("> >", ">>")
- name = name.replace("> >", ">>") # catch consecutive
-
- return name
-
- def compress_function_name(self, name):
- """Compress function name according to the user preferences."""
-
- if self.options.strip:
- name = self.strip_function_name(name)
-
- if self.options.wrap:
- name = self.wrap_function_name(name)
-
- # TODO: merge functions with same resulting name
-
- return name
-
- def write_graph(self):
- dot = DotWriter(self.output)
- profile = self.profile
- profile.prune(self.options.node_thres/100.0, self.options.edge_thres/100.0)
-
- for function in profile.functions.itervalues():
- function.name = self.compress_function_name(function.name)
-
- dot.graph(profile, self.theme)
-
-
-if __name__ == '__main__':
- Main().main()
diff --git a/site_utils/dashboard/external/io.py b/site_utils/dashboard/external/io.py
deleted file mode 100644
index ec93c66..0000000
--- a/site_utils/dashboard/external/io.py
+++ /dev/null
@@ -1,1168 +0,0 @@
-# Copyright (c) 2000-2007 Gary Strangman. All rights reserved
-#
-# Disclaimer
-#
-# This software is provided "as-is". There are no expressed or implied
-# warranties of any kind, including, but not limited to, the warranties
-# of merchantability and fittness for a given application. In no event
-# shall Gary Strangman be liable for any direct, indirect, incidental,
-# special, exemplary or consequential damages (including, but not limited
-# to, loss of use, data or profits, or business interruption) however
-# caused and on any theory of liability, whether in contract, strict
-# liability or tort (including negligence or otherwise) arising in any way
-# out of the use of this software, even if advised of the possibility of
-# such damage.
-#
-# Comments and/or additions are welcome (send e-mail to:
-# strang@nmr.mgh.harvard.edu).
-#
-"""
-Defines a number of functions for pseudo-command-line OS functionality.
-
- cd(directory)
- pwd <-- can be used WITHOUT parens
- ls(d='.')
- rename(from,to)
- get(namepatterns,verbose=1)
- getstrings(namepatterns,verbose=1)
- put(outlist,filename,writetype='w')
- aget(namepatterns,verbose=1)
- aput(outarray,filename,writetype='w')
- bget(filename,numslices=1,xsize=64,ysize=64)
- braw(filename,btype)
- bput(outarray,filename,writeheader=0,packstring='h',writetype='wb')
- mrget(filename)
- find_dirs(sourcedir)
-"""
-
-## CHANGES:
-## =======
-## 07-11-26 ... more numpy conversion work
-## 06-08-07 ... converted to numpy, changed version to 0.6
-## 06-02-03 ... added add2afnihistory() to load modify afni HISTORY_NOTEs,
-## and added that option to array2afni output
-## 04-06-14 ... added getafniparam() to load in afni values from HEAD files
-## 03-04-09 ... fixed brikget to load based on datatype, changed array2afni
-## so that the default sliceorder is altplus, not seqplus
-## 02-11-20 ... added binget(), binput(), array2afni(), version 0.5
-## 02-10-20 ... added find_dirs() function, changed version to 0.4
-## 01-11-15 ... changed aput() and put() to accept a delimiter
-## 01-04-19 ... added oneperline option to put() function
-## 99-11-07 ... added DAs quick flat-text-file loaders, load() and fload()
-## 99-11-01 ... added version number (0.1) for distribution
-## 99-08-30 ... Put quickload in here
-## 99-06-27 ... Changed bget thing back ... confused ...
-## 99-06-24 ... exchanged xsize and ysize in bget for non-square images (NT??)
-## modified bget to raise an IOError when file not found
-## 99-06-12 ... added load() and save() aliases for aget() and aput() (resp.)
-## 99-04-13 ... changed aget() to ignore (!!!!) lines beginning with # or %
-## 99-01-17 ... changed get() so ints come in as ints (not floats)
-##
-
-
-
-try:
- import mmapfile
-except:
- pass
-
-import pstat
-import glob, re, string, types, os, struct, copy, time, tempfile, sys
-from types import *
-import numpy as N
-
-__version__ = 0.6
-
-def wrap(f):
- """
-Wraps a function so that if it's entered *by itself*
-in the interpreter without ()'s, it gets called anyway
-"""
- class W:
- def __init__(self, f):
- self.f = f
- def __repr__(self):
- x =apply(self.f)
- if x:
- return repr(x)
- else:
- return ''
- return W(f)
-
-
-def cd (directory):
- """
-Changes the working python directory for the interpreter.
-
-Usage: cd(directory) where 'directory' is a string
-"""
- os.chdir(directory)
- return
-
-
-def pwd():
- """
-Changes the working python directory for the interpreter.
-
-Usage: pwd (no parens needed)
-"""
- return os.getcwd()
-pwd = wrap(pwd)
-
-
-def ls(d='.'):
- """
-Produces a directory listing. Default is the current directory.
-
-Usage: ls(d='.')
-"""
- os.system('ls '+d)
- return None
-
-
-def rename(source, dest):
- """
-Renames files specified by UNIX inpattern to those specified by UNIX
-outpattern. Can only handle a single '*' in the two patterns!!!
-
-Usage: rename (source, dest) e.g., rename('*.txt', '*.c')
-"""
- infiles = glob.glob(source)
- outfiles = []
- incutindex = string.index(source,'*')
- outcutindex = string.index(source,'*')
- findpattern1 = source[0:incutindex]
- findpattern2 = source[incutindex+1:]
- replpattern1 = dest[0:incutindex]
- replpattern2 = dest[incutindex+1:]
- for fname in infiles:
- if incutindex > 0:
- newname = re.sub(findpattern1,replpattern1,fname,1)
- if outcutindex < len(dest)-1:
- if incutindex > 0:
- lastone = string.rfind(newname,replpattern2)
- newname = newname[0:lastone] + re.sub(findpattern2,replpattern2,fname[lastone:],1)
- else:
- lastone = string.rfind(fname,findpattern2)
- if lastone <> -1:
- newname = fname[0:lastone]
- newname = newname + re.sub(findpattern2,replpattern2,fname[lastone:],1)
- print fname, newname
- os.rename(fname,newname)
- return
-
-
-def get (namepatterns,verbose=1):
- """
-Loads a list of lists from text files (specified by a UNIX-style
-wildcard filename pattern) and converts all numeric values to floats.
-Uses the glob module for filename pattern conversion. Loaded filename
-is printed if verbose=1.
-
-Usage: get (namepatterns,verbose=1)
-Returns: a 1D or 2D list of lists from whitespace delimited text files
- specified by namepatterns; numbers that can be converted to floats
- are so converted
-"""
- fnames = []
- if type(namepatterns) in [ListType,TupleType]:
- for item in namepatterns:
- fnames = fnames + glob.glob(item)
- else:
- fnames = glob.glob(namepatterns)
-
- if len(fnames) == 0:
- if verbose:
- print 'NO FILENAMES MATCH ('+namepatterns+') !!'
- return None
-
- if verbose:
- print fnames # so user knows what has been loaded
- elements = []
- for i in range(len(fnames)):
- file = open(fnames[i])
- newelements = map(string.split,file.readlines())
- for i in range(len(newelements)):
- for j in range(len(newelements[i])):
- try:
- newelements[i][j] = string.atoi(newelements[i][j])
- except ValueError:
- try:
- newelements[i][j] = string.atof(newelements[i][j])
- except:
- pass
- elements = elements + newelements
- if len(elements)==1: elements = elements[0]
- return elements
-
-
-def getstrings (namepattern,verbose=1):
- """
-Loads a (set of) text file(s), with all elements left as string type.
-Uses UNIX-style wildcards (i.e., function uses glob). Loaded filename
-is printed if verbose=1.
-
-Usage: getstrings (namepattern, verbose=1)
-Returns: a list of strings, one per line in each text file specified by
- namepattern
-"""
- fnames = glob.glob(namepattern)
- if len(fnames) == 0:
- if verbose:
- print 'NO FILENAMES MATCH ('+namepattern+') !!'
- return None
- if verbose:
- print fnames
- elements = []
- for filename in fnames:
- file = open(filename)
- newelements = map(string.split,file.readlines())
- elements = elements + newelements
- return elements
-
-
-def put (outlist,fname,writetype='w',oneperline=0,delimit=' '):
- """
-Writes a passed mixed-type list (str and/or numbers) to an output
-file, and then closes the file. Default is overwrite the destination
-file.
-
-Usage: put (outlist,fname,writetype='w',oneperline=0,delimit=' ')
-Returns: None
-"""
- if type(outlist) in [N.ndarray]:
- aput(outlist,fname,writetype)
- return
- if type(outlist[0]) not in [ListType,TupleType]: # 1D list
- outfile = open(fname,writetype)
- if not oneperline:
- outlist = pstat.list2string(outlist,delimit)
- outfile.write(outlist)
- outfile.write('\n')
- else: # they want one element from the list on each file line
- for item in outlist:
- outfile.write(str(item)+'\n')
- outfile.close()
- else: # 2D list (list-of-lists)
- outfile = open(fname,writetype)
- for row in outlist:
- outfile.write(pstat.list2string(row,delimit))
- outfile.write('\n')
- outfile.close()
- return None
-
-
-def isstring(x):
- if type(x)==StringType:
- return 1
- else:
- return 0
-
-
-
-def aget (namepattern,verbose=1):
- """
-Loads an array from 2D text files (specified by a UNIX-style wildcard
-filename pattern). ONLY 'GET' FILES WITH EQUAL NUMBERS OF COLUMNS
-ON EVERY ROW (otherwise returned array will be zero-dimensional).
-
-Usage: aget (namepattern)
-Returns: an array of integers, floats or objects (type='O'), depending on the
- contents of the files specified by namepattern
-"""
- fnames = glob.glob(namepattern)
- if len(fnames) == 0:
- if verbose:
- print 'NO FILENAMES MATCH ('+namepattern+') !!'
- return None
- if verbose:
- print fnames
- elements = []
- for filename in fnames:
- file = open(filename)
- newelements = file.readlines()
- del_list = []
- for row in range(len(newelements)):
- if (newelements[row][0]=='%' or newelements[row][0]=='#'
- or len(newelements[row])==1 or newelements[row][0]=='\r'):
- del_list.append(row)
- del_list.reverse()
- for i in del_list:
- newelements.pop(i)
- newelements = map(string.split,newelements)
- for i in range(len(newelements)):
- for j in range(len(newelements[i])):
- try:
- newelements[i][j] = string.atof(newelements[i][j])
- except:
- pass
- elements = elements + newelements
- for row in range(len(elements)):
- if N.add.reduce(N.array(map(isstring,elements[row])))==len(elements[row]):
- print "A row of strings was found. Returning a LIST."
- return elements
- try:
- elements = N.array(elements)
- except TypeError:
- elements = N.array(elements,dtype='O')
- return elements
-
-
-def aput (outarray,fname,writetype='w',delimit=' '):
- """
-Sends passed 1D or 2D array to an output file and closes the file.
-
-Usage: aput (outarray,fname,writetype='w',delimit=' ')
-Returns: None
-"""
- outfile = open(fname,writetype)
- if len(outarray.shape) == 1:
- outarray = outarray[N.newaxis,:]
- if len(outarray.shape) > 2:
- raise TypeError, "put() and aput() require 1D or 2D arrays. Otherwise use some kind of pickling."
- else: # must be a 2D array
- for row in outarray:
- outfile.write(string.join(map(str,row),delimit))
- outfile.write('\n')
- outfile.close()
- return None
-
-
-def bget(imfile,shp=None,unpackstr=N.int16,bytesperpixel=2.0,sliceinit=0):
- """
-Reads in a binary file, typically with a .bshort or .bfloat extension.
-If so, the last 3 parameters are set appropriately. If not, the last 3
-parameters default to reading .bshort files (2-byte integers in big-endian
-binary format).
-
-Usage: bget(imfile,shp=None,unpackstr=N.int16,bytesperpixel=2.0,sliceinit=0)
-"""
- if imfile[:3] == 'COR':
- return CORget(imfile)
- if imfile[-2:] == 'MR':
- return mrget(imfile,unpackstr)
- if imfile[-4:] == 'BRIK':
- return brikget(imfile,unpackstr,shp)
- if imfile[-3:] in ['mnc','MNC','inc','INC']:
- return mincget(imfile,unpackstr,shp)
- if imfile[-3:] == 'img':
- return mghbget(imfile,unpackstr,shp)
- if imfile[-6:] == 'bshort' or imfile[-6:] == 'bfloat':
- if shp == None:
- return mghbget(imfile,unpackstr=unpackstr,bytesperpixel=bytesperpixel,sliceinit=sliceinit)
- else:
- return mghbget(imfile,shp[0],shp[1],shp[2],unpackstr,bytesperpixel,sliceinit)
-
-
-def CORget(infile):
- """
-Reads a binary COR-nnn file (flattening file).
-
-Usage: CORget(imfile)
-Returns: 2D array of 16-bit ints
-"""
- d=braw(infile,N.int8)
- d.shape = (256,256)
- d = N.where(d>=0,d,256+d)
- return d
-
-
-def mincget(imfile,unpackstr=N.int16,shp=None):
- """
-Loads in a .MNC file.
-
-Usage: mincget(imfile,unpackstr=N.int16,shp=None) default shp = -1,20,64,64
-"""
- if shp == None:
- shp = (-1,20,64,64)
- os.system('mincextract -short -range 0 4095 -image_range 0 4095 ' +
- imfile+' > minctemp.bshort')
- try:
- d = braw('minctemp.bshort',unpackstr)
- except:
- print "Couldn't find file: "+imfile
- raise IOError, "Couldn't find file in mincget()"
-
- print shp, d.shape
- d.shape = shp
- os.system('rm minctemp.bshort')
- return d
-
-
-def brikget(imfile,unpackstr=N.int16,shp=None):
- """
-Gets an AFNI BRIK file.
-
-Usage: brikget(imfile,unpackstr=N.int16,shp=None) default shp: (-1,48,61,51)
-"""
- if shp == None:
- shp = (-1,48,61,51)
- try:
- file = open(imfile, "rb")
- except:
- print "Couldn't find file: "+imfile
- raise IOError, "Couldn't find file in brikget()"
- try:
- header = imfile[0:-4]+'HEAD'
- lines = open(header).readlines()
- for i in range(len(lines)):
- if string.find(lines[i],'DATASET_DIMENSIONS') <> -1:
- dims = string.split(lines[i+2][0:string.find(lines[i+2],' 0')])
- dims = map(string.atoi,dims)
- if string.find(lines[i],'BRICK_FLOAT_FACS') <> -1:
- count = string.atoi(string.split(lines[i+1])[2])
- mults = []
- for j in range(int(N.ceil(count/5.))):
- mults += map(string.atof,string.split(lines[i+2+j]))
- mults = N.array(mults)
- if string.find(lines[i],'BRICK_TYPES') <> -1:
- first5 = lines[i+2]
- first5 = map(string.atoi,string.split(first5))
- if first5[0] == 0:
- unpackstr = N.uint8
- elif first5[0] == 1:
- unpackstr = N.int16
- elif first5[0] == 3:
- unpackstr = N.float32
- elif first5[0] == 5:
- unpackstr = N.complex32
- dims.reverse()
- shp = [-1]+dims
- except IOError:
- print "No header file. Continuing ..."
- lines = None
-
- print shp
- print 'Using unpackstr:',unpackstr #,', bytesperpixel=',bytesperpixel
-
- file = open(imfile, "rb")
- bdata = file.read()
-
- # the > forces big-endian (for or from Sun/SGI)
- bdata = N.fromstring(bdata,unpackstr)
-# littleEndian = ( struct.pack('i',1)==struct.pack('<i',1) )
- if (max(bdata)>1e30):
- bdata = bdata.byteswap()
- try:
- bdata.shape = shp
- except:
- print 'Incorrect shape ...',shp,len(bdata)
- raise ValueError, 'Incorrect shape for file size'
- if len(bdata) == 1:
- bdata = bdata[0]
-
- if N.sum(mults) == 0:
- return bdata
- try:
- multshape = [1]*len(bdata.shape)
- for i in range(len(bdata.shape)):
- if len(mults) == bdata.shape[i]:
- multshape[i] = len(mults)
- break
- mults.shape = multshape
- return bdata*mults
- except:
- return bdata
-
-def mghbget(imfile,numslices=-1,xsize=64,ysize=64,
- unpackstr=N.int16,bytesperpixel=2.0,sliceinit=0):
- """
-Reads in a binary file, typically with a .bshort or .bfloat extension.
-If so, the last 3 parameters are set appropriately. If not, the last 3
-parameters default to reading .bshort files (2-byte integers in big-endian
-binary format).
-
-Usage: mghbget(imfile, numslices=-1, xsize=64, ysize=64,
- unpackstr=N.int16, bytesperpixel=2.0, sliceinit=0)
-"""
- try:
- file = open(imfile, "rb")
- except:
- print "Couldn't find file: "+imfile
- raise IOError, "Couldn't find file in bget()"
- try:
- header = imfile[0:-6]+'hdr'
- vals = get(header,0) # '0' means no missing-file warning msg
- if type(vals[0]) == ListType: # it's an extended header
- xsize = int(vals[0][0])
- ysize = int(vals[0][1])
- numslices = int(vals[0][2])
- else:
- xsize = int(vals[0])
- ysize = int(vals[1])
- numslices = int(vals[2])
- except:
- print "No header file. Continuing ..."
-
- suffix = imfile[-6:]
- if suffix == 'bshort':
- pass
- elif suffix[-3:] == 'img':
- pass
- elif suffix == 'bfloat':
- unpackstr = N.float32
- bytesperpixel = 4.0
- sliceinit = 0.0
- else:
- print 'Not a bshort, bfloat or img file.'
- print 'Using unpackstr:',unpackstr,', bytesperpixel=',bytesperpixel
-
- imsize = xsize*ysize
- file = open(imfile, "rb")
- bdata = file.read()
-
- numpixels = len(bdata) / bytesperpixel
- if numpixels%1 != 0:
- raise ValueError, "Incorrect file size in fmri.bget()"
- else: # the > forces big-endian (for or from Sun/SGI)
- bdata = N.fromstring(bdata,unpackstr)
-# littleEndian = ( struct.pack('i',1)==struct.pack('<i',1) )
-# if littleEndian:
-# bdata = bdata.byteswap()
- if (max(bdata)>1e30):
- bdata = bdata.byteswap()
- if suffix[-3:] == 'img':
- if numslices == -1:
- numslices = len(bdata)/8200 # 8200=(64*64*2)+8 bytes per image
- xsize = 64
- ysize = 128
- slices = N.zeros((numslices,xsize,ysize),N.int32)
- for i in range(numslices):
- istart = i*8 + i*xsize*ysize
- iend = i*8 + (i+1)*xsize*ysize
- print i, istart,iend
- slices[i] = N.reshape(N.array(bdata[istart:iend]),(xsize,ysize))
- else:
- if numslices == 1:
- slices = N.reshape(N.array(bdata),[xsize,ysize])
- else:
- slices = N.reshape(N.array(bdata),[numslices,xsize,ysize])
- if len(slices) == 1:
- slices = slices[0]
- return slices
-
-
-def braw(fname,btype,shp=None):
- """
-Opens a binary file, unpacks it, and returns a flat array of the
-type specified. Use Numeric types ... N.float32, N.int64, etc.
-
-Usage: braw(fname,btype,shp=None)
-Returns: flat array of floats, or ints (if btype=N.int16)
-"""
- file = open(fname,'rb')
- bdata = file.read()
- bdata = N.fromstring(bdata,btype)
-# littleEndian = ( struct.pack('i',1)==struct.pack('<i',1) )
-# if littleEndian:
-# bdata = bdata.byteswap() # didn't used to need this with '>' above
- if (max(bdata)>1e30):
- bdata = bdata.byteswap()
- if shp:
- try:
- bdata.shape = shp
- return bdata
- except:
- pass
- return N.array(bdata)
-
-
-def glget(fname,btype):
- """
-Load in a file containing pixels from glReadPixels dump.
-
-Usage: glget(fname,btype)
-Returns: array of 'btype elements with shape 'shape', suitable for im.ashow()
-"""
- d = braw(fname,btype)
- d = d[8:]
- f = open(fname,'rb')
- shp = f.read(8)
- f.close()
- shp = N.fromstring(shp,N.int32)
- shp[0],shp[1] = shp[1],shp[0]
- try:
- carray = N.reshape(d,shp)
- return
- except:
- pass
- try:
- r = d[0::3]+0
- g = d[1::3]+0
- b = d[2::3]+0
- r.shape = shp
- g.shape = shp
- b.shape = shp
- carray = N.array([r,g,b])
- except:
- outstr = "glget: shape not correct for data of length "+str(len(d))
- raise ValueError, outstr
- return carray
-
-
-def mget(fname,btype):
- """
-Load in a file that was saved from matlab
-
-Usage: mget(fname,btype)
-"""
- d = braw(fname,btype)
- try:
- header = fname[0:-6]+'hdr'
- vals = get(header,0) # '0' means no missing-file warning msg
- if type(vals[0]) == ListType: # it's an extended header
- xsize = int(vals[0][0])
- ysize = int(vals[0][1])
- numslices = int(vals[0][2])
- else:
- xsize = int(vals[0])
- ysize = int(vals[1])
- numslices = int(vals[2])
- print xsize,ysize,numslices, d.shape
- except:
- print "No header file. Continuing ..."
- if numslices == 1:
- d.shape = [ysize,xsize]
- return N.transpose(d)*1
- else:
- d.shape = [numslices,ysize,xsize]
- return N.transpose(d)*1
-
-
-def mput(outarray,fname,writeheader=0,btype=N.int16):
- """
-Save a file for use in matlab.
-"""
- outarray = N.transpose(outarray)
- outdata = N.ravel(outarray).astype(btype)
- outdata = outdata.tostring()
- outfile = open(fname,'wb')
- outfile.write(outdata)
- outfile.close()
- if writeheader == 1:
- try:
- suffixindex = string.rfind(fname,'.')
- hdrname = fname[0:suffixindex]
- except ValueError:
- hdrname = fname
- if len(outarray.shape) == 2:
- hdr = [outarray.shape[1],outarray.shape[0], 1, 0]
- else:
- hdr = [outarray.shape[2],outarray.shape[1],outarray.shape[0], 0,'\n']
- print hdrname+'.hdr'
- outfile = open(hdrname+'.hdr','w')
- outfile.write(pstat.list2string(hdr))
- outfile.close()
- return None
-
-
-def bput(outarray,fname,writeheader=0,packtype=N.int16,writetype='wb'):
- """
-Writes the passed array to a binary output file, and then closes
-the file. Default is overwrite the destination file.
-
-Usage: bput (outarray,filename,writeheader=0,packtype=N.int16,writetype='wb')
-"""
- suffix = fname[-6:]
- if suffix == 'bshort':
- packtype = N.int16
- elif suffix == 'bfloat':
- packtype = N.float32
- else:
- print 'Not a bshort or bfloat file. Using packtype=',packtype
-
- outdata = N.ravel(outarray).astype(packtype)
-# littleEndian = ( struct.pack('i',1)==struct.pack('<i',1) )
-# if littleEndian:
-# outdata = outdata.byteswap()
- outdata = outdata.tostring()
- outfile = open(fname,writetype)
- outfile.write(outdata)
- outfile.close()
- if writeheader == 1:
- try:
- suffixindex = string.rfind(fname,'.')
- hdrname = fname[0:suffixindex]
- except ValueError:
- hdrname = fname
- if len(outarray.shape) == 2:
- hdr = [outarray.shape[0],outarray.shape[1], 1, 0]
- else:
- hdr = [outarray.shape[1],outarray.shape[2],outarray.shape[0], 0,'\n']
- print hdrname+'.hdr'
- outfile = open(hdrname+'.hdr','w')
- outfile.write(pstat.list2string(hdr))
- outfile.close()
- return None
-
-
-def mrget(fname,datatype=N.int16):
- """
-Opens a binary .MR file and clips off the tail data portion of it, returning
-the result as an array.
-
-Usage: mrget(fname,datatype=N.int16)
-"""
- d = braw(fname,datatype)
- if len(d) > 512*512:
- return N.reshape(d[-512*512:],(512,512))
- elif len(d) > 320*320:
- return N.reshape(d[-320*320:],(320,320))
- elif len(d) > 256*256:
- return N.reshape(d[-256*256:],(256,256))
- elif len(d) > 128*128:
- return N.reshape(d[-128*128:],(128,128))
- elif len(d) > 64*64:
- return N.reshape(d[-64*64:],(64,64))
- else:
- return N.reshape(d[-32*32:],(32,32))
-
-
-def quickload(fname,linestocut=4):
- """
-Quickly loads in a long text file, chopping off first n 'linestocut'.
-
-Usage: quickload(fname,linestocut=4)
-Returns: array filled with data in fname
-"""
- f = open(fname,'r')
- d = f.readlines()
- f.close()
- print fname,'read in.'
- d = d[linestocut:]
- d = map(string.split,d)
- print 'Done with string.split on lines.'
- for i in range(len(d)):
- d[i] = map(string.atoi,d[i])
- print 'Conversion to ints done.'
- return N.array(d)
-
-def writedelimited (listoflists, delimiter, file, writetype='w'):
- """
-Writes a list of lists in columns, separated by character(s) delimiter
-to specified file. File-overwrite is the default.
-
-Usage: writedelimited (listoflists,delimiter,filename,writetype='w')
-Returns: None
-"""
- if type(listoflists[0]) not in [ListType,TupleType]:
- listoflists = [listoflists]
- outfile = open(file,writetype)
- rowstokill = []
- list2print = copy.deepcopy(listoflists)
- for i in range(len(listoflists)):
- if listoflists[i] == ['\n'] or listoflists[i]=='\n' or listoflists[i]=='dashes':
- rowstokill = rowstokill + [i]
- rowstokill.reverse()
- for row in rowstokill:
- del list2print[row]
- maxsize = [0]*len(list2print[0])
- for row in listoflists:
- if row == ['\n'] or row == '\n':
- outfile.write('\n')
- elif row == ['dashes'] or row == 'dashes':
- dashes = [0]*len(maxsize)
- for j in range(len(maxsize)):
- dashes[j] = '------'
- outfile.write(pstat.linedelimited(dashes,delimiter))
- else:
- outfile.write(pstat.linedelimited(row,delimiter))
- outfile.write('\n')
- outfile.close()
- return None
-
-def writecc (listoflists,file,writetype='w',extra=2):
- """
-Writes a list of lists to a file in columns, customized by the max
-size of items within the columns (max size of items in col, +2 characters)
-to specified file. File-overwrite is the default.
-
-Usage: writecc (listoflists,file,writetype='w',extra=2)
-Returns: None
-"""
- if type(listoflists[0]) not in [ListType,TupleType]:
- listoflists = [listoflists]
- outfile = open(file,writetype)
- rowstokill = []
- list2print = copy.deepcopy(listoflists)
- for i in range(len(listoflists)):
- if listoflists[i] == ['\n'] or listoflists[i]=='\n' or listoflists[i]=='dashes':
- rowstokill = rowstokill + [i]
- rowstokill.reverse()
- for row in rowstokill:
- del list2print[row]
- maxsize = [0]*len(list2print[0])
- for col in range(len(list2print[0])):
- items = pstat.colex(list2print,col)
- items = map(pstat.makestr,items)
- maxsize[col] = max(map(len,items)) + extra
- for row in listoflists:
- if row == ['\n'] or row == '\n':
- outfile.write('\n')
- elif row == ['dashes'] or row == 'dashes':
- dashes = [0]*len(maxsize)
- for j in range(len(maxsize)):
- dashes[j] = '-'*(maxsize[j]-2)
- outfile.write(pstat.lineincustcols(dashes,maxsize))
- else:
- outfile.write(pstat.lineincustcols(row,maxsize))
- outfile.write('\n')
- outfile.close()
- return None
-
-
-def writefc (listoflists,colsize,file,writetype='w'):
- """
-Writes a list of lists to a file in columns of fixed size. File-overwrite
-is the default.
-
-Usage: writefc (listoflists,colsize,file,writetype='w')
-Returns: None
-"""
- if type(listoflists) == N.ndarray:
- listoflists = listoflists.tolist()
- if type(listoflists[0]) not in [ListType,TupleType]:
- listoflists = [listoflists]
- outfile = open(file,writetype)
- rowstokill = []
- list2print = copy.deepcopy(listoflists)
- for i in range(len(listoflists)):
- if listoflists[i] == ['\n'] or listoflists[i]=='\n' or listoflists[i]=='dashes':
- rowstokill = rowstokill + [i]
- rowstokill.reverse()
- for row in rowstokill:
- del list2print[row]
- n = [0]*len(list2print[0])
- for row in listoflists:
- if row == ['\n'] or row == '\n':
- outfile.write('\n')
- elif row == ['dashes'] or row == 'dashes':
- dashes = [0]*colsize
- for j in range(len(n)):
- dashes[j] = '-'*(colsize)
- outfile.write(pstat.lineincols(dashes,colsize))
- else:
- outfile.write(pstat.lineincols(row,colsize))
- outfile.write('\n')
- outfile.close()
- return None
-
-
-def load(fname,lines_to_ignore=4,type='i'):
- """
-Load in huge, flat, 2D text files. Can handle differing line-lengths AND
-can strip #/% on UNIX (or with a better NT grep). Requires wc, grep, and
-mmapfile.lib/.pyd. Type can be 'i', 'f' or 'd', for ints, floats or doubles,
-respectively. Lines_to_ignore determines how many lines at the start of the
-file to ignore (required for non-working grep).
-
-Usage: load(fname,lines_to_ignore=4,type='i')
-Returns: numpy array of specified type
-"""
- start = time.time() ## START TIMER
- if type == 'i':
- intype = int
- elif type in ['f','d']:
- intype = float
- else:
- raise ValueError, "type can be 'i', 'f' or 'd' in load()"
-
- ## STRIP OUT % AND # LINES
- tmpname = tempfile.mktemp()
- if sys.platform == 'win32':
- # NT VERSION OF GREP DOESN'T DO THE STRIPPING ... SIGH
- cmd = "grep.exe -v \'%\' "+fname+" > "+tmpname
- print cmd
- os.system(cmd)
- else:
- # UNIX SIDE SHOULD WORK
- cmd = "cat "+fname+" | grep -v \'%\' |grep -v \'#\' > "+tmpname
- print cmd
- os.system(cmd)
-
- ## GET NUMBER OF ROWS, COLUMNS AND LINE-LENGTH, USING WC
- wc = string.split(os.popen("wc "+tmpname).read())
- numlines = int(wc[0]) - lines_to_ignore
- tfp = open(tmpname)
- if lines_to_ignore <> 0:
- for i in range(lines_to_ignore):
- junk = tfp.readline()
- numcols = len(string.split(tfp.readline())) #int(float(wc[1])/numlines)
- tfp.close()
-
- ## PREPARE INPUT SPACE
- a = N.zeros((numlines*numcols), type)
- block = 65536 # chunk to read, in bytes
- data = mmapfile.mmapfile(tmpname, '', 0)
- if lines_to_ignore <> 0 and sys.platform == 'win32':
- for i in range(lines_to_ignore):
- junk = data.readline()
- i = 0
- d = ' '
- carryover = ''
- while len(d) <> 0:
- d = carryover + data.read(block)
- cutindex = string.rfind(d,'\n')
- carryover = d[cutindex+1:]
- d = d[:cutindex+1]
- d = map(intype,string.split(d))
- a[i:i+len(d)] = d
- i = i + len(d)
- end = time.time()
- print "%d sec" % round(end-start,2)
- data.close()
- os.remove(tmpname)
- return N.reshape(a,[numlines,numcols])
-
-
-def find_dirs(sourcedir):
- """Finds and returns all directories in sourcedir
-
-Usage: find_dirs(sourcedir)
-Returns: list of directory names (potentially empty)
-"""
- files = os.listdir(sourcedir)
- dirs = []
- for fname in files:
- if os.path.isdir(os.path.join(sourcedir,fname)):
- dirs.append(fname)
- return dirs
-
-
-# ALIASES ...
-save = aput
-
-
-
-def binget(fname,btype=None):
- """
-Loads a binary file from disk. Assumes associated hdr file is in same
-location. You can force an unpacking type, or else it tries to figure
-it out from the filename (4th-to-last character). Hence, readable file
-formats are ...
-
-1bin=int8, sbin=int16, ibin=int32, fbin=float32, dbin=float64, etc.
-
-Usage: binget(fname,btype=None)
-Returns: data in file fname of type btype
-"""
- file = open(fname,'rb')
- bdata = file.read()
- file.close()
-
- # if none given, assume character preceeding 'bin' is the unpacktype
- if not btype:
- btype = fname[-4]
- try:
- bdata = N.fromstring(bdata,btype)
- except:
- raise ValueError, "Bad unpacking type."
-
- # force the data on disk to be LittleEndian (for more efficient PC/Linux use)
- if not N.little_endian:
- bdata = bdata.byteswap()
-
- try:
- header = fname[:-3]+'hdr'
- vals = get(header,0) # '0' means no missing-file warning msg
- print vals
- if type(vals[0]) == ListType: # it's an extended header
- xsize = int(vals[0][0])
- ysize = int(vals[0][1])
- numslices = int(vals[0][2])
- else:
- bdata.shape = vals
- except:
- print "No (or bad) header file. Returning unshaped array."
- return N.array(bdata)
-
-
-
-def binput(outarray,fname,packtype=None,writetype='wb'):
- """
-Unravels outarray and writes the data to a file, always in LittleEndian
-format, along with a header file containing the original data shape. Default
-is overwrite the destination file. Tries to figure out packtype from
-4th-to-last character in filename. Thus, the routine understands these
-file formats ...
-
-1bin=int8, sbin=int16, ibin=int32, fbin=float32, dbin=float64, etc.
-
-Usage: binput(outarray,filename,packtype=None,writetype='wb')
-"""
- if not packtype:
- packtype = fname[-4]
-
- # a speck of error checking
- if packtype == N.int16 and outarray.dtype.char == 'f':
- # check to see if there's data loss
- if max(N.ravel(outarray)) > 32767 or min(N.ravel(outarray))<-32768:
- print "*** WARNING: CONVERTING FLOAT DATA TO OUT-OF RANGE INT16 DATA"
- outdata = N.ravel(outarray).astype(packtype)
-
- # force the data on disk to be little_endian (for more efficient PC/Linux use)
- if not N.little_endian:
- outdata = outdata.byteswap()
- outdata = outdata.tostring()
- outfile = open(fname,writetype)
- outfile.write(outdata)
- outfile.close()
-
- # Now, write the header file
- try:
- suffixindex = string.rfind(fname,'.')
- hdrname = fname[0:suffixindex+2]+'hdr' # include .s or .f or .1 or whatever
- except ValueError:
- hdrname = fname
- hdr = outarray.shape
- print hdrname
- outfile = open(hdrname,'w')
- outfile.write(pstat.list2string(hdr))
- outfile.close()
- return None
-
-def getafniparam(headfilename,paramname):
- """
-Loads in an AFNI header file, and returns the values of 'paramname'.
-
-Usage: getafniparam(headfile,paramname)
-Returns: appropriate "type" for params, or None if fails
-"""
- if headfilename[-4:] == 'BRIK': # if asked for BRIK, change it to HEAD
- headfilename = headfilename[:-4]+'HEAD'
- d = get(headfilename)
- lines = open(headfilename,'r').readlines()
- for i in range(len(lines)):
- if string.find(lines[i],paramname) <> -1:
- count = d[i+1][-1]
- gotten = 0
- result = []
- for j in range(i+2,len(lines)):
- for k in range(len(d[j])):
- if type(d[j][k]) == StringType:
- result = d[j][k][1:count]
- return result
- else:
- result.append(d[j][k])
- gotten += 1
- if gotten == count:
- break
- return result
- return None
-
-
-def add2afnihistory(headfilename,newtext):
- """
-Adds 'newtext' to HISTORY_NOTE in afni file specified in headfilename.
-
-Usage: add2afnihistory(headfile,newtext)
-Returns: None
-"""
- if headfilename[-4:] == 'BRIK': # if asked for BRIK, change it to HEAD
- headfilename = headfilename[:-4]+'HEAD'
- d = get(headfilename)
- lines = open(headfilename,'r').readlines()
- for i in range(len(lines)):
- if string.find(lines[i],'HISTORY_NOTE') <> -1:
- bytecount = d[i+1][-1]
- oldstr = lines[i+2][:-2]
- date = '[python:*** %s] ' %time.asctime()
- lines[i+2] = oldstr +'\\n' +date +newtext +'~\n'
- lines[i+1] = ' count = %s\n' %str(len(lines[i+2]))
- f = open(headfilename,'w')
- f.writelines(lines)
- f.close()
- return
-
-
-def array2afni(d,brikprefix,voltype=None,TR=2000,sliceorder='seqplus',geomparent=None,view=None,corrlength=1,briklabels=None,historytext=None):
- """
-Converts an array 'd' to an AFNI BRIK/HEAD combo via putbin and to3d. Tries to
-guess the AFNI volume type
-
-voltype = {'-anat','-epan','-fim'}
-geomparent = filename of the afni BRIK file with the same geometry
-view = {'tlrc', 'acpc' or 'orig'}
-corrlength = # of images used in the (single-timeseries) correlation (for fico)
-briklabels = list of names (strings) to use for brick labels
-historytext = string to be appended to the history file, if any
-
-Usage: array2afni(d,brikprefix,voltype=None,TR=2000,
- sliceorder='seqplus',geomparent=None,view=None,
- corrlength=1,briklabels=None,historytext=None)
-Returns: None
-"""
- # converts numpy typecode()s into appropriate strings for to3d command line
- typecodemapping = {'c':'b', # character
- 'B':'b', # UnsignedInt8
- 'f':'f', # float0, float8, float16, float32
- 'd':'f', # float64
- 'b':'b', # int0, int8
- 'h':'', # int16
- 'i':'i', # int32
- 'l':'i'} # int
-
- # Verify that the data is proper size (3- or 4-D)
- if len(d.shape) not in [3,4]:
- raise ValueError, "A 3D or 4D array is required for array2afni() ... %s" %d.shape
-
- # Save out the array to a binary file, homebrew style
- if d.dtype.char == N.float64:
- outcode = 'f'
- else:
- outcode = d.dtype.char
- tmpoutname = 'afnitmp.%sbin' % outcode
- binput(d.astype(outcode),tmpoutname)
- if not voltype:
- if len(d.shape) == 3: # either anatomy or functional
- if d.dtype.char in ['s','i','l']: # if floats, assume functional
- voltype = '-anat'
- else:
- voltype = '-fim'
- else: # 4D dataset, must be anatomical timeseries (epan)
- voltype = '-anat'
- if voltype[0] != '-':
- voltype = '-'+voltype
- if len(d.shape) == 3: # either anatomy or functional
- timepts = 1
- slices = d.shape[0]
- timestr = ''
- elif len(d.shape) == 4:
- if voltype=='-fico':
- timepts = 1
- d = N.reshape(d,[d.shape[0]*d.shape[1],d.shape[2],d.shape[3]])
- slices = d.shape[0]
- timestr = '-statpar %s 1 1 ' % corrlength
- else:
- timepts = d.shape[0]
- slices = d.shape[1]
- timestr = '-time:zt %d %d %0.3f %s ' % (slices,timepts,TR,sliceorder)
-
- cmd = 'to3d %s -prefix %s -session . ' % (voltype, brikprefix)
- if not view:
- view = 'orig'
- cmd += '-view %s ' % view
- if geomparent:
- cmd += '-geomparent %s ' % geomparent
- cmd += timestr
- cmd += '3D%s:0:0:%d:%d:%d:%s' % (typecodemapping[d.dtype.char],d.shape[-1],d.shape[-2],slices*timepts,tmpoutname)
- print cmd
- os.system(cmd)
- os.remove(tmpoutname)
- os.remove(tmpoutname[:-3]+'hdr')
-
- if len(d.shape)==4 and briklabels:
- names = ''
- for label in briklabels:
- names += str(label)+'~'
- count = len(names)
- appendstr = """\n\ntype = string-attribute
-name = BRICK_LABS
-count = %s
-'%s""" % (count, names)
- f = open('%s+%s.HEAD' %(brikprefix,view), 'a')
- f.write(appendstr)
- f.close()
-
- if historytext:
- add2afnihistory('%s+%s.HEAD'%(brikprefix,view),historytext)
diff --git a/site_utils/dashboard/external/pstat.py b/site_utils/dashboard/external/pstat.py
deleted file mode 100644
index ae0e76b..0000000
--- a/site_utils/dashboard/external/pstat.py
+++ /dev/null
@@ -1,1066 +0,0 @@
-# Copyright (c) 1999-2007 Gary Strangman; All Rights Reserved.
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# Comments and/or additions are welcome (send e-mail to:
-# strang@nmr.mgh.harvard.edu).
-#
-"""
-pstat.py module
-
-#################################################
-####### Written by: Gary Strangman ###########
-####### Last modified: Dec 18, 2007 ###########
-#################################################
-
-This module provides some useful list and array manipulation routines
-modeled after those found in the |Stat package by Gary Perlman, plus a
-number of other useful list/file manipulation functions. The list-based
-functions include:
-
- abut (source,*args)
- simpleabut (source, addon)
- colex (listoflists,cnums)
- collapse (listoflists,keepcols,collapsecols,fcn1=None,fcn2=None,cfcn=None)
- dm (listoflists,criterion)
- flat (l)
- linexand (listoflists,columnlist,valuelist)
- linexor (listoflists,columnlist,valuelist)
- linedelimited (inlist,delimiter)
- lineincols (inlist,colsize)
- lineincustcols (inlist,colsizes)
- list2string (inlist)
- makelol(inlist)
- makestr(x)
- printcc (lst,extra=2)
- printincols (listoflists,colsize)
- pl (listoflists)
- printl(listoflists)
- replace (lst,oldval,newval)
- recode (inlist,listmap,cols='all')
- remap (listoflists,criterion)
- roundlist (inlist,num_digits_to_round_floats_to)
- sortby(listoflists,sortcols)
- unique (inlist)
- duplicates(inlist)
- writedelimited (listoflists, delimiter, file, writetype='w')
-
-Some of these functions have alternate versions which are defined only if
-Numeric (NumPy) can be imported. These functions are generally named as
-above, with an 'a' prefix.
-
- aabut (source, *args)
- acolex (a,indices,axis=1)
- acollapse (a,keepcols,collapsecols,sterr=0,ns=0)
- adm (a,criterion)
- alinexand (a,columnlist,valuelist)
- alinexor (a,columnlist,valuelist)
- areplace (a,oldval,newval)
- arecode (a,listmap,col='all')
- arowcompare (row1, row2)
- arowsame (row1, row2)
- asortrows(a,axis=0)
- aunique(inarray)
- aduplicates(inarray)
-
-Currently, the code is all but completely un-optimized. In many cases, the
-array versions of functions amount simply to aliases to built-in array
-functions/methods. Their inclusion here is for function name consistency.
-"""
-
-## CHANGE LOG:
-## ==========
-## 07-11-26 ... edited to work with numpy
-## 01-11-15 ... changed list2string() to accept a delimiter
-## 01-06-29 ... converted exec()'s to eval()'s to make compatible with Py2.1
-## 01-05-31 ... added duplicates() and aduplicates() functions
-## 00-12-28 ... license made GPL, docstring and import requirements
-## 99-11-01 ... changed version to 0.3
-## 99-08-30 ... removed get, getstrings, put, aget, aput (into io.py)
-## 03/27/99 ... added areplace function, made replace fcn recursive
-## 12/31/98 ... added writefc function for ouput to fixed column sizes
-## 12/07/98 ... fixed import problem (failed on collapse() fcn)
-## added __version__ variable (now 0.2)
-## 12/05/98 ... updated doc-strings
-## added features to collapse() function
-## added flat() function for lists
-## fixed a broken asortrows()
-## 11/16/98 ... fixed minor bug in aput for 1D arrays
-##
-## 11/08/98 ... fixed aput to output large arrays correctly
-
-import stats # required 3rd party module
-import string, copy
-from types import *
-
-__version__ = 0.4
-
-###=========================== LIST FUNCTIONS ==========================
-###
-### Here are the list functions, DEFINED FOR ALL SYSTEMS.
-### Array functions (for NumPy-enabled computers) appear below.
-###
-
-def abut (source,*args):
- """
-Like the |Stat abut command. It concatenates two lists side-by-side
-and returns the result. '2D' lists are also accomodated for either argument
-(source or addon). CAUTION: If one list is shorter, it will be repeated
-until it is as long as the longest list. If this behavior is not desired,
-use pstat.simpleabut().
-
-Usage: abut(source, args) where args=any # of lists
-Returns: a list of lists as long as the LONGEST list past, source on the
- 'left', lists in <args> attached consecutively on the 'right'
-"""
-
- if type(source) not in [ListType,TupleType]:
- source = [source]
- for addon in args:
- if type(addon) not in [ListType,TupleType]:
- addon = [addon]
- if len(addon) < len(source): # is source list longer?
- if len(source) % len(addon) == 0: # are they integer multiples?
- repeats = len(source)/len(addon) # repeat addon n times
- origadd = copy.deepcopy(addon)
- for i in range(repeats-1):
- addon = addon + origadd
- else:
- repeats = len(source)/len(addon)+1 # repeat addon x times,
- origadd = copy.deepcopy(addon) # x is NOT an integer
- for i in range(repeats-1):
- addon = addon + origadd
- addon = addon[0:len(source)]
- elif len(source) < len(addon): # is addon list longer?
- if len(addon) % len(source) == 0: # are they integer multiples?
- repeats = len(addon)/len(source) # repeat source n times
- origsour = copy.deepcopy(source)
- for i in range(repeats-1):
- source = source + origsour
- else:
- repeats = len(addon)/len(source)+1 # repeat source x times,
- origsour = copy.deepcopy(source) # x is NOT an integer
- for i in range(repeats-1):
- source = source + origsour
- source = source[0:len(addon)]
-
- source = simpleabut(source,addon)
- return source
-
-
-def simpleabut (source, addon):
- """
-Concatenates two lists as columns and returns the result. '2D' lists
-are also accomodated for either argument (source or addon). This DOES NOT
-repeat either list to make the 2 lists of equal length. Beware of list pairs
-with different lengths ... the resulting list will be the length of the
-FIRST list passed.
-
-Usage: simpleabut(source,addon) where source, addon=list (or list-of-lists)
-Returns: a list of lists as long as source, with source on the 'left' and
- addon on the 'right'
-"""
- if type(source) not in [ListType,TupleType]:
- source = [source]
- if type(addon) not in [ListType,TupleType]:
- addon = [addon]
- minlen = min(len(source),len(addon))
- list = copy.deepcopy(source) # start abut process
- if type(source[0]) not in [ListType,TupleType]:
- if type(addon[0]) not in [ListType,TupleType]:
- for i in range(minlen):
- list[i] = [source[i]] + [addon[i]] # source/addon = column
- else:
- for i in range(minlen):
- list[i] = [source[i]] + addon[i] # addon=list-of-lists
- else:
- if type(addon[0]) not in [ListType,TupleType]:
- for i in range(minlen):
- list[i] = source[i] + [addon[i]] # source=list-of-lists
- else:
- for i in range(minlen):
- list[i] = source[i] + addon[i] # source/addon = list-of-lists
- source = list
- return source
-
-
-def colex (listoflists,cnums):
- """
-Extracts from listoflists the columns specified in the list 'cnums'
-(cnums can be an integer, a sequence of integers, or a string-expression that
-corresponds to a slice operation on the variable x ... e.g., 'x[3:]' will colex
-columns 3 onward from the listoflists).
-
-Usage: colex (listoflists,cnums)
-Returns: a list-of-lists corresponding to the columns from listoflists
- specified by cnums, in the order the column numbers appear in cnums
-"""
- global index
- column = 0
- if type(cnums) in [ListType,TupleType]: # if multiple columns to get
- index = cnums[0]
- column = map(lambda x: x[index], listoflists)
- for col in cnums[1:]:
- index = col
- column = abut(column,map(lambda x: x[index], listoflists))
- elif type(cnums) == StringType: # if an 'x[3:]' type expr.
- evalstring = 'map(lambda x: x'+cnums+', listoflists)'
- column = eval(evalstring)
- else: # else it's just 1 col to get
- index = cnums
- column = map(lambda x: x[index], listoflists)
- return column
-
-
-def collapse (listoflists,keepcols,collapsecols,fcn1=None,fcn2=None,cfcn=None):
- """
-Averages data in collapsecol, keeping all unique items in keepcols
-(using unique, which keeps unique LISTS of column numbers), retaining the
-unique sets of values in keepcols, the mean for each. Setting fcn1
-and/or fcn2 to point to a function rather than None (e.g., stats.sterr, len)
-will append those results (e.g., the sterr, N) after each calculated mean.
-cfcn is the collapse function to apply (defaults to mean, defined here in the
-pstat module to avoid circular imports with stats.py, but harmonicmean or
-others could be passed).
-
-Usage: collapse (listoflists,keepcols,collapsecols,fcn1=None,fcn2=None,cfcn=None)
-Returns: a list of lists with all unique permutations of entries appearing in
- columns ("conditions") specified by keepcols, abutted with the result of
- cfcn (if cfcn=None, defaults to the mean) of each column specified by
- collapsecols.
-"""
- def collmean (inlist):
- s = 0
- for item in inlist:
- s = s + item
- return s/float(len(inlist))
-
- if type(keepcols) not in [ListType,TupleType]:
- keepcols = [keepcols]
- if type(collapsecols) not in [ListType,TupleType]:
- collapsecols = [collapsecols]
- if cfcn == None:
- cfcn = collmean
- if keepcols == []:
- means = [0]*len(collapsecols)
- for i in range(len(collapsecols)):
- avgcol = colex(listoflists,collapsecols[i])
- means[i] = cfcn(avgcol)
- if fcn1:
- try:
- test = fcn1(avgcol)
- except:
- test = 'N/A'
- means[i] = [means[i], test]
- if fcn2:
- try:
- test = fcn2(avgcol)
- except:
- test = 'N/A'
- try:
- means[i] = means[i] + [len(avgcol)]
- except TypeError:
- means[i] = [means[i],len(avgcol)]
- return means
- else:
- values = colex(listoflists,keepcols)
- uniques = unique(values)
- uniques.sort()
- newlist = []
- if type(keepcols) not in [ListType,TupleType]: keepcols = [keepcols]
- for item in uniques:
- if type(item) not in [ListType,TupleType]: item =[item]
- tmprows = linexand(listoflists,keepcols,item)
- for col in collapsecols:
- avgcol = colex(tmprows,col)
- item.append(cfcn(avgcol))
- if fcn1 <> None:
- try:
- test = fcn1(avgcol)
- except:
- test = 'N/A'
- item.append(test)
- if fcn2 <> None:
- try:
- test = fcn2(avgcol)
- except:
- test = 'N/A'
- item.append(test)
- newlist.append(item)
- return newlist
-
-
-def dm (listoflists,criterion):
- """
-Returns rows from the passed list of lists that meet the criteria in
-the passed criterion expression (a string as a function of x; e.g., 'x[3]>=9'
-will return all rows where the 4th column>=9 and "x[2]=='N'" will return rows
-with column 2 equal to the string 'N').
-
-Usage: dm (listoflists, criterion)
-Returns: rows from listoflists that meet the specified criterion.
-"""
- function = 'filter(lambda x: '+criterion+',listoflists)'
- lines = eval(function)
- return lines
-
-
-def flat(l):
- """
-Returns the flattened version of a '2D' list. List-correlate to the a.ravel()()
-method of NumPy arrays.
-
-Usage: flat(l)
-"""
- newl = []
- for i in range(len(l)):
- for j in range(len(l[i])):
- newl.append(l[i][j])
- return newl
-
-
-def linexand (listoflists,columnlist,valuelist):
- """
-Returns the rows of a list of lists where col (from columnlist) = val
-(from valuelist) for EVERY pair of values (columnlist[i],valuelists[i]).
-len(columnlist) must equal len(valuelist).
-
-Usage: linexand (listoflists,columnlist,valuelist)
-Returns: the rows of listoflists where columnlist[i]=valuelist[i] for ALL i
-"""
- if type(columnlist) not in [ListType,TupleType]:
- columnlist = [columnlist]
- if type(valuelist) not in [ListType,TupleType]:
- valuelist = [valuelist]
- criterion = ''
- for i in range(len(columnlist)):
- if type(valuelist[i])==StringType:
- critval = '\'' + valuelist[i] + '\''
- else:
- critval = str(valuelist[i])
- criterion = criterion + ' x['+str(columnlist[i])+']=='+critval+' and'
- criterion = criterion[0:-3] # remove the "and" after the last crit
- function = 'filter(lambda x: '+criterion+',listoflists)'
- lines = eval(function)
- return lines
-
-
-def linexor (listoflists,columnlist,valuelist):
- """
-Returns the rows of a list of lists where col (from columnlist) = val
-(from valuelist) for ANY pair of values (colunmlist[i],valuelist[i[).
-One value is required for each column in columnlist. If only one value
-exists for columnlist but multiple values appear in valuelist, the
-valuelist values are all assumed to pertain to the same column.
-
-Usage: linexor (listoflists,columnlist,valuelist)
-Returns: the rows of listoflists where columnlist[i]=valuelist[i] for ANY i
-"""
- if type(columnlist) not in [ListType,TupleType]:
- columnlist = [columnlist]
- if type(valuelist) not in [ListType,TupleType]:
- valuelist = [valuelist]
- criterion = ''
- if len(columnlist) == 1 and len(valuelist) > 1:
- columnlist = columnlist*len(valuelist)
- for i in range(len(columnlist)): # build an exec string
- if type(valuelist[i])==StringType:
- critval = '\'' + valuelist[i] + '\''
- else:
- critval = str(valuelist[i])
- criterion = criterion + ' x['+str(columnlist[i])+']=='+critval+' or'
- criterion = criterion[0:-2] # remove the "or" after the last crit
- function = 'filter(lambda x: '+criterion+',listoflists)'
- lines = eval(function)
- return lines
-
-
-def linedelimited (inlist,delimiter):
- """
-Returns a string composed of elements in inlist, with each element
-separated by 'delimiter.' Used by function writedelimited. Use '\t'
-for tab-delimiting.
-
-Usage: linedelimited (inlist,delimiter)
-"""
- outstr = ''
- for item in inlist:
- if type(item) <> StringType:
- item = str(item)
- outstr = outstr + item + delimiter
- outstr = outstr[0:-1]
- return outstr
-
-
-def lineincols (inlist,colsize):
- """
-Returns a string composed of elements in inlist, with each element
-right-aligned in columns of (fixed) colsize.
-
-Usage: lineincols (inlist,colsize) where colsize is an integer
-"""
- outstr = ''
- for item in inlist:
- if type(item) <> StringType:
- item = str(item)
- size = len(item)
- if size <= colsize:
- for i in range(colsize-size):
- outstr = outstr + ' '
- outstr = outstr + item
- else:
- outstr = outstr + item[0:colsize+1]
- return outstr
-
-
-def lineincustcols (inlist,colsizes):
- """
-Returns a string composed of elements in inlist, with each element
-right-aligned in a column of width specified by a sequence colsizes. The
-length of colsizes must be greater than or equal to the number of columns
-in inlist.
-
-Usage: lineincustcols (inlist,colsizes)
-Returns: formatted string created from inlist
-"""
- outstr = ''
- for i in range(len(inlist)):
- if type(inlist[i]) <> StringType:
- item = str(inlist[i])
- else:
- item = inlist[i]
- size = len(item)
- if size <= colsizes[i]:
- for j in range(colsizes[i]-size):
- outstr = outstr + ' '
- outstr = outstr + item
- else:
- outstr = outstr + item[0:colsizes[i]+1]
- return outstr
-
-
-def list2string (inlist,delimit=' '):
- """
-Converts a 1D list to a single long string for file output, using
-the string.join function.
-
-Usage: list2string (inlist,delimit=' ')
-Returns: the string created from inlist
-"""
- stringlist = map(makestr,inlist)
- return string.join(stringlist,delimit)
-
-
-def makelol(inlist):
- """
-Converts a 1D list to a 2D list (i.e., a list-of-lists). Useful when you
-want to use put() to write a 1D list one item per line in the file.
-
-Usage: makelol(inlist)
-Returns: if l = [1,2,'hi'] then returns [[1],[2],['hi']] etc.
-"""
- x = []
- for item in inlist:
- x.append([item])
- return x
-
-
-def makestr (x):
- if type(x) <> StringType:
- x = str(x)
- return x
-
-
-def printcc (lst,extra=2):
- """
-Prints a list of lists in columns, customized by the max size of items
-within the columns (max size of items in col, plus 'extra' number of spaces).
-Use 'dashes' or '\\n' in the list-of-lists to print dashes or blank lines,
-respectively.
-
-Usage: printcc (lst,extra=2)
-Returns: None
-"""
- if type(lst[0]) not in [ListType,TupleType]:
- lst = [lst]
- rowstokill = []
- list2print = copy.deepcopy(lst)
- for i in range(len(lst)):
- if lst[i] == ['\n'] or lst[i]=='\n' or lst[i]=='dashes' or lst[i]=='' or lst[i]==['']:
- rowstokill = rowstokill + [i]
- rowstokill.reverse() # delete blank rows from the end
- for row in rowstokill:
- del list2print[row]
- maxsize = [0]*len(list2print[0])
- for col in range(len(list2print[0])):
- items = colex(list2print,col)
- items = map(makestr,items)
- maxsize[col] = max(map(len,items)) + extra
- for row in lst:
- if row == ['\n'] or row == '\n' or row == '' or row == ['']:
- print
- elif row == ['dashes'] or row == 'dashes':
- dashes = [0]*len(maxsize)
- for j in range(len(maxsize)):
- dashes[j] = '-'*(maxsize[j]-2)
- print lineincustcols(dashes,maxsize)
- else:
- print lineincustcols(row,maxsize)
- return None
-
-
-def printincols (listoflists,colsize):
- """
-Prints a list of lists in columns of (fixed) colsize width, where
-colsize is an integer.
-
-Usage: printincols (listoflists,colsize)
-Returns: None
-"""
- for row in listoflists:
- print lineincols(row,colsize)
- return None
-
-
-def pl (listoflists):
- """
-Prints a list of lists, 1 list (row) at a time.
-
-Usage: pl(listoflists)
-Returns: None
-"""
- for row in listoflists:
- if row[-1] == '\n':
- print row,
- else:
- print row
- return None
-
-
-def printl(listoflists):
- """Alias for pl."""
- pl(listoflists)
- return
-
-
-def replace (inlst,oldval,newval):
- """
-Replaces all occurrences of 'oldval' with 'newval', recursively.
-
-Usage: replace (inlst,oldval,newval)
-"""
- lst = inlst*1
- for i in range(len(lst)):
- if type(lst[i]) not in [ListType,TupleType]:
- if lst[i]==oldval: lst[i]=newval
- else:
- lst[i] = replace(lst[i],oldval,newval)
- return lst
-
-
-def recode (inlist,listmap,cols=None):
- """
-Changes the values in a list to a new set of values (useful when
-you need to recode data from (e.g.) strings to numbers. cols defaults
-to None (meaning all columns are recoded).
-
-Usage: recode (inlist,listmap,cols=None) cols=recode cols, listmap=2D list
-Returns: inlist with the appropriate values replaced with new ones
-"""
- lst = copy.deepcopy(inlist)
- if cols != None:
- if type(cols) not in [ListType,TupleType]:
- cols = [cols]
- for col in cols:
- for row in range(len(lst)):
- try:
- idx = colex(listmap,0).index(lst[row][col])
- lst[row][col] = listmap[idx][1]
- except ValueError:
- pass
- else:
- for row in range(len(lst)):
- for col in range(len(lst)):
- try:
- idx = colex(listmap,0).index(lst[row][col])
- lst[row][col] = listmap[idx][1]
- except ValueError:
- pass
- return lst
-
-
-def remap (listoflists,criterion):
- """
-Remaps values in a given column of a 2D list (listoflists). This requires
-a criterion as a function of 'x' so that the result of the following is
-returned ... map(lambda x: 'criterion',listoflists).
-
-Usage: remap(listoflists,criterion) criterion=string
-Returns: remapped version of listoflists
-"""
- function = 'map(lambda x: '+criterion+',listoflists)'
- lines = eval(function)
- return lines
-
-
-def roundlist (inlist,digits):
- """
-Goes through each element in a 1D or 2D inlist, and applies the following
-function to all elements of FloatType ... round(element,digits).
-
-Usage: roundlist(inlist,digits)
-Returns: list with rounded floats
-"""
- if type(inlist[0]) in [IntType, FloatType]:
- inlist = [inlist]
- l = inlist*1
- for i in range(len(l)):
- for j in range(len(l[i])):
- if type(l[i][j])==FloatType:
- l[i][j] = round(l[i][j],digits)
- return l
-
-
-def sortby(listoflists,sortcols):
- """
-Sorts a list of lists on the column(s) specified in the sequence
-sortcols.
-
-Usage: sortby(listoflists,sortcols)
-Returns: sorted list, unchanged column ordering
-"""
- newlist = abut(colex(listoflists,sortcols),listoflists)
- newlist.sort()
- try:
- numcols = len(sortcols)
- except TypeError:
- numcols = 1
- crit = '[' + str(numcols) + ':]'
- newlist = colex(newlist,crit)
- return newlist
-
-
-def unique (inlist):
- """
-Returns all unique items in the passed list. If the a list-of-lists
-is passed, unique LISTS are found (i.e., items in the first dimension are
-compared).
-
-Usage: unique (inlist)
-Returns: the unique elements (or rows) in inlist
-"""
- uniques = []
- for item in inlist:
- if item not in uniques:
- uniques.append(item)
- return uniques
-
-def duplicates(inlist):
- """
-Returns duplicate items in the FIRST dimension of the passed list.
-
-Usage: duplicates (inlist)
-"""
- dups = []
- for i in range(len(inlist)):
- if inlist[i] in inlist[i+1:]:
- dups.append(inlist[i])
- return dups
-
-
-def nonrepeats(inlist):
- """
-Returns items that are NOT duplicated in the first dim of the passed list.
-
-Usage: nonrepeats (inlist)
-"""
- nonrepeats = []
- for i in range(len(inlist)):
- if inlist.count(inlist[i]) == 1:
- nonrepeats.append(inlist[i])
- return nonrepeats
-
-
-#=================== PSTAT ARRAY FUNCTIONS =====================
-#=================== PSTAT ARRAY FUNCTIONS =====================
-#=================== PSTAT ARRAY FUNCTIONS =====================
-#=================== PSTAT ARRAY FUNCTIONS =====================
-#=================== PSTAT ARRAY FUNCTIONS =====================
-#=================== PSTAT ARRAY FUNCTIONS =====================
-#=================== PSTAT ARRAY FUNCTIONS =====================
-#=================== PSTAT ARRAY FUNCTIONS =====================
-#=================== PSTAT ARRAY FUNCTIONS =====================
-#=================== PSTAT ARRAY FUNCTIONS =====================
-#=================== PSTAT ARRAY FUNCTIONS =====================
-#=================== PSTAT ARRAY FUNCTIONS =====================
-#=================== PSTAT ARRAY FUNCTIONS =====================
-#=================== PSTAT ARRAY FUNCTIONS =====================
-#=================== PSTAT ARRAY FUNCTIONS =====================
-#=================== PSTAT ARRAY FUNCTIONS =====================
-
-try: # DEFINE THESE *ONLY* IF numpy IS AVAILABLE
- import numpy as N
-
- def aabut (source, *args):
- """
-Like the |Stat abut command. It concatenates two arrays column-wise
-and returns the result. CAUTION: If one array is shorter, it will be
-repeated until it is as long as the other.
-
-Usage: aabut (source, args) where args=any # of arrays
-Returns: an array as long as the LONGEST array past, source appearing on the
- 'left', arrays in <args> attached on the 'right'.
-"""
- if len(source.shape)==1:
- width = 1
- source = N.resize(source,[source.shape[0],width])
- else:
- width = source.shape[1]
- for addon in args:
- if len(addon.shape)==1:
- width = 1
- addon = N.resize(addon,[source.shape[0],width])
- else:
- width = source.shape[1]
- if len(addon) < len(source):
- addon = N.resize(addon,[source.shape[0],addon.shape[1]])
- elif len(source) < len(addon):
- source = N.resize(source,[addon.shape[0],source.shape[1]])
- source = N.concatenate((source,addon),1)
- return source
-
-
- def acolex (a,indices,axis=1):
- """
-Extracts specified indices (a list) from passed array, along passed
-axis (column extraction is default). BEWARE: A 1D array is presumed to be a
-column-array (and that the whole array will be returned as a column).
-
-Usage: acolex (a,indices,axis=1)
-Returns: the columns of a specified by indices
-"""
- if type(indices) not in [ListType,TupleType,N.ndarray]:
- indices = [indices]
- if len(N.shape(a)) == 1:
- cols = N.resize(a,[a.shape[0],1])
- else:
- cols = N.take(a,indices,axis)
- return cols
-
-
- def acollapse (a,keepcols,collapsecols,fcn1=None,fcn2=None,cfcn=None):
- """
-Averages data in collapsecol, keeping all unique items in keepcols
-(using unique, which keeps unique LISTS of column numbers), retaining
-the unique sets of values in keepcols, the mean for each. If stderror or
-N of the mean are desired, set either or both parameters to 1.
-
-Usage: acollapse (a,keepcols,collapsecols,fcn1=None,fcn2=None,cfcn=None)
-Returns: unique 'conditions' specified by the contents of columns specified
- by keepcols, abutted with the mean(s) of column(s) specified by
- collapsecols
-"""
- def acollmean (inarray):
- return N.sum(N.ravel(inarray))
-
- if type(keepcols) not in [ListType,TupleType,N.ndarray]:
- keepcols = [keepcols]
- if type(collapsecols) not in [ListType,TupleType,N.ndarray]:
- collapsecols = [collapsecols]
-
- if cfcn == None:
- cfcn = acollmean
- if keepcols == []:
- avgcol = acolex(a,collapsecols)
- means = N.sum(avgcol)/float(len(avgcol))
- if fcn1<>None:
- try:
- test = fcn1(avgcol)
- except:
- test = N.array(['N/A']*len(means))
- means = aabut(means,test)
- if fcn2<>None:
- try:
- test = fcn2(avgcol)
- except:
- test = N.array(['N/A']*len(means))
- means = aabut(means,test)
- return means
- else:
- if type(keepcols) not in [ListType,TupleType,N.ndarray]:
- keepcols = [keepcols]
- values = colex(a,keepcols) # so that "item" can be appended (below)
- uniques = unique(values) # get a LIST, so .sort keeps rows intact
- uniques.sort()
- newlist = []
- for item in uniques:
- if type(item) not in [ListType,TupleType,N.ndarray]:
- item =[item]
- tmprows = alinexand(a,keepcols,item)
- for col in collapsecols:
- avgcol = acolex(tmprows,col)
- item.append(acollmean(avgcol))
- if fcn1<>None:
- try:
- test = fcn1(avgcol)
- except:
- test = 'N/A'
- item.append(test)
- if fcn2<>None:
- try:
- test = fcn2(avgcol)
- except:
- test = 'N/A'
- item.append(test)
- newlist.append(item)
- try:
- new_a = N.array(newlist)
- except TypeError:
- new_a = N.array(newlist,'O')
- return new_a
-
-
- def adm (a,criterion):
- """
-Returns rows from the passed list of lists that meet the criteria in
-the passed criterion expression (a string as a function of x).
-
-Usage: adm (a,criterion) where criterion is like 'x[2]==37'
-"""
- function = 'filter(lambda x: '+criterion+',a)'
- lines = eval(function)
- try:
- lines = N.array(lines)
- except:
- lines = N.array(lines,dtype='O')
- return lines
-
-
- def isstring(x):
- if type(x)==StringType:
- return 1
- else:
- return 0
-
-
- def alinexand (a,columnlist,valuelist):
- """
-Returns the rows of an array where col (from columnlist) = val
-(from valuelist). One value is required for each column in columnlist.
-
-Usage: alinexand (a,columnlist,valuelist)
-Returns: the rows of a where columnlist[i]=valuelist[i] for ALL i
-"""
- if type(columnlist) not in [ListType,TupleType,N.ndarray]:
- columnlist = [columnlist]
- if type(valuelist) not in [ListType,TupleType,N.ndarray]:
- valuelist = [valuelist]
- criterion = ''
- for i in range(len(columnlist)):
- if type(valuelist[i])==StringType:
- critval = '\'' + valuelist[i] + '\''
- else:
- critval = str(valuelist[i])
- criterion = criterion + ' x['+str(columnlist[i])+']=='+critval+' and'
- criterion = criterion[0:-3] # remove the "and" after the last crit
- return adm(a,criterion)
-
-
- def alinexor (a,columnlist,valuelist):
- """
-Returns the rows of an array where col (from columnlist) = val (from
-valuelist). One value is required for each column in columnlist.
-The exception is if either columnlist or valuelist has only 1 value,
-in which case that item will be expanded to match the length of the
-other list.
-
-Usage: alinexor (a,columnlist,valuelist)
-Returns: the rows of a where columnlist[i]=valuelist[i] for ANY i
-"""
- if type(columnlist) not in [ListType,TupleType,N.ndarray]:
- columnlist = [columnlist]
- if type(valuelist) not in [ListType,TupleType,N.ndarray]:
- valuelist = [valuelist]
- criterion = ''
- if len(columnlist) == 1 and len(valuelist) > 1:
- columnlist = columnlist*len(valuelist)
- elif len(valuelist) == 1 and len(columnlist) > 1:
- valuelist = valuelist*len(columnlist)
- for i in range(len(columnlist)):
- if type(valuelist[i])==StringType:
- critval = '\'' + valuelist[i] + '\''
- else:
- critval = str(valuelist[i])
- criterion = criterion + ' x['+str(columnlist[i])+']=='+critval+' or'
- criterion = criterion[0:-2] # remove the "or" after the last crit
- return adm(a,criterion)
-
-
- def areplace (a,oldval,newval):
- """
-Replaces all occurrences of oldval with newval in array a.
-
-Usage: areplace(a,oldval,newval)
-"""
- return N.where(a==oldval,newval,a)
-
-
- def arecode (a,listmap,col='all'):
- """
-Remaps the values in an array to a new set of values (useful when
-you need to recode data from (e.g.) strings to numbers as most stats
-packages require. Can work on SINGLE columns, or 'all' columns at once.
-@@@BROKEN 2007-11-26
-
-Usage: arecode (a,listmap,col='all')
-Returns: a version of array a where listmap[i][0] = (instead) listmap[i][1]
-"""
- ashape = a.shape
- if col == 'all':
- work = a.ravel()
- else:
- work = acolex(a,col)
- work = work.ravel()
- for pair in listmap:
- if type(pair[1]) == StringType or work.dtype.char=='O' or a.dtype.char=='O':
- work = N.array(work,dtype='O')
- a = N.array(a,dtype='O')
- for i in range(len(work)):
- if work[i]==pair[0]:
- work[i] = pair[1]
- if col == 'all':
- return N.reshape(work,ashape)
- else:
- return N.concatenate([a[:,0:col],work[:,N.newaxis],a[:,col+1:]],1)
- else: # must be a non-Object type array and replacement
- work = N.where(work==pair[0],pair[1],work)
- return N.concatenate([a[:,0:col],work[:,N.newaxis],a[:,col+1:]],1)
-
-
- def arowcompare(row1, row2):
- """
-Compares two rows from an array, regardless of whether it is an
-array of numbers or of python objects (which requires the cmp function).
-@@@PURPOSE? 2007-11-26
-
-Usage: arowcompare(row1,row2)
-Returns: an array of equal length containing 1s where the two rows had
- identical elements and 0 otherwise
-"""
- return
- if row1.dtype.char=='O' or row2.dtype=='O':
- cmpvect = N.logical_not(abs(N.array(map(cmp,row1,row2)))) # cmp fcn gives -1,0,1
- else:
- cmpvect = N.equal(row1,row2)
- return cmpvect
-
-
- def arowsame(row1, row2):
- """
-Compares two rows from an array, regardless of whether it is an
-array of numbers or of python objects (which requires the cmp function).
-
-Usage: arowsame(row1,row2)
-Returns: 1 if the two rows are identical, 0 otherwise.
-"""
- cmpval = N.alltrue(arowcompare(row1,row2))
- return cmpval
-
-
- def asortrows(a,axis=0):
- """
-Sorts an array "by rows". This differs from the Numeric.sort() function,
-which sorts elements WITHIN the given axis. Instead, this function keeps
-the elements along the given axis intact, but shifts them 'up or down'
-relative to one another.
-
-Usage: asortrows(a,axis=0)
-Returns: sorted version of a
-"""
- return N.sort(a,axis=axis,kind='mergesort')
-
-
- def aunique(inarray):
- """
-Returns unique items in the FIRST dimension of the passed array. Only
-works on arrays NOT including string items.
-
-Usage: aunique (inarray)
-"""
- uniques = N.array([inarray[0]])
- if len(uniques.shape) == 1: # IF IT'S A 1D ARRAY
- for item in inarray[1:]:
- if N.add.reduce(N.equal(uniques,item).ravel()) == 0:
- try:
- uniques = N.concatenate([uniques,N.array[N.newaxis,:]])
- except TypeError:
- uniques = N.concatenate([uniques,N.array([item])])
- else: # IT MUST BE A 2+D ARRAY
- if inarray.dtype.char != 'O': # not an Object array
- for item in inarray[1:]:
- if not N.sum(N.alltrue(N.equal(uniques,item),1)):
- try:
- uniques = N.concatenate( [uniques,item[N.newaxis,:]] )
- except TypeError: # the item to add isn't a list
- uniques = N.concatenate([uniques,N.array([item])])
- else:
- pass # this item is already in the uniques array
- else: # must be an Object array, alltrue/equal functions don't work
- for item in inarray[1:]:
- newflag = 1
- for unq in uniques: # NOTE: cmp --> 0=same, -1=<, 1=>
- test = N.sum(abs(N.array(map(cmp,item,unq))))
- if test == 0: # if item identical to any 1 row in uniques
- newflag = 0 # then not a novel item to add
- break
- if newflag == 1:
- try:
- uniques = N.concatenate( [uniques,item[N.newaxis,:]] )
- except TypeError: # the item to add isn't a list
- uniques = N.concatenate([uniques,N.array([item])])
- return uniques
-
-
- def aduplicates(inarray):
- """
-Returns duplicate items in the FIRST dimension of the passed array. Only
-works on arrays NOT including string items.
-
-Usage: aunique (inarray)
-"""
- inarray = N.array(inarray)
- if len(inarray.shape) == 1: # IF IT'S A 1D ARRAY
- dups = []
- inarray = inarray.tolist()
- for i in range(len(inarray)):
- if inarray[i] in inarray[i+1:]:
- dups.append(inarray[i])
- dups = aunique(dups)
- else: # IT MUST BE A 2+D ARRAY
- dups = []
- aslist = inarray.tolist()
- for i in range(len(aslist)):
- if aslist[i] in aslist[i+1:]:
- dups.append(aslist[i])
- dups = unique(dups)
- dups = N.array(dups)
- return dups
-
-except ImportError: # IF NUMERIC ISN'T AVAILABLE, SKIP ALL arrayfuncs
- pass
diff --git a/site_utils/dashboard/external/stats.py b/site_utils/dashboard/external/stats.py
deleted file mode 100644
index ed53105..0000000
--- a/site_utils/dashboard/external/stats.py
+++ /dev/null
@@ -1,4524 +0,0 @@
-# Copyright (c) 1999-2007 Gary Strangman; All Rights Reserved.
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# Comments and/or additions are welcome (send e-mail to:
-# strang@nmr.mgh.harvard.edu).
-#
-"""
-stats.py module
-
-(Requires pstat.py module.)
-
-#################################################
-####### Written by: Gary Strangman ###########
-####### Last modified: Dec 18, 2007 ###########
-#################################################
-
-A collection of basic statistical functions for python. The function
-names appear below.
-
-IMPORTANT: There are really *3* sets of functions. The first set has an 'l'
-prefix, which can be used with list or tuple arguments. The second set has
-an 'a' prefix, which can accept NumPy array arguments. These latter
-functions are defined only when NumPy is available on the system. The third
-type has NO prefix (i.e., has the name that appears below). Functions of
-this set are members of a "Dispatch" class, c/o David Ascher. This class
-allows different functions to be called depending on the type of the passed
-arguments. Thus, stats.mean is a member of the Dispatch class and
-stats.mean(range(20)) will call stats.lmean(range(20)) while
-stats.mean(Numeric.arange(20)) will call stats.amean(Numeric.arange(20)).
-This is a handy way to keep consistent function names when different
-argument types require different functions to be called. Having
-implementated the Dispatch class, however, means that to get info on
-a given function, you must use the REAL function name ... that is
-"print stats.lmean.__doc__" or "print stats.amean.__doc__" work fine,
-while "print stats.mean.__doc__" will print the doc for the Dispatch
-class. NUMPY FUNCTIONS ('a' prefix) generally have more argument options
-but should otherwise be consistent with the corresponding list functions.
-
-Disclaimers: The function list is obviously incomplete and, worse, the
-functions are not optimized. All functions have been tested (some more
-so than others), but they are far from bulletproof. Thus, as with any
-free software, no warranty or guarantee is expressed or implied. :-) A
-few extra functions that don't appear in the list below can be found by
-interested treasure-hunters. These functions don't necessarily have
-both list and array versions but were deemed useful
-
-CENTRAL TENDENCY: geometricmean
- harmonicmean
- mean
- median
- medianscore
- mode
-
-MOMENTS: moment
- variation
- skew
- kurtosis
- skewtest (for Numpy arrays only)
- kurtosistest (for Numpy arrays only)
- normaltest (for Numpy arrays only)
-
-ALTERED VERSIONS: tmean (for Numpy arrays only)
- tvar (for Numpy arrays only)
- tmin (for Numpy arrays only)
- tmax (for Numpy arrays only)
- tstdev (for Numpy arrays only)
- tsem (for Numpy arrays only)
- describe
-
-FREQUENCY STATS: itemfreq
- scoreatpercentile
- percentileofscore
- histogram
- cumfreq
- relfreq
-
-VARIABILITY: obrientransform
- samplevar
- samplestdev
- signaltonoise (for Numpy arrays only)
- var
- stdev
- sterr
- sem
- z
- zs
- zmap (for Numpy arrays only)
-
-TRIMMING FCNS: threshold (for Numpy arrays only)
- trimboth
- trim1
- round (round all vals to 'n' decimals; Numpy only)
-
-CORRELATION FCNS: covariance (for Numpy arrays only)
- correlation (for Numpy arrays only)
- paired
- pearsonr
- spearmanr
- pointbiserialr
- kendalltau
- linregress
-
-INFERENTIAL STATS: ttest_1samp
- ttest_ind
- ttest_rel
- chisquare
- ks_2samp
- mannwhitneyu
- ranksums
- wilcoxont
- kruskalwallish
- friedmanchisquare
-
-PROBABILITY CALCS: chisqprob
- erfcc
- zprob
- ksprob
- fprob
- betacf
- gammln
- betai
-
-ANOVA FUNCTIONS: F_oneway
- F_value
-
-SUPPORT FUNCTIONS: writecc
- incr
- sign (for Numpy arrays only)
- sum
- cumsum
- ss
- summult
- sumdiffsquared
- square_of_sums
- shellsort
- rankdata
- outputpairedstats
- findwithin
-"""
-## CHANGE LOG:
-## ===========
-## 07-11.26 ... conversion for numpy started
-## 07-05-16 ... added Lin's Concordance Correlation Coefficient (alincc) and acov
-## 05-08-21 ... added "Dice's coefficient"
-## 04-10-26 ... added ap2t(), an ugly fcn for converting p-vals to T-vals
-## 04-04-03 ... added amasslinregress() function to do regression on N-D arrays
-## 03-01-03 ... CHANGED VERSION TO 0.6
-## fixed atsem() to properly handle limits=None case
-## improved histogram and median functions (estbinwidth) and
-## fixed atvar() function (wrong answers for neg numbers?!?)
-## 02-11-19 ... fixed attest_ind and attest_rel for div-by-zero Overflows
-## 02-05-10 ... fixed lchisqprob indentation (failed when df=even)
-## 00-12-28 ... removed aanova() to separate module, fixed licensing to
-## match Python License, fixed doc string & imports
-## 00-04-13 ... pulled all "global" statements, except from aanova()
-## added/fixed lots of documentation, removed io.py dependency
-## changed to version 0.5
-## 99-11-13 ... added asign() function
-## 99-11-01 ... changed version to 0.4 ... enough incremental changes now
-## 99-10-25 ... added acovariance and acorrelation functions
-## 99-10-10 ... fixed askew/akurtosis to avoid divide-by-zero errors
-## added aglm function (crude, but will be improved)
-## 99-10-04 ... upgraded acumsum, ass, asummult, asamplevar, avar, etc. to
-## all handle lists of 'dimension's and keepdims
-## REMOVED ar0, ar2, ar3, ar4 and replaced them with around
-## reinserted fixes for abetai to avoid math overflows
-## 99-09-05 ... rewrote achisqprob/aerfcc/aksprob/afprob/abetacf/abetai to
-## handle multi-dimensional arrays (whew!)
-## 99-08-30 ... fixed l/amoment, l/askew, l/akurtosis per D'Agostino (1990)
-## added anormaltest per same reference
-## re-wrote azprob to calc arrays of probs all at once
-## 99-08-22 ... edited attest_ind printing section so arrays could be rounded
-## 99-08-19 ... fixed amean and aharmonicmean for non-error(!) overflow on
-## short/byte arrays (mean of #s btw 100-300 = -150??)
-## 99-08-09 ... fixed asum so that the None case works for Byte arrays
-## 99-08-08 ... fixed 7/3 'improvement' to handle t-calcs on N-D arrays
-## 99-07-03 ... improved attest_ind, attest_rel (zero-division errortrap)
-## 99-06-24 ... fixed bug(?) in attest_ind (n1=a.shape[0])
-## 04/11/99 ... added asignaltonoise, athreshold functions, changed all
-## max/min in array section to N.maximum/N.minimum,
-## fixed square_of_sums to prevent integer overflow
-## 04/10/99 ... !!! Changed function name ... sumsquared ==> square_of_sums
-## 03/18/99 ... Added ar0, ar2, ar3 and ar4 rounding functions
-## 02/28/99 ... Fixed aobrientransform to return an array rather than a list
-## 01/15/99 ... Essentially ceased updating list-versions of functions (!!!)
-## 01/13/99 ... CHANGED TO VERSION 0.3
-## fixed bug in a/lmannwhitneyu p-value calculation
-## 12/31/98 ... fixed variable-name bug in ldescribe
-## 12/19/98 ... fixed bug in findwithin (fcns needed pstat. prefix)
-## 12/16/98 ... changed amedianscore to return float (not array) for 1 score
-## 12/14/98 ... added atmin and atmax functions
-## removed umath from import line (not needed)
-## l/ageometricmean modified to reduce chance of overflows (take
-## nth root first, then multiply)
-## 12/07/98 ... added __version__variable (now 0.2)
-## removed all 'stats.' from anova() fcn
-## 12/06/98 ... changed those functions (except shellsort) that altered
-## arguments in-place ... cumsum, ranksort, ...
-## updated (and fixed some) doc-strings
-## 12/01/98 ... added anova() function (requires NumPy)
-## incorporated Dispatch class
-## 11/12/98 ... added functionality to amean, aharmonicmean, ageometricmean
-## added 'asum' function (added functionality to N.add.reduce)
-## fixed both moment and amoment (two errors)
-## changed name of skewness and askewness to skew and askew
-## fixed (a)histogram (which sometimes counted points <lowerlimit)
-
-import pstat # required 3rd party module
-import math, string, copy # required python modules
-from types import *
-
-__version__ = 0.6
-
-############# DISPATCH CODE ##############
-
-
-class Dispatch:
- """
-The Dispatch class, care of David Ascher, allows different functions to
-be called depending on the argument types. This way, there can be one
-function name regardless of the argument type. To access function doc
-in stats.py module, prefix the function with an 'l' or 'a' for list or
-array arguments, respectively. That is, print stats.lmean.__doc__ or
-print stats.amean.__doc__ or whatever.
-"""
-
- def __init__(self, *tuples):
- self._dispatch = {}
- for func, types in tuples:
- for t in types:
- if t in self._dispatch.keys():
- raise ValueError, "can't have two dispatches on "+str(t)
- self._dispatch[t] = func
- self._types = self._dispatch.keys()
-
- def __call__(self, arg1, *args, **kw):
- if type(arg1) not in self._types:
- raise TypeError, "don't know how to dispatch %s arguments" % type(arg1)
- return apply(self._dispatch[type(arg1)], (arg1,) + args, kw)
-
-
-##########################################################################
-######################## LIST-BASED FUNCTIONS ########################
-##########################################################################
-
-### Define these regardless
-
-####################################
-####### CENTRAL TENDENCY #########
-####################################
-
-def lgeometricmean (inlist):
- """
-Calculates the geometric mean of the values in the passed list.
-That is: n-th root of (x1 * x2 * ... * xn). Assumes a '1D' list.
-
-Usage: lgeometricmean(inlist)
-"""
- mult = 1.0
- one_over_n = 1.0/len(inlist)
- for item in inlist:
- mult = mult * pow(item,one_over_n)
- return mult
-
-
-def lharmonicmean (inlist):
- """
-Calculates the harmonic mean of the values in the passed list.
-That is: n / (1/x1 + 1/x2 + ... + 1/xn). Assumes a '1D' list.
-
-Usage: lharmonicmean(inlist)
-"""
- sum = 0
- for item in inlist:
- sum = sum + 1.0/item
- return len(inlist) / sum
-
-
-def lmean (inlist):
- """
-Returns the arithematic mean of the values in the passed list.
-Assumes a '1D' list, but will function on the 1st dim of an array(!).
-
-Usage: lmean(inlist)
-"""
- sum = 0
- for item in inlist:
- sum = sum + item
- return sum/float(len(inlist))
-
-
-def lmedian (inlist,numbins=1000):
- """
-Returns the computed median value of a list of numbers, given the
-number of bins to use for the histogram (more bins brings the computed value
-closer to the median score, default number of bins = 1000). See G.W.
-Heiman's Basic Stats (1st Edition), or CRC Probability & Statistics.
-
-Usage: lmedian (inlist, numbins=1000)
-"""
- (hist, smallest, binsize, extras) = histogram(inlist,numbins,[min(inlist),max(inlist)]) # make histog
- cumhist = cumsum(hist) # make cumulative histogram
- for i in range(len(cumhist)): # get 1st(!) index holding 50%ile score
- if cumhist[i]>=len(inlist)/2.0:
- cfbin = i
- break
- LRL = smallest + binsize*cfbin # get lower read limit of that bin
- cfbelow = cumhist[cfbin-1]
- freq = float(hist[cfbin]) # frequency IN the 50%ile bin
- median = LRL + ((len(inlist)/2.0 - cfbelow)/float(freq))*binsize # median formula
- return median
-
-
-def lmedianscore (inlist):
- """
-Returns the 'middle' score of the passed list. If there is an even
-number of scores, the mean of the 2 middle scores is returned.
-
-Usage: lmedianscore(inlist)
-"""
-
- newlist = copy.deepcopy(inlist)
- newlist.sort()
- if len(newlist) % 2 == 0: # if even number of scores, average middle 2
- index = len(newlist)/2 # integer division correct
- median = float(newlist[index] + newlist[index-1]) /2
- else:
- index = len(newlist)/2 # int divsion gives mid value when count from 0
- median = newlist[index]
- return median
-
-
-def lmode(inlist):
- """
-Returns a list of the modal (most common) score(s) in the passed
-list. If there is more than one such score, all are returned. The
-bin-count for the mode(s) is also returned.
-
-Usage: lmode(inlist)
-Returns: bin-count for mode(s), a list of modal value(s)
-"""
-
- scores = pstat.unique(inlist)
- scores.sort()
- freq = []
- for item in scores:
- freq.append(inlist.count(item))
- maxfreq = max(freq)
- mode = []
- stillmore = 1
- while stillmore:
- try:
- indx = freq.index(maxfreq)
- mode.append(scores[indx])
- del freq[indx]
- del scores[indx]
- except ValueError:
- stillmore=0
- return maxfreq, mode
-
-
-####################################
-############ MOMENTS #############
-####################################
-
-def lmoment(inlist,moment=1):
- """
-Calculates the nth moment about the mean for a sample (defaults to
-the 1st moment). Used to calculate coefficients of skewness and kurtosis.
-
-Usage: lmoment(inlist,moment=1)
-Returns: appropriate moment (r) from ... 1/n * SUM((inlist(i)-mean)**r)
-"""
- if moment == 1:
- return 0.0
- else:
- mn = mean(inlist)
- n = len(inlist)
- s = 0
- for x in inlist:
- s = s + (x-mn)**moment
- return s/float(n)
-
-
-def lvariation(inlist):
- """
-Returns the coefficient of variation, as defined in CRC Standard
-Probability and Statistics, p.6.
-
-Usage: lvariation(inlist)
-"""
- return 100.0*samplestdev(inlist)/float(mean(inlist))
-
-
-def lskew(inlist):
- """
-Returns the skewness of a distribution, as defined in Numerical
-Recipies (alternate defn in CRC Standard Probability and Statistics, p.6.)
-
-Usage: lskew(inlist)
-"""
- return moment(inlist,3)/pow(moment(inlist,2),1.5)
-
-
-def lkurtosis(inlist):
- """
-Returns the kurtosis of a distribution, as defined in Numerical
-Recipies (alternate defn in CRC Standard Probability and Statistics, p.6.)
-
-Usage: lkurtosis(inlist)
-"""
- return moment(inlist,4)/pow(moment(inlist,2),2.0)
-
-
-def ldescribe(inlist):
- """
-Returns some descriptive statistics of the passed list (assumed to be 1D).
-
-Usage: ldescribe(inlist)
-Returns: n, mean, standard deviation, skew, kurtosis
-"""
- n = len(inlist)
- mm = (min(inlist),max(inlist))
- m = mean(inlist)
- sd = stdev(inlist)
- sk = skew(inlist)
- kurt = kurtosis(inlist)
- return n, mm, m, sd, sk, kurt
-
-
-####################################
-####### FREQUENCY STATS ##########
-####################################
-
-def litemfreq(inlist):
- """
-Returns a list of pairs. Each pair consists of one of the scores in inlist
-and it's frequency count. Assumes a 1D list is passed.
-
-Usage: litemfreq(inlist)
-Returns: a 2D frequency table (col [0:n-1]=scores, col n=frequencies)
-"""
- scores = pstat.unique(inlist)
- scores.sort()
- freq = []
- for item in scores:
- freq.append(inlist.count(item))
- return pstat.abut(scores, freq)
-
-
-def lscoreatpercentile (inlist, percent):
- """
-Returns the score at a given percentile relative to the distribution
-given by inlist.
-
-Usage: lscoreatpercentile(inlist,percent)
-"""
- if percent > 1:
- print "\nDividing percent>1 by 100 in lscoreatpercentile().\n"
- percent = percent / 100.0
- targetcf = percent*len(inlist)
- h, lrl, binsize, extras = histogram(inlist)
- cumhist = cumsum(copy.deepcopy(h))
- for i in range(len(cumhist)):
- if cumhist[i] >= targetcf:
- break
- score = binsize * ((targetcf - cumhist[i-1]) / float(h[i])) + (lrl+binsize*i)
- return score
-
-
-def lpercentileofscore (inlist, score,histbins=10,defaultlimits=None):
- """
-Returns the percentile value of a score relative to the distribution
-given by inlist. Formula depends on the values used to histogram the data(!).
-
-Usage: lpercentileofscore(inlist,score,histbins=10,defaultlimits=None)
-"""
-
- h, lrl, binsize, extras = histogram(inlist,histbins,defaultlimits)
- cumhist = cumsum(copy.deepcopy(h))
- i = int((score - lrl)/float(binsize))
- pct = (cumhist[i-1]+((score-(lrl+binsize*i))/float(binsize))*h[i])/float(len(inlist)) * 100
- return pct
-
-
-def lhistogram (inlist,numbins=10,defaultreallimits=None,printextras=0):
- """
-Returns (i) a list of histogram bin counts, (ii) the smallest value
-of the histogram binning, and (iii) the bin width (the last 2 are not
-necessarily integers). Default number of bins is 10. If no sequence object
-is given for defaultreallimits, the routine picks (usually non-pretty) bins
-spanning all the numbers in the inlist.
-
-Usage: lhistogram (inlist, numbins=10, defaultreallimits=None,suppressoutput=0)
-Returns: list of bin values, lowerreallimit, binsize, lowpoints, highpoints
-"""
- if (defaultreallimits <> None):
- if type(defaultreallimits) not in [ListType,TupleType] or len(defaultreallimits)==1: # only one limit given, assumed to be lower one & upper is calc'd
- lowerreallimit = defaultreallimits
- upperreallimit = 1.000001 * max(inlist)
- else: # assume both limits given
- lowerreallimit = defaultreallimits[0]
- upperreallimit = defaultreallimits[1]
- binsize = (upperreallimit-lowerreallimit)/float(numbins)
- else: # no limits given for histogram, both must be calc'd
- estbinwidth=(max(inlist)-min(inlist))/float(numbins) +1e-6 #1=>cover all
- binsize = ((max(inlist)-min(inlist)+estbinwidth))/float(numbins)
- lowerreallimit = min(inlist) - binsize/2 #lower real limit,1st bin
- bins = [0]*(numbins)
- lowpoints = 0
- highpoints = 0
- for num in inlist:
- try:
- if (num-lowerreallimit) < 0:
- lowpoints += 1
- else:
- bintoincrement = int((num-lowerreallimit)/float(binsize))
- bins[bintoincrement] = bins[bintoincrement] + 1
- except:
- highpoints += 1
- if (printextras == 1) and ((lowpoints + highpoints) > 0):
- extrapoints = lowpoints + highpoints
- print '\nPoints outside given histogram range =',extrapoints
- return (bins, lowerreallimit, binsize, lowpoints, highpoints)
-
-
-def lcumfreq(inlist,numbins=10,defaultreallimits=None):
- """
-Returns a cumulative frequency histogram, using the histogram function.
-
-Usage: lcumfreq(inlist,numbins=10,defaultreallimits=None)
-Returns: list of cumfreq bin values, lowerreallimit, binsize, extrapoints
-"""
- h,l,b,e = histogram(inlist,numbins,defaultreallimits)
- cumhist = cumsum(copy.deepcopy(h))
- return cumhist,l,b,e
-
-
-def lrelfreq(inlist,numbins=10,defaultreallimits=None):
- """
-Returns a relative frequency histogram, using the histogram function.
-
-Usage: lrelfreq(inlist,numbins=10,defaultreallimits=None)
-Returns: list of cumfreq bin values, lowerreallimit, binsize, extrapoints
-"""
- h,l,b,e = histogram(inlist,numbins,defaultreallimits)
- for i in range(len(h)):
- h[i] = h[i]/float(len(inlist))
- return h,l,b,e
-
-
-####################################
-##### VARIABILITY FUNCTIONS ######
-####################################
-
-def lobrientransform(*args):
- """
-Computes a transform on input data (any number of columns). Used to
-test for homogeneity of variance prior to running one-way stats. From
-Maxwell and Delaney, p.112.
-
-Usage: lobrientransform(*args)
-Returns: transformed data for use in an ANOVA
-"""
- TINY = 1e-10
- k = len(args)
- n = [0.0]*k
- v = [0.0]*k
- m = [0.0]*k
- nargs = []
- for i in range(k):
- nargs.append(copy.deepcopy(args[i]))
- n[i] = float(len(nargs[i]))
- v[i] = var(nargs[i])
- m[i] = mean(nargs[i])
- for j in range(k):
- for i in range(n[j]):
- t1 = (n[j]-1.5)*n[j]*(nargs[j][i]-m[j])**2
- t2 = 0.5*v[j]*(n[j]-1.0)
- t3 = (n[j]-1.0)*(n[j]-2.0)
- nargs[j][i] = (t1-t2) / float(t3)
- check = 1
- for j in range(k):
- if v[j] - mean(nargs[j]) > TINY:
- check = 0
- if check <> 1:
- raise ValueError, 'Problem in obrientransform.'
- else:
- return nargs
-
-
-def lsamplevar (inlist):
- """
-Returns the variance of the values in the passed list using
-N for the denominator (i.e., DESCRIBES the sample variance only).
-
-Usage: lsamplevar(inlist)
-"""
- n = len(inlist)
- mn = mean(inlist)
- deviations = []
- for item in inlist:
- deviations.append(item-mn)
- return ss(deviations)/float(n)
-
-
-def lsamplestdev (inlist):
- """
-Returns the standard deviation of the values in the passed list using
-N for the denominator (i.e., DESCRIBES the sample stdev only).
-
-Usage: lsamplestdev(inlist)
-"""
- return math.sqrt(samplevar(inlist))
-
-
-def lcov (x,y, keepdims=0):
- """
-Returns the estimated covariance of the values in the passed
-array (i.e., N-1). Dimension can equal None (ravel array first), an
-integer (the dimension over which to operate), or a sequence (operate
-over multiple dimensions). Set keepdims=1 to return an array with the
-same number of dimensions as inarray.
-
-Usage: lcov(x,y,keepdims=0)
-"""
-
- n = len(x)
- xmn = mean(x)
- ymn = mean(y)
- xdeviations = [0]*len(x)
- ydeviations = [0]*len(y)
- for i in range(len(x)):
- xdeviations[i] = x[i] - xmn
- ydeviations[i] = y[i] - ymn
- ss = 0.0
- for i in range(len(xdeviations)):
- ss = ss + xdeviations[i]*ydeviations[i]
- return ss/float(n-1)
-
-
-def lvar (inlist):
- """
-Returns the variance of the values in the passed list using N-1
-for the denominator (i.e., for estimating population variance).
-
-Usage: lvar(inlist)
-"""
- n = len(inlist)
- mn = mean(inlist)
- deviations = [0]*len(inlist)
- for i in range(len(inlist)):
- deviations[i] = inlist[i] - mn
- return ss(deviations)/float(n-1)
-
-
-def lstdev (inlist):
- """
-Returns the standard deviation of the values in the passed list
-using N-1 in the denominator (i.e., to estimate population stdev).
-
-Usage: lstdev(inlist)
-"""
- return math.sqrt(var(inlist))
-
-
-def lsterr(inlist):
- """
-Returns the standard error of the values in the passed list using N-1
-in the denominator (i.e., to estimate population standard error).
-
-Usage: lsterr(inlist)
-"""
- return stdev(inlist) / float(math.sqrt(len(inlist)))
-
-
-def lsem (inlist):
- """
-Returns the estimated standard error of the mean (sx-bar) of the
-values in the passed list. sem = stdev / sqrt(n)
-
-Usage: lsem(inlist)
-"""
- sd = stdev(inlist)
- n = len(inlist)
- return sd/math.sqrt(n)
-
-
-def lz (inlist, score):
- """
-Returns the z-score for a given input score, given that score and the
-list from which that score came. Not appropriate for population calculations.
-
-Usage: lz(inlist, score)
-"""
- z = (score-mean(inlist))/samplestdev(inlist)
- return z
-
-
-def lzs (inlist):
- """
-Returns a list of z-scores, one for each score in the passed list.
-
-Usage: lzs(inlist)
-"""
- zscores = []
- for item in inlist:
- zscores.append(z(inlist,item))
- return zscores
-
-
-####################################
-####### TRIMMING FUNCTIONS #######
-####################################
-
-def ltrimboth (l,proportiontocut):
- """
-Slices off the passed proportion of items from BOTH ends of the passed
-list (i.e., with proportiontocut=0.1, slices 'leftmost' 10% AND 'rightmost'
-10% of scores. Assumes list is sorted by magnitude. Slices off LESS if
-proportion results in a non-integer slice index (i.e., conservatively
-slices off proportiontocut).
-
-Usage: ltrimboth (l,proportiontocut)
-Returns: trimmed version of list l
-"""
- lowercut = int(proportiontocut*len(l))
- uppercut = len(l) - lowercut
- return l[lowercut:uppercut]
-
-
-def ltrim1 (l,proportiontocut,tail='right'):
- """
-Slices off the passed proportion of items from ONE end of the passed
-list (i.e., if proportiontocut=0.1, slices off 'leftmost' or 'rightmost'
-10% of scores). Slices off LESS if proportion results in a non-integer
-slice index (i.e., conservatively slices off proportiontocut).
-
-Usage: ltrim1 (l,proportiontocut,tail='right') or set tail='left'
-Returns: trimmed version of list l
-"""
- if tail == 'right':
- lowercut = 0
- uppercut = len(l) - int(proportiontocut*len(l))
- elif tail == 'left':
- lowercut = int(proportiontocut*len(l))
- uppercut = len(l)
- return l[lowercut:uppercut]
-
-
-####################################
-##### CORRELATION FUNCTIONS ######
-####################################
-
-def lpaired(x,y):
- """
-Interactively determines the type of data and then runs the
-appropriated statistic for paired group data.
-
-Usage: lpaired(x,y)
-Returns: appropriate statistic name, value, and probability
-"""
- samples = ''
- while samples not in ['i','r','I','R','c','C']:
- print '\nIndependent or related samples, or correlation (i,r,c): ',
- samples = raw_input()
-
- if samples in ['i','I','r','R']:
- print '\nComparing variances ...',
-# USE O'BRIEN'S TEST FOR HOMOGENEITY OF VARIANCE, Maxwell & delaney, p.112
- r = obrientransform(x,y)
- f,p = F_oneway(pstat.colex(r,0),pstat.colex(r,1))
- if p<0.05:
- vartype='unequal, p='+str(round(p,4))
- else:
- vartype='equal'
- print vartype
- if samples in ['i','I']:
- if vartype[0]=='e':
- t,p = ttest_ind(x,y,0)
- print '\nIndependent samples t-test: ', round(t,4),round(p,4)
- else:
- if len(x)>20 or len(y)>20:
- z,p = ranksums(x,y)
- print '\nRank Sums test (NONparametric, n>20): ', round(z,4),round(p,4)
- else:
- u,p = mannwhitneyu(x,y)
- print '\nMann-Whitney U-test (NONparametric, ns<20): ', round(u,4),round(p,4)
-
- else: # RELATED SAMPLES
- if vartype[0]=='e':
- t,p = ttest_rel(x,y,0)
- print '\nRelated samples t-test: ', round(t,4),round(p,4)
- else:
- t,p = ranksums(x,y)
- print '\nWilcoxon T-test (NONparametric): ', round(t,4),round(p,4)
- else: # CORRELATION ANALYSIS
- corrtype = ''
- while corrtype not in ['c','C','r','R','d','D']:
- print '\nIs the data Continuous, Ranked, or Dichotomous (c,r,d): ',
- corrtype = raw_input()
- if corrtype in ['c','C']:
- m,b,r,p,see = linregress(x,y)
- print '\nLinear regression for continuous variables ...'
- lol = [['Slope','Intercept','r','Prob','SEestimate'],[round(m,4),round(b,4),round(r,4),round(p,4),round(see,4)]]
- pstat.printcc(lol)
- elif corrtype in ['r','R']:
- r,p = spearmanr(x,y)
- print '\nCorrelation for ranked variables ...'
- print "Spearman's r: ",round(r,4),round(p,4)
- else: # DICHOTOMOUS
- r,p = pointbiserialr(x,y)
- print '\nAssuming x contains a dichotomous variable ...'
- print 'Point Biserial r: ',round(r,4),round(p,4)
- print '\n\n'
- return None
-
-
-def lpearsonr(x,y):
- """
-Calculates a Pearson correlation coefficient and the associated
-probability value. Taken from Heiman's Basic Statistics for the Behav.
-Sci (2nd), p.195.
-
-Usage: lpearsonr(x,y) where x and y are equal-length lists
-Returns: Pearson's r value, two-tailed p-value
-"""
- TINY = 1.0e-30
- if len(x) <> len(y):
- raise ValueError, 'Input values not paired in pearsonr. Aborting.'
- n = len(x)
- x = map(float,x)
- y = map(float,y)
- xmean = mean(x)
- ymean = mean(y)
- r_num = n*(summult(x,y)) - sum(x)*sum(y)
- r_den = math.sqrt((n*ss(x) - square_of_sums(x))*(n*ss(y)-square_of_sums(y)))
- r = (r_num / r_den) # denominator already a float
- df = n-2
- t = r*math.sqrt(df/((1.0-r+TINY)*(1.0+r+TINY)))
- prob = betai(0.5*df,0.5,df/float(df+t*t))
- return r, prob
-
-
-def llincc(x,y):
- """
-Calculates Lin's concordance correlation coefficient.
-
-Usage: alincc(x,y) where x, y are equal-length arrays
-Returns: Lin's CC
-"""
- covar = lcov(x,y)*(len(x)-1)/float(len(x)) # correct denom to n
- xvar = lvar(x)*(len(x)-1)/float(len(x)) # correct denom to n
- yvar = lvar(y)*(len(y)-1)/float(len(y)) # correct denom to n
- lincc = (2 * covar) / ((xvar+yvar) +((amean(x)-amean(y))**2))
- return lincc
-
-
-def lspearmanr(x,y):
- """
-Calculates a Spearman rank-order correlation coefficient. Taken
-from Heiman's Basic Statistics for the Behav. Sci (1st), p.192.
-
-Usage: lspearmanr(x,y) where x and y are equal-length lists
-Returns: Spearman's r, two-tailed p-value
-"""
- TINY = 1e-30
- if len(x) <> len(y):
- raise ValueError, 'Input values not paired in spearmanr. Aborting.'
- n = len(x)
- rankx = rankdata(x)
- ranky = rankdata(y)
- dsq = sumdiffsquared(rankx,ranky)
- rs = 1 - 6*dsq / float(n*(n**2-1))
- t = rs * math.sqrt((n-2) / ((rs+1.0)*(1.0-rs)))
- df = n-2
- probrs = betai(0.5*df,0.5,df/(df+t*t)) # t already a float
-# probability values for rs are from part 2 of the spearman function in
-# Numerical Recipies, p.510. They are close to tables, but not exact. (?)
- return rs, probrs
-
-
-def lpointbiserialr(x,y):
- """
-Calculates a point-biserial correlation coefficient and the associated
-probability value. Taken from Heiman's Basic Statistics for the Behav.
-Sci (1st), p.194.
-
-Usage: lpointbiserialr(x,y) where x,y are equal-length lists
-Returns: Point-biserial r, two-tailed p-value
-"""
- TINY = 1e-30
- if len(x) <> len(y):
- raise ValueError, 'INPUT VALUES NOT PAIRED IN pointbiserialr. ABORTING.'
- data = pstat.abut(x,y)
- categories = pstat.unique(x)
- if len(categories) <> 2:
- raise ValueError, "Exactly 2 categories required for pointbiserialr()."
- else: # there are 2 categories, continue
- codemap = pstat.abut(categories,range(2))
- recoded = pstat.recode(data,codemap,0)
- x = pstat.linexand(data,0,categories[0])
- y = pstat.linexand(data,0,categories[1])
- xmean = mean(pstat.colex(x,1))
- ymean = mean(pstat.colex(y,1))
- n = len(data)
- adjust = math.sqrt((len(x)/float(n))*(len(y)/float(n)))
- rpb = (ymean - xmean)/samplestdev(pstat.colex(data,1))*adjust
- df = n-2
- t = rpb*math.sqrt(df/((1.0-rpb+TINY)*(1.0+rpb+TINY)))
- prob = betai(0.5*df,0.5,df/(df+t*t)) # t already a float
- return rpb, prob
-
-
-def lkendalltau(x,y):
- """
-Calculates Kendall's tau ... correlation of ordinal data. Adapted
-from function kendl1 in Numerical Recipies. Needs good test-routine.@@@
-
-Usage: lkendalltau(x,y)
-Returns: Kendall's tau, two-tailed p-value
-"""
- n1 = 0
- n2 = 0
- iss = 0
- for j in range(len(x)-1):
- for k in range(j,len(y)):
- a1 = x[j] - x[k]
- a2 = y[j] - y[k]
- aa = a1 * a2
- if (aa): # neither list has a tie
- n1 = n1 + 1
- n2 = n2 + 1
- if aa > 0:
- iss = iss + 1
- else:
- iss = iss -1
- else:
- if (a1):
- n1 = n1 + 1
- else:
- n2 = n2 + 1
- tau = iss / math.sqrt(n1*n2)
- svar = (4.0*len(x)+10.0) / (9.0*len(x)*(len(x)-1))
- z = tau / math.sqrt(svar)
- prob = erfcc(abs(z)/1.4142136)
- return tau, prob
-
-
-def llinregress(x,y):
- """
-Calculates a regression line on x,y pairs.
-
-Usage: llinregress(x,y) x,y are equal-length lists of x-y coordinates
-Returns: slope, intercept, r, two-tailed prob, sterr-of-estimate
-"""
- TINY = 1.0e-20
- if len(x) <> len(y):
- raise ValueError, 'Input values not paired in linregress. Aborting.'
- n = len(x)
- x = map(float,x)
- y = map(float,y)
- xmean = mean(x)
- ymean = mean(y)
- r_num = float(n*(summult(x,y)) - sum(x)*sum(y))
- r_den = math.sqrt((n*ss(x) - square_of_sums(x))*(n*ss(y)-square_of_sums(y)))
- r = r_num / r_den
- z = 0.5*math.log((1.0+r+TINY)/(1.0-r+TINY))
- df = n-2
- t = r*math.sqrt(df/((1.0-r+TINY)*(1.0+r+TINY)))
- prob = betai(0.5*df,0.5,df/(df+t*t))
- slope = r_num / float(n*ss(x) - square_of_sums(x))
- intercept = ymean - slope*xmean
- sterrest = math.sqrt(1-r*r)*samplestdev(y)
- return slope, intercept, r, prob, sterrest
-
-
-####################################
-##### INFERENTIAL STATISTICS #####
-####################################
-
-def lttest_1samp(a,popmean,printit=0,name='Sample',writemode='a'):
- """
-Calculates the t-obtained for the independent samples T-test on ONE group
-of scores a, given a population mean. If printit=1, results are printed
-to the screen. If printit='filename', the results are output to 'filename'
-using the given writemode (default=append). Returns t-value, and prob.
-
-Usage: lttest_1samp(a,popmean,Name='Sample',printit=0,writemode='a')
-Returns: t-value, two-tailed prob
-"""
- x = mean(a)
- v = var(a)
- n = len(a)
- df = n-1
- svar = ((n-1)*v)/float(df)
- t = (x-popmean)/math.sqrt(svar*(1.0/n))
- prob = betai(0.5*df,0.5,float(df)/(df+t*t))
-
- if printit <> 0:
- statname = 'Single-sample T-test.'
- outputpairedstats(printit,writemode,
- 'Population','--',popmean,0,0,0,
- name,n,x,v,min(a),max(a),
- statname,t,prob)
- return t,prob
-
-
-def lttest_ind (a, b, printit=0, name1='Samp1', name2='Samp2', writemode='a'):
- """
-Calculates the t-obtained T-test on TWO INDEPENDENT samples of
-scores a, and b. From Numerical Recipies, p.483. If printit=1, results
-are printed to the screen. If printit='filename', the results are output
-to 'filename' using the given writemode (default=append). Returns t-value,
-and prob.
-
-Usage: lttest_ind(a,b,printit=0,name1='Samp1',name2='Samp2',writemode='a')
-Returns: t-value, two-tailed prob
-"""
- x1 = mean(a)
- x2 = mean(b)
- v1 = stdev(a)**2
- v2 = stdev(b)**2
- n1 = len(a)
- n2 = len(b)
- df = n1+n2-2
- svar = ((n1-1)*v1+(n2-1)*v2)/float(df)
- t = (x1-x2)/math.sqrt(svar*(1.0/n1 + 1.0/n2))
- prob = betai(0.5*df,0.5,df/(df+t*t))
-
- if printit <> 0:
- statname = 'Independent samples T-test.'
- outputpairedstats(printit,writemode,
- name1,n1,x1,v1,min(a),max(a),
- name2,n2,x2,v2,min(b),max(b),
- statname,t,prob)
- return t,prob
-
-
-def lttest_rel (a,b,printit=0,name1='Sample1',name2='Sample2',writemode='a'):
- """
-Calculates the t-obtained T-test on TWO RELATED samples of scores,
-a and b. From Numerical Recipies, p.483. If printit=1, results are
-printed to the screen. If printit='filename', the results are output to
-'filename' using the given writemode (default=append). Returns t-value,
-and prob.
-
-Usage: lttest_rel(a,b,printit=0,name1='Sample1',name2='Sample2',writemode='a')
-Returns: t-value, two-tailed prob
-"""
- if len(a)<>len(b):
- raise ValueError, 'Unequal length lists in ttest_rel.'
- x1 = mean(a)
- x2 = mean(b)
- v1 = var(a)
- v2 = var(b)
- n = len(a)
- cov = 0
- for i in range(len(a)):
- cov = cov + (a[i]-x1) * (b[i]-x2)
- df = n-1
- cov = cov / float(df)
- sd = math.sqrt((v1+v2 - 2.0*cov)/float(n))
- t = (x1-x2)/sd
- prob = betai(0.5*df,0.5,df/(df+t*t))
-
- if printit <> 0:
- statname = 'Related samples T-test.'
- outputpairedstats(printit,writemode,
- name1,n,x1,v1,min(a),max(a),
- name2,n,x2,v2,min(b),max(b),
- statname,t,prob)
- return t, prob
-
-
-def lchisquare(f_obs,f_exp=None):
- """
-Calculates a one-way chi square for list of observed frequencies and returns
-the result. If no expected frequencies are given, the total N is assumed to
-be equally distributed across all groups.
-
-Usage: lchisquare(f_obs, f_exp=None) f_obs = list of observed cell freq.
-Returns: chisquare-statistic, associated p-value
-"""
- k = len(f_obs) # number of groups
- if f_exp == None:
- f_exp = [sum(f_obs)/float(k)] * len(f_obs) # create k bins with = freq.
- chisq = 0
- for i in range(len(f_obs)):
- chisq = chisq + (f_obs[i]-f_exp[i])**2 / float(f_exp[i])
- return chisq, chisqprob(chisq, k-1)
-
-
-def lks_2samp (data1,data2):
- """
-Computes the Kolmogorov-Smirnof statistic on 2 samples. From
-Numerical Recipies in C, page 493.
-
-Usage: lks_2samp(data1,data2) data1&2 are lists of values for 2 conditions
-Returns: KS D-value, associated p-value
-"""
- j1 = 0
- j2 = 0
- fn1 = 0.0
- fn2 = 0.0
- n1 = len(data1)
- n2 = len(data2)
- en1 = n1
- en2 = n2
- d = 0.0
- data1.sort()
- data2.sort()
- while j1 < n1 and j2 < n2:
- d1=data1[j1]
- d2=data2[j2]
- if d1 <= d2:
- fn1 = (j1)/float(en1)
- j1 = j1 + 1
- if d2 <= d1:
- fn2 = (j2)/float(en2)
- j2 = j2 + 1
- dt = (fn2-fn1)
- if math.fabs(dt) > math.fabs(d):
- d = dt
- try:
- en = math.sqrt(en1*en2/float(en1+en2))
- prob = ksprob((en+0.12+0.11/en)*abs(d))
- except:
- prob = 1.0
- return d, prob
-
-
-def lmannwhitneyu(x,y):
- """
-Calculates a Mann-Whitney U statistic on the provided scores and
-returns the result. Use only when the n in each condition is < 20 and
-you have 2 independent samples of ranks. NOTE: Mann-Whitney U is
-significant if the u-obtained is LESS THAN or equal to the critical
-value of U found in the tables. Equivalent to Kruskal-Wallis H with
-just 2 groups.
-
-Usage: lmannwhitneyu(data)
-Returns: u-statistic, one-tailed p-value (i.e., p(z(U)))
-"""
- n1 = len(x)
- n2 = len(y)
- ranked = rankdata(x+y)
- rankx = ranked[0:n1] # get the x-ranks
- ranky = ranked[n1:] # the rest are y-ranks
- u1 = n1*n2 + (n1*(n1+1))/2.0 - sum(rankx) # calc U for x
- u2 = n1*n2 - u1 # remainder is U for y
- bigu = max(u1,u2)
- smallu = min(u1,u2)
- T = math.sqrt(tiecorrect(ranked)) # correction factor for tied scores
- if T == 0:
- raise ValueError, 'All numbers are identical in lmannwhitneyu'
- sd = math.sqrt(T*n1*n2*(n1+n2+1)/12.0)
- z = abs((bigu-n1*n2/2.0) / sd) # normal approximation for prob calc
- return smallu, 1.0 - zprob(z)
-
-
-def ltiecorrect(rankvals):
- """
-Corrects for ties in Mann Whitney U and Kruskal Wallis H tests. See
-Siegel, S. (1956) Nonparametric Statistics for the Behavioral Sciences.
-New York: McGraw-Hill. Code adapted from |Stat rankind.c code.
-
-Usage: ltiecorrect(rankvals)
-Returns: T correction factor for U or H
-"""
- sorted,posn = shellsort(rankvals)
- n = len(sorted)
- T = 0.0
- i = 0
- while (i<n-1):
- if sorted[i] == sorted[i+1]:
- nties = 1
- while (i<n-1) and (sorted[i] == sorted[i+1]):
- nties = nties +1
- i = i +1
- T = T + nties**3 - nties
- i = i+1
- T = T / float(n**3-n)
- return 1.0 - T
-
-
-def lranksums(x,y):
- """
-Calculates the rank sums statistic on the provided scores and
-returns the result. Use only when the n in each condition is > 20 and you
-have 2 independent samples of ranks.
-
-Usage: lranksums(x,y)
-Returns: a z-statistic, two-tailed p-value
-"""
- n1 = len(x)
- n2 = len(y)
- alldata = x+y
- ranked = rankdata(alldata)
- x = ranked[:n1]
- y = ranked[n1:]
- s = sum(x)
- expected = n1*(n1+n2+1) / 2.0
- z = (s - expected) / math.sqrt(n1*n2*(n1+n2+1)/12.0)
- prob = 2*(1.0 -zprob(abs(z)))
- return z, prob
-
-
-def lwilcoxont(x,y):
- """
-Calculates the Wilcoxon T-test for related samples and returns the
-result. A non-parametric T-test.
-
-Usage: lwilcoxont(x,y)
-Returns: a t-statistic, two-tail probability estimate
-"""
- if len(x) <> len(y):
- raise ValueError, 'Unequal N in wilcoxont. Aborting.'
- d=[]
- for i in range(len(x)):
- diff = x[i] - y[i]
- if diff <> 0:
- d.append(diff)
- count = len(d)
- absd = map(abs,d)
- absranked = rankdata(absd)
- r_plus = 0.0
- r_minus = 0.0
- for i in range(len(absd)):
- if d[i] < 0:
- r_minus = r_minus + absranked[i]
- else:
- r_plus = r_plus + absranked[i]
- wt = min(r_plus, r_minus)
- mn = count * (count+1) * 0.25
- se = math.sqrt(count*(count+1)*(2.0*count+1.0)/24.0)
- z = math.fabs(wt-mn) / se
- prob = 2*(1.0 -zprob(abs(z)))
- return wt, prob
-
-
-def lkruskalwallish(*args):
- """
-The Kruskal-Wallis H-test is a non-parametric ANOVA for 3 or more
-groups, requiring at least 5 subjects in each group. This function
-calculates the Kruskal-Wallis H-test for 3 or more independent samples
-and returns the result.
-
-Usage: lkruskalwallish(*args)
-Returns: H-statistic (corrected for ties), associated p-value
-"""
- args = list(args)
- n = [0]*len(args)
- all = []
- n = map(len,args)
- for i in range(len(args)):
- all = all + args[i]
- ranked = rankdata(all)
- T = tiecorrect(ranked)
- for i in range(len(args)):
- args[i] = ranked[0:n[i]]
- del ranked[0:n[i]]
- rsums = []
- for i in range(len(args)):
- rsums.append(sum(args[i])**2)
- rsums[i] = rsums[i] / float(n[i])
- ssbn = sum(rsums)
- totaln = sum(n)
- h = 12.0 / (totaln*(totaln+1)) * ssbn - 3*(totaln+1)
- df = len(args) - 1
- if T == 0:
- raise ValueError, 'All numbers are identical in lkruskalwallish'
- h = h / float(T)
- return h, chisqprob(h,df)
-
-
-def lfriedmanchisquare(*args):
- """
-Friedman Chi-Square is a non-parametric, one-way within-subjects
-ANOVA. This function calculates the Friedman Chi-square test for repeated
-measures and returns the result, along with the associated probability
-value. It assumes 3 or more repeated measures. Only 3 levels requires a
-minimum of 10 subjects in the study. Four levels requires 5 subjects per
-level(??).
-
-Usage: lfriedmanchisquare(*args)
-Returns: chi-square statistic, associated p-value
-"""
- k = len(args)
- if k < 3:
- raise ValueError, 'Less than 3 levels. Friedman test not appropriate.'
- n = len(args[0])
- data = apply(pstat.abut,tuple(args))
- for i in range(len(data)):
- data[i] = rankdata(data[i])
- ssbn = 0
- for i in range(k):
- ssbn = ssbn + sum(args[i])**2
- chisq = 12.0 / (k*n*(k+1)) * ssbn - 3*n*(k+1)
- return chisq, chisqprob(chisq,k-1)
-
-
-####################################
-#### PROBABILITY CALCULATIONS ####
-####################################
-
-def lchisqprob(chisq,df):
- """
-Returns the (1-tailed) probability value associated with the provided
-chi-square value and df. Adapted from chisq.c in Gary Perlman's |Stat.
-
-Usage: lchisqprob(chisq,df)
-"""
- BIG = 20.0
- def ex(x):
- BIG = 20.0
- if x < -BIG:
- return 0.0
- else:
- return math.exp(x)
-
- if chisq <=0 or df < 1:
- return 1.0
- a = 0.5 * chisq
- if df%2 == 0:
- even = 1
- else:
- even = 0
- if df > 1:
- y = ex(-a)
- if even:
- s = y
- else:
- s = 2.0 * zprob(-math.sqrt(chisq))
- if (df > 2):
- chisq = 0.5 * (df - 1.0)
- if even:
- z = 1.0
- else:
- z = 0.5
- if a > BIG:
- if even:
- e = 0.0
- else:
- e = math.log(math.sqrt(math.pi))
- c = math.log(a)
- while (z <= chisq):
- e = math.log(z) + e
- s = s + ex(c*z-a-e)
- z = z + 1.0
- return s
- else:
- if even:
- e = 1.0
- else:
- e = 1.0 / math.sqrt(math.pi) / math.sqrt(a)
- c = 0.0
- while (z <= chisq):
- e = e * (a/float(z))
- c = c + e
- z = z + 1.0
- return (c*y+s)
- else:
- return s
-
-
-def lerfcc(x):
- """
-Returns the complementary error function erfc(x) with fractional
-error everywhere less than 1.2e-7. Adapted from Numerical Recipies.
-
-Usage: lerfcc(x)
-"""
- z = abs(x)
- t = 1.0 / (1.0+0.5*z)
- ans = t * math.exp(-z*z-1.26551223 + t*(1.00002368+t*(0.37409196+t*(0.09678418+t*(-0.18628806+t*(0.27886807+t*(-1.13520398+t*(1.48851587+t*(-0.82215223+t*0.17087277)))))))))
- if x >= 0:
- return ans
- else:
- return 2.0 - ans
-
-
-def lzprob(z):
- """
-Returns the area under the normal curve 'to the left of' the given z value.
-Thus,
- for z<0, zprob(z) = 1-tail probability
- for z>0, 1.0-zprob(z) = 1-tail probability
- for any z, 2.0*(1.0-zprob(abs(z))) = 2-tail probability
-Adapted from z.c in Gary Perlman's |Stat.
-
-Usage: lzprob(z)
-"""
- Z_MAX = 6.0 # maximum meaningful z-value
- if z == 0.0:
- x = 0.0
- else:
- y = 0.5 * math.fabs(z)
- if y >= (Z_MAX*0.5):
- x = 1.0
- elif (y < 1.0):
- w = y*y
- x = ((((((((0.000124818987 * w
- -0.001075204047) * w +0.005198775019) * w
- -0.019198292004) * w +0.059054035642) * w
- -0.151968751364) * w +0.319152932694) * w
- -0.531923007300) * w +0.797884560593) * y * 2.0
- else:
- y = y - 2.0
- x = (((((((((((((-0.000045255659 * y
- +0.000152529290) * y -0.000019538132) * y
- -0.000676904986) * y +0.001390604284) * y
- -0.000794620820) * y -0.002034254874) * y
- +0.006549791214) * y -0.010557625006) * y
- +0.011630447319) * y -0.009279453341) * y
- +0.005353579108) * y -0.002141268741) * y
- +0.000535310849) * y +0.999936657524
- if z > 0.0:
- prob = ((x+1.0)*0.5)
- else:
- prob = ((1.0-x)*0.5)
- return prob
-
-
-def lksprob(alam):
- """
-Computes a Kolmolgorov-Smirnov t-test significance level. Adapted from
-Numerical Recipies.
-
-Usage: lksprob(alam)
-"""
- fac = 2.0
- sum = 0.0
- termbf = 0.0
- a2 = -2.0*alam*alam
- for j in range(1,201):
- term = fac*math.exp(a2*j*j)
- sum = sum + term
- if math.fabs(term) <= (0.001*termbf) or math.fabs(term) < (1.0e-8*sum):
- return sum
- fac = -fac
- termbf = math.fabs(term)
- return 1.0 # Get here only if fails to converge; was 0.0!!
-
-
-def lfprob (dfnum, dfden, F):
- """
-Returns the (1-tailed) significance level (p-value) of an F
-statistic given the degrees of freedom for the numerator (dfR-dfF) and
-the degrees of freedom for the denominator (dfF).
-
-Usage: lfprob(dfnum, dfden, F) where usually dfnum=dfbn, dfden=dfwn
-"""
- p = betai(0.5*dfden, 0.5*dfnum, dfden/float(dfden+dfnum*F))
- return p
-
-
-def lbetacf(a,b,x):
- """
-This function evaluates the continued fraction form of the incomplete
-Beta function, betai. (Adapted from: Numerical Recipies in C.)
-
-Usage: lbetacf(a,b,x)
-"""
- ITMAX = 200
- EPS = 3.0e-7
-
- bm = az = am = 1.0
- qab = a+b
- qap = a+1.0
- qam = a-1.0
- bz = 1.0-qab*x/qap
- for i in range(ITMAX+1):
- em = float(i+1)
- tem = em + em
- d = em*(b-em)*x/((qam+tem)*(a+tem))
- ap = az + d*am
- bp = bz+d*bm
- d = -(a+em)*(qab+em)*x/((qap+tem)*(a+tem))
- app = ap+d*az
- bpp = bp+d*bz
- aold = az
- am = ap/bpp
- bm = bp/bpp
- az = app/bpp
- bz = 1.0
- if (abs(az-aold)<(EPS*abs(az))):
- return az
- print 'a or b too big, or ITMAX too small in Betacf.'
-
-
-def lgammln(xx):
- """
-Returns the gamma function of xx.
- Gamma(z) = Integral(0,infinity) of t^(z-1)exp(-t) dt.
-(Adapted from: Numerical Recipies in C.)
-
-Usage: lgammln(xx)
-"""
-
- coeff = [76.18009173, -86.50532033, 24.01409822, -1.231739516,
- 0.120858003e-2, -0.536382e-5]
- x = xx - 1.0
- tmp = x + 5.5
- tmp = tmp - (x+0.5)*math.log(tmp)
- ser = 1.0
- for j in range(len(coeff)):
- x = x + 1
- ser = ser + coeff[j]/x
- return -tmp + math.log(2.50662827465*ser)
-
-
-def lbetai(a,b,x):
- """
-Returns the incomplete beta function:
-
- I-sub-x(a,b) = 1/B(a,b)*(Integral(0,x) of t^(a-1)(1-t)^(b-1) dt)
-
-where a,b>0 and B(a,b) = G(a)*G(b)/(G(a+b)) where G(a) is the gamma
-function of a. The continued fraction formulation is implemented here,
-using the betacf function. (Adapted from: Numerical Recipies in C.)
-
-Usage: lbetai(a,b,x)
-"""
- if (x<0.0 or x>1.0):
- raise ValueError, 'Bad x in lbetai'
- if (x==0.0 or x==1.0):
- bt = 0.0
- else:
- bt = math.exp(gammln(a+b)-gammln(a)-gammln(b)+a*math.log(x)+b*
- math.log(1.0-x))
- if (x<(a+1.0)/(a+b+2.0)):
- return bt*betacf(a,b,x)/float(a)
- else:
- return 1.0-bt*betacf(b,a,1.0-x)/float(b)
-
-
-####################################
-####### ANOVA CALCULATIONS #######
-####################################
-
-def lF_oneway(*lists):
- """
-Performs a 1-way ANOVA, returning an F-value and probability given
-any number of groups. From Heiman, pp.394-7.
-
-Usage: F_oneway(*lists) where *lists is any number of lists, one per
- treatment group
-Returns: F value, one-tailed p-value
-"""
- a = len(lists) # ANOVA on 'a' groups, each in it's own list
- means = [0]*a
- vars = [0]*a
- ns = [0]*a
- alldata = []
- tmp = map(N.array,lists)
- means = map(amean,tmp)
- vars = map(avar,tmp)
- ns = map(len,lists)
- for i in range(len(lists)):
- alldata = alldata + lists[i]
- alldata = N.array(alldata)
- bign = len(alldata)
- sstot = ass(alldata)-(asquare_of_sums(alldata)/float(bign))
- ssbn = 0
- for list in lists:
- ssbn = ssbn + asquare_of_sums(N.array(list))/float(len(list))
- ssbn = ssbn - (asquare_of_sums(alldata)/float(bign))
- sswn = sstot-ssbn
- dfbn = a-1
- dfwn = bign - a
- msb = ssbn/float(dfbn)
- msw = sswn/float(dfwn)
- f = msb/msw
- prob = fprob(dfbn,dfwn,f)
- return f, prob
-
-
-def lF_value (ER,EF,dfnum,dfden):
- """
-Returns an F-statistic given the following:
- ER = error associated with the null hypothesis (the Restricted model)
- EF = error associated with the alternate hypothesis (the Full model)
- dfR-dfF = degrees of freedom of the numerator
- dfF = degrees of freedom associated with the denominator/Full model
-
-Usage: lF_value(ER,EF,dfnum,dfden)
-"""
- return ((ER-EF)/float(dfnum) / (EF/float(dfden)))
-
-
-####################################
-######## SUPPORT FUNCTIONS #######
-####################################
-
-def writecc (listoflists,file,writetype='w',extra=2):
- """
-Writes a list of lists to a file in columns, customized by the max
-size of items within the columns (max size of items in col, +2 characters)
-to specified file. File-overwrite is the default.
-
-Usage: writecc (listoflists,file,writetype='w',extra=2)
-Returns: None
-"""
- if type(listoflists[0]) not in [ListType,TupleType]:
- listoflists = [listoflists]
- outfile = open(file,writetype)
- rowstokill = []
- list2print = copy.deepcopy(listoflists)
- for i in range(len(listoflists)):
- if listoflists[i] == ['\n'] or listoflists[i]=='\n' or listoflists[i]=='dashes':
- rowstokill = rowstokill + [i]
- rowstokill.reverse()
- for row in rowstokill:
- del list2print[row]
- maxsize = [0]*len(list2print[0])
- for col in range(len(list2print[0])):
- items = pstat.colex(list2print,col)
- items = map(pstat.makestr,items)
- maxsize[col] = max(map(len,items)) + extra
- for row in listoflists:
- if row == ['\n'] or row == '\n':
- outfile.write('\n')
- elif row == ['dashes'] or row == 'dashes':
- dashes = [0]*len(maxsize)
- for j in range(len(maxsize)):
- dashes[j] = '-'*(maxsize[j]-2)
- outfile.write(pstat.lineincustcols(dashes,maxsize))
- else:
- outfile.write(pstat.lineincustcols(row,maxsize))
- outfile.write('\n')
- outfile.close()
- return None
-
-
-def lincr(l,cap): # to increment a list up to a max-list of 'cap'
- """
-Simulate a counting system from an n-dimensional list.
-
-Usage: lincr(l,cap) l=list to increment, cap=max values for each list pos'n
-Returns: next set of values for list l, OR -1 (if overflow)
-"""
- l[0] = l[0] + 1 # e.g., [0,0,0] --> [2,4,3] (=cap)
- for i in range(len(l)):
- if l[i] > cap[i] and i < len(l)-1: # if carryover AND not done
- l[i] = 0
- l[i+1] = l[i+1] + 1
- elif l[i] > cap[i] and i == len(l)-1: # overflow past last column, must be finished
- l = -1
- return l
-
-
-def lsum (inlist):
- """
-Returns the sum of the items in the passed list.
-
-Usage: lsum(inlist)
-"""
- s = 0
- for item in inlist:
- s = s + item
- return s
-
-
-def lcumsum (inlist):
- """
-Returns a list consisting of the cumulative sum of the items in the
-passed list.
-
-Usage: lcumsum(inlist)
-"""
- newlist = copy.deepcopy(inlist)
- for i in range(1,len(newlist)):
- newlist[i] = newlist[i] + newlist[i-1]
- return newlist
-
-
-def lss(inlist):
- """
-Squares each value in the passed list, adds up these squares and
-returns the result.
-
-Usage: lss(inlist)
-"""
- ss = 0
- for item in inlist:
- ss = ss + item*item
- return ss
-
-
-def lsummult (list1,list2):
- """
-Multiplies elements in list1 and list2, element by element, and
-returns the sum of all resulting multiplications. Must provide equal
-length lists.
-
-Usage: lsummult(list1,list2)
-"""
- if len(list1) <> len(list2):
- raise ValueError, "Lists not equal length in summult."
- s = 0
- for item1,item2 in pstat.abut(list1,list2):
- s = s + item1*item2
- return s
-
-
-def lsumdiffsquared(x,y):
- """
-Takes pairwise differences of the values in lists x and y, squares
-these differences, and returns the sum of these squares.
-
-Usage: lsumdiffsquared(x,y)
-Returns: sum[(x[i]-y[i])**2]
-"""
- sds = 0
- for i in range(len(x)):
- sds = sds + (x[i]-y[i])**2
- return sds
-
-
-def lsquare_of_sums(inlist):
- """
-Adds the values in the passed list, squares the sum, and returns
-the result.
-
-Usage: lsquare_of_sums(inlist)
-Returns: sum(inlist[i])**2
-"""
- s = sum(inlist)
- return float(s)*s
-
-
-def lshellsort(inlist):
- """
-Shellsort algorithm. Sorts a 1D-list.
-
-Usage: lshellsort(inlist)
-Returns: sorted-inlist, sorting-index-vector (for original list)
-"""
- n = len(inlist)
- svec = copy.deepcopy(inlist)
- ivec = range(n)
- gap = n/2 # integer division needed
- while gap >0:
- for i in range(gap,n):
- for j in range(i-gap,-1,-gap):
- while j>=0 and svec[j]>svec[j+gap]:
- temp = svec[j]
- svec[j] = svec[j+gap]
- svec[j+gap] = temp
- itemp = ivec[j]
- ivec[j] = ivec[j+gap]
- ivec[j+gap] = itemp
- gap = gap / 2 # integer division needed
-# svec is now sorted inlist, and ivec has the order svec[i] = vec[ivec[i]]
- return svec, ivec
-
-
-def lrankdata(inlist):
- """
-Ranks the data in inlist, dealing with ties appropritely. Assumes
-a 1D inlist. Adapted from Gary Perlman's |Stat ranksort.
-
-Usage: lrankdata(inlist)
-Returns: a list of length equal to inlist, containing rank scores
-"""
- n = len(inlist)
- svec, ivec = shellsort(inlist)
- sumranks = 0
- dupcount = 0
- newlist = [0]*n
- for i in range(n):
- sumranks = sumranks + i
- dupcount = dupcount + 1
- if i==n-1 or svec[i] <> svec[i+1]:
- averank = sumranks / float(dupcount) + 1
- for j in range(i-dupcount+1,i+1):
- newlist[ivec[j]] = averank
- sumranks = 0
- dupcount = 0
- return newlist
-
-
-def outputpairedstats(fname,writemode,name1,n1,m1,se1,min1,max1,name2,n2,m2,se2,min2,max2,statname,stat,prob):
- """
-Prints or write to a file stats for two groups, using the name, n,
-mean, sterr, min and max for each group, as well as the statistic name,
-its value, and the associated p-value.
-
-Usage: outputpairedstats(fname,writemode,
- name1,n1,mean1,stderr1,min1,max1,
- name2,n2,mean2,stderr2,min2,max2,
- statname,stat,prob)
-Returns: None
-"""
- suffix = '' # for *s after the p-value
- try:
- x = prob.shape
- prob = prob[0]
- except:
- pass
- if prob < 0.001: suffix = ' ***'
- elif prob < 0.01: suffix = ' **'
- elif prob < 0.05: suffix = ' *'
- title = [['Name','N','Mean','SD','Min','Max']]
- lofl = title+[[name1,n1,round(m1,3),round(math.sqrt(se1),3),min1,max1],
- [name2,n2,round(m2,3),round(math.sqrt(se2),3),min2,max2]]
- if type(fname)<>StringType or len(fname)==0:
- print
- print statname
- print
- pstat.printcc(lofl)
- print
- try:
- if stat.shape == ():
- stat = stat[0]
- if prob.shape == ():
- prob = prob[0]
- except:
- pass
- print 'Test statistic = ',round(stat,3),' p = ',round(prob,3),suffix
- print
- else:
- file = open(fname,writemode)
- file.write('\n'+statname+'\n\n')
- file.close()
- writecc(lofl,fname,'a')
- file = open(fname,'a')
- try:
- if stat.shape == ():
- stat = stat[0]
- if prob.shape == ():
- prob = prob[0]
- except:
- pass
- file.write(pstat.list2string(['\nTest statistic = ',round(stat,4),' p = ',round(prob,4),suffix,'\n\n']))
- file.close()
- return None
-
-
-def lfindwithin (data):
- """
-Returns an integer representing a binary vector, where 1=within-
-subject factor, 0=between. Input equals the entire data 2D list (i.e.,
-column 0=random factor, column -1=measured values (those two are skipped).
-Note: input data is in |Stat format ... a list of lists ("2D list") with
-one row per measured value, first column=subject identifier, last column=
-score, one in-between column per factor (these columns contain level
-designations on each factor). See also stats.anova.__doc__.
-
-Usage: lfindwithin(data) data in |Stat format
-"""
-
- numfact = len(data[0])-1
- withinvec = 0
- for col in range(1,numfact):
- examplelevel = pstat.unique(pstat.colex(data,col))[0]
- rows = pstat.linexand(data,col,examplelevel) # get 1 level of this factor
- factsubjs = pstat.unique(pstat.colex(rows,0))
- allsubjs = pstat.unique(pstat.colex(data,0))
- if len(factsubjs) == len(allsubjs): # fewer Ss than scores on this factor?
- withinvec = withinvec + (1 << col)
- return withinvec
-
-
-#########################################################
-#########################################################
-####### DISPATCH LISTS AND TUPLES TO ABOVE FCNS #########
-#########################################################
-#########################################################
-
-## CENTRAL TENDENCY:
-geometricmean = Dispatch ( (lgeometricmean, (ListType, TupleType)), )
-harmonicmean = Dispatch ( (lharmonicmean, (ListType, TupleType)), )
-mean = Dispatch ( (lmean, (ListType, TupleType)), )
-median = Dispatch ( (lmedian, (ListType, TupleType)), )
-medianscore = Dispatch ( (lmedianscore, (ListType, TupleType)), )
-mode = Dispatch ( (lmode, (ListType, TupleType)), )
-
-## MOMENTS:
-moment = Dispatch ( (lmoment, (ListType, TupleType)), )
-variation = Dispatch ( (lvariation, (ListType, TupleType)), )
-skew = Dispatch ( (lskew, (ListType, TupleType)), )
-kurtosis = Dispatch ( (lkurtosis, (ListType, TupleType)), )
-describe = Dispatch ( (ldescribe, (ListType, TupleType)), )
-
-## FREQUENCY STATISTICS:
-itemfreq = Dispatch ( (litemfreq, (ListType, TupleType)), )
-scoreatpercentile = Dispatch ( (lscoreatpercentile, (ListType, TupleType)), )
-percentileofscore = Dispatch ( (lpercentileofscore, (ListType, TupleType)), )
-histogram = Dispatch ( (lhistogram, (ListType, TupleType)), )
-cumfreq = Dispatch ( (lcumfreq, (ListType, TupleType)), )
-relfreq = Dispatch ( (lrelfreq, (ListType, TupleType)), )
-
-## VARIABILITY:
-obrientransform = Dispatch ( (lobrientransform, (ListType, TupleType)), )
-samplevar = Dispatch ( (lsamplevar, (ListType, TupleType)), )
-samplestdev = Dispatch ( (lsamplestdev, (ListType, TupleType)), )
-var = Dispatch ( (lvar, (ListType, TupleType)), )
-stdev = Dispatch ( (lstdev, (ListType, TupleType)), )
-sterr = Dispatch ( (lsterr, (ListType, TupleType)), )
-sem = Dispatch ( (lsem, (ListType, TupleType)), )
-z = Dispatch ( (lz, (ListType, TupleType)), )
-zs = Dispatch ( (lzs, (ListType, TupleType)), )
-
-## TRIMMING FCNS:
-trimboth = Dispatch ( (ltrimboth, (ListType, TupleType)), )
-trim1 = Dispatch ( (ltrim1, (ListType, TupleType)), )
-
-## CORRELATION FCNS:
-paired = Dispatch ( (lpaired, (ListType, TupleType)), )
-pearsonr = Dispatch ( (lpearsonr, (ListType, TupleType)), )
-spearmanr = Dispatch ( (lspearmanr, (ListType, TupleType)), )
-pointbiserialr = Dispatch ( (lpointbiserialr, (ListType, TupleType)), )
-kendalltau = Dispatch ( (lkendalltau, (ListType, TupleType)), )
-linregress = Dispatch ( (llinregress, (ListType, TupleType)), )
-
-## INFERENTIAL STATS:
-ttest_1samp = Dispatch ( (lttest_1samp, (ListType, TupleType)), )
-ttest_ind = Dispatch ( (lttest_ind, (ListType, TupleType)), )
-ttest_rel = Dispatch ( (lttest_rel, (ListType, TupleType)), )
-chisquare = Dispatch ( (lchisquare, (ListType, TupleType)), )
-ks_2samp = Dispatch ( (lks_2samp, (ListType, TupleType)), )
-mannwhitneyu = Dispatch ( (lmannwhitneyu, (ListType, TupleType)), )
-ranksums = Dispatch ( (lranksums, (ListType, TupleType)), )
-tiecorrect = Dispatch ( (ltiecorrect, (ListType, TupleType)), )
-wilcoxont = Dispatch ( (lwilcoxont, (ListType, TupleType)), )
-kruskalwallish = Dispatch ( (lkruskalwallish, (ListType, TupleType)), )
-friedmanchisquare = Dispatch ( (lfriedmanchisquare, (ListType, TupleType)), )
-
-## PROBABILITY CALCS:
-chisqprob = Dispatch ( (lchisqprob, (IntType, FloatType)), )
-zprob = Dispatch ( (lzprob, (IntType, FloatType)), )
-ksprob = Dispatch ( (lksprob, (IntType, FloatType)), )
-fprob = Dispatch ( (lfprob, (IntType, FloatType)), )
-betacf = Dispatch ( (lbetacf, (IntType, FloatType)), )
-betai = Dispatch ( (lbetai, (IntType, FloatType)), )
-erfcc = Dispatch ( (lerfcc, (IntType, FloatType)), )
-gammln = Dispatch ( (lgammln, (IntType, FloatType)), )
-
-## ANOVA FUNCTIONS:
-F_oneway = Dispatch ( (lF_oneway, (ListType, TupleType)), )
-F_value = Dispatch ( (lF_value, (ListType, TupleType)), )
-
-## SUPPORT FUNCTIONS:
-incr = Dispatch ( (lincr, (ListType, TupleType)), )
-sum = Dispatch ( (lsum, (ListType, TupleType)), )
-cumsum = Dispatch ( (lcumsum, (ListType, TupleType)), )
-ss = Dispatch ( (lss, (ListType, TupleType)), )
-summult = Dispatch ( (lsummult, (ListType, TupleType)), )
-square_of_sums = Dispatch ( (lsquare_of_sums, (ListType, TupleType)), )
-sumdiffsquared = Dispatch ( (lsumdiffsquared, (ListType, TupleType)), )
-shellsort = Dispatch ( (lshellsort, (ListType, TupleType)), )
-rankdata = Dispatch ( (lrankdata, (ListType, TupleType)), )
-findwithin = Dispatch ( (lfindwithin, (ListType, TupleType)), )
-
-
-#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS ===============
-#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS ===============
-#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS ===============
-#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS ===============
-#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS ===============
-#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS ===============
-#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS ===============
-#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS ===============
-#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS ===============
-#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS ===============
-#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS ===============
-#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS ===============
-#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS ===============
-#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS ===============
-#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS ===============
-#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS ===============
-#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS ===============
-#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS ===============
-#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS ===============
-
-try: # DEFINE THESE *ONLY* IF NUMERIC IS AVAILABLE
- import numpy as N
- import numpy.linalg as LA
-
-
-#####################################
-######## ACENTRAL TENDENCY ########
-#####################################
-
- def ageometricmean (inarray,dimension=None,keepdims=0):
- """
-Calculates the geometric mean of the values in the passed array.
-That is: n-th root of (x1 * x2 * ... * xn). Defaults to ALL values in
-the passed array. Use dimension=None to flatten array first. REMEMBER: if
-dimension=0, it collapses over dimension 0 ('rows' in a 2D array) only, and
-if dimension is a sequence, it collapses over all specified dimensions. If
-keepdims is set to 1, the resulting array will have as many dimensions as
-inarray, with only 1 'level' per dim that was collapsed over.
-
-Usage: ageometricmean(inarray,dimension=None,keepdims=0)
-Returns: geometric mean computed over dim(s) listed in dimension
-"""
- inarray = N.array(inarray,N.float_)
- if dimension == None:
- inarray = N.ravel(inarray)
- size = len(inarray)
- mult = N.power(inarray,1.0/size)
- mult = N.multiply.reduce(mult)
- elif type(dimension) in [IntType,FloatType]:
- size = inarray.shape[dimension]
- mult = N.power(inarray,1.0/size)
- mult = N.multiply.reduce(mult,dimension)
- if keepdims == 1:
- shp = list(inarray.shape)
- shp[dimension] = 1
- sum = N.reshape(sum,shp)
- else: # must be a SEQUENCE of dims to average over
- dims = list(dimension)
- dims.sort()
- dims.reverse()
- size = N.array(N.multiply.reduce(N.take(inarray.shape,dims)),N.float_)
- mult = N.power(inarray,1.0/size)
- for dim in dims:
- mult = N.multiply.reduce(mult,dim)
- if keepdims == 1:
- shp = list(inarray.shape)
- for dim in dims:
- shp[dim] = 1
- mult = N.reshape(mult,shp)
- return mult
-
-
- def aharmonicmean (inarray,dimension=None,keepdims=0):
- """
-Calculates the harmonic mean of the values in the passed array.
-That is: n / (1/x1 + 1/x2 + ... + 1/xn). Defaults to ALL values in
-the passed array. Use dimension=None to flatten array first. REMEMBER: if
-dimension=0, it collapses over dimension 0 ('rows' in a 2D array) only, and
-if dimension is a sequence, it collapses over all specified dimensions. If
-keepdims is set to 1, the resulting array will have as many dimensions as
-inarray, with only 1 'level' per dim that was collapsed over.
-
-Usage: aharmonicmean(inarray,dimension=None,keepdims=0)
-Returns: harmonic mean computed over dim(s) in dimension
-"""
- inarray = inarray.astype(N.float_)
- if dimension == None:
- inarray = N.ravel(inarray)
- size = len(inarray)
- s = N.add.reduce(1.0 / inarray)
- elif type(dimension) in [IntType,FloatType]:
- size = float(inarray.shape[dimension])
- s = N.add.reduce(1.0/inarray, dimension)
- if keepdims == 1:
- shp = list(inarray.shape)
- shp[dimension] = 1
- s = N.reshape(s,shp)
- else: # must be a SEQUENCE of dims to average over
- dims = list(dimension)
- dims.sort()
- nondims = []
- for i in range(len(inarray.shape)):
- if i not in dims:
- nondims.append(i)
- tinarray = N.transpose(inarray,nondims+dims) # put keep-dims first
- idx = [0] *len(nondims)
- if idx == []:
- size = len(N.ravel(inarray))
- s = asum(1.0 / inarray)
- if keepdims == 1:
- s = N.reshape([s],N.ones(len(inarray.shape)))
- else:
- idx[0] = -1
- loopcap = N.array(tinarray.shape[0:len(nondims)]) -1
- s = N.zeros(loopcap+1,N.float_)
- while incr(idx,loopcap) <> -1:
- s[idx] = asum(1.0/tinarray[idx])
- size = N.multiply.reduce(N.take(inarray.shape,dims))
- if keepdims == 1:
- shp = list(inarray.shape)
- for dim in dims:
- shp[dim] = 1
- s = N.reshape(s,shp)
- return size / s
-
-
- def amean (inarray,dimension=None,keepdims=0):
- """
-Calculates the arithmatic mean of the values in the passed array.
-That is: 1/n * (x1 + x2 + ... + xn). Defaults to ALL values in the
-passed array. Use dimension=None to flatten array first. REMEMBER: if
-dimension=0, it collapses over dimension 0 ('rows' in a 2D array) only, and
-if dimension is a sequence, it collapses over all specified dimensions. If
-keepdims is set to 1, the resulting array will have as many dimensions as
-inarray, with only 1 'level' per dim that was collapsed over.
-
-Usage: amean(inarray,dimension=None,keepdims=0)
-Returns: arithematic mean calculated over dim(s) in dimension
-"""
- if inarray.dtype in [N.int_, N.short,N.ubyte]:
- inarray = inarray.astype(N.float_)
- if dimension == None:
- inarray = N.ravel(inarray)
- sum = N.add.reduce(inarray)
- denom = float(len(inarray))
- elif type(dimension) in [IntType,FloatType]:
- sum = asum(inarray,dimension)
- denom = float(inarray.shape[dimension])
- if keepdims == 1:
- shp = list(inarray.shape)
- shp[dimension] = 1
- sum = N.reshape(sum,shp)
- else: # must be a TUPLE of dims to average over
- dims = list(dimension)
- dims.sort()
- dims.reverse()
- sum = inarray *1.0
- for dim in dims:
- sum = N.add.reduce(sum,dim)
- denom = N.array(N.multiply.reduce(N.take(inarray.shape,dims)),N.float_)
- if keepdims == 1:
- shp = list(inarray.shape)
- for dim in dims:
- shp[dim] = 1
- sum = N.reshape(sum,shp)
- return sum/denom
-
-
- def amedian (inarray,numbins=1000):
- """
-Calculates the COMPUTED median value of an array of numbers, given the
-number of bins to use for the histogram (more bins approaches finding the
-precise median value of the array; default number of bins = 1000). From
-G.W. Heiman's Basic Stats, or CRC Probability & Statistics.
-NOTE: THIS ROUTINE ALWAYS uses the entire passed array (flattens it first).
-
-Usage: amedian(inarray,numbins=1000)
-Returns: median calculated over ALL values in inarray
-"""
- inarray = N.ravel(inarray)
- (hist, smallest, binsize, extras) = ahistogram(inarray,numbins,[min(inarray),max(inarray)])
- cumhist = N.cumsum(hist) # make cumulative histogram
- otherbins = N.greater_equal(cumhist,len(inarray)/2.0)
- otherbins = list(otherbins) # list of 0/1s, 1s start at median bin
- cfbin = otherbins.index(1) # get 1st(!) index holding 50%ile score
- LRL = smallest + binsize*cfbin # get lower read limit of that bin
- cfbelow = N.add.reduce(hist[0:cfbin]) # cum. freq. below bin
- freq = hist[cfbin] # frequency IN the 50%ile bin
- median = LRL + ((len(inarray)/2.0-cfbelow)/float(freq))*binsize # MEDIAN
- return median
-
-
- def amedianscore (inarray,dimension=None):
- """
-Returns the 'middle' score of the passed array. If there is an even
-number of scores, the mean of the 2 middle scores is returned. Can function
-with 1D arrays, or on the FIRST dimension of 2D arrays (i.e., dimension can
-be None, to pre-flatten the array, or else dimension must equal 0).
-
-Usage: amedianscore(inarray,dimension=None)
-Returns: 'middle' score of the array, or the mean of the 2 middle scores
-"""
- if dimension == None:
- inarray = N.ravel(inarray)
- dimension = 0
- inarray = N.sort(inarray,dimension)
- if inarray.shape[dimension] % 2 == 0: # if even number of elements
- indx = inarray.shape[dimension]/2 # integer division correct
- median = N.asarray(inarray[indx]+inarray[indx-1]) / 2.0
- else:
- indx = inarray.shape[dimension] / 2 # integer division correct
- median = N.take(inarray,[indx],dimension)
- if median.shape == (1,):
- median = median[0]
- return median
-
-
- def amode(a, dimension=None):
- """
-Returns an array of the modal (most common) score in the passed array.
-If there is more than one such score, ONLY THE FIRST is returned.
-The bin-count for the modal values is also returned. Operates on whole
-array (dimension=None), or on a given dimension.
-
-Usage: amode(a, dimension=None)
-Returns: array of bin-counts for mode(s), array of corresponding modal values
-"""
-
- if dimension == None:
- a = N.ravel(a)
- dimension = 0
- scores = pstat.aunique(N.ravel(a)) # get ALL unique values
- testshape = list(a.shape)
- testshape[dimension] = 1
- oldmostfreq = N.zeros(testshape)
- oldcounts = N.zeros(testshape)
- for score in scores:
- template = N.equal(a,score)
- counts = asum(template,dimension,1)
- mostfrequent = N.where(counts>oldcounts,score,oldmostfreq)
- oldcounts = N.where(counts>oldcounts,counts,oldcounts)
- oldmostfreq = mostfrequent
- return oldcounts, mostfrequent
-
-
- def atmean(a,limits=None,inclusive=(1,1)):
- """
-Returns the arithmetic mean of all values in an array, ignoring values
-strictly outside the sequence passed to 'limits'. Note: either limit
-in the sequence, or the value of limits itself, can be set to None. The
-inclusive list/tuple determines whether the lower and upper limiting bounds
-(respectively) are open/exclusive (0) or closed/inclusive (1).
-
-Usage: atmean(a,limits=None,inclusive=(1,1))
-"""
- if a.dtype in [N.int_, N.short,N.ubyte]:
- a = a.astype(N.float_)
- if limits == None:
- return mean(a)
- assert type(limits) in [ListType,TupleType,N.ndarray], "Wrong type for limits in atmean"
- if inclusive[0]: lowerfcn = N.greater_equal
- else: lowerfcn = N.greater
- if inclusive[1]: upperfcn = N.less_equal
- else: upperfcn = N.less
- if limits[0] > N.maximum.reduce(N.ravel(a)) or limits[1] < N.minimum.reduce(N.ravel(a)):
- raise ValueError, "No array values within given limits (atmean)."
- elif limits[0]==None and limits[1]<>None:
- mask = upperfcn(a,limits[1])
- elif limits[0]<>None and limits[1]==None:
- mask = lowerfcn(a,limits[0])
- elif limits[0]<>None and limits[1]<>None:
- mask = lowerfcn(a,limits[0])*upperfcn(a,limits[1])
- s = float(N.add.reduce(N.ravel(a*mask)))
- n = float(N.add.reduce(N.ravel(mask)))
- return s/n
-
-
- def atvar(a,limits=None,inclusive=(1,1)):
- """
-Returns the sample variance of values in an array, (i.e., using N-1),
-ignoring values strictly outside the sequence passed to 'limits'.
-Note: either limit in the sequence, or the value of limits itself,
-can be set to None. The inclusive list/tuple determines whether the lower
-and upper limiting bounds (respectively) are open/exclusive (0) or
-closed/inclusive (1). ASSUMES A FLAT ARRAY (OR ELSE PREFLATTENS).
-
-Usage: atvar(a,limits=None,inclusive=(1,1))
-"""
- a = a.astype(N.float_)
- if limits == None or limits == [None,None]:
- return avar(a)
- assert type(limits) in [ListType,TupleType,N.ndarray], "Wrong type for limits in atvar"
- if inclusive[0]: lowerfcn = N.greater_equal
- else: lowerfcn = N.greater
- if inclusive[1]: upperfcn = N.less_equal
- else: upperfcn = N.less
- if limits[0] > N.maximum.reduce(N.ravel(a)) or limits[1] < N.minimum.reduce(N.ravel(a)):
- raise ValueError, "No array values within given limits (atvar)."
- elif limits[0]==None and limits[1]<>None:
- mask = upperfcn(a,limits[1])
- elif limits[0]<>None and limits[1]==None:
- mask = lowerfcn(a,limits[0])
- elif limits[0]<>None and limits[1]<>None:
- mask = lowerfcn(a,limits[0])*upperfcn(a,limits[1])
-
- a = N.compress(mask,a) # squish out excluded values
- return avar(a)
-
-
- def atmin(a,lowerlimit=None,dimension=None,inclusive=1):
- """
-Returns the minimum value of a, along dimension, including only values less
-than (or equal to, if inclusive=1) lowerlimit. If the limit is set to None,
-all values in the array are used.
-
-Usage: atmin(a,lowerlimit=None,dimension=None,inclusive=1)
-"""
- if inclusive: lowerfcn = N.greater
- else: lowerfcn = N.greater_equal
- if dimension == None:
- a = N.ravel(a)
- dimension = 0
- if lowerlimit == None:
- lowerlimit = N.minimum.reduce(N.ravel(a))-11
- biggest = N.maximum.reduce(N.ravel(a))
- ta = N.where(lowerfcn(a,lowerlimit),a,biggest)
- return N.minimum.reduce(ta,dimension)
-
-
- def atmax(a,upperlimit,dimension=None,inclusive=1):
- """
-Returns the maximum value of a, along dimension, including only values greater
-than (or equal to, if inclusive=1) upperlimit. If the limit is set to None,
-a limit larger than the max value in the array is used.
-
-Usage: atmax(a,upperlimit,dimension=None,inclusive=1)
-"""
- if inclusive: upperfcn = N.less
- else: upperfcn = N.less_equal
- if dimension == None:
- a = N.ravel(a)
- dimension = 0
- if upperlimit == None:
- upperlimit = N.maximum.reduce(N.ravel(a))+1
- smallest = N.minimum.reduce(N.ravel(a))
- ta = N.where(upperfcn(a,upperlimit),a,smallest)
- return N.maximum.reduce(ta,dimension)
-
-
- def atstdev(a,limits=None,inclusive=(1,1)):
- """
-Returns the standard deviation of all values in an array, ignoring values
-strictly outside the sequence passed to 'limits'. Note: either limit
-in the sequence, or the value of limits itself, can be set to None. The
-inclusive list/tuple determines whether the lower and upper limiting bounds
-(respectively) are open/exclusive (0) or closed/inclusive (1).
-
-Usage: atstdev(a,limits=None,inclusive=(1,1))
-"""
- return N.sqrt(tvar(a,limits,inclusive))
-
-
- def atsem(a,limits=None,inclusive=(1,1)):
- """
-Returns the standard error of the mean for the values in an array,
-(i.e., using N for the denominator), ignoring values strictly outside
-the sequence passed to 'limits'. Note: either limit in the sequence,
-or the value of limits itself, can be set to None. The inclusive list/tuple
-determines whether the lower and upper limiting bounds (respectively) are
-open/exclusive (0) or closed/inclusive (1).
-
-Usage: atsem(a,limits=None,inclusive=(1,1))
-"""
- sd = tstdev(a,limits,inclusive)
- if limits == None or limits == [None,None]:
- n = float(len(N.ravel(a)))
- limits = [min(a)-1, max(a)+1]
- assert type(limits) in [ListType,TupleType,N.ndarray], "Wrong type for limits in atsem"
- if inclusive[0]: lowerfcn = N.greater_equal
- else: lowerfcn = N.greater
- if inclusive[1]: upperfcn = N.less_equal
- else: upperfcn = N.less
- if limits[0] > N.maximum.reduce(N.ravel(a)) or limits[1] < N.minimum.reduce(N.ravel(a)):
- raise ValueError, "No array values within given limits (atsem)."
- elif limits[0]==None and limits[1]<>None:
- mask = upperfcn(a,limits[1])
- elif limits[0]<>None and limits[1]==None:
- mask = lowerfcn(a,limits[0])
- elif limits[0]<>None and limits[1]<>None:
- mask = lowerfcn(a,limits[0])*upperfcn(a,limits[1])
- term1 = N.add.reduce(N.ravel(a*a*mask))
- n = float(N.add.reduce(N.ravel(mask)))
- return sd/math.sqrt(n)
-
-
-#####################################
-############ AMOMENTS #############
-#####################################
-
- def amoment(a,moment=1,dimension=None):
- """
-Calculates the nth moment about the mean for a sample (defaults to the
-1st moment). Generally used to calculate coefficients of skewness and
-kurtosis. Dimension can equal None (ravel array first), an integer
-(the dimension over which to operate), or a sequence (operate over
-multiple dimensions).
-
-Usage: amoment(a,moment=1,dimension=None)
-Returns: appropriate moment along given dimension
-"""
- if dimension == None:
- a = N.ravel(a)
- dimension = 0
- if moment == 1:
- return 0.0
- else:
- mn = amean(a,dimension,1) # 1=keepdims
- s = N.power((a-mn),moment)
- return amean(s,dimension)
-
-
- def avariation(a,dimension=None):
- """
-Returns the coefficient of variation, as defined in CRC Standard
-Probability and Statistics, p.6. Dimension can equal None (ravel array
-first), an integer (the dimension over which to operate), or a
-sequence (operate over multiple dimensions).
-
-Usage: avariation(a,dimension=None)
-"""
- return 100.0*asamplestdev(a,dimension)/amean(a,dimension)
-
-
- def askew(a,dimension=None):
- """
-Returns the skewness of a distribution (normal ==> 0.0; >0 means extra
-weight in left tail). Use askewtest() to see if it's close enough.
-Dimension can equal None (ravel array first), an integer (the
-dimension over which to operate), or a sequence (operate over multiple
-dimensions).
-
-Usage: askew(a, dimension=None)
-Returns: skew of vals in a along dimension, returning ZERO where all vals equal
-"""
- denom = N.power(amoment(a,2,dimension),1.5)
- zero = N.equal(denom,0)
- if type(denom) == N.ndarray and asum(zero) <> 0:
- print "Number of zeros in askew: ",asum(zero)
- denom = denom + zero # prevent divide-by-zero
- return N.where(zero, 0, amoment(a,3,dimension)/denom)
-
-
- def akurtosis(a,dimension=None):
- """
-Returns the kurtosis of a distribution (normal ==> 3.0; >3 means
-heavier in the tails, and usually more peaked). Use akurtosistest()
-to see if it's close enough. Dimension can equal None (ravel array
-first), an integer (the dimension over which to operate), or a
-sequence (operate over multiple dimensions).
-
-Usage: akurtosis(a,dimension=None)
-Returns: kurtosis of values in a along dimension, and ZERO where all vals equal
-"""
- denom = N.power(amoment(a,2,dimension),2)
- zero = N.equal(denom,0)
- if type(denom) == N.ndarray and asum(zero) <> 0:
- print "Number of zeros in akurtosis: ",asum(zero)
- denom = denom + zero # prevent divide-by-zero
- return N.where(zero,0,amoment(a,4,dimension)/denom)
-
-
- def adescribe(inarray,dimension=None):
- """
-Returns several descriptive statistics of the passed array. Dimension
-can equal None (ravel array first), an integer (the dimension over
-which to operate), or a sequence (operate over multiple dimensions).
-
-Usage: adescribe(inarray,dimension=None)
-Returns: n, (min,max), mean, standard deviation, skew, kurtosis
-"""
- if dimension == None:
- inarray = N.ravel(inarray)
- dimension = 0
- n = inarray.shape[dimension]
- mm = (N.minimum.reduce(inarray),N.maximum.reduce(inarray))
- m = amean(inarray,dimension)
- sd = astdev(inarray,dimension)
- skew = askew(inarray,dimension)
- kurt = akurtosis(inarray,dimension)
- return n, mm, m, sd, skew, kurt
-
-
-#####################################
-######## NORMALITY TESTS ##########
-#####################################
-
- def askewtest(a,dimension=None):
- """
-Tests whether the skew is significantly different from a normal
-distribution. Dimension can equal None (ravel array first), an
-integer (the dimension over which to operate), or a sequence (operate
-over multiple dimensions).
-
-Usage: askewtest(a,dimension=None)
-Returns: z-score and 2-tail z-probability
-"""
- if dimension == None:
- a = N.ravel(a)
- dimension = 0
- b2 = askew(a,dimension)
- n = float(a.shape[dimension])
- y = b2 * N.sqrt(((n+1)*(n+3)) / (6.0*(n-2)) )
- beta2 = ( 3.0*(n*n+27*n-70)*(n+1)*(n+3) ) / ( (n-2.0)*(n+5)*(n+7)*(n+9) )
- W2 = -1 + N.sqrt(2*(beta2-1))
- delta = 1/N.sqrt(N.log(N.sqrt(W2)))
- alpha = N.sqrt(2/(W2-1))
- y = N.where(y==0,1,y)
- Z = delta*N.log(y/alpha + N.sqrt((y/alpha)**2+1))
- return Z, (1.0-zprob(Z))*2
-
-
- def akurtosistest(a,dimension=None):
- """
-Tests whether a dataset has normal kurtosis (i.e.,
-kurtosis=3(n-1)/(n+1)) Valid only for n>20. Dimension can equal None
-(ravel array first), an integer (the dimension over which to operate),
-or a sequence (operate over multiple dimensions).
-
-Usage: akurtosistest(a,dimension=None)
-Returns: z-score and 2-tail z-probability, returns 0 for bad pixels
-"""
- if dimension == None:
- a = N.ravel(a)
- dimension = 0
- n = float(a.shape[dimension])
- if n<20:
- print "akurtosistest only valid for n>=20 ... continuing anyway, n=",n
- b2 = akurtosis(a,dimension)
- E = 3.0*(n-1) /(n+1)
- varb2 = 24.0*n*(n-2)*(n-3) / ((n+1)*(n+1)*(n+3)*(n+5))
- x = (b2-E)/N.sqrt(varb2)
- sqrtbeta1 = 6.0*(n*n-5*n+2)/((n+7)*(n+9)) * N.sqrt((6.0*(n+3)*(n+5))/
- (n*(n-2)*(n-3)))
- A = 6.0 + 8.0/sqrtbeta1 *(2.0/sqrtbeta1 + N.sqrt(1+4.0/(sqrtbeta1**2)))
- term1 = 1 -2/(9.0*A)
- denom = 1 +x*N.sqrt(2/(A-4.0))
- denom = N.where(N.less(denom,0), 99, denom)
- term2 = N.where(N.equal(denom,0), term1, N.power((1-2.0/A)/denom,1/3.0))
- Z = ( term1 - term2 ) / N.sqrt(2/(9.0*A))
- Z = N.where(N.equal(denom,99), 0, Z)
- return Z, (1.0-zprob(Z))*2
-
-
- def anormaltest(a,dimension=None):
- """
-Tests whether skew and/OR kurtosis of dataset differs from normal
-curve. Can operate over multiple dimensions. Dimension can equal
-None (ravel array first), an integer (the dimension over which to
-operate), or a sequence (operate over multiple dimensions).
-
-Usage: anormaltest(a,dimension=None)
-Returns: z-score and 2-tail probability
-"""
- if dimension == None:
- a = N.ravel(a)
- dimension = 0
- s,p = askewtest(a,dimension)
- k,p = akurtosistest(a,dimension)
- k2 = N.power(s,2) + N.power(k,2)
- return k2, achisqprob(k2,2)
-
-
-#####################################
-###### AFREQUENCY FUNCTIONS #######
-#####################################
-
- def aitemfreq(a):
- """
-Returns a 2D array of item frequencies. Column 1 contains item values,
-column 2 contains their respective counts. Assumes a 1D array is passed.
-@@@sorting OK?
-
-Usage: aitemfreq(a)
-Returns: a 2D frequency table (col [0:n-1]=scores, col n=frequencies)
-"""
- scores = pstat.aunique(a)
- scores = N.sort(scores)
- freq = N.zeros(len(scores))
- for i in range(len(scores)):
- freq[i] = N.add.reduce(N.equal(a,scores[i]))
- return N.array(pstat.aabut(scores, freq))
-
-
- def ascoreatpercentile (inarray, percent):
- """
-Usage: ascoreatpercentile(inarray,percent) 0<percent<100
-Returns: score at given percentile, relative to inarray distribution
-"""
- percent = percent / 100.0
- targetcf = percent*len(inarray)
- h, lrl, binsize, extras = histogram(inarray)
- cumhist = cumsum(h*1)
- for i in range(len(cumhist)):
- if cumhist[i] >= targetcf:
- break
- score = binsize * ((targetcf - cumhist[i-1]) / float(h[i])) + (lrl+binsize*i)
- return score
-
-
- def apercentileofscore (inarray,score,histbins=10,defaultlimits=None):
- """
-Note: result of this function depends on the values used to histogram
-the data(!).
-
-Usage: apercentileofscore(inarray,score,histbins=10,defaultlimits=None)
-Returns: percentile-position of score (0-100) relative to inarray
-"""
- h, lrl, binsize, extras = histogram(inarray,histbins,defaultlimits)
- cumhist = cumsum(h*1)
- i = int((score - lrl)/float(binsize))
- pct = (cumhist[i-1]+((score-(lrl+binsize*i))/float(binsize))*h[i])/float(len(inarray)) * 100
- return pct
-
-
- def ahistogram (inarray,numbins=10,defaultlimits=None,printextras=1):
- """
-Returns (i) an array of histogram bin counts, (ii) the smallest value
-of the histogram binning, and (iii) the bin width (the last 2 are not
-necessarily integers). Default number of bins is 10. Defaultlimits
-can be None (the routine picks bins spanning all the numbers in the
-inarray) or a 2-sequence (lowerlimit, upperlimit). Returns all of the
-following: array of bin values, lowerreallimit, binsize, extrapoints.
-
-Usage: ahistogram(inarray,numbins=10,defaultlimits=None,printextras=1)
-Returns: (array of bin counts, bin-minimum, min-width, #-points-outside-range)
-"""
- inarray = N.ravel(inarray) # flatten any >1D arrays
- if (defaultlimits <> None):
- lowerreallimit = defaultlimits[0]
- upperreallimit = defaultlimits[1]
- binsize = (upperreallimit-lowerreallimit) / float(numbins)
- else:
- Min = N.minimum.reduce(inarray)
- Max = N.maximum.reduce(inarray)
- estbinwidth = float(Max - Min)/float(numbins) + 1e-6
- binsize = (Max-Min+estbinwidth)/float(numbins)
- lowerreallimit = Min - binsize/2.0 #lower real limit,1st bin
- bins = N.zeros(numbins)
- extrapoints = 0
- for num in inarray:
- try:
- if (num-lowerreallimit) < 0:
- extrapoints = extrapoints + 1
- else:
- bintoincrement = int((num-lowerreallimit) / float(binsize))
- bins[bintoincrement] = bins[bintoincrement] + 1
- except: # point outside lower/upper limits
- extrapoints = extrapoints + 1
- if (extrapoints > 0 and printextras == 1):
- print '\nPoints outside given histogram range =',extrapoints
- return (bins, lowerreallimit, binsize, extrapoints)
-
-
- def acumfreq(a,numbins=10,defaultreallimits=None):
- """
-Returns a cumulative frequency histogram, using the histogram function.
-Defaultreallimits can be None (use all data), or a 2-sequence containing
-lower and upper limits on values to include.
-
-Usage: acumfreq(a,numbins=10,defaultreallimits=None)
-Returns: array of cumfreq bin values, lowerreallimit, binsize, extrapoints
-"""
- h,l,b,e = histogram(a,numbins,defaultreallimits)
- cumhist = cumsum(h*1)
- return cumhist,l,b,e
-
-
- def arelfreq(a,numbins=10,defaultreallimits=None):
- """
-Returns a relative frequency histogram, using the histogram function.
-Defaultreallimits can be None (use all data), or a 2-sequence containing
-lower and upper limits on values to include.
-
-Usage: arelfreq(a,numbins=10,defaultreallimits=None)
-Returns: array of cumfreq bin values, lowerreallimit, binsize, extrapoints
-"""
- h,l,b,e = histogram(a,numbins,defaultreallimits)
- h = N.array(h/float(a.shape[0]))
- return h,l,b,e
-
-
-#####################################
-###### AVARIABILITY FUNCTIONS #####
-#####################################
-
- def aobrientransform(*args):
- """
-Computes a transform on input data (any number of columns). Used to
-test for homogeneity of variance prior to running one-way stats. Each
-array in *args is one level of a factor. If an F_oneway() run on the
-transformed data and found significant, variances are unequal. From
-Maxwell and Delaney, p.112.
-
-Usage: aobrientransform(*args) *args = 1D arrays, one per level of factor
-Returns: transformed data for use in an ANOVA
-"""
- TINY = 1e-10
- k = len(args)
- n = N.zeros(k,N.float_)
- v = N.zeros(k,N.float_)
- m = N.zeros(k,N.float_)
- nargs = []
- for i in range(k):
- nargs.append(args[i].astype(N.float_))
- n[i] = float(len(nargs[i]))
- v[i] = var(nargs[i])
- m[i] = mean(nargs[i])
- for j in range(k):
- for i in range(n[j]):
- t1 = (n[j]-1.5)*n[j]*(nargs[j][i]-m[j])**2
- t2 = 0.5*v[j]*(n[j]-1.0)
- t3 = (n[j]-1.0)*(n[j]-2.0)
- nargs[j][i] = (t1-t2) / float(t3)
- check = 1
- for j in range(k):
- if v[j] - mean(nargs[j]) > TINY:
- check = 0
- if check <> 1:
- raise ValueError, 'Lack of convergence in obrientransform.'
- else:
- return N.array(nargs)
-
-
- def asamplevar (inarray,dimension=None,keepdims=0):
- """
-Returns the sample standard deviation of the values in the passed
-array (i.e., using N). Dimension can equal None (ravel array first),
-an integer (the dimension over which to operate), or a sequence
-(operate over multiple dimensions). Set keepdims=1 to return an array
-with the same number of dimensions as inarray.
-
-Usage: asamplevar(inarray,dimension=None,keepdims=0)
-"""
- if dimension == None:
- inarray = N.ravel(inarray)
- dimension = 0
- if dimension == 1:
- mn = amean(inarray,dimension)[:,N.NewAxis]
- else:
- mn = amean(inarray,dimension,keepdims=1)
- deviations = inarray - mn
- if type(dimension) == ListType:
- n = 1
- for d in dimension:
- n = n*inarray.shape[d]
- else:
- n = inarray.shape[dimension]
- svar = ass(deviations,dimension,keepdims) / float(n)
- return svar
-
-
- def asamplestdev (inarray, dimension=None, keepdims=0):
- """
-Returns the sample standard deviation of the values in the passed
-array (i.e., using N). Dimension can equal None (ravel array first),
-an integer (the dimension over which to operate), or a sequence
-(operate over multiple dimensions). Set keepdims=1 to return an array
-with the same number of dimensions as inarray.
-
-Usage: asamplestdev(inarray,dimension=None,keepdims=0)
-"""
- return N.sqrt(asamplevar(inarray,dimension,keepdims))
-
-
- def asignaltonoise(instack,dimension=0):
- """
-Calculates signal-to-noise. Dimension can equal None (ravel array
-first), an integer (the dimension over which to operate), or a
-sequence (operate over multiple dimensions).
-
-Usage: asignaltonoise(instack,dimension=0):
-Returns: array containing the value of (mean/stdev) along dimension,
- or 0 when stdev=0
-"""
- m = mean(instack,dimension)
- sd = stdev(instack,dimension)
- return N.where(sd==0,0,m/sd)
-
-
- def acov (x,y, dimension=None,keepdims=0):
- """
-Returns the estimated covariance of the values in the passed
-array (i.e., N-1). Dimension can equal None (ravel array first), an
-integer (the dimension over which to operate), or a sequence (operate
-over multiple dimensions). Set keepdims=1 to return an array with the
-same number of dimensions as inarray.
-
-Usage: acov(x,y,dimension=None,keepdims=0)
-"""
- if dimension == None:
- x = N.ravel(x)
- y = N.ravel(y)
- dimension = 0
- xmn = amean(x,dimension,1) # keepdims
- xdeviations = x - xmn
- ymn = amean(y,dimension,1) # keepdims
- ydeviations = y - ymn
- if type(dimension) == ListType:
- n = 1
- for d in dimension:
- n = n*x.shape[d]
- else:
- n = x.shape[dimension]
- covar = N.sum(xdeviations*ydeviations)/float(n-1)
- return covar
-
-
- def avar (inarray, dimension=None,keepdims=0):
- """
-Returns the estimated population variance of the values in the passed
-array (i.e., N-1). Dimension can equal None (ravel array first), an
-integer (the dimension over which to operate), or a sequence (operate
-over multiple dimensions). Set keepdims=1 to return an array with the
-same number of dimensions as inarray.
-
-Usage: avar(inarray,dimension=None,keepdims=0)
-"""
- if dimension == None:
- inarray = N.ravel(inarray)
- dimension = 0
- mn = amean(inarray,dimension,1)
- deviations = inarray - mn
- if type(dimension) == ListType:
- n = 1
- for d in dimension:
- n = n*inarray.shape[d]
- else:
- n = inarray.shape[dimension]
- var = ass(deviations,dimension,keepdims)/float(n-1)
- return var
-
-
- def astdev (inarray, dimension=None, keepdims=0):
- """
-Returns the estimated population standard deviation of the values in
-the passed array (i.e., N-1). Dimension can equal None (ravel array
-first), an integer (the dimension over which to operate), or a
-sequence (operate over multiple dimensions). Set keepdims=1 to return
-an array with the same number of dimensions as inarray.
-
-Usage: astdev(inarray,dimension=None,keepdims=0)
-"""
- return N.sqrt(avar(inarray,dimension,keepdims))
-
-
- def asterr (inarray, dimension=None, keepdims=0):
- """
-Returns the estimated population standard error of the values in the
-passed array (i.e., N-1). Dimension can equal None (ravel array
-first), an integer (the dimension over which to operate), or a
-sequence (operate over multiple dimensions). Set keepdims=1 to return
-an array with the same number of dimensions as inarray.
-
-Usage: asterr(inarray,dimension=None,keepdims=0)
-"""
- if dimension == None:
- inarray = N.ravel(inarray)
- dimension = 0
- return astdev(inarray,dimension,keepdims) / float(N.sqrt(inarray.shape[dimension]))
-
-
- def asem (inarray, dimension=None, keepdims=0):
- """
-Returns the standard error of the mean (i.e., using N) of the values
-in the passed array. Dimension can equal None (ravel array first), an
-integer (the dimension over which to operate), or a sequence (operate
-over multiple dimensions). Set keepdims=1 to return an array with the
-same number of dimensions as inarray.
-
-Usage: asem(inarray,dimension=None, keepdims=0)
-"""
- if dimension == None:
- inarray = N.ravel(inarray)
- dimension = 0
- if type(dimension) == ListType:
- n = 1
- for d in dimension:
- n = n*inarray.shape[d]
- else:
- n = inarray.shape[dimension]
- s = asamplestdev(inarray,dimension,keepdims) / N.sqrt(n-1)
- return s
-
-
- def az (a, score):
- """
-Returns the z-score of a given input score, given thearray from which
-that score came. Not appropriate for population calculations, nor for
-arrays > 1D.
-
-Usage: az(a, score)
-"""
- z = (score-amean(a)) / asamplestdev(a)
- return z
-
-
- def azs (a):
- """
-Returns a 1D array of z-scores, one for each score in the passed array,
-computed relative to the passed array.
-
-Usage: azs(a)
-"""
- zscores = []
- for item in a:
- zscores.append(z(a,item))
- return N.array(zscores)
-
-
- def azmap (scores, compare, dimension=0):
- """
-Returns an array of z-scores the shape of scores (e.g., [x,y]), compared to
-array passed to compare (e.g., [time,x,y]). Assumes collapsing over dim 0
-of the compare array.
-
-Usage: azs(scores, compare, dimension=0)
-"""
- mns = amean(compare,dimension)
- sstd = asamplestdev(compare,0)
- return (scores - mns) / sstd
-
-
-#####################################
-####### ATRIMMING FUNCTIONS #######
-#####################################
-
-## deleted around() as it's in numpy now
-
- def athreshold(a,threshmin=None,threshmax=None,newval=0):
- """
-Like Numeric.clip() except that values <threshmid or >threshmax are replaced
-by newval instead of by threshmin/threshmax (respectively).
-
-Usage: athreshold(a,threshmin=None,threshmax=None,newval=0)
-Returns: a, with values <threshmin or >threshmax replaced with newval
-"""
- mask = N.zeros(a.shape)
- if threshmin <> None:
- mask = mask + N.where(a<threshmin,1,0)
- if threshmax <> None:
- mask = mask + N.where(a>threshmax,1,0)
- mask = N.clip(mask,0,1)
- return N.where(mask,newval,a)
-
-
- def atrimboth (a,proportiontocut):
- """
-Slices off the passed proportion of items from BOTH ends of the passed
-array (i.e., with proportiontocut=0.1, slices 'leftmost' 10% AND
-'rightmost' 10% of scores. You must pre-sort the array if you want
-"proper" trimming. Slices off LESS if proportion results in a
-non-integer slice index (i.e., conservatively slices off
-proportiontocut).
-
-Usage: atrimboth (a,proportiontocut)
-Returns: trimmed version of array a
-"""
- lowercut = int(proportiontocut*len(a))
- uppercut = len(a) - lowercut
- return a[lowercut:uppercut]
-
-
- def atrim1 (a,proportiontocut,tail='right'):
- """
-Slices off the passed proportion of items from ONE end of the passed
-array (i.e., if proportiontocut=0.1, slices off 'leftmost' or 'rightmost'
-10% of scores). Slices off LESS if proportion results in a non-integer
-slice index (i.e., conservatively slices off proportiontocut).
-
-Usage: atrim1(a,proportiontocut,tail='right') or set tail='left'
-Returns: trimmed version of array a
-"""
- if string.lower(tail) == 'right':
- lowercut = 0
- uppercut = len(a) - int(proportiontocut*len(a))
- elif string.lower(tail) == 'left':
- lowercut = int(proportiontocut*len(a))
- uppercut = len(a)
- return a[lowercut:uppercut]
-
-
-#####################################
-##### ACORRELATION FUNCTIONS ######
-#####################################
-
- def acovariance(X):
- """
-Computes the covariance matrix of a matrix X. Requires a 2D matrix input.
-
-Usage: acovariance(X)
-Returns: covariance matrix of X
-"""
- if len(X.shape) <> 2:
- raise TypeError, "acovariance requires 2D matrices"
- n = X.shape[0]
- mX = amean(X,0)
- return N.dot(N.transpose(X),X) / float(n) - N.multiply.outer(mX,mX)
-
-
- def acorrelation(X):
- """
-Computes the correlation matrix of a matrix X. Requires a 2D matrix input.
-
-Usage: acorrelation(X)
-Returns: correlation matrix of X
-"""
- C = acovariance(X)
- V = N.diagonal(C)
- return C / N.sqrt(N.multiply.outer(V,V))
-
-
- def apaired(x,y):
- """
-Interactively determines the type of data in x and y, and then runs the
-appropriated statistic for paired group data.
-
-Usage: apaired(x,y) x,y = the two arrays of values to be compared
-Returns: appropriate statistic name, value, and probability
-"""
- samples = ''
- while samples not in ['i','r','I','R','c','C']:
- print '\nIndependent or related samples, or correlation (i,r,c): ',
- samples = raw_input()
-
- if samples in ['i','I','r','R']:
- print '\nComparing variances ...',
-# USE O'BRIEN'S TEST FOR HOMOGENEITY OF VARIANCE, Maxwell & delaney, p.112
- r = obrientransform(x,y)
- f,p = F_oneway(pstat.colex(r,0),pstat.colex(r,1))
- if p<0.05:
- vartype='unequal, p='+str(round(p,4))
- else:
- vartype='equal'
- print vartype
- if samples in ['i','I']:
- if vartype[0]=='e':
- t,p = ttest_ind(x,y,None,0)
- print '\nIndependent samples t-test: ', round(t,4),round(p,4)
- else:
- if len(x)>20 or len(y)>20:
- z,p = ranksums(x,y)
- print '\nRank Sums test (NONparametric, n>20): ', round(z,4),round(p,4)
- else:
- u,p = mannwhitneyu(x,y)
- print '\nMann-Whitney U-test (NONparametric, ns<20): ', round(u,4),round(p,4)
-
- else: # RELATED SAMPLES
- if vartype[0]=='e':
- t,p = ttest_rel(x,y,0)
- print '\nRelated samples t-test: ', round(t,4),round(p,4)
- else:
- t,p = ranksums(x,y)
- print '\nWilcoxon T-test (NONparametric): ', round(t,4),round(p,4)
- else: # CORRELATION ANALYSIS
- corrtype = ''
- while corrtype not in ['c','C','r','R','d','D']:
- print '\nIs the data Continuous, Ranked, or Dichotomous (c,r,d): ',
- corrtype = raw_input()
- if corrtype in ['c','C']:
- m,b,r,p,see = linregress(x,y)
- print '\nLinear regression for continuous variables ...'
- lol = [['Slope','Intercept','r','Prob','SEestimate'],[round(m,4),round(b,4),round(r,4),round(p,4),round(see,4)]]
- pstat.printcc(lol)
- elif corrtype in ['r','R']:
- r,p = spearmanr(x,y)
- print '\nCorrelation for ranked variables ...'
- print "Spearman's r: ",round(r,4),round(p,4)
- else: # DICHOTOMOUS
- r,p = pointbiserialr(x,y)
- print '\nAssuming x contains a dichotomous variable ...'
- print 'Point Biserial r: ',round(r,4),round(p,4)
- print '\n\n'
- return None
-
-
- def dices(x,y):
- """
-Calculates Dice's coefficient ... (2*number of common terms)/(number of terms in x +
-number of terms in y). Returns a value between 0 (orthogonal) and 1.
-
-Usage: dices(x,y)
-"""
- import sets
- x = sets.Set(x)
- y = sets.Set(y)
- common = len(x.intersection(y))
- total = float(len(x) + len(y))
- return 2*common/total
-
-
- def icc(x,y=None,verbose=0):
- """
-Calculates intraclass correlation coefficients using simple, Type I sums of squares.
-If only one variable is passed, assumed it's an Nx2 matrix
-
-Usage: icc(x,y=None,verbose=0)
-Returns: icc rho, prob ####PROB IS A GUESS BASED ON PEARSON
-"""
- TINY = 1.0e-20
- if y:
- all = N.concatenate([x,y],0)
- else:
- all = x+0
- x = all[:,0]
- y = all[:,1]
- totalss = ass(all-mean(all))
- pairmeans = (x+y)/2.
- withinss = ass(x-pairmeans) + ass(y-pairmeans)
- withindf = float(len(x))
- betwdf = float(len(x)-1)
- withinms = withinss / withindf
- betweenms = (totalss-withinss) / betwdf
- rho = (betweenms-withinms)/(withinms+betweenms)
- t = rho*math.sqrt(betwdf/((1.0-rho+TINY)*(1.0+rho+TINY)))
- prob = abetai(0.5*betwdf,0.5,betwdf/(betwdf+t*t),verbose)
- return rho, prob
-
-
- def alincc(x,y):
- """
-Calculates Lin's concordance correlation coefficient.
-
-Usage: alincc(x,y) where x, y are equal-length arrays
-Returns: Lin's CC
-"""
- x = N.ravel(x)
- y = N.ravel(y)
- covar = acov(x,y)*(len(x)-1)/float(len(x)) # correct denom to n
- xvar = avar(x)*(len(x)-1)/float(len(x)) # correct denom to n
- yvar = avar(y)*(len(y)-1)/float(len(y)) # correct denom to n
- lincc = (2 * covar) / ((xvar+yvar) +((amean(x)-amean(y))**2))
- return lincc
-
-
- def apearsonr(x,y,verbose=1):
- """
-Calculates a Pearson correlation coefficient and returns p. Taken
-from Heiman's Basic Statistics for the Behav. Sci (2nd), p.195.
-
-Usage: apearsonr(x,y,verbose=1) where x,y are equal length arrays
-Returns: Pearson's r, two-tailed p-value
-"""
- TINY = 1.0e-20
- n = len(x)
- xmean = amean(x)
- ymean = amean(y)
- r_num = n*(N.add.reduce(x*y)) - N.add.reduce(x)*N.add.reduce(y)
- r_den = math.sqrt((n*ass(x) - asquare_of_sums(x))*(n*ass(y)-asquare_of_sums(y)))
- r = (r_num / r_den)
- df = n-2
- t = r*math.sqrt(df/((1.0-r+TINY)*(1.0+r+TINY)))
- prob = abetai(0.5*df,0.5,df/(df+t*t),verbose)
- return r,prob
-
-
- def aspearmanr(x,y):
- """
-Calculates a Spearman rank-order correlation coefficient. Taken
-from Heiman's Basic Statistics for the Behav. Sci (1st), p.192.
-
-Usage: aspearmanr(x,y) where x,y are equal-length arrays
-Returns: Spearman's r, two-tailed p-value
-"""
- TINY = 1e-30
- n = len(x)
- rankx = rankdata(x)
- ranky = rankdata(y)
- dsq = N.add.reduce((rankx-ranky)**2)
- rs = 1 - 6*dsq / float(n*(n**2-1))
- t = rs * math.sqrt((n-2) / ((rs+1.0)*(1.0-rs)))
- df = n-2
- probrs = abetai(0.5*df,0.5,df/(df+t*t))
-# probability values for rs are from part 2 of the spearman function in
-# Numerical Recipies, p.510. They close to tables, but not exact.(?)
- return rs, probrs
-
-
- def apointbiserialr(x,y):
- """
-Calculates a point-biserial correlation coefficient and the associated
-probability value. Taken from Heiman's Basic Statistics for the Behav.
-Sci (1st), p.194.
-
-Usage: apointbiserialr(x,y) where x,y are equal length arrays
-Returns: Point-biserial r, two-tailed p-value
-"""
- TINY = 1e-30
- categories = pstat.aunique(x)
- data = pstat.aabut(x,y)
- if len(categories) <> 2:
- raise ValueError, "Exactly 2 categories required (in x) for pointbiserialr()."
- else: # there are 2 categories, continue
- codemap = pstat.aabut(categories,N.arange(2))
- recoded = pstat.arecode(data,codemap,0)
- x = pstat.alinexand(data,0,categories[0])
- y = pstat.alinexand(data,0,categories[1])
- xmean = amean(pstat.acolex(x,1))
- ymean = amean(pstat.acolex(y,1))
- n = len(data)
- adjust = math.sqrt((len(x)/float(n))*(len(y)/float(n)))
- rpb = (ymean - xmean)/asamplestdev(pstat.acolex(data,1))*adjust
- df = n-2
- t = rpb*math.sqrt(df/((1.0-rpb+TINY)*(1.0+rpb+TINY)))
- prob = abetai(0.5*df,0.5,df/(df+t*t))
- return rpb, prob
-
-
- def akendalltau(x,y):
- """
-Calculates Kendall's tau ... correlation of ordinal data. Adapted
-from function kendl1 in Numerical Recipies. Needs good test-cases.@@@
-
-Usage: akendalltau(x,y)
-Returns: Kendall's tau, two-tailed p-value
-"""
- n1 = 0
- n2 = 0
- iss = 0
- for j in range(len(x)-1):
- for k in range(j,len(y)):
- a1 = x[j] - x[k]
- a2 = y[j] - y[k]
- aa = a1 * a2
- if (aa): # neither array has a tie
- n1 = n1 + 1
- n2 = n2 + 1
- if aa > 0:
- iss = iss + 1
- else:
- iss = iss -1
- else:
- if (a1):
- n1 = n1 + 1
- else:
- n2 = n2 + 1
- tau = iss / math.sqrt(n1*n2)
- svar = (4.0*len(x)+10.0) / (9.0*len(x)*(len(x)-1))
- z = tau / math.sqrt(svar)
- prob = erfcc(abs(z)/1.4142136)
- return tau, prob
-
-
- def alinregress(*args):
- """
-Calculates a regression line on two arrays, x and y, corresponding to x,y
-pairs. If a single 2D array is passed, alinregress finds dim with 2 levels
-and splits data into x,y pairs along that dim.
-
-Usage: alinregress(*args) args=2 equal-length arrays, or one 2D array
-Returns: slope, intercept, r, two-tailed prob, sterr-of-the-estimate, n
-"""
- TINY = 1.0e-20
- if len(args) == 1: # more than 1D array?
- args = args[0]
- if len(args) == 2:
- x = args[0]
- y = args[1]
- else:
- x = args[:,0]
- y = args[:,1]
- else:
- x = args[0]
- y = args[1]
- n = len(x)
- xmean = amean(x)
- ymean = amean(y)
- r_num = n*(N.add.reduce(x*y)) - N.add.reduce(x)*N.add.reduce(y)
- r_den = math.sqrt((n*ass(x) - asquare_of_sums(x))*(n*ass(y)-asquare_of_sums(y)))
- r = r_num / r_den
- z = 0.5*math.log((1.0+r+TINY)/(1.0-r+TINY))
- df = n-2
- t = r*math.sqrt(df/((1.0-r+TINY)*(1.0+r+TINY)))
- prob = abetai(0.5*df,0.5,df/(df+t*t))
- slope = r_num / (float(n)*ass(x) - asquare_of_sums(x))
- intercept = ymean - slope*xmean
- sterrest = math.sqrt(1-r*r)*asamplestdev(y)
- return slope, intercept, r, prob, sterrest, n
-
- def amasslinregress(*args):
- """
-Calculates a regression line on one 1D array (x) and one N-D array (y).
-
-Returns: slope, intercept, r, two-tailed prob, sterr-of-the-estimate, n
-"""
- TINY = 1.0e-20
- if len(args) == 1: # more than 1D array?
- args = args[0]
- if len(args) == 2:
- x = N.ravel(args[0])
- y = args[1]
- else:
- x = N.ravel(args[:,0])
- y = args[:,1]
- else:
- x = args[0]
- y = args[1]
- x = x.astype(N.float_)
- y = y.astype(N.float_)
- n = len(x)
- xmean = amean(x)
- ymean = amean(y,0)
- shp = N.ones(len(y.shape))
- shp[0] = len(x)
- x.shape = shp
- print x.shape, y.shape
- r_num = n*(N.add.reduce(x*y,0)) - N.add.reduce(x)*N.add.reduce(y,0)
- r_den = N.sqrt((n*ass(x) - asquare_of_sums(x))*(n*ass(y,0)-asquare_of_sums(y,0)))
- zerodivproblem = N.equal(r_den,0)
- r_den = N.where(zerodivproblem,1,r_den) # avoid zero-division in 1st place
- r = r_num / r_den # need to do this nicely for matrix division
- r = N.where(zerodivproblem,0.0,r)
- z = 0.5*N.log((1.0+r+TINY)/(1.0-r+TINY))
- df = n-2
- t = r*N.sqrt(df/((1.0-r+TINY)*(1.0+r+TINY)))
- prob = abetai(0.5*df,0.5,df/(df+t*t))
-
- ss = float(n)*ass(x)-asquare_of_sums(x)
- s_den = N.where(ss==0,1,ss) # avoid zero-division in 1st place
- slope = r_num / s_den
- intercept = ymean - slope*xmean
- sterrest = N.sqrt(1-r*r)*asamplestdev(y,0)
- return slope, intercept, r, prob, sterrest, n
-
-
-#####################################
-##### AINFERENTIAL STATISTICS #####
-#####################################
-
- def attest_1samp(a,popmean,printit=0,name='Sample',writemode='a'):
- """
-Calculates the t-obtained for the independent samples T-test on ONE group
-of scores a, given a population mean. If printit=1, results are printed
-to the screen. If printit='filename', the results are output to 'filename'
-using the given writemode (default=append). Returns t-value, and prob.
-
-Usage: attest_1samp(a,popmean,Name='Sample',printit=0,writemode='a')
-Returns: t-value, two-tailed prob
-"""
- if type(a) != N.ndarray:
- a = N.array(a)
- x = amean(a)
- v = avar(a)
- n = len(a)
- df = n-1
- svar = ((n-1)*v) / float(df)
- t = (x-popmean)/math.sqrt(svar*(1.0/n))
- prob = abetai(0.5*df,0.5,df/(df+t*t))
-
- if printit <> 0:
- statname = 'Single-sample T-test.'
- outputpairedstats(printit,writemode,
- 'Population','--',popmean,0,0,0,
- name,n,x,v,N.minimum.reduce(N.ravel(a)),
- N.maximum.reduce(N.ravel(a)),
- statname,t,prob)
- return t,prob
-
-
- def attest_ind (a, b, dimension=None, printit=0, name1='Samp1', name2='Samp2',writemode='a'):
- """
-Calculates the t-obtained T-test on TWO INDEPENDENT samples of scores
-a, and b. From Numerical Recipies, p.483. If printit=1, results are
-printed to the screen. If printit='filename', the results are output
-to 'filename' using the given writemode (default=append). Dimension
-can equal None (ravel array first), or an integer (the dimension over
-which to operate on a and b).
-
-Usage: attest_ind (a,b,dimension=None,printit=0,
- Name1='Samp1',Name2='Samp2',writemode='a')
-Returns: t-value, two-tailed p-value
-"""
- if dimension == None:
- a = N.ravel(a)
- b = N.ravel(b)
- dimension = 0
- x1 = amean(a,dimension)
- x2 = amean(b,dimension)
- v1 = avar(a,dimension)
- v2 = avar(b,dimension)
- n1 = a.shape[dimension]
- n2 = b.shape[dimension]
- df = n1+n2-2
- svar = ((n1-1)*v1+(n2-1)*v2) / float(df)
- zerodivproblem = N.equal(svar,0)
- svar = N.where(zerodivproblem,1,svar) # avoid zero-division in 1st place
- t = (x1-x2)/N.sqrt(svar*(1.0/n1 + 1.0/n2)) # N-D COMPUTATION HERE!!!!!!
- t = N.where(zerodivproblem,1.0,t) # replace NaN/wrong t-values with 1.0
- probs = abetai(0.5*df,0.5,float(df)/(df+t*t))
-
- if type(t) == N.ndarray:
- probs = N.reshape(probs,t.shape)
- if probs.shape == (1,):
- probs = probs[0]
-
- if printit <> 0:
- if type(t) == N.ndarray:
- t = t[0]
- if type(probs) == N.ndarray:
- probs = probs[0]
- statname = 'Independent samples T-test.'
- outputpairedstats(printit,writemode,
- name1,n1,x1,v1,N.minimum.reduce(N.ravel(a)),
- N.maximum.reduce(N.ravel(a)),
- name2,n2,x2,v2,N.minimum.reduce(N.ravel(b)),
- N.maximum.reduce(N.ravel(b)),
- statname,t,probs)
- return
- return t, probs
-
- def ap2t(pval,df):
- """
-Tries to compute a t-value from a p-value (or pval array) and associated df.
-SLOW for large numbers of elements(!) as it re-computes p-values 20 times
-(smaller step-sizes) at which point it decides it's done. Keeps the signs
-of the input array. Returns 1000 (or -1000) if t>100.
-
-Usage: ap2t(pval,df)
-Returns: an array of t-values with the shape of pval
- """
- pval = N.array(pval)
- signs = N.sign(pval)
- pval = abs(pval)
- t = N.ones(pval.shape,N.float_)*50
- step = N.ones(pval.shape,N.float_)*25
- print "Initial ap2t() prob calc"
- prob = abetai(0.5*df,0.5,float(df)/(df+t*t))
- print 'ap2t() iter: ',
- for i in range(10):
- print i,' ',
- t = N.where(pval<prob,t+step,t-step)
- prob = abetai(0.5*df,0.5,float(df)/(df+t*t))
- step = step/2
- print
- # since this is an ugly hack, we get ugly boundaries
- t = N.where(t>99.9,1000,t) # hit upper-boundary
- t = t+signs
- return t #, prob, pval
-
-
- def attest_rel (a,b,dimension=None,printit=0,name1='Samp1',name2='Samp2',writemode='a'):
- """
-Calculates the t-obtained T-test on TWO RELATED samples of scores, a
-and b. From Numerical Recipies, p.483. If printit=1, results are
-printed to the screen. If printit='filename', the results are output
-to 'filename' using the given writemode (default=append). Dimension
-can equal None (ravel array first), or an integer (the dimension over
-which to operate on a and b).
-
-Usage: attest_rel(a,b,dimension=None,printit=0,
- name1='Samp1',name2='Samp2',writemode='a')
-Returns: t-value, two-tailed p-value
-"""
- if dimension == None:
- a = N.ravel(a)
- b = N.ravel(b)
- dimension = 0
- if len(a)<>len(b):
- raise ValueError, 'Unequal length arrays.'
- x1 = amean(a,dimension)
- x2 = amean(b,dimension)
- v1 = avar(a,dimension)
- v2 = avar(b,dimension)
- n = a.shape[dimension]
- df = float(n-1)
- d = (a-b).astype('d')
-
- denom = N.sqrt((n*N.add.reduce(d*d,dimension) - N.add.reduce(d,dimension)**2) /df)
- zerodivproblem = N.equal(denom,0)
- denom = N.where(zerodivproblem,1,denom) # avoid zero-division in 1st place
- t = N.add.reduce(d,dimension) / denom # N-D COMPUTATION HERE!!!!!!
- t = N.where(zerodivproblem,1.0,t) # replace NaN/wrong t-values with 1.0
- probs = abetai(0.5*df,0.5,float(df)/(df+t*t))
- if type(t) == N.ndarray:
- probs = N.reshape(probs,t.shape)
- if probs.shape == (1,):
- probs = probs[0]
-
- if printit <> 0:
- statname = 'Related samples T-test.'
- outputpairedstats(printit,writemode,
- name1,n,x1,v1,N.minimum.reduce(N.ravel(a)),
- N.maximum.reduce(N.ravel(a)),
- name2,n,x2,v2,N.minimum.reduce(N.ravel(b)),
- N.maximum.reduce(N.ravel(b)),
- statname,t,probs)
- return
- return t, probs
-
-
- def achisquare(f_obs,f_exp=None):
- """
-Calculates a one-way chi square for array of observed frequencies and returns
-the result. If no expected frequencies are given, the total N is assumed to
-be equally distributed across all groups.
-@@@NOT RIGHT??
-
-Usage: achisquare(f_obs, f_exp=None) f_obs = array of observed cell freq.
-Returns: chisquare-statistic, associated p-value
-"""
-
- k = len(f_obs)
- if f_exp == None:
- f_exp = N.array([sum(f_obs)/float(k)] * len(f_obs),N.float_)
- f_exp = f_exp.astype(N.float_)
- chisq = N.add.reduce((f_obs-f_exp)**2 / f_exp)
- return chisq, achisqprob(chisq, k-1)
-
-
- def aks_2samp (data1,data2):
- """
-Computes the Kolmogorov-Smirnof statistic on 2 samples. Modified from
-Numerical Recipies in C, page 493. Returns KS D-value, prob. Not ufunc-
-like.
-
-Usage: aks_2samp(data1,data2) where data1 and data2 are 1D arrays
-Returns: KS D-value, p-value
-"""
- j1 = 0 # N.zeros(data1.shape[1:]) TRIED TO MAKE THIS UFUNC-LIKE
- j2 = 0 # N.zeros(data2.shape[1:])
- fn1 = 0.0 # N.zeros(data1.shape[1:],N.float_)
- fn2 = 0.0 # N.zeros(data2.shape[1:],N.float_)
- n1 = data1.shape[0]
- n2 = data2.shape[0]
- en1 = n1*1
- en2 = n2*1
- d = N.zeros(data1.shape[1:],N.float_)
- data1 = N.sort(data1,0)
- data2 = N.sort(data2,0)
- while j1 < n1 and j2 < n2:
- d1=data1[j1]
- d2=data2[j2]
- if d1 <= d2:
- fn1 = (j1)/float(en1)
- j1 = j1 + 1
- if d2 <= d1:
- fn2 = (j2)/float(en2)
- j2 = j2 + 1
- dt = (fn2-fn1)
- if abs(dt) > abs(d):
- d = dt
-# try:
- en = math.sqrt(en1*en2/float(en1+en2))
- prob = aksprob((en+0.12+0.11/en)*N.fabs(d))
-# except:
-# prob = 1.0
- return d, prob
-
-
- def amannwhitneyu(x,y):
- """
-Calculates a Mann-Whitney U statistic on the provided scores and
-returns the result. Use only when the n in each condition is < 20 and
-you have 2 independent samples of ranks. REMEMBER: Mann-Whitney U is
-significant if the u-obtained is LESS THAN or equal to the critical
-value of U.
-
-Usage: amannwhitneyu(x,y) where x,y are arrays of values for 2 conditions
-Returns: u-statistic, one-tailed p-value (i.e., p(z(U)))
-"""
- n1 = len(x)
- n2 = len(y)
- ranked = rankdata(N.concatenate((x,y)))
- rankx = ranked[0:n1] # get the x-ranks
- ranky = ranked[n1:] # the rest are y-ranks
- u1 = n1*n2 + (n1*(n1+1))/2.0 - sum(rankx) # calc U for x
- u2 = n1*n2 - u1 # remainder is U for y
- bigu = max(u1,u2)
- smallu = min(u1,u2)
- T = math.sqrt(tiecorrect(ranked)) # correction factor for tied scores
- if T == 0:
- raise ValueError, 'All numbers are identical in amannwhitneyu'
- sd = math.sqrt(T*n1*n2*(n1+n2+1)/12.0)
- z = abs((bigu-n1*n2/2.0) / sd) # normal approximation for prob calc
- return smallu, 1.0 - azprob(z)
-
-
- def atiecorrect(rankvals):
- """
-Tie-corrector for ties in Mann Whitney U and Kruskal Wallis H tests.
-See Siegel, S. (1956) Nonparametric Statistics for the Behavioral
-Sciences. New York: McGraw-Hill. Code adapted from |Stat rankind.c
-code.
-
-Usage: atiecorrect(rankvals)
-Returns: T correction factor for U or H
-"""
- sorted,posn = ashellsort(N.array(rankvals))
- n = len(sorted)
- T = 0.0
- i = 0
- while (i<n-1):
- if sorted[i] == sorted[i+1]:
- nties = 1
- while (i<n-1) and (sorted[i] == sorted[i+1]):
- nties = nties +1
- i = i +1
- T = T + nties**3 - nties
- i = i+1
- T = T / float(n**3-n)
- return 1.0 - T
-
-
- def aranksums(x,y):
- """
-Calculates the rank sums statistic on the provided scores and returns
-the result.
-
-Usage: aranksums(x,y) where x,y are arrays of values for 2 conditions
-Returns: z-statistic, two-tailed p-value
-"""
- n1 = len(x)
- n2 = len(y)
- alldata = N.concatenate((x,y))
- ranked = arankdata(alldata)
- x = ranked[:n1]
- y = ranked[n1:]
- s = sum(x)
- expected = n1*(n1+n2+1) / 2.0
- z = (s - expected) / math.sqrt(n1*n2*(n1+n2+1)/12.0)
- prob = 2*(1.0 - azprob(abs(z)))
- return z, prob
-
-
- def awilcoxont(x,y):
- """
-Calculates the Wilcoxon T-test for related samples and returns the
-result. A non-parametric T-test.
-
-Usage: awilcoxont(x,y) where x,y are equal-length arrays for 2 conditions
-Returns: t-statistic, two-tailed p-value
-"""
- if len(x) <> len(y):
- raise ValueError, 'Unequal N in awilcoxont. Aborting.'
- d = x-y
- d = N.compress(N.not_equal(d,0),d) # Keep all non-zero differences
- count = len(d)
- absd = abs(d)
- absranked = arankdata(absd)
- r_plus = 0.0
- r_minus = 0.0
- for i in range(len(absd)):
- if d[i] < 0:
- r_minus = r_minus + absranked[i]
- else:
- r_plus = r_plus + absranked[i]
- wt = min(r_plus, r_minus)
- mn = count * (count+1) * 0.25
- se = math.sqrt(count*(count+1)*(2.0*count+1.0)/24.0)
- z = math.fabs(wt-mn) / se
- z = math.fabs(wt-mn) / se
- prob = 2*(1.0 -zprob(abs(z)))
- return wt, prob
-
-
- def akruskalwallish(*args):
- """
-The Kruskal-Wallis H-test is a non-parametric ANOVA for 3 or more
-groups, requiring at least 5 subjects in each group. This function
-calculates the Kruskal-Wallis H and associated p-value for 3 or more
-independent samples.
-
-Usage: akruskalwallish(*args) args are separate arrays for 3+ conditions
-Returns: H-statistic (corrected for ties), associated p-value
-"""
- assert len(args) == 3, "Need at least 3 groups in stats.akruskalwallish()"
- args = list(args)
- n = [0]*len(args)
- n = map(len,args)
- all = []
- for i in range(len(args)):
- all = all + args[i].tolist()
- ranked = rankdata(all)
- T = tiecorrect(ranked)
- for i in range(len(args)):
- args[i] = ranked[0:n[i]]
- del ranked[0:n[i]]
- rsums = []
- for i in range(len(args)):
- rsums.append(sum(args[i])**2)
- rsums[i] = rsums[i] / float(n[i])
- ssbn = sum(rsums)
- totaln = sum(n)
- h = 12.0 / (totaln*(totaln+1)) * ssbn - 3*(totaln+1)
- df = len(args) - 1
- if T == 0:
- raise ValueError, 'All numbers are identical in akruskalwallish'
- h = h / float(T)
- return h, chisqprob(h,df)
-
-
- def afriedmanchisquare(*args):
- """
-Friedman Chi-Square is a non-parametric, one-way within-subjects
-ANOVA. This function calculates the Friedman Chi-square test for
-repeated measures and returns the result, along with the associated
-probability value. It assumes 3 or more repeated measures. Only 3
-levels requires a minimum of 10 subjects in the study. Four levels
-requires 5 subjects per level(??).
-
-Usage: afriedmanchisquare(*args) args are separate arrays for 2+ conditions
-Returns: chi-square statistic, associated p-value
-"""
- k = len(args)
- if k < 3:
- raise ValueError, '\nLess than 3 levels. Friedman test not appropriate.\n'
- n = len(args[0])
- data = apply(pstat.aabut,args)
- data = data.astype(N.float_)
- for i in range(len(data)):
- data[i] = arankdata(data[i])
- ssbn = asum(asum(args,1)**2)
- chisq = 12.0 / (k*n*(k+1)) * ssbn - 3*n*(k+1)
- return chisq, achisqprob(chisq,k-1)
-
-
-#####################################
-#### APROBABILITY CALCULATIONS ####
-#####################################
-
- def achisqprob(chisq,df):
- """
-Returns the (1-tail) probability value associated with the provided chi-square
-value and df. Heavily modified from chisq.c in Gary Perlman's |Stat. Can
-handle multiple dimensions.
-
-Usage: achisqprob(chisq,df) chisq=chisquare stat., df=degrees of freedom
-"""
- BIG = 200.0
- def ex(x):
- BIG = 200.0
- exponents = N.where(N.less(x,-BIG),-BIG,x)
- return N.exp(exponents)
-
- if type(chisq) == N.ndarray:
- arrayflag = 1
- else:
- arrayflag = 0
- chisq = N.array([chisq])
- if df < 1:
- return N.ones(chisq.shape,N.float)
- probs = N.zeros(chisq.shape,N.float_)
- probs = N.where(N.less_equal(chisq,0),1.0,probs) # set prob=1 for chisq<0
- a = 0.5 * chisq
- if df > 1:
- y = ex(-a)
- if df%2 == 0:
- even = 1
- s = y*1
- s2 = s*1
- else:
- even = 0
- s = 2.0 * azprob(-N.sqrt(chisq))
- s2 = s*1
- if (df > 2):
- chisq = 0.5 * (df - 1.0)
- if even:
- z = N.ones(probs.shape,N.float_)
- else:
- z = 0.5 *N.ones(probs.shape,N.float_)
- if even:
- e = N.zeros(probs.shape,N.float_)
- else:
- e = N.log(N.sqrt(N.pi)) *N.ones(probs.shape,N.float_)
- c = N.log(a)
- mask = N.zeros(probs.shape)
- a_big = N.greater(a,BIG)
- a_big_frozen = -1 *N.ones(probs.shape,N.float_)
- totalelements = N.multiply.reduce(N.array(probs.shape))
- while asum(mask)<>totalelements:
- e = N.log(z) + e
- s = s + ex(c*z-a-e)
- z = z + 1.0
-# print z, e, s
- newmask = N.greater(z,chisq)
- a_big_frozen = N.where(newmask*N.equal(mask,0)*a_big, s, a_big_frozen)
- mask = N.clip(newmask+mask,0,1)
- if even:
- z = N.ones(probs.shape,N.float_)
- e = N.ones(probs.shape,N.float_)
- else:
- z = 0.5 *N.ones(probs.shape,N.float_)
- e = 1.0 / N.sqrt(N.pi) / N.sqrt(a) * N.ones(probs.shape,N.float_)
- c = 0.0
- mask = N.zeros(probs.shape)
- a_notbig_frozen = -1 *N.ones(probs.shape,N.float_)
- while asum(mask)<>totalelements:
- e = e * (a/z.astype(N.float_))
- c = c + e
- z = z + 1.0
-# print '#2', z, e, c, s, c*y+s2
- newmask = N.greater(z,chisq)
- a_notbig_frozen = N.where(newmask*N.equal(mask,0)*(1-a_big),
- c*y+s2, a_notbig_frozen)
- mask = N.clip(newmask+mask,0,1)
- probs = N.where(N.equal(probs,1),1,
- N.where(N.greater(a,BIG),a_big_frozen,a_notbig_frozen))
- return probs
- else:
- return s
-
-
- def aerfcc(x):
- """
-Returns the complementary error function erfc(x) with fractional error
-everywhere less than 1.2e-7. Adapted from Numerical Recipies. Can
-handle multiple dimensions.
-
-Usage: aerfcc(x)
-"""
- z = abs(x)
- t = 1.0 / (1.0+0.5*z)
- ans = t * N.exp(-z*z-1.26551223 + t*(1.00002368+t*(0.37409196+t*(0.09678418+t*(-0.18628806+t*(0.27886807+t*(-1.13520398+t*(1.48851587+t*(-0.82215223+t*0.17087277)))))))))
- return N.where(N.greater_equal(x,0), ans, 2.0-ans)
-
-
- def azprob(z):
- """
-Returns the area under the normal curve 'to the left of' the given z value.
-Thus,
- for z<0, zprob(z) = 1-tail probability
- for z>0, 1.0-zprob(z) = 1-tail probability
- for any z, 2.0*(1.0-zprob(abs(z))) = 2-tail probability
-Adapted from z.c in Gary Perlman's |Stat. Can handle multiple dimensions.
-
-Usage: azprob(z) where z is a z-value
-"""
- def yfunc(y):
- x = (((((((((((((-0.000045255659 * y
- +0.000152529290) * y -0.000019538132) * y
- -0.000676904986) * y +0.001390604284) * y
- -0.000794620820) * y -0.002034254874) * y
- +0.006549791214) * y -0.010557625006) * y
- +0.011630447319) * y -0.009279453341) * y
- +0.005353579108) * y -0.002141268741) * y
- +0.000535310849) * y +0.999936657524
- return x
-
- def wfunc(w):
- x = ((((((((0.000124818987 * w
- -0.001075204047) * w +0.005198775019) * w
- -0.019198292004) * w +0.059054035642) * w
- -0.151968751364) * w +0.319152932694) * w
- -0.531923007300) * w +0.797884560593) * N.sqrt(w) * 2.0
- return x
-
- Z_MAX = 6.0 # maximum meaningful z-value
- x = N.zeros(z.shape,N.float_) # initialize
- y = 0.5 * N.fabs(z)
- x = N.where(N.less(y,1.0),wfunc(y*y),yfunc(y-2.0)) # get x's
- x = N.where(N.greater(y,Z_MAX*0.5),1.0,x) # kill those with big Z
- prob = N.where(N.greater(z,0),(x+1)*0.5,(1-x)*0.5)
- return prob
-
-
- def aksprob(alam):
- """
-Returns the probability value for a K-S statistic computed via ks_2samp.
-Adapted from Numerical Recipies. Can handle multiple dimensions.
-
-Usage: aksprob(alam)
-"""
- if type(alam) == N.ndarray:
- frozen = -1 *N.ones(alam.shape,N.float64)
- alam = alam.astype(N.float64)
- arrayflag = 1
- else:
- frozen = N.array(-1.)
- alam = N.array(alam,N.float64)
- arrayflag = 1
- mask = N.zeros(alam.shape)
- fac = 2.0 *N.ones(alam.shape,N.float_)
- sum = N.zeros(alam.shape,N.float_)
- termbf = N.zeros(alam.shape,N.float_)
- a2 = N.array(-2.0*alam*alam,N.float64)
- totalelements = N.multiply.reduce(N.array(mask.shape))
- for j in range(1,201):
- if asum(mask) == totalelements:
- break
- exponents = (a2*j*j)
- overflowmask = N.less(exponents,-746)
- frozen = N.where(overflowmask,0,frozen)
- mask = mask+overflowmask
- term = fac*N.exp(exponents)
- sum = sum + term
- newmask = N.where(N.less_equal(abs(term),(0.001*termbf)) +
- N.less(abs(term),1.0e-8*sum), 1, 0)
- frozen = N.where(newmask*N.equal(mask,0), sum, frozen)
- mask = N.clip(mask+newmask,0,1)
- fac = -fac
- termbf = abs(term)
- if arrayflag:
- return N.where(N.equal(frozen,-1), 1.0, frozen) # 1.0 if doesn't converge
- else:
- return N.where(N.equal(frozen,-1), 1.0, frozen)[0] # 1.0 if doesn't converge
-
-
- def afprob (dfnum, dfden, F):
- """
-Returns the 1-tailed significance level (p-value) of an F statistic
-given the degrees of freedom for the numerator (dfR-dfF) and the degrees
-of freedom for the denominator (dfF). Can handle multiple dims for F.
-
-Usage: afprob(dfnum, dfden, F) where usually dfnum=dfbn, dfden=dfwn
-"""
- if type(F) == N.ndarray:
- return abetai(0.5*dfden, 0.5*dfnum, dfden/(1.0*dfden+dfnum*F))
- else:
- return abetai(0.5*dfden, 0.5*dfnum, dfden/float(dfden+dfnum*F))
-
-
- def abetacf(a,b,x,verbose=1):
- """
-Evaluates the continued fraction form of the incomplete Beta function,
-betai. (Adapted from: Numerical Recipies in C.) Can handle multiple
-dimensions for x.
-
-Usage: abetacf(a,b,x,verbose=1)
-"""
- ITMAX = 200
- EPS = 3.0e-7
-
- arrayflag = 1
- if type(x) == N.ndarray:
- frozen = N.ones(x.shape,N.float_) *-1 #start out w/ -1s, should replace all
- else:
- arrayflag = 0
- frozen = N.array([-1])
- x = N.array([x])
- mask = N.zeros(x.shape)
- bm = az = am = 1.0
- qab = a+b
- qap = a+1.0
- qam = a-1.0
- bz = 1.0-qab*x/qap
- for i in range(ITMAX+1):
- if N.sum(N.ravel(N.equal(frozen,-1)))==0:
- break
- em = float(i+1)
- tem = em + em
- d = em*(b-em)*x/((qam+tem)*(a+tem))
- ap = az + d*am
- bp = bz+d*bm
- d = -(a+em)*(qab+em)*x/((qap+tem)*(a+tem))
- app = ap+d*az
- bpp = bp+d*bz
- aold = az*1
- am = ap/bpp
- bm = bp/bpp
- az = app/bpp
- bz = 1.0
- newmask = N.less(abs(az-aold),EPS*abs(az))
- frozen = N.where(newmask*N.equal(mask,0), az, frozen)
- mask = N.clip(mask+newmask,0,1)
- noconverge = asum(N.equal(frozen,-1))
- if noconverge <> 0 and verbose:
- print 'a or b too big, or ITMAX too small in Betacf for ',noconverge,' elements'
- if arrayflag:
- return frozen
- else:
- return frozen[0]
-
-
- def agammln(xx):
- """
-Returns the gamma function of xx.
- Gamma(z) = Integral(0,infinity) of t^(z-1)exp(-t) dt.
-Adapted from: Numerical Recipies in C. Can handle multiple dims ... but
-probably doesn't normally have to.
-
-Usage: agammln(xx)
-"""
- coeff = [76.18009173, -86.50532033, 24.01409822, -1.231739516,
- 0.120858003e-2, -0.536382e-5]
- x = xx - 1.0
- tmp = x + 5.5
- tmp = tmp - (x+0.5)*N.log(tmp)
- ser = 1.0
- for j in range(len(coeff)):
- x = x + 1
- ser = ser + coeff[j]/x
- return -tmp + N.log(2.50662827465*ser)
-
-
- def abetai(a,b,x,verbose=1):
- """
-Returns the incomplete beta function:
-
- I-sub-x(a,b) = 1/B(a,b)*(Integral(0,x) of t^(a-1)(1-t)^(b-1) dt)
-
-where a,b>0 and B(a,b) = G(a)*G(b)/(G(a+b)) where G(a) is the gamma
-function of a. The continued fraction formulation is implemented
-here, using the betacf function. (Adapted from: Numerical Recipies in
-C.) Can handle multiple dimensions.
-
-Usage: abetai(a,b,x,verbose=1)
-"""
- TINY = 1e-15
- if type(a) == N.ndarray:
- if asum(N.less(x,0)+N.greater(x,1)) <> 0:
- raise ValueError, 'Bad x in abetai'
- x = N.where(N.equal(x,0),TINY,x)
- x = N.where(N.equal(x,1.0),1-TINY,x)
-
- bt = N.where(N.equal(x,0)+N.equal(x,1), 0, -1)
- exponents = ( gammln(a+b)-gammln(a)-gammln(b)+a*N.log(x)+b*
- N.log(1.0-x) )
- # 746 (below) is the MAX POSSIBLE BEFORE OVERFLOW
- exponents = N.where(N.less(exponents,-740),-740,exponents)
- bt = N.exp(exponents)
- if type(x) == N.ndarray:
- ans = N.where(N.less(x,(a+1)/(a+b+2.0)),
- bt*abetacf(a,b,x,verbose)/float(a),
- 1.0-bt*abetacf(b,a,1.0-x,verbose)/float(b))
- else:
- if x<(a+1)/(a+b+2.0):
- ans = bt*abetacf(a,b,x,verbose)/float(a)
- else:
- ans = 1.0-bt*abetacf(b,a,1.0-x,verbose)/float(b)
- return ans
-
-
-#####################################
-####### AANOVA CALCULATIONS #######
-#####################################
-
- import LinearAlgebra, operator
- LA = LinearAlgebra
-
- def aglm(data,para):
- """
-Calculates a linear model fit ... anova/ancova/lin-regress/t-test/etc. Taken
-from:
- Peterson et al. Statistical limitations in functional neuroimaging
- I. Non-inferential methods and statistical models. Phil Trans Royal Soc
- Lond B 354: 1239-1260.
-
-Usage: aglm(data,para)
-Returns: statistic, p-value ???
-"""
- if len(para) <> len(data):
- print "data and para must be same length in aglm"
- return
- n = len(para)
- p = pstat.aunique(para)
- x = N.zeros((n,len(p))) # design matrix
- for l in range(len(p)):
- x[:,l] = N.equal(para,p[l])
- b = N.dot(N.dot(LA.inv(N.dot(N.transpose(x),x)), # i.e., b=inv(X'X)X'Y
- N.transpose(x)),
- data)
- diffs = (data - N.dot(x,b))
- s_sq = 1./(n-len(p)) * N.dot(N.transpose(diffs), diffs)
-
- if len(p) == 2: # ttest_ind
- c = N.array([1,-1])
- df = n-2
- fact = asum(1.0/asum(x,0)) # i.e., 1/n1 + 1/n2 + 1/n3 ...
- t = N.dot(c,b) / N.sqrt(s_sq*fact)
- probs = abetai(0.5*df,0.5,float(df)/(df+t*t))
- return t, probs
-
-
- def aF_oneway(*args):
- """
-Performs a 1-way ANOVA, returning an F-value and probability given
-any number of groups. From Heiman, pp.394-7.
-
-Usage: aF_oneway (*args) where *args is 2 or more arrays, one per
- treatment group
-Returns: f-value, probability
-"""
- na = len(args) # ANOVA on 'na' groups, each in it's own array
- means = [0]*na
- vars = [0]*na
- ns = [0]*na
- alldata = []
- tmp = map(N.array,args)
- means = map(amean,tmp)
- vars = map(avar,tmp)
- ns = map(len,args)
- alldata = N.concatenate(args)
- bign = len(alldata)
- sstot = ass(alldata)-(asquare_of_sums(alldata)/float(bign))
- ssbn = 0
- for a in args:
- ssbn = ssbn + asquare_of_sums(N.array(a))/float(len(a))
- ssbn = ssbn - (asquare_of_sums(alldata)/float(bign))
- sswn = sstot-ssbn
- dfbn = na-1
- dfwn = bign - na
- msb = ssbn/float(dfbn)
- msw = sswn/float(dfwn)
- f = msb/msw
- prob = fprob(dfbn,dfwn,f)
- return f, prob
-
-
- def aF_value (ER,EF,dfR,dfF):
- """
-Returns an F-statistic given the following:
- ER = error associated with the null hypothesis (the Restricted model)
- EF = error associated with the alternate hypothesis (the Full model)
- dfR = degrees of freedom the Restricted model
- dfF = degrees of freedom associated with the Restricted model
-"""
- return ((ER-EF)/float(dfR-dfF) / (EF/float(dfF)))
-
-
- def outputfstats(Enum, Eden, dfnum, dfden, f, prob):
- Enum = round(Enum,3)
- Eden = round(Eden,3)
- dfnum = round(Enum,3)
- dfden = round(dfden,3)
- f = round(f,3)
- prob = round(prob,3)
- suffix = '' # for *s after the p-value
- if prob < 0.001: suffix = ' ***'
- elif prob < 0.01: suffix = ' **'
- elif prob < 0.05: suffix = ' *'
- title = [['EF/ER','DF','Mean Square','F-value','prob','']]
- lofl = title+[[Enum, dfnum, round(Enum/float(dfnum),3), f, prob, suffix],
- [Eden, dfden, round(Eden/float(dfden),3),'','','']]
- pstat.printcc(lofl)
- return
-
-
- def F_value_multivariate(ER, EF, dfnum, dfden):
- """
-Returns an F-statistic given the following:
- ER = error associated with the null hypothesis (the Restricted model)
- EF = error associated with the alternate hypothesis (the Full model)
- dfR = degrees of freedom the Restricted model
- dfF = degrees of freedom associated with the Restricted model
-where ER and EF are matrices from a multivariate F calculation.
-"""
- if type(ER) in [IntType, FloatType]:
- ER = N.array([[ER]])
- if type(EF) in [IntType, FloatType]:
- EF = N.array([[EF]])
- n_um = (LA.det(ER) - LA.det(EF)) / float(dfnum)
- d_en = LA.det(EF) / float(dfden)
- return n_um / d_en
-
-
-#####################################
-####### ASUPPORT FUNCTIONS ########
-#####################################
-
- def asign(a):
- """
-Usage: asign(a)
-Returns: array shape of a, with -1 where a<0 and +1 where a>=0
-"""
- a = N.asarray(a)
- if ((type(a) == type(1.4)) or (type(a) == type(1))):
- return a-a-N.less(a,0)+N.greater(a,0)
- else:
- return N.zeros(N.shape(a))-N.less(a,0)+N.greater(a,0)
-
-
- def asum (a, dimension=None,keepdims=0):
- """
-An alternative to the Numeric.add.reduce function, which allows one to
-(1) collapse over multiple dimensions at once, and/or (2) to retain
-all dimensions in the original array (squashing one down to size.
-Dimension can equal None (ravel array first), an integer (the
-dimension over which to operate), or a sequence (operate over multiple
-dimensions). If keepdims=1, the resulting array will have as many
-dimensions as the input array.
-
-Usage: asum(a, dimension=None, keepdims=0)
-Returns: array summed along 'dimension'(s), same _number_ of dims if keepdims=1
-"""
- if type(a) == N.ndarray and a.dtype in [N.int_, N.short, N.ubyte]:
- a = a.astype(N.float_)
- if dimension == None:
- s = N.sum(N.ravel(a))
- elif type(dimension) in [IntType,FloatType]:
- s = N.add.reduce(a, dimension)
- if keepdims == 1:
- shp = list(a.shape)
- shp[dimension] = 1
- s = N.reshape(s,shp)
- else: # must be a SEQUENCE of dims to sum over
- dims = list(dimension)
- dims.sort()
- dims.reverse()
- s = a *1.0
- for dim in dims:
- s = N.add.reduce(s,dim)
- if keepdims == 1:
- shp = list(a.shape)
- for dim in dims:
- shp[dim] = 1
- s = N.reshape(s,shp)
- return s
-
-
- def acumsum (a,dimension=None):
- """
-Returns an array consisting of the cumulative sum of the items in the
-passed array. Dimension can equal None (ravel array first), an
-integer (the dimension over which to operate), or a sequence (operate
-over multiple dimensions, but this last one just barely makes sense).
-
-Usage: acumsum(a,dimension=None)
-"""
- if dimension == None:
- a = N.ravel(a)
- dimension = 0
- if type(dimension) in [ListType, TupleType, N.ndarray]:
- dimension = list(dimension)
- dimension.sort()
- dimension.reverse()
- for d in dimension:
- a = N.add.accumulate(a,d)
- return a
- else:
- return N.add.accumulate(a,dimension)
-
-
- def ass(inarray, dimension=None, keepdims=0):
- """
-Squares each value in the passed array, adds these squares & returns
-the result. Unfortunate function name. :-) Defaults to ALL values in
-the array. Dimension can equal None (ravel array first), an integer
-(the dimension over which to operate), or a sequence (operate over
-multiple dimensions). Set keepdims=1 to maintain the original number
-of dimensions.
-
-Usage: ass(inarray, dimension=None, keepdims=0)
-Returns: sum-along-'dimension' for (inarray*inarray)
-"""
- if dimension == None:
- inarray = N.ravel(inarray)
- dimension = 0
- return asum(inarray*inarray,dimension,keepdims)
-
-
- def asummult (array1,array2,dimension=None,keepdims=0):
- """
-Multiplies elements in array1 and array2, element by element, and
-returns the sum (along 'dimension') of all resulting multiplications.
-Dimension can equal None (ravel array first), an integer (the
-dimension over which to operate), or a sequence (operate over multiple
-dimensions). A trivial function, but included for completeness.
-
-Usage: asummult(array1,array2,dimension=None,keepdims=0)
-"""
- if dimension == None:
- array1 = N.ravel(array1)
- array2 = N.ravel(array2)
- dimension = 0
- return asum(array1*array2,dimension,keepdims)
-
-
- def asquare_of_sums(inarray, dimension=None, keepdims=0):
- """
-Adds the values in the passed array, squares that sum, and returns the
-result. Dimension can equal None (ravel array first), an integer (the
-dimension over which to operate), or a sequence (operate over multiple
-dimensions). If keepdims=1, the returned array will have the same
-NUMBER of dimensions as the original.
-
-Usage: asquare_of_sums(inarray, dimension=None, keepdims=0)
-Returns: the square of the sum over dim(s) in dimension
-"""
- if dimension == None:
- inarray = N.ravel(inarray)
- dimension = 0
- s = asum(inarray,dimension,keepdims)
- if type(s) == N.ndarray:
- return s.astype(N.float_)*s
- else:
- return float(s)*s
-
-
- def asumdiffsquared(a,b, dimension=None, keepdims=0):
- """
-Takes pairwise differences of the values in arrays a and b, squares
-these differences, and returns the sum of these squares. Dimension
-can equal None (ravel array first), an integer (the dimension over
-which to operate), or a sequence (operate over multiple dimensions).
-keepdims=1 means the return shape = len(a.shape) = len(b.shape)
-
-Usage: asumdiffsquared(a,b)
-Returns: sum[ravel(a-b)**2]
-"""
- if dimension == None:
- inarray = N.ravel(a)
- dimension = 0
- return asum((a-b)**2,dimension,keepdims)
-
-
- def ashellsort(inarray):
- """
-Shellsort algorithm. Sorts a 1D-array.
-
-Usage: ashellsort(inarray)
-Returns: sorted-inarray, sorting-index-vector (for original array)
-"""
- n = len(inarray)
- svec = inarray *1.0
- ivec = range(n)
- gap = n/2 # integer division needed
- while gap >0:
- for i in range(gap,n):
- for j in range(i-gap,-1,-gap):
- while j>=0 and svec[j]>svec[j+gap]:
- temp = svec[j]
- svec[j] = svec[j+gap]
- svec[j+gap] = temp
- itemp = ivec[j]
- ivec[j] = ivec[j+gap]
- ivec[j+gap] = itemp
- gap = gap / 2 # integer division needed
-# svec is now sorted input vector, ivec has the order svec[i] = vec[ivec[i]]
- return svec, ivec
-
-
- def arankdata(inarray):
- """
-Ranks the data in inarray, dealing with ties appropritely. Assumes
-a 1D inarray. Adapted from Gary Perlman's |Stat ranksort.
-
-Usage: arankdata(inarray)
-Returns: array of length equal to inarray, containing rank scores
-"""
- n = len(inarray)
- svec, ivec = ashellsort(inarray)
- sumranks = 0
- dupcount = 0
- newarray = N.zeros(n,N.float_)
- for i in range(n):
- sumranks = sumranks + i
- dupcount = dupcount + 1
- if i==n-1 or svec[i] <> svec[i+1]:
- averank = sumranks / float(dupcount) + 1
- for j in range(i-dupcount+1,i+1):
- newarray[ivec[j]] = averank
- sumranks = 0
- dupcount = 0
- return newarray
-
-
- def afindwithin(data):
- """
-Returns a binary vector, 1=within-subject factor, 0=between. Input
-equals the entire data array (i.e., column 1=random factor, last
-column = measured values.
-
-Usage: afindwithin(data) data in |Stat format
-"""
- numfact = len(data[0])-2
- withinvec = [0]*numfact
- for col in range(1,numfact+1):
- rows = pstat.linexand(data,col,pstat.unique(pstat.colex(data,1))[0]) # get 1 level of this factor
- if len(pstat.unique(pstat.colex(rows,0))) < len(rows): # if fewer subjects than scores on this factor
- withinvec[col-1] = 1
- return withinvec
-
-
- #########################################################
- #########################################################
- ###### RE-DEFINE DISPATCHES TO INCLUDE ARRAYS #########
- #########################################################
- #########################################################
-
-## CENTRAL TENDENCY:
- geometricmean = Dispatch ( (lgeometricmean, (ListType, TupleType)),
- (ageometricmean, (N.ndarray,)) )
- harmonicmean = Dispatch ( (lharmonicmean, (ListType, TupleType)),
- (aharmonicmean, (N.ndarray,)) )
- mean = Dispatch ( (lmean, (ListType, TupleType)),
- (amean, (N.ndarray,)) )
- median = Dispatch ( (lmedian, (ListType, TupleType)),
- (amedian, (N.ndarray,)) )
- medianscore = Dispatch ( (lmedianscore, (ListType, TupleType)),
- (amedianscore, (N.ndarray,)) )
- mode = Dispatch ( (lmode, (ListType, TupleType)),
- (amode, (N.ndarray,)) )
- tmean = Dispatch ( (atmean, (N.ndarray,)) )
- tvar = Dispatch ( (atvar, (N.ndarray,)) )
- tstdev = Dispatch ( (atstdev, (N.ndarray,)) )
- tsem = Dispatch ( (atsem, (N.ndarray,)) )
-
-## VARIATION:
- moment = Dispatch ( (lmoment, (ListType, TupleType)),
- (amoment, (N.ndarray,)) )
- variation = Dispatch ( (lvariation, (ListType, TupleType)),
- (avariation, (N.ndarray,)) )
- skew = Dispatch ( (lskew, (ListType, TupleType)),
- (askew, (N.ndarray,)) )
- kurtosis = Dispatch ( (lkurtosis, (ListType, TupleType)),
- (akurtosis, (N.ndarray,)) )
- describe = Dispatch ( (ldescribe, (ListType, TupleType)),
- (adescribe, (N.ndarray,)) )
-
-## DISTRIBUTION TESTS
-
- skewtest = Dispatch ( (askewtest, (ListType, TupleType)),
- (askewtest, (N.ndarray,)) )
- kurtosistest = Dispatch ( (akurtosistest, (ListType, TupleType)),
- (akurtosistest, (N.ndarray,)) )
- normaltest = Dispatch ( (anormaltest, (ListType, TupleType)),
- (anormaltest, (N.ndarray,)) )
-
-## FREQUENCY STATS:
- itemfreq = Dispatch ( (litemfreq, (ListType, TupleType)),
- (aitemfreq, (N.ndarray,)) )
- scoreatpercentile = Dispatch ( (lscoreatpercentile, (ListType, TupleType)),
- (ascoreatpercentile, (N.ndarray,)) )
- percentileofscore = Dispatch ( (lpercentileofscore, (ListType, TupleType)),
- (apercentileofscore, (N.ndarray,)) )
- histogram = Dispatch ( (lhistogram, (ListType, TupleType)),
- (ahistogram, (N.ndarray,)) )
- cumfreq = Dispatch ( (lcumfreq, (ListType, TupleType)),
- (acumfreq, (N.ndarray,)) )
- relfreq = Dispatch ( (lrelfreq, (ListType, TupleType)),
- (arelfreq, (N.ndarray,)) )
-
-## VARIABILITY:
- obrientransform = Dispatch ( (lobrientransform, (ListType, TupleType)),
- (aobrientransform, (N.ndarray,)) )
- samplevar = Dispatch ( (lsamplevar, (ListType, TupleType)),
- (asamplevar, (N.ndarray,)) )
- samplestdev = Dispatch ( (lsamplestdev, (ListType, TupleType)),
- (asamplestdev, (N.ndarray,)) )
- signaltonoise = Dispatch( (asignaltonoise, (N.ndarray,)),)
- var = Dispatch ( (lvar, (ListType, TupleType)),
- (avar, (N.ndarray,)) )
- stdev = Dispatch ( (lstdev, (ListType, TupleType)),
- (astdev, (N.ndarray,)) )
- sterr = Dispatch ( (lsterr, (ListType, TupleType)),
- (asterr, (N.ndarray,)) )
- sem = Dispatch ( (lsem, (ListType, TupleType)),
- (asem, (N.ndarray,)) )
- z = Dispatch ( (lz, (ListType, TupleType)),
- (az, (N.ndarray,)) )
- zs = Dispatch ( (lzs, (ListType, TupleType)),
- (azs, (N.ndarray,)) )
-
-## TRIMMING FCNS:
- threshold = Dispatch( (athreshold, (N.ndarray,)),)
- trimboth = Dispatch ( (ltrimboth, (ListType, TupleType)),
- (atrimboth, (N.ndarray,)) )
- trim1 = Dispatch ( (ltrim1, (ListType, TupleType)),
- (atrim1, (N.ndarray,)) )
-
-## CORRELATION FCNS:
- paired = Dispatch ( (lpaired, (ListType, TupleType)),
- (apaired, (N.ndarray,)) )
- lincc = Dispatch ( (llincc, (ListType, TupleType)),
- (alincc, (N.ndarray,)) )
- pearsonr = Dispatch ( (lpearsonr, (ListType, TupleType)),
- (apearsonr, (N.ndarray,)) )
- spearmanr = Dispatch ( (lspearmanr, (ListType, TupleType)),
- (aspearmanr, (N.ndarray,)) )
- pointbiserialr = Dispatch ( (lpointbiserialr, (ListType, TupleType)),
- (apointbiserialr, (N.ndarray,)) )
- kendalltau = Dispatch ( (lkendalltau, (ListType, TupleType)),
- (akendalltau, (N.ndarray,)) )
- linregress = Dispatch ( (llinregress, (ListType, TupleType)),
- (alinregress, (N.ndarray,)) )
-
-## INFERENTIAL STATS:
- ttest_1samp = Dispatch ( (lttest_1samp, (ListType, TupleType)),
- (attest_1samp, (N.ndarray,)) )
- ttest_ind = Dispatch ( (lttest_ind, (ListType, TupleType)),
- (attest_ind, (N.ndarray,)) )
- ttest_rel = Dispatch ( (lttest_rel, (ListType, TupleType)),
- (attest_rel, (N.ndarray,)) )
- chisquare = Dispatch ( (lchisquare, (ListType, TupleType)),
- (achisquare, (N.ndarray,)) )
- ks_2samp = Dispatch ( (lks_2samp, (ListType, TupleType)),
- (aks_2samp, (N.ndarray,)) )
- mannwhitneyu = Dispatch ( (lmannwhitneyu, (ListType, TupleType)),
- (amannwhitneyu, (N.ndarray,)) )
- tiecorrect = Dispatch ( (ltiecorrect, (ListType, TupleType)),
- (atiecorrect, (N.ndarray,)) )
- ranksums = Dispatch ( (lranksums, (ListType, TupleType)),
- (aranksums, (N.ndarray,)) )
- wilcoxont = Dispatch ( (lwilcoxont, (ListType, TupleType)),
- (awilcoxont, (N.ndarray,)) )
- kruskalwallish = Dispatch ( (lkruskalwallish, (ListType, TupleType)),
- (akruskalwallish, (N.ndarray,)) )
- friedmanchisquare = Dispatch ( (lfriedmanchisquare, (ListType, TupleType)),
- (afriedmanchisquare, (N.ndarray,)) )
-
-## PROBABILITY CALCS:
- chisqprob = Dispatch ( (lchisqprob, (IntType, FloatType)),
- (achisqprob, (N.ndarray,)) )
- zprob = Dispatch ( (lzprob, (IntType, FloatType)),
- (azprob, (N.ndarray,)) )
- ksprob = Dispatch ( (lksprob, (IntType, FloatType)),
- (aksprob, (N.ndarray,)) )
- fprob = Dispatch ( (lfprob, (IntType, FloatType)),
- (afprob, (N.ndarray,)) )
- betacf = Dispatch ( (lbetacf, (IntType, FloatType)),
- (abetacf, (N.ndarray,)) )
- betai = Dispatch ( (lbetai, (IntType, FloatType)),
- (abetai, (N.ndarray,)) )
- erfcc = Dispatch ( (lerfcc, (IntType, FloatType)),
- (aerfcc, (N.ndarray,)) )
- gammln = Dispatch ( (lgammln, (IntType, FloatType)),
- (agammln, (N.ndarray,)) )
-
-## ANOVA FUNCTIONS:
- F_oneway = Dispatch ( (lF_oneway, (ListType, TupleType)),
- (aF_oneway, (N.ndarray,)) )
- F_value = Dispatch ( (lF_value, (ListType, TupleType)),
- (aF_value, (N.ndarray,)) )
-
-## SUPPORT FUNCTIONS:
- incr = Dispatch ( (lincr, (ListType, TupleType, N.ndarray)), )
- sum = Dispatch ( (lsum, (ListType, TupleType)),
- (asum, (N.ndarray,)) )
- cumsum = Dispatch ( (lcumsum, (ListType, TupleType)),
- (acumsum, (N.ndarray,)) )
- ss = Dispatch ( (lss, (ListType, TupleType)),
- (ass, (N.ndarray,)) )
- summult = Dispatch ( (lsummult, (ListType, TupleType)),
- (asummult, (N.ndarray,)) )
- square_of_sums = Dispatch ( (lsquare_of_sums, (ListType, TupleType)),
- (asquare_of_sums, (N.ndarray,)) )
- sumdiffsquared = Dispatch ( (lsumdiffsquared, (ListType, TupleType)),
- (asumdiffsquared, (N.ndarray,)) )
- shellsort = Dispatch ( (lshellsort, (ListType, TupleType)),
- (ashellsort, (N.ndarray,)) )
- rankdata = Dispatch ( (lrankdata, (ListType, TupleType)),
- (arankdata, (N.ndarray,)) )
- findwithin = Dispatch ( (lfindwithin, (ListType, TupleType)),
- (afindwithin, (N.ndarray,)) )
-
-###################### END OF NUMERIC FUNCTION BLOCK #####################
-
-###################### END OF STATISTICAL FUNCTIONS ######################
-
-except ImportError:
- pass
diff --git a/site_utils/dashboard/gviz_api.py b/site_utils/dashboard/gviz_api.py
deleted file mode 100644
index 67fd9eb..0000000
--- a/site_utils/dashboard/gviz_api.py
+++ /dev/null
@@ -1,1054 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright (C) 2009 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Converts Python data into data for Google Visualization API clients.
-
-This library can be used to create a google.visualization.DataTable usable by
-visualizations built on the Google Visualization API. Output formats are raw
-JSON, JSON response, and JavaScript.
-
-See http://code.google.com/apis/visualization/ for documentation on the
-Google Visualization API.
-"""
-
-__author__ = "Amit Weinstein, Misha Seltzer"
-
-import cgi
-import datetime
-import types
-
-
-class DataTableException(Exception):
- """The general exception object thrown by DataTable."""
- pass
-
-
-class DataTable(object):
- """Wraps the data to convert to a Google Visualization API DataTable.
-
- Create this object, populate it with data, then call one of the ToJS...
- methods to return a string representation of the data in the format described.
-
- You can clear all data from the object to reuse it, but you cannot clear
- individual cells, rows, or columns. You also cannot modify the table schema
- specified in the class constructor.
-
- You can add new data one or more rows at a time. All data added to an
- instantiated DataTable must conform to the schema passed in to __init__().
-
- You can reorder the columns in the output table, and also specify row sorting
- order by column. The default column order is according to the original
- table_description parameter. Default row sort order is ascending, by column
- 1 values. For a dictionary, we sort the keys for order.
-
- The data and the table_description are closely tied, as described here:
-
- The table schema is defined in the class constructor's table_description
- parameter. The user defines each column using a tuple of
- (id[, type[, label[, custom_properties]]]). The default value for type is
- string, label is the same as ID if not specified, and custom properties is
- an empty dictionary if not specified.
-
- table_description is a dictionary or list, containing one or more column
- descriptor tuples, nested dictionaries, and lists. Each dictionary key, list
- element, or dictionary element must eventually be defined as
- a column description tuple. Here's an example of a dictionary where the key
- is a tuple, and the value is a list of two tuples:
- {('a', 'number'): [('b', 'number'), ('c', 'string')]}
-
- This flexibility in data entry enables you to build and manipulate your data
- in a Python structure that makes sense for your program.
-
- Add data to the table using the same nested design as the table's
- table_description, replacing column descriptor tuples with cell data, and
- each row is an element in the top level collection. This will be a bit
- clearer after you look at the following examples showing the
- table_description, matching data, and the resulting table:
-
- Columns as list of tuples [col1, col2, col3]
- table_description: [('a', 'number'), ('b', 'string')]
- AppendData( [[1, 'z'], [2, 'w'], [4, 'o'], [5, 'k']] )
- Table:
- a b <--- these are column ids/labels
- 1 z
- 2 w
- 4 o
- 5 k
-
- Dictionary of columns, where key is a column, and value is a list of
- columns {col1: [col2, col3]}
- table_description: {('a', 'number'): [('b', 'number'), ('c', 'string')]}
- AppendData( data: {1: [2, 'z'], 3: [4, 'w']}
- Table:
- a b c
- 1 2 z
- 3 4 w
-
- Dictionary where key is a column, and the value is itself a dictionary of
- columns {col1: {col2, col3}}
- table_description: {('a', 'number'): {'b': 'number', 'c': 'string'}}
- AppendData( data: {1: {'b': 2, 'c': 'z'}, 3: {'b': 4, 'c': 'w'}}
- Table:
- a b c
- 1 2 z
- 3 4 w
- """
-
- def __init__(self, table_description, data=None, custom_properties=None):
- """Initialize the data table from a table schema and (optionally) data.
-
- See the class documentation for more information on table schema and data
- values.
-
- Args:
- table_description: A table schema, following one of the formats described
- in TableDescriptionParser(). Schemas describe the
- column names, data types, and labels. See
- TableDescriptionParser() for acceptable formats.
- data: Optional. If given, fills the table with the given data. The data
- structure must be consistent with schema in table_description. See
- the class documentation for more information on acceptable data. You
- can add data later by calling AppendData().
- custom_properties: Optional. A dictionary from string to string that
- goes into the table's custom properties. This can be
- later changed by changing self.custom_properties.
-
- Raises:
- DataTableException: Raised if the data and the description did not match,
- or did not use the supported formats.
- """
- self.__columns = self.TableDescriptionParser(table_description)
- self.__data = []
- self.custom_properties = {}
- if custom_properties is not None:
- self.custom_properties = custom_properties
- if data:
- self.LoadData(data)
-
- @staticmethod
- def _EscapeValueForCsv(v):
- """Escapes the value for use in a CSV file.
-
- Puts the string in double-quotes, and escapes any inner double-quotes by
- doubling them.
-
- Args:
- v: The value to escape.
-
- Returns:
- The escaped values.
- """
- return '"%s"' % v.replace('"', '""')
-
- @staticmethod
- def _EscapeValue(v):
- """Puts the string in quotes, and escapes any inner quotes and slashes."""
- if isinstance(v, unicode):
- # Here we use repr as in the usual case, but on unicode strings, it
- # also escapes the unicode characters (which we want to leave as is).
- # So, after repr() we decode using raw-unicode-escape, which decodes
- # only the unicode characters, and leaves all the rest (", ', \n and
- # more) escaped.
- # We don't take the first character, because repr adds a u in the
- # beginning of the string (usual repr output for unicode is u'...').
- return repr(v).decode("raw-unicode-escape")[1:]
- # Here we use python built-in escaping mechanism for string using repr.
- return repr(str(v))
-
- @staticmethod
- def _EscapeCustomProperties(custom_properties):
- """Escapes the custom properties dictionary."""
- l = []
- for key, value in custom_properties.iteritems():
- l.append("%s:%s" % (DataTable._EscapeValue(key),
- DataTable._EscapeValue(value)))
- return "{%s}" % ",".join(l)
-
- @staticmethod
- def SingleValueToJS(value, value_type, escape_func=None):
- """Translates a single value and type into a JS value.
-
- Internal helper method.
-
- Args:
- value: The value which should be converted
- value_type: One of "string", "number", "boolean", "date", "datetime" or
- "timeofday".
- escape_func: The function to use for escaping strings.
-
- Returns:
- The proper JS format (as string) of the given value according to the
- given value_type. For None, we simply return "null".
- If a tuple is given, it should be in one of the following forms:
- - (value, formatted value)
- - (value, formatted value, custom properties)
- where the formatted value is a string, and custom properties is a
- dictionary of the custom properties for this cell.
- To specify custom properties without specifying formatted value, one can
- pass None as the formatted value.
- One can also have a null-valued cell with formatted value and/or custom
- properties by specifying None for the value.
- This method ignores the custom properties except for checking that it is a
- dictionary. The custom properties are handled in the ToJSon and ToJSCode
- methods.
- The real type of the given value is not strictly checked. For example,
- any type can be used for string - as we simply take its str( ) and for
- boolean value we just check "if value".
- Examples:
- SingleValueToJS(None, "boolean") returns "null"
- SingleValueToJS(False, "boolean") returns "false"
- SingleValueToJS((5, "5$"), "number") returns ("5", "'5$'")
- SingleValueToJS((None, "5$"), "number") returns ("null", "'5$'")
-
- Raises:
- DataTableException: The value and type did not match in a not-recoverable
- way, for example given value 'abc' for type 'number'.
- """
- if escape_func is None:
- escape_func = DataTable._EscapeValue
- if isinstance(value, tuple):
- # In case of a tuple, we run the same function on the value itself and
- # add the formatted value.
- if (len(value) not in [2, 3] or
- (len(value) == 3 and not isinstance(value[2], dict))):
- raise DataTableException("Wrong format for value and formatting - %s." %
- str(value))
- if not isinstance(value[1], types.StringTypes + (types.NoneType,)):
- raise DataTableException("Formatted value is not string, given %s." %
- type(value[1]))
- js_value = DataTable.SingleValueToJS(value[0], value_type)
- if value[1] is None:
- return (js_value, None)
- return (js_value, escape_func(value[1]))
-
- # The standard case - no formatting.
- t_value = type(value)
- if value is None:
- return "null"
- if value_type == "boolean":
- if value:
- return "true"
- return "false"
-
- elif value_type == "number":
- if isinstance(value, (int, long, float)):
- return str(value)
- raise DataTableException("Wrong type %s when expected number" % t_value)
-
- elif value_type == "string":
- if isinstance(value, tuple):
- raise DataTableException("Tuple is not allowed as string value.")
- return escape_func(value)
-
- elif value_type == "date":
- if not isinstance(value, (datetime.date, datetime.datetime)):
- raise DataTableException("Wrong type %s when expected date" % t_value)
- # We need to shift the month by 1 to match JS Date format
- return "new Date(%d,%d,%d)" % (value.year, value.month - 1, value.day)
-
- elif value_type == "timeofday":
- if not isinstance(value, (datetime.time, datetime.datetime)):
- raise DataTableException("Wrong type %s when expected time" % t_value)
- return "[%d,%d,%d]" % (value.hour, value.minute, value.second)
-
- elif value_type == "datetime":
- if not isinstance(value, datetime.datetime):
- raise DataTableException("Wrong type %s when expected datetime" %
- t_value)
- return "new Date(%d,%d,%d,%d,%d,%d)" % (value.year,
- value.month - 1, # To match JS
- value.day,
- value.hour,
- value.minute,
- value.second)
- # If we got here, it means the given value_type was not one of the
- # supported types.
- raise DataTableException("Unsupported type %s" % value_type)
-
- @staticmethod
- def ColumnTypeParser(description):
- """Parses a single column description. Internal helper method.
-
- Args:
- description: a column description in the possible formats:
- 'id'
- ('id',)
- ('id', 'type')
- ('id', 'type', 'label')
- ('id', 'type', 'label', {'custom_prop1': 'custom_val1'})
- Returns:
- Dictionary with the following keys: id, label, type, and
- custom_properties where:
- - If label not given, it equals the id.
- - If type not given, string is used by default.
- - If custom properties are not given, an empty dictionary is used by
- default.
-
- Raises:
- DataTableException: The column description did not match the RE, or
- unsupported type was passed.
- """
- if not description:
- raise DataTableException("Description error: empty description given")
-
- if not isinstance(description, (types.StringTypes, tuple)):
- raise DataTableException("Description error: expected either string or "
- "tuple, got %s." % type(description))
-
- if isinstance(description, types.StringTypes):
- description = (description,)
-
- # According to the tuple's length, we fill the keys
- # We verify everything is of type string
- for elem in description[:3]:
- if not isinstance(elem, types.StringTypes):
- raise DataTableException("Description error: expected tuple of "
- "strings, current element of type %s." %
- type(elem))
- desc_dict = {"id": description[0],
- "label": description[0],
- "type": "string",
- "custom_properties": {}}
- if len(description) > 1:
- desc_dict["type"] = description[1].lower()
- if len(description) > 2:
- desc_dict["label"] = description[2]
- if len(description) > 3:
- if not isinstance(description[3], dict):
- raise DataTableException("Description error: expected custom "
- "properties of type dict, current element "
- "of type %s." % type(description[3]))
- desc_dict["custom_properties"] = description[3]
- if len(description) > 4:
- raise DataTableException("Description error: tuple of length > 4")
- if desc_dict["type"] not in ["string", "number", "boolean",
- "date", "datetime", "timeofday"]:
- raise DataTableException(
- "Description error: unsupported type '%s'" % desc_dict["type"])
- return desc_dict
-
- @staticmethod
- def TableDescriptionParser(table_description, depth=0):
- """Parses the table_description object for internal use.
-
- Parses the user-submitted table description into an internal format used
- by the Python DataTable class. Returns the flat list of parsed columns.
-
- Args:
- table_description: A description of the table which should comply
- with one of the formats described below.
- depth: Optional. The depth of the first level in the current description.
- Used by recursive calls to this function.
-
- Returns:
- List of columns, where each column represented by a dictionary with the
- keys: id, label, type, depth, container which means the following:
- - id: the id of the column
- - name: The name of the column
- - type: The datatype of the elements in this column. Allowed types are
- described in ColumnTypeParser().
- - depth: The depth of this column in the table description
- - container: 'dict', 'iter' or 'scalar' for parsing the format easily.
- - custom_properties: The custom properties for this column.
- The returned description is flattened regardless of how it was given.
-
- Raises:
- DataTableException: Error in a column description or in the description
- structure.
-
- Examples:
- A column description can be of the following forms:
- 'id'
- ('id',)
- ('id', 'type')
- ('id', 'type', 'label')
- ('id', 'type', 'label', {'custom_prop1': 'custom_val1'})
- or as a dictionary:
- 'id': 'type'
- 'id': ('type',)
- 'id': ('type', 'label')
- 'id': ('type', 'label', {'custom_prop1': 'custom_val1'})
- If the type is not specified, we treat it as string.
- If no specific label is given, the label is simply the id.
- If no custom properties are given, we use an empty dictionary.
-
- input: [('a', 'date'), ('b', 'timeofday', 'b', {'foo': 'bar'})]
- output: [{'id': 'a', 'label': 'a', 'type': 'date',
- 'depth': 0, 'container': 'iter', 'custom_properties': {}},
- {'id': 'b', 'label': 'b', 'type': 'timeofday',
- 'depth': 0, 'container': 'iter',
- 'custom_properties': {'foo': 'bar'}}]
-
- input: {'a': [('b', 'number'), ('c', 'string', 'column c')]}
- output: [{'id': 'a', 'label': 'a', 'type': 'string',
- 'depth': 0, 'container': 'dict', 'custom_properties': {}},
- {'id': 'b', 'label': 'b', 'type': 'number',
- 'depth': 1, 'container': 'iter', 'custom_properties': {}},
- {'id': 'c', 'label': 'column c', 'type': 'string',
- 'depth': 1, 'container': 'iter', 'custom_properties': {}}]
-
- input: {('a', 'number', 'column a'): { 'b': 'number', 'c': 'string'}}
- output: [{'id': 'a', 'label': 'column a', 'type': 'number',
- 'depth': 0, 'container': 'dict', 'custom_properties': {}},
- {'id': 'b', 'label': 'b', 'type': 'number',
- 'depth': 1, 'container': 'dict', 'custom_properties': {}},
- {'id': 'c', 'label': 'c', 'type': 'string',
- 'depth': 1, 'container': 'dict', 'custom_properties': {}}]
-
- input: { ('w', 'string', 'word'): ('c', 'number', 'count') }
- output: [{'id': 'w', 'label': 'word', 'type': 'string',
- 'depth': 0, 'container': 'dict', 'custom_properties': {}},
- {'id': 'c', 'label': 'count', 'type': 'number',
- 'depth': 1, 'container': 'scalar', 'custom_properties': {}}]
-
- input: {'a': ('number', 'column a'), 'b': ('string', 'column b')}
- output: [{'id': 'a', 'label': 'column a', 'type': 'number', 'depth': 0,
- 'container': 'dict', 'custom_properties': {}},
- {'id': 'b', 'label': 'column b', 'type': 'string', 'depth': 0,
- 'container': 'dict', 'custom_properties': {}}
-
- NOTE: there might be ambiguity in the case of a dictionary representation
- of a single column. For example, the following description can be parsed
- in 2 different ways: {'a': ('b', 'c')} can be thought of a single column
- with the id 'a', of type 'b' and the label 'c', or as 2 columns: one named
- 'a', and the other named 'b' of type 'c'. We choose the first option by
- default, and in case the second option is the right one, it is possible to
- make the key into a tuple (i.e. {('a',): ('b', 'c')}) or add more info
- into the tuple, thus making it look like this: {'a': ('b', 'c', 'b', {})}
- -- second 'b' is the label, and {} is the custom properties field.
- """
- # For the recursion step, we check for a scalar object (string or tuple)
- if isinstance(table_description, (types.StringTypes, tuple)):
- parsed_col = DataTable.ColumnTypeParser(table_description)
- parsed_col["depth"] = depth
- parsed_col["container"] = "scalar"
- return [parsed_col]
-
- # Since it is not scalar, table_description must be iterable.
- if not hasattr(table_description, "__iter__"):
- raise DataTableException("Expected an iterable object, got %s" %
- type(table_description))
- if not isinstance(table_description, dict):
- # We expects a non-dictionary iterable item.
- columns = []
- for desc in table_description:
- parsed_col = DataTable.ColumnTypeParser(desc)
- parsed_col["depth"] = depth
- parsed_col["container"] = "iter"
- columns.append(parsed_col)
- if not columns:
- raise DataTableException("Description iterable objects should not"
- " be empty.")
- return columns
- # The other case is a dictionary
- if not table_description:
- raise DataTableException("Empty dictionaries are not allowed inside"
- " description")
-
- # To differentiate between the two cases of more levels below or this is
- # the most inner dictionary, we consider the number of keys (more then one
- # key is indication for most inner dictionary) and the type of the key and
- # value in case of only 1 key (if the type of key is string and the type of
- # the value is a tuple of 0-3 items, we assume this is the most inner
- # dictionary).
- # NOTE: this way of differentiating might create ambiguity. See docs.
- if (len(table_description) != 1 or
- (isinstance(table_description.keys()[0], types.StringTypes) and
- isinstance(table_description.values()[0], tuple) and
- len(table_description.values()[0]) < 4)):
- # This is the most inner dictionary. Parsing types.
- columns = []
- # We sort the items, equivalent to sort the keys since they are unique
- for key, value in sorted(table_description.items()):
- # We parse the column type as (key, type) or (key, type, label) using
- # ColumnTypeParser.
- if isinstance(value, tuple):
- parsed_col = DataTable.ColumnTypeParser((key,) + value)
- else:
- parsed_col = DataTable.ColumnTypeParser((key, value))
- parsed_col["depth"] = depth
- parsed_col["container"] = "dict"
- columns.append(parsed_col)
- return columns
- # This is an outer dictionary, must have at most one key.
- parsed_col = DataTable.ColumnTypeParser(table_description.keys()[0])
- parsed_col["depth"] = depth
- parsed_col["container"] = "dict"
- return ([parsed_col] +
- DataTable.TableDescriptionParser(table_description.values()[0],
- depth=depth + 1))
-
- @property
- def columns(self):
- """Returns the parsed table description."""
- return self.__columns
-
- def NumberOfRows(self):
- """Returns the number of rows in the current data stored in the table."""
- return len(self.__data)
-
- def SetRowsCustomProperties(self, rows, custom_properties):
- """Sets the custom properties for given row(s).
-
- Can accept a single row or an iterable of rows.
- Sets the given custom properties for all specified rows.
-
- Args:
- rows: The row, or rows, to set the custom properties for.
- custom_properties: A string to string dictionary of custom properties to
- set for all rows.
- """
- if not hasattr(rows, "__iter__"):
- rows = [rows]
- for row in rows:
- self.__data[row] = (self.__data[row][0], custom_properties)
-
- def LoadData(self, data, custom_properties=None):
- """Loads new rows to the data table, clearing existing rows.
-
- May also set the custom_properties for the added rows. The given custom
- properties dictionary specifies the dictionary that will be used for *all*
- given rows.
-
- Args:
- data: The rows that the table will contain.
- custom_properties: A dictionary of string to string to set as the custom
- properties for all rows.
- """
- self.__data = []
- self.AppendData(data, custom_properties)
-
- def AppendData(self, data, custom_properties=None):
- """Appends new data to the table.
-
- Data is appended in rows. Data must comply with
- the table schema passed in to __init__(). See SingleValueToJS() for a list
- of acceptable data types. See the class documentation for more information
- and examples of schema and data values.
-
- Args:
- data: The row to add to the table. The data must conform to the table
- description format.
- custom_properties: A dictionary of string to string, representing the
- custom properties to add to all the rows.
-
- Raises:
- DataTableException: The data structure does not match the description.
- """
- # If the maximal depth is 0, we simply iterate over the data table
- # lines and insert them using _InnerAppendData. Otherwise, we simply
- # let the _InnerAppendData handle all the levels.
- if not self.__columns[-1]["depth"]:
- for row in data:
- self._InnerAppendData(({}, custom_properties), row, 0)
- else:
- self._InnerAppendData(({}, custom_properties), data, 0)
-
- def _InnerAppendData(self, prev_col_values, data, col_index):
- """Inner function to assist LoadData."""
- # We first check that col_index has not exceeded the columns size
- if col_index >= len(self.__columns):
- raise DataTableException("The data does not match description, too deep")
-
- # Dealing with the scalar case, the data is the last value.
- if self.__columns[col_index]["container"] == "scalar":
- prev_col_values[0][self.__columns[col_index]["id"]] = data
- self.__data.append(prev_col_values)
- return
-
- if self.__columns[col_index]["container"] == "iter":
- if not hasattr(data, "__iter__") or isinstance(data, dict):
- raise DataTableException("Expected iterable object, got %s" %
- type(data))
- # We only need to insert the rest of the columns
- # If there are less items than expected, we only add what there is.
- for value in data:
- if col_index >= len(self.__columns):
- raise DataTableException("Too many elements given in data")
- prev_col_values[0][self.__columns[col_index]["id"]] = value
- col_index += 1
- self.__data.append(prev_col_values)
- return
-
- # We know the current level is a dictionary, we verify the type.
- if not isinstance(data, dict):
- raise DataTableException("Expected dictionary at current level, got %s" %
- type(data))
- # We check if this is the last level
- if self.__columns[col_index]["depth"] == self.__columns[-1]["depth"]:
- # We need to add the keys in the dictionary as they are
- for col in self.__columns[col_index:]:
- if col["id"] in data:
- prev_col_values[0][col["id"]] = data[col["id"]]
- self.__data.append(prev_col_values)
- return
-
- # We have a dictionary in an inner depth level.
- if not data.keys():
- # In case this is an empty dictionary, we add a record with the columns
- # filled only until this point.
- self.__data.append(prev_col_values)
- else:
- for key in sorted(data):
- col_values = dict(prev_col_values[0])
- col_values[self.__columns[col_index]["id"]] = key
- self._InnerAppendData((col_values, prev_col_values[1]),
- data[key], col_index + 1)
-
- def _PreparedData(self, order_by=()):
- """Prepares the data for enumeration - sorting it by order_by.
-
- Args:
- order_by: Optional. Specifies the name of the column(s) to sort by, and
- (optionally) which direction to sort in. Default sort direction
- is asc. Following formats are accepted:
- "string_col_name" -- For a single key in default (asc) order.
- ("string_col_name", "asc|desc") -- For a single key.
- [("col_1","asc|desc"), ("col_2","asc|desc")] -- For more than
- one column, an array of tuples of (col_name, "asc|desc").
-
- Returns:
- The data sorted by the keys given.
-
- Raises:
- DataTableException: Sort direction not in 'asc' or 'desc'
- """
- if not order_by:
- return self.__data
-
- proper_sort_keys = []
- if isinstance(order_by, types.StringTypes) or (
- isinstance(order_by, tuple) and len(order_by) == 2 and
- order_by[1].lower() in ["asc", "desc"]):
- order_by = (order_by,)
- for key in order_by:
- if isinstance(key, types.StringTypes):
- proper_sort_keys.append((key, 1))
- elif (isinstance(key, (list, tuple)) and len(key) == 2 and
- key[1].lower() in ("asc", "desc")):
- proper_sort_keys.append((key[0], key[1].lower() == "asc" and 1 or -1))
- else:
- raise DataTableException("Expected tuple with second value: "
- "'asc' or 'desc'")
-
- def SortCmpFunc(row1, row2):
- """cmp function for sorted. Compares by keys and 'asc'/'desc' keywords."""
- for key, asc_mult in proper_sort_keys:
- cmp_result = asc_mult * cmp(row1[0].get(key), row2[0].get(key))
- if cmp_result:
- return cmp_result
- return 0
-
- return sorted(self.__data, cmp=SortCmpFunc)
-
- def ToJSCode(self, name, columns_order=None, order_by=()):
- """Writes the data table as a JS code string.
-
- This method writes a string of JS code that can be run to
- generate a DataTable with the specified data. Typically used for debugging
- only.
-
- Args:
- name: The name of the table. The name would be used as the DataTable's
- variable name in the created JS code.
- columns_order: Optional. Specifies the order of columns in the
- output table. Specify a list of all column IDs in the order
- in which you want the table created.
- Note that you must list all column IDs in this parameter,
- if you use it.
- order_by: Optional. Specifies the name of the column(s) to sort by.
- Passed as is to _PreparedData.
-
- Returns:
- A string of JS code that, when run, generates a DataTable with the given
- name and the data stored in the DataTable object.
- Example result:
- "var tab1 = new google.visualization.DataTable();
- tab1.addColumn('string', 'a', 'a');
- tab1.addColumn('number', 'b', 'b');
- tab1.addColumn('boolean', 'c', 'c');
- tab1.addRows(10);
- tab1.setCell(0, 0, 'a');
- tab1.setCell(0, 1, 1, null, {'foo': 'bar'});
- tab1.setCell(0, 2, true);
- ...
- tab1.setCell(9, 0, 'c');
- tab1.setCell(9, 1, 3, '3$');
- tab1.setCell(9, 2, false);"
-
- Raises:
- DataTableException: The data does not match the type.
- """
- if columns_order is None:
- columns_order = [col["id"] for col in self.__columns]
- col_dict = dict([(col["id"], col) for col in self.__columns])
-
- # We first create the table with the given name
- jscode = "var %s = new google.visualization.DataTable();\n" % name
- if self.custom_properties:
- jscode += "%s.setTableProperties(%s);\n" % (
- name, DataTable._EscapeCustomProperties(self.custom_properties))
-
- # We add the columns to the table
- for i, col in enumerate(columns_order):
- jscode += "%s.addColumn('%s', %s, %s);\n" % (
- name,
- col_dict[col]["type"],
- DataTable._EscapeValue(col_dict[col]["label"]),
- DataTable._EscapeValue(col_dict[col]["id"]))
- if col_dict[col]["custom_properties"]:
- jscode += "%s.setColumnProperties(%d, %s);\n" % (
- name, i, DataTable._EscapeCustomProperties(
- col_dict[col]["custom_properties"]))
- jscode += "%s.addRows(%d);\n" % (name, len(self.__data))
-
- # We now go over the data and add each row
- for (i, (row, cp)) in enumerate(self._PreparedData(order_by)):
- # We add all the elements of this row by their order
- for (j, col) in enumerate(columns_order):
- if col not in row or row[col] is None:
- continue
- cell_cp = ""
- if isinstance(row[col], tuple) and len(row[col]) == 3:
- cell_cp = ", %s" % DataTable._EscapeCustomProperties(row[col][2])
- value = self.SingleValueToJS(row[col], col_dict[col]["type"])
- if isinstance(value, tuple):
- # We have a formatted value or custom property as well
- if value[1] is None:
- value = (value[0], "null")
- jscode += ("%s.setCell(%d, %d, %s, %s%s);\n" %
- (name, i, j, value[0], value[1], cell_cp))
- else:
- jscode += "%s.setCell(%d, %d, %s);\n" % (name, i, j, value)
- if cp:
- jscode += "%s.setRowProperties(%d, %s);\n" % (
- name, i, DataTable._EscapeCustomProperties(cp))
- return jscode
-
- def ToHtml(self, columns_order=None, order_by=()):
- """Writes the data table as an HTML table code string.
-
- Args:
- columns_order: Optional. Specifies the order of columns in the
- output table. Specify a list of all column IDs in the order
- in which you want the table created.
- Note that you must list all column IDs in this parameter,
- if you use it.
- order_by: Optional. Specifies the name of the column(s) to sort by.
- Passed as is to _PreparedData.
-
- Returns:
- An HTML table code string.
- Example result (the result is without the newlines):
- <html><body><table border='1'>
- <thead><tr><th>a</th><th>b</th><th>c</th></tr></thead>
- <tbody>
- <tr><td>1</td><td>"z"</td><td>2</td></tr>
- <tr><td>"3$"</td><td>"w"</td><td></td></tr>
- </tbody>
- </table></body></html>
-
- Raises:
- DataTableException: The data does not match the type.
- """
- table_template = "<html><body><table border='1'>%s</table></body></html>"
- columns_template = "<thead><tr>%s</tr></thead>"
- rows_template = "<tbody>%s</tbody>"
- row_template = "<tr>%s</tr>"
- header_cell_template = "<th>%s</th>"
- cell_template = "<td>%s</td>"
-
- if columns_order is None:
- columns_order = [col["id"] for col in self.__columns]
- col_dict = dict([(col["id"], col) for col in self.__columns])
-
- columns_list = []
- for col in columns_order:
- columns_list.append(header_cell_template %
- cgi.escape(col_dict[col]["label"]))
- columns_html = columns_template % "".join(columns_list)
-
- rows_list = []
- # We now go over the data and add each row
- for row, unused_cp in self._PreparedData(order_by):
- cells_list = []
- # We add all the elements of this row by their order
- for col in columns_order:
- # For empty string we want empty quotes ("").
- value = ""
- if col in row and row[col] is not None:
- value = self.SingleValueToJS(row[col], col_dict[col]["type"])
- if isinstance(value, tuple):
- # We have a formatted value and we're going to use it
- cells_list.append(cell_template % cgi.escape(value[1]))
- else:
- cells_list.append(cell_template % cgi.escape(value))
- rows_list.append(row_template % "".join(cells_list))
- rows_html = rows_template % "".join(rows_list)
-
- return table_template % (columns_html + rows_html)
-
- def ToCsv(self, columns_order=None, order_by=(), separator=", "):
- """Writes the data table as a CSV string.
-
- Args:
- columns_order: Optional. Specifies the order of columns in the
- output table. Specify a list of all column IDs in the order
- in which you want the table created.
- Note that you must list all column IDs in this parameter,
- if you use it.
- order_by: Optional. Specifies the name of the column(s) to sort by.
- Passed as is to _PreparedData.
- separator: Optional. The separator to use between the values.
-
- Returns:
- A CSV string representing the table.
- Example result:
- 'a', 'b', 'c'
- 1, 'z', 2
- 3, 'w', ''
-
- Raises:
- DataTableException: The data does not match the type.
- """
- if columns_order is None:
- columns_order = [col["id"] for col in self.__columns]
- col_dict = dict([(col["id"], col) for col in self.__columns])
-
- columns_list = []
- for col in columns_order:
- columns_list.append(DataTable._EscapeValueForCsv(col_dict[col]["label"]))
- columns_line = separator.join(columns_list)
-
- rows_list = []
- # We now go over the data and add each row
- for row, unused_cp in self._PreparedData(order_by):
- cells_list = []
- # We add all the elements of this row by their order
- for col in columns_order:
- value = '""'
- if col in row and row[col] is not None:
- value = self.SingleValueToJS(row[col], col_dict[col]["type"],
- DataTable._EscapeValueForCsv)
- if isinstance(value, tuple):
- # We have a formatted value. Using it only for date/time types.
- if col_dict[col]["type"] in ["date", "datetime", "timeofday"]:
- cells_list.append(value[1])
- else:
- cells_list.append(value[0])
- else:
- # We need to quote date types, because they contain commas.
- if (col_dict[col]["type"] in ["date", "datetime", "timeofday"] and
- value != '""'):
- value = '"%s"' % value
- cells_list.append(value)
- rows_list.append(separator.join(cells_list))
- rows = "\n".join(rows_list)
-
- return "%s\n%s" % (columns_line, rows)
-
- def ToTsvExcel(self, columns_order=None, order_by=()):
- """Returns a file in tab-separated-format readable by MS Excel.
-
- Returns a file in UTF-16 little endian encoding, with tabs separating the
- values.
-
- Args:
- columns_order: Delegated to ToCsv.
- order_by: Delegated to ToCsv.
-
- Returns:
- A tab-separated little endian UTF16 file representing the table.
- """
- return self.ToCsv(
- columns_order, order_by, separator="\t").encode("UTF-16LE")
-
- def ToJSon(self, columns_order=None, order_by=()):
- """Writes a JSON string that can be used in a JS DataTable constructor.
-
- This method writes a JSON string that can be passed directly into a Google
- Visualization API DataTable constructor. Use this output if you are
- hosting the visualization HTML on your site, and want to code the data
- table in Python. Pass this string into the
- google.visualization.DataTable constructor, e.g,:
- ... on my page that hosts my visualization ...
- google.setOnLoadCallback(drawTable);
- function drawTable() {
- var data = new google.visualization.DataTable(_my_JSon_string, 0.6);
- myTable.draw(data);
- }
-
- Args:
- columns_order: Optional. Specifies the order of columns in the
- output table. Specify a list of all column IDs in the order
- in which you want the table created.
- Note that you must list all column IDs in this parameter,
- if you use it.
- order_by: Optional. Specifies the name of the column(s) to sort by.
- Passed as is to _PreparedData().
-
- Returns:
- A JSon constructor string to generate a JS DataTable with the data
- stored in the DataTable object.
- Example result (the result is without the newlines):
- {cols: [{id:'a',label:'a',type:'number'},
- {id:'b',label:'b',type:'string'},
- {id:'c',label:'c',type:'number'}],
- rows: [{c:[{v:1},{v:'z'},{v:2}]}, c:{[{v:3,f:'3$'},{v:'w'},{v:null}]}],
- p: {'foo': 'bar'}}
-
- Raises:
- DataTableException: The data does not match the type.
- """
- if columns_order is None:
- columns_order = [col["id"] for col in self.__columns]
- col_dict = dict([(col["id"], col) for col in self.__columns])
-
- # Creating the columns jsons
- cols_jsons = []
- for col_id in columns_order:
- d = dict(col_dict[col_id])
- d["id"] = DataTable._EscapeValue(d["id"])
- d["label"] = DataTable._EscapeValue(d["label"])
- d["cp"] = ""
- if col_dict[col_id]["custom_properties"]:
- d["cp"] = ",p:%s" % DataTable._EscapeCustomProperties(
- col_dict[col_id]["custom_properties"])
- cols_jsons.append(
- "{id:%(id)s,label:%(label)s,type:'%(type)s'%(cp)s}" % d)
-
- # Creating the rows jsons
- rows_jsons = []
- for row, cp in self._PreparedData(order_by):
- cells_jsons = []
- for col in columns_order:
- # We omit the {v:null} for a None value of the not last column
- value = row.get(col, None)
- if value is None and col != columns_order[-1]:
- cells_jsons.append("")
- else:
- value = self.SingleValueToJS(value, col_dict[col]["type"])
- if isinstance(value, tuple):
- # We have a formatted value or custom property as well
- if len(row.get(col)) == 3:
- if value[1] is None:
- cells_jsons.append("{v:%s,p:%s}" % (
- value[0],
- DataTable._EscapeCustomProperties(row.get(col)[2])))
- else:
- cells_jsons.append("{v:%s,f:%s,p:%s}" % (value + (
- DataTable._EscapeCustomProperties(row.get(col)[2]),)))
- else:
- cells_jsons.append("{v:%s,f:%s}" % value)
- else:
- cells_jsons.append("{v:%s}" % value)
- if cp:
- rows_jsons.append("{c:[%s],p:%s}" % (
- ",".join(cells_jsons), DataTable._EscapeCustomProperties(cp)))
- else:
- rows_jsons.append("{c:[%s]}" % ",".join(cells_jsons))
-
- general_custom_properties = ""
- if self.custom_properties:
- general_custom_properties = (
- ",p:%s" % DataTable._EscapeCustomProperties(self.custom_properties))
-
- # We now join the columns jsons and the rows jsons
- json = "{cols:[%s],rows:[%s]%s}" % (",".join(cols_jsons),
- ",".join(rows_jsons),
- general_custom_properties)
- return json
-
- def ToJSonResponse(self, columns_order=None, order_by=(), req_id=0,
- response_handler="google.visualization.Query.setResponse"):
- """Writes a table as a JSON response that can be returned as-is to a client.
-
- This method writes a JSON response to return to a client in response to a
- Google Visualization API query. This string can be processed by the calling
- page, and is used to deliver a data table to a visualization hosted on
- a different page.
-
- Args:
- columns_order: Optional. Passed straight to self.ToJSon().
- order_by: Optional. Passed straight to self.ToJSon().
- req_id: Optional. The response id, as retrieved by the request.
- response_handler: Optional. The response handler, as retrieved by the
- request.
-
- Returns:
- A JSON response string to be received by JS the visualization Query
- object. This response would be translated into a DataTable on the
- client side.
- Example result (newlines added for readability):
- google.visualization.Query.setResponse({
- 'version':'0.6', 'reqId':'0', 'status':'OK',
- 'table': {cols: [...], rows: [...]}});
-
- Note: The URL returning this string can be used as a data source by Google
- Visualization Gadgets or from JS code.
- """
- table = self.ToJSon(columns_order, order_by)
- return ("%s({'version':'0.6', 'reqId':'%s', 'status':'OK', "
- "'table': %s});") % (response_handler, req_id, table)
-
- def ToResponse(self, columns_order=None, order_by=(), tqx=""):
- """Writes the right response according to the request string passed in tqx.
-
- This method parses the tqx request string (format of which is defined in
- the documentation for implementing a data source of Google Visualization),
- and returns the right response according to the request.
- It parses out the "out" parameter of tqx, calls the relevant response
- (ToJSonResponse() for "json", ToCsv() for "csv", ToHtml() for "html",
- ToTsvExcel() for "tsv-excel") and passes the response function the rest of
- the relevant request keys.
-
- Args:
- columns_order: Optional. Passed as is to the relevant response function.
- order_by: Optional. Passed as is to the relevant response function.
- tqx: Optional. The request string as received by HTTP GET. Should be in
- the format "key1:value1;key2:value2...". All keys have a default
- value, so an empty string will just do the default (which is calling
- ToJSonResponse() with no extra parameters).
-
- Returns:
- A response string, as returned by the relevant response function.
-
- Raises:
- DataTableException: One of the parameters passed in tqx is not supported.
- """
- tqx_dict = {}
- if tqx:
- tqx_dict = dict(opt.split(":") for opt in tqx.split(";"))
- if tqx_dict.get("version", "0.6") != "0.6":
- raise DataTableException(
- "Version (%s) passed by request is not supported."
- % tqx_dict["version"])
-
- if tqx_dict.get("out", "json") == "json":
- response_handler = tqx_dict.get("responseHandler",
- "google.visualization.Query.setResponse")
- return self.ToJSonResponse(columns_order, order_by,
- req_id=tqx_dict.get("reqId", 0),
- response_handler=response_handler)
- elif tqx_dict["out"] == "html":
- return self.ToHtml(columns_order, order_by)
- elif tqx_dict["out"] == "csv":
- return self.ToCsv(columns_order, order_by)
- elif tqx_dict["out"] == "tsv-excel":
- return self.ToTsvExcel(columns_order, order_by)
- else:
- raise DataTableException(
- "'out' parameter: '%s' is not supported" % tqx_dict["out"])
diff --git a/site_utils/dashboard/monitoring_view.py b/site_utils/dashboard/monitoring_view.py
deleted file mode 100644
index 9788c0d..0000000
--- a/site_utils/dashboard/monitoring_view.py
+++ /dev/null
@@ -1,58 +0,0 @@
-# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Class for tracking a few monitoring statistics.
-
-This class allows time-based stats to be queried and updated into a local
-db (json file) to track and notice unexpected trends in database data.
-The initial example is to periodically record the count of the django_session
-table over time and graph it to notice unexpected spikes.
-
-Includes: class MonitoringView(object)
-"""
-
-import json
-import os
-
-MONITORING_DB_FILE = "monitoring_db.json"
-
-
-class MonitoringView(object):
- """View used to show monitoring information in summary views.
-
- Initially, this will be used to watch for specific undesirable activities
- in the database. Perhaps this will get re-abstracted into something more
- substantial.
- """
-
- def __init__(self, dash_base_dir):
- """Retrieve the contents of the current monitoring db file.
-
- No locking protects this file.
-
- Monitoring File should be of the following format:
- {
- "django_session": [{"time": "Wed Jun 6 10:33:02 PDT 2012",
- "sessions": 170},
- {"time": "Wed Jun 6 13:14:47 PDT 2012",
- "sessions": 52422}, ...]
- }
- """
- self._monitoring_path = os.path.join(dash_base_dir, MONITORING_DB_FILE)
- if not os.path.exists(self._monitoring_path):
- self.monitoring_db = {'django_session': []}
- else:
- self.monitoring_db = json.load(open(self._monitoring_path))
-
- def UpdateMonitoringDB(self, db_key, updated_value_dict):
- """Add a newly retrieved data value to the in-memory db."""
- self.monitoring_db[db_key].append(updated_value_dict)
-
- def WriteMonitoringDB(self):
- """Write the updated monitoring db file.
-
- Not protected by locking code.
- """
- with open(self._monitoring_path, 'w') as f:
- f.write(json.dumps(self.monitoring_db))
diff --git a/site_utils/dashboard/plot_gen.py b/site_utils/dashboard/plot_gen.py
deleted file mode 100755
index 9d044e8..0000000
--- a/site_utils/dashboard/plot_gen.py
+++ /dev/null
@@ -1,75 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Build index.html and plot .png/.html files for desired perf plots."""
-
-import datetime
-import logging
-import os
-import re
-
-from django.shortcuts import render_to_response
-
-import dash_util
-
-from dash_view import AutotestDashView
-from monitoring_view import MonitoringView
-
-# String resources.
-from dash_strings import DASHBOARD_MAIN
-from dash_strings import PERF_INDEX_FILE
-from dash_strings import PLOT_FILE
-from dash_strings import PLOT_MONITORING_FILE
-
-
-def PlotAllNetbook(
- base_dir, dash_view, tpl_netbook, tpl_board):
- """Invoke plot function on all requested plots to create output html files."""
- logging.info('build %s plots into %s', tpl_netbook, base_dir)
-
- # Produce the main test results + plots combo page for each netbook.
- tpl_last_updated = dash_view.GetFormattedLastUpdated()
- dash_util.SaveHTML(
- os.path.join(base_dir, PLOT_FILE),
- render_to_response(
- os.path.join('tables/details', PLOT_FILE),
- locals()).content)
- # Produce a performance landing page + plots combo page for each netbook.
- dash_util.SaveHTML(
- os.path.join(base_dir, PERF_INDEX_FILE),
- render_to_response(
- os.path.join('tables/details', PERF_INDEX_FILE),
- locals()).content)
-
-
-def PlotMonitoringViews(base_dir, dash_view):
- """Build plot pages of monitored values such as django_session."""
- monitoring_view = MonitoringView(base_dir)
- current_time = datetime.datetime.now().isoformat()
- entry = {'time': current_time.split('.')[0],
- 'sessions': dash_view.QueryDjangoSession()}
- monitoring_view.UpdateMonitoringDB('django_session', entry)
- monitoring_view.WriteMonitoringDB()
- tpl_last_updated = dash_view.GetFormattedLastUpdated()
- tpl_monitoring = monitoring_view.monitoring_db
- dash_util.SaveHTML(
- os.path.join(base_dir, PLOT_MONITORING_FILE),
- render_to_response(
- os.path.join('tables', PLOT_MONITORING_FILE),
- locals()).content)
-
-
-def BuildAllPlots(dash_base_dir, dash_view):
- """Build all plots for each netbook and board."""
- for netbook in dash_view.netbooks:
- for board in dash_view.GetNetbookBoardTypes(netbook):
- base_dir = os.path.join(dash_base_dir, netbook, board)
- if not os.path.exists(base_dir):
- dash_util.MakeChmodDirs(base_dir)
- PlotAllNetbook(base_dir, dash_view, netbook, board)
- PlotMonitoringViews(dash_base_dir, dash_view)
-
-
-if __name__ == '__main__':
- print 'Run %s with --plot-generate.' % DASHBOARD_MAIN
diff --git a/site_utils/dashboard/preprocess_functions.py b/site_utils/dashboard/preprocess_functions.py
deleted file mode 100644
index 800be80..0000000
--- a/site_utils/dashboard/preprocess_functions.py
+++ /dev/null
@@ -1,180 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Functions called to preprocess perf data for regressions."""
-
-import logging
-import operator
-
-import dash_util
-
-# String resources.
-from dash_strings import PREPROCESSED_TAG
-from dash_strings import EMAIL_ALERT_DELTA_TABLE_SKELETON
-from dash_strings import EMAIL_ALERT_DELTA_TABLE_ROW
-
-PREFIX_LEN = 9
-
-class PreprocessFunctions(object):
- """Class to contain and invoke preprocessing functions."""
-
- def Invoke(self, function_name, params, keyvals, checks):
- """Dispatch function and return True if failed."""
- if not hasattr(self, function_name):
- logging.debug('Preprocessing function %s not found.', function_name)
- return False
- return getattr(self, function_name)(params, keyvals, checks)
-
- def _MakePreprocessedKey(self, key, seq=None):
- """Helper to distinguish created preprocessing keys from regulars."""
- new_key = '%s%s' % (key, PREPROCESSED_TAG)
- if seq:
- new_key = '%s%s%s' % (seq, PREPROCESSED_TAG, new_key)
- return new_key
-
- def _IsPreprocessedKey(self, key):
- """Helper to decide if a key was produced by us."""
- key_len = len(key)
- pp_len = len(PREPROCESSED_TAG)
- return key[key_len-pp_len:] == PREPROCESSED_TAG, pp_len, key_len
-
- def _GetOriginalKeySeq(self, key):
- """Helper to distinguish created preprocessing keys from regulars."""
- new_key = ''
- seq = 0
- is_pp, pp_len, key_len = self._IsPreprocessedKey(key)
- if is_pp:
- new_key = key[:key_len-pp_len]
- n = new_key.find(PREPROCESSED_TAG)
- if n > -1:
- seq = int(new_key[:n])
- new_key = new_key[n+pp_len:]
- return new_key, seq
-
- def IsPreprocessedKey(self, key):
- is_pp, _, _ = self._IsPreprocessedKey(key)
- return is_pp
-
- def PreprocessorHTML(self, test_name, regressed_keys):
- """Process preprocessed keys and related details into a summary."""
- # Preserve order using seq hints.
- preprocessed_in_order = []
- preprocessed_details = {}
- for test_key, regressed_stats in regressed_keys.iteritems():
- build_val = regressed_stats['build_mean']
- build_npoints = regressed_stats['build_samples']
- expected_val = regressed_stats['historical_mean']
- expected_npoints = regressed_stats['historical_samples']
- is_pp, _, _ = self._IsPreprocessedKey(test_key)
- if is_pp:
- orig_key, seq = self._GetOriginalKeySeq(test_key)
- details_row = preprocessed_details.setdefault(
- orig_key, [0.0, 0.0, 0.0, 0.0])
- details_row[0] = build_val
- details_row[1] = expected_val
- preprocessed_in_order.append((orig_key, seq))
- else:
- details_row = preprocessed_details.setdefault(
- test_key, [0.0, 0.0, 0.0, 0.0])
- details_row[2] = build_val
- details_row[3] = expected_val
- preprocessed_in_order.sort(key=operator.itemgetter(1))
-
- # Build html.
- converter = dash_util.HumanReadableFloat()
- current_table = []
- table_list = [current_table]
- previous_key = None
- for one_key, seq in preprocessed_in_order:
- if previous_key and (previous_key[:PREFIX_LEN] != one_key[:PREFIX_LEN]):
- current_table = []
- table_list.append(current_table)
- pp_build_val, pp_expected_val, build_val, expected_val = (
- preprocessed_details[one_key])
- current_table.append(
- EMAIL_ALERT_DELTA_TABLE_ROW % {
- 'key': one_key,
- 'pp_latest': converter.Convert(pp_build_val),
- 'pp_average': converter.Convert(pp_expected_val),
- 'latest': converter.Convert(build_val),
- 'average': converter.Convert(expected_val)})
- previous_key = one_key
-
- preprocessed_html = []
- for current_table in table_list:
- preprocessed_html.append(
- EMAIL_ALERT_DELTA_TABLE_SKELETON % {
- 'test_name': test_name,
- 'body': ''.join(current_table)})
- return preprocessed_html
-
- def GroupDeltas(self, params, keyvals, checks):
- """Create new keyvals using deltas based on existing keyvals."""
- # Average the values for each checked key and build into a structure
- # just like the existing keyvals.
- stub_test_id = 0
- key_build_averages = {}
- build_key_counts = {}
- for one_key in checks:
- key_build_averages[one_key] = {}
- for build, values in keyvals[one_key].iteritems():
- key_set = build_key_counts.setdefault(build, set())
- key_set.add(one_key)
- key_build_averages[one_key][build] = (
- [sum(values[0], 0.0) / len(values[0])], [stub_test_id])
-
- # Figure out the relative order of the keys in increasing
- # order of one build's average values. Use the build with the
- # most keys as a reference.
- high_water_build = None
- high_water_count = 0
- for build, key_set in build_key_counts.iteritems():
- if len(key_set) > high_water_count:
- high_water_count = len(key_set)
- high_water_build = build
- averages = []
- for one_key in checks:
- if (not high_water_build or
- not high_water_build in key_build_averages[one_key]):
- logging.warning(
- 'Key %s is missing build %s in GroupDeltas().',
- one_key, high_water_build)
- else:
- averages.append((
- one_key, key_build_averages[one_key][high_water_build][0]))
- averages.sort(key=operator.itemgetter(1))
-
- # Generate the new keys that use deltas as values.
- # Group them according to a prefix on each key.
- prefix_groups = {}
- for one_key, _ in averages:
- key_list = prefix_groups.setdefault(one_key[:PREFIX_LEN], [])
- key_list.append(one_key)
-
- i = 1 # For later sorting of the group by value.
- delta_prefix_groups = prefix_groups.keys()
- delta_prefix_groups.sort()
- for one_key_group in delta_prefix_groups:
- previous_key = None
- for one_key in prefix_groups[one_key_group]:
- new_key_name = self._MakePreprocessedKey(one_key, i)
- # Add the new key to the checks.
- checks[new_key_name] = checks[one_key]
-
- # Add the new key and data into keyvals.
- if previous_key:
- # Calculate the deltas of calculated averages.
- for build in key_build_averages[one_key].iterkeys():
- if (build in key_build_averages[one_key] and
- build in key_build_averages[previous_key]):
- new_keyval = keyvals.setdefault(new_key_name, {})
- new_keyval[build] = ([
- key_build_averages[one_key][build][0][0] -
- key_build_averages[previous_key][build][0][0]],
- [stub_test_id])
- else:
- # Copy the structure from the averages calculated.
- keyvals[new_key_name] = key_build_averages[one_key]
- previous_key = one_key
- i += 1
diff --git a/site_utils/dashboard/run_copy_dashboard.sh b/site_utils/dashboard/run_copy_dashboard.sh
deleted file mode 100755
index ff85097..0000000
--- a/site_utils/dashboard/run_copy_dashboard.sh
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/bin/bash
-#
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-#
-# Author: truty@google.com (Mike Truty)
-#
-# This script for building dash and copying resulting files.
-
-declare -r SCRIPT_DIR="$(cd $(dirname $0); pwd)"
-
-declare -r EXEC_BASE=$(dirname "$0")
-declare -r RESULTS_BASE="/usr/local/autotest/results/dashboard"
-
-declare -r RESULTS_SERVER="cautotest.corp.google.com"
-declare -r ROLE_ACCOUNT="chromeos-test"
-
-set -e
-
-function create_copy_dash() {
- local result_base=$1
- local result_parent=$(dirname ${result_base})
- local job_limit=$2
- local extra_options=$3
-
- ${EXEC_BASE}/run_generate.py \
- --config-file=${EXEC_BASE}/dash_config.json \
- -d ${result_base} \
- -j ${job_limit} \
- ${extra_options} &> /dev/null
-}
-
-if [[ $1 != "dashboard" && $1 != "email" ]]; then
- echo "Usage: `basename $0` [dashboard | email]"
- exit $E_BADARGS
-fi
-
-if [[ $1 == "dashboard" ]]; then
- # Create and copy regular dash.
- create_copy_dash ${RESULTS_BASE} 10000 "-t -p"
- # Generate alerts.
- create_copy_dash ${RESULTS_BASE} 3000 "-a"
-elif [[ $1 == "email" ]]; then
- # Create and copy regular dash.
- create_copy_dash ${RESULTS_BASE} 1500 "-m"
-fi
diff --git a/site_utils/dashboard/run_generate.py b/site_utils/dashboard/run_generate.py
deleted file mode 100755
index bdd28e3..0000000
--- a/site_utils/dashboard/run_generate.py
+++ /dev/null
@@ -1,208 +0,0 @@
-#!/usr/bin/python
-# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Master chromeos test dashboard/email driver - called by cron."""
-
-import commands
-import getpass
-import json
-import logging
-import optparse
-import os
-import sys
-
-# dash_common and dash_view needed first for anything db or django related.
-import dash_common
-import dash_util
-from dash_view import AutotestDashView
-
-from alert_email import AlertAll
-from build_info import BuildInfo
-from dash_email import EmailAllFailures
-from dash_email import EmailFromConfig
-from plot_gen import BuildAllPlots
-from table_gen import BuildAllTables
-
-# String resources.
-from dash_strings import AUTOTEST_USER
-from dash_strings import LAST_N_JOBS_LIMIT
-from dash_strings import SUMMARY_TABLE_ROW_LIMIT
-
-
-def ParseArgs(argv):
- base_dir = os.path.dirname(os.path.abspath(argv[0]))
-
- parser = optparse.OptionParser()
- parser.add_option('-a', '--alert-generate',
- help='regression alert email [default: %default]',
- dest='alertgenerate', action='store_true', default=False)
- parser.add_option('-c', '--config-file',
- help='config file [default: %default]',
- dest='configfile', default="dash_config.json")
- parser.add_option('-d', '--dash-dir',
- help='base dashboar dir [default: %default]',
- dest='dashdir',
- default='/usr/local/autotest/results/dashboard')
- parser.add_option('-j', '--job-limit',
- help='limit to last n jobs [default: %default]',
- dest='joblimit', type='int', default=LAST_N_JOBS_LIMIT)
- parser.add_option('-k', '--keep-build-cache',
- help='avoid pruning cache [default: %default]',
- dest='keepbuildcache', action='store_true', default=False)
- parser.add_option('-m', '--mail-generate',
- help='send failure emails [default: %default]',
- dest='mailgenerate', action='store_true', default=False)
- parser.add_option('-n', '--no-execute',
- help='Do not execute subcommands [default: %default]',
- dest='noexecute', action='store_true', default=False)
- parser.add_option('-p', '--plot-generate',
- help='build dash test plots [default: %default]',
- dest='plotgenerate', action='store_true', default=False)
- parser.add_option('-t', '--table-generate',
- help='build dash test tables [default: %default]',
- dest='tablegenerate', action='store_true', default=False)
- parser.add_option('', '--summary-limit',
- help='max rows in summaries [default: %default]',
- dest='summarylimit', type='int',
- default=SUMMARY_TABLE_ROW_LIMIT)
- parser.add_option('', '--waterfall-limit',
- help='max rows in waterfall summaries [default: %default]',
- dest='waterfalllimit', type='int',
- default=SUMMARY_TABLE_ROW_LIMIT)
- parser.add_option('', '--profile',
- help='Enable profiling of execution [default: %default]',
- dest='profiler', action='store_true', default=False)
- parser.add_option('-v', '--verbose',
- help='Show more output [default: %default]',
- dest='verbose', action='store_true', default=False)
- options, args = parser.parse_args()
-
- logging_level = logging.INFO
- if options.verbose:
- logging_level = logging.DEBUG
-
- logging.basicConfig(level=logging_level)
-
- return options, base_dir
-
-
-def CheckOptions(options):
- dash_base_dir = ''
- if (not options.alertgenerate and
- not options.mailgenerate and
- not options.plotgenerate and
- not options.tablegenerate):
- logging.fatal(
- 'Must supply at least 1 command from: '
- '--alert-generate, --mail-generate, '
- '--plot-generate or --table-generate.')
- sys.exit(1)
-
- me = getpass.getuser()
- if options.dashdir:
- dash_base_dir = options.dashdir
- elif me == AUTOTEST_USER:
- dash_base_dir = '/usr/local/autotest/results/dashboard'
- else:
- dash_base_dir = '/home/%s/www/dash' % me
-
- if not os.path.exists(dash_base_dir):
- dash_util.MakeChmodDirs(dash_base_dir)
-
- logging.info("Using dir: %s.", dash_base_dir)
- return dash_base_dir
-
-
-def GetJSONOptions(current_dir, json_file):
- if not os.path.exists(json_file):
- json_file = os.path.join(current_dir, json_file)
- if not os.path.exists(json_file):
- return None
- return json.load(open(json_file))
-
-
-def DoWork(options, base_dir):
- diag = dash_util.DebugTiming()
- dash_base_dir = CheckOptions(options)
- # Umask needed for permissions on created dirs.
- prev_dir = os.getcwd()
- os.chdir(dash_base_dir)
- prev_umask = os.umask(0)
-
- # Dash config file sets things up.
- dash_options = GetJSONOptions(base_dir, options.configfile)
- if not dash_options:
- logging.fatal('Missing config.')
- sys.exit(1)
-
- # Populate data model.
- dash_view = AutotestDashView()
- dash_view.CrashSetup(dash_base_dir)
- dash_view.SetDashConfig(dash_options)
- if not options.noexecute:
- dash_view.LoadFromDB(options.joblimit)
-
- # Build info singleton cache for performance improvement.
- build_info = BuildInfo()
-
- if options.tablegenerate:
- logging.info("Generating tables.")
- if not options.noexecute:
- BuildAllTables(dash_base_dir, dash_view, dash_options,
- options.summarylimit, options.waterfalllimit)
-
- if options.mailgenerate:
- logging.info("Generating email.")
- if not options.noexecute:
- EmailFromConfig(dash_base_dir, dash_view, dash_options)
- if not options.noexecute:
- EmailAllFailures(dash_base_dir, dash_view)
-
- if options.plotgenerate:
- logging.info("Generating plots.")
- if not options.noexecute:
- BuildAllPlots(dash_base_dir, dash_view)
-
- if options.alertgenerate:
- logging.info("Generating alerts.")
- if not options.noexecute:
- dash_view.LoadPerfFromDB(options.joblimit)
- AlertAll(dash_base_dir, dash_view, dash_options)
-
- if not options.keepbuildcache:
- if not options.noexecute:
- build_info.PruneTmpFiles(dash_view)
-
- os.umask(prev_umask)
- os.chdir(prev_dir)
- del diag
-
-
-def main(argv):
- """Can generate tables, plots and email."""
- options, base_dir = ParseArgs(argv)
- do_work = 'DoWork(options, base_dir)'
- if options.profiler:
- logging.info('Profiling...')
- import tempfile, cProfile, pstats
- base_filename = os.path.basename(os.path.abspath(argv[0]))
- pname = os.path.join(tempfile.gettempdir(),
- '%s.profiler.out' % base_filename)
- logging.debug('Using profile file: %s.', pname)
- cProfile.runctx(do_work, globals=globals(), locals=locals(),
- filename=pname)
- p = pstats.Stats(pname)
- p.sort_stats('cumulative').print_stats(20)
- pngname = os.path.join(tempfile.gettempdir(), '%s.png' % base_filename)
- png_command = 'python %s -f pstats %s | dot -Tpng -o %s' % (
- os.path.join(base_dir, 'external', 'gprof2dot.py'), pname, pngname)
- logging.debug(png_command)
- commands.getoutput(png_command)
- else:
- exec(do_work)
-
-
-if __name__ == '__main__':
- main(sys.argv)
diff --git a/site_utils/dashboard/stats_functions.py b/site_utils/dashboard/stats_functions.py
deleted file mode 100644
index 2157a43..0000000
--- a/site_utils/dashboard/stats_functions.py
+++ /dev/null
@@ -1,160 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Functions called to analyze perf data for regressions."""
-
-import logging
-
-import external.stats as stats
-
-from dash_view import AutotestDashView
-
-
-class StatsFunctions(object):
- """Class to contain and invoke statistics functions."""
-
- def __init__(self):
- self._dash_view = AutotestDashView()
-
- def Invoke(self, function_name, params, vals, build):
- """Dispatch function and return True if failed."""
- if not hasattr(self, function_name):
- logging.debug('Stats function %s not found.', function_name)
- return False
- if build not in vals:
- logging.debug('Build %s not found in vals %s.', build, vals.keys())
- return False
- return getattr(self, function_name)(params, vals, build)
-
- def _Averages(self, vals, build):
- """Calculate averages for one build and all days."""
- build_mean = 0.0
- build_nsamples = 0
- previous_mean = 0.0
- previous_nsamples = 0
- mean_list = []
- sum_nsamples = 0
- # Loop through each build with values.
- for seq in vals:
- value_list = vals[seq][0]
- mean_value = stats.lmean(value_list)
- if build == seq:
- build_mean = mean_value
- build_nsamples = len(value_list)
- else:
- mean_list.append(mean_value)
- sum_nsamples += len(value_list)
- # Average over all builds prior to and not including this build.
- if mean_list:
- historical_mean = stats.lmean(mean_list)
- historical_samples = sum_nsamples / len(mean_list)
- else:
- historical_mean = 0.0
- historical_samples = 0
- results = {
- 'build_mean': build_mean,
- 'build_samples': build_nsamples,
- 'historical_mean': historical_mean,
- 'historical_samples': historical_samples}
- return results
-
- def PrintAverages(self, params, vals, build):
- """Always returns True - for regular summaries."""
- data_statistics = self._Averages(vals, build)
- return True, data_statistics
-
- def PrintStats(self, params, vals, build):
- """Always returns True - for detailed summaries."""
- value_list = vals[build][0]
- stats_lstdev = 0.0
- stats_lmed = value_list[0]
- if len(value_list) > 1:
- stats_lstdev = stats.lstdev(value_list)
- stats_lmed = stats.lmedianscore(value_list)
- # This is a 'sample standard deviation'.
- data_statistics = {
- 'build_sample_stdev': stats_lstdev,
- 'build_median_value': stats_lmed}
- return True, data_statistics
-
- def PrintHistogram(self, params, vals, build):
- """Always returns True - for detailed summaries."""
- numbins = params['numbins']
- limits = params['limits']
- bins, lowbin, binsize, lowpoints, highpoints = stats.lhistogram(
- vals[build][0], numbins, limits)
- data_array = []
- if lowpoints:
- data_array.append((lowpoints, 0, round(lowbin, 2)))
- for i in xrange(len(bins)):
- data_array.append((
- bins[i],
- round(lowbin+(binsize*i), 2),
- round(lowbin+(binsize*i)+binsize, 2)))
- if highpoints:
- data_array.append((highpoints, lowbin+binsize*len(bins), '...'))
- data_statistics = {
- 'histogram': {
- 'data': data_array,
- 'height': params['height'],
- 'width': params['width']}}
- return True, data_statistics
-
- def PrintIterations(self, params, vals, build):
- """Always returns True - for detailed summaries."""
- value_list = vals[build][0]
- test_idxes = vals[build][1]
- if len(value_list) <= 1:
- return False, {}
- iterations = vals[build][2]
- list_len = len(value_list)
- if not list_len == len(iterations):
- logging.warning('KeyVals without matching iterations on build %s.', build)
- return False, {}
- previous_iteration = 0 # Autotest iterations are 1-based.
- i = 1
- column_array = [(
- vals[build][3][0],
- self._dash_view.GetTestFromIdx(test_idxes[0])['tag'])]
- value_array = []
- known_iterations = set()
- for j in xrange(list_len):
- iteration = iterations[j]
- value = value_list[j]
- if iteration <= previous_iteration:
- i += 1
- column_array.append((
- vals[build][3][j],
- self._dash_view.GetTestFromIdx(test_idxes[j])['tag']))
- if not iteration in known_iterations:
- value_array.append((iteration-1, 0, iteration))
- known_iterations.add(iteration)
- value_array.append((iteration-1, i, value))
- previous_iteration = iteration
- data_statistics = {
- 'values': {
- 'column_names': column_array,
- 'rowcount': len(known_iterations),
- 'data': value_array,
- 'height': params['height'],
- 'width': params['width']}}
- return True, data_statistics
-
- def OverAverage(self, params, vals, build):
- """Returns True if build average is > overall average."""
- data_statistics = self._Averages(vals, build)
- build_avg, _, overall_avg, _ = data_statistics
- return build_avg > overall_avg, data_statistics
-
- def OverThreshhold(self, params, vals, build):
- """Returns True if latest build is > threshhold."""
- data_statistics = self._Averages(vals, build)
- build_avg, _, _, _ = data_statistics
- return build_avg > float(params), data_statistics
-
- def UnderThreshhold(self, params, vals, build):
- """Returns True if latest build is < threshhold."""
- data_statistics = self._Averages(vals, build)
- build_avg, _, _, _ = data_statistics
- return build_avg < float(params), data_statistics
diff --git a/site_utils/dashboard/table_builder.py b/site_utils/dashboard/table_builder.py
deleted file mode 100644
index 91de01f..0000000
--- a/site_utils/dashboard/table_builder.py
+++ /dev/null
@@ -1,166 +0,0 @@
-# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Class to wrap building detail table for board/netbook/testcategory."""
-
-import os
-
-import dash_util
-
-from build_info import BuildInfo
-from dash_view import AutotestDashView
-
-# String resources.
-from dash_strings import BVT_TAG
-
-# Constants
-DEFAULT_TEST_NAME_LENGTH = 18
-
-
-class TableBuilder(object):
- """Class manages building header and body for details tables."""
-
- def __init__(self, base_dir, netbook, board_type, category):
- self._netbook = netbook
- self._board_type = board_type
- self._category = category
- self._build_info = BuildInfo()
- self._dash_view = AutotestDashView()
- self._test_list = self._dash_view.GetTestNames(
- netbook, board_type, category)
- self._test_list.sort()
- self._build_numbers = self._dash_view.GetBuilds(
- netbook, board_type, category)
- self._good_set, self._failed_list = self._SplitTestList()
-
- def _SplitTestList(self):
- """Determine and list the tests that passed and failed."""
- good_set = set()
- failed_list = [] # Maintains order discovered.
-
- for build in self._build_numbers:
- for test_name in self._test_list:
- test_details = self._GetTestDetails(test_name, build)
- if test_details:
- for t in test_details:
- good_status = (t['status'] == 'GOOD')
- test_tuple = (test_name, t['experimental'])
- if test_tuple in failed_list:
- continue
- if good_status:
- good_set.add(test_tuple)
- else:
- failed_list.append(test_tuple)
- good_set.discard(test_tuple)
- return sorted(good_set), failed_list
-
- def _BuildTableHeader(self, test_list):
- """Generate header with test names for columns."""
- table_header = []
- for test_name, experimental in test_list:
- test_path = self._dash_view.GetAutotestInfo(test_name)[1]
- if len(test_name) > DEFAULT_TEST_NAME_LENGTH:
- test_alias = test_name[:DEFAULT_TEST_NAME_LENGTH] + '...'
- else:
- test_alias = test_name
- table_header.append((test_path, test_alias.replace('_', ' '),
- test_name, experimental))
- return table_header
-
- def _GetBuildMetadata(self, build):
- """Retrieve info used to populate build header popups."""
- started, finished, elapsed = self._dash_view.GetFormattedJobTimes(
- self._netbook, self._board_type, self._category, build)
- fstarted, ffinished, felapsed, ffinished_short = (
- self._build_info.GetFormattedBuildTimes(self._board_type, build))
- return (fstarted, ffinished, felapsed,
- started, finished, elapsed,
- self._dash_view.GetFormattedLastUpdated())
-
- def _GetTestDetails(self, test_name, build):
- return self._dash_view.GetTestDetails(
- self._netbook, self._board_type, self._category, test_name, build)
-
- def _BuildTableBody(self, test_list):
- """Generate table body with test results in cells."""
- table_body = []
-
- for build in self._build_numbers:
- chrome_version = self._build_info.GetChromeVersion(self._board_type,
- build)
- test_status_list = []
- for test_name, experimental in test_list:
- # Include either the good details or the details of the
- # first failure in the list (last chronological failure).
- cell_content = []
- test_details = self._GetTestDetails(test_name, build)
- if test_details:
- total_tests = len(test_details)
- passed_tests = 0
- failed_tests = 0
- for t in test_details:
- current_fail = False
- test_status = t['status']
- if test_status == 'GOOD':
- passed_tests += 1
- query = '%(tag)s' % t
- else:
- failed_tests += 1
- current_fail = True
- query = '%(tag)s/%(test_name)s' % t
- # Populate the detailed cell popups prudently.
- host_info = []
- chrome_version_attr = chrome_version
- if chrome_version and len(chrome_version) == 2:
- chrome_version_attr = '%s (%s)' % (chrome_version[0],
- chrome_version[1])
- priority_attrs = [
- ('Chrome Version', 'chrome-version', chrome_version_attr),
- ('ChromeOS Version', 'CHROMEOS_RELEASE_DESCRIPTION', None),
- ('Platform', 'host-platform', None),
- ('Kernel Version', 'sysinfo-uname', None),
- ('Reason', 'reason', None)]
- for popup_header, attr_key, default in priority_attrs:
- attr_value = t['attr'].get(attr_key, default)
- if attr_value:
- host_info.append((popup_header, attr_value))
- test_status = test_status[0].upper()
- # Treat TEST_NA as WARNING.
- if test_status == 'T':
- test_status = 'W'
- if (not cell_content) or (current_fail and failed_tests == 1):
- cell_content = [test_name, t['hostname'], host_info, query,
- test_status]
- if cell_content:
- test_summaries = [passed_tests, total_tests]
- test_summaries.extend(
- self._dash_view.GetCrashes().GetBuildTestCrashSummary(
- self._netbook, self._board_type, build, test_name))
- cell_content.extend(test_summaries)
- test_status_list.append(cell_content)
- popup = self._GetBuildMetadata(build)
- table_body.append(('', build, popup, test_status_list, chrome_version))
- return table_body
-
- def BuildTables(self):
- """Generate table body with test results in cells."""
- good_table_header = self._BuildTableHeader(self._good_set)
- good_table_body = self._BuildTableBody(self._good_set)
- result = [{'label': 'Good Tests',
- 'header': good_table_header,
- 'body': good_table_body}]
- if self._failed_list:
- failed_table_header = self._BuildTableHeader(self._failed_list)
- failed_table_body = self._BuildTableBody(self._failed_list)
- result.insert(0, {'label': 'Failed Tests',
- 'header': failed_table_header,
- 'body': failed_table_body})
- return result
-
- def CountTestList(self):
- """Count the number of tests and failed ones."""
- if self._build_numbers:
- return len(self._good_set)+len(self._failed_list), len(self._failed_list)
- else:
- return 0, 0
diff --git a/site_utils/dashboard/table_gen.py b/site_utils/dashboard/table_gen.py
deleted file mode 100755
index 23e47f3..0000000
--- a/site_utils/dashboard/table_gen.py
+++ /dev/null
@@ -1,318 +0,0 @@
-# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Generate detail table .html files for board-netbook-testcategories."""
-
-import datetime
-import logging
-import os
-
-from django.shortcuts import render_to_response
-
-import dash_util
-
-from build_info import BuildInfo
-from dash_view import SummaryRanges
-from table_builder import TableBuilder
-
-# String resources.
-from dash_strings import BVT_TAG
-from dash_strings import DASHBOARD_MAIN
-from dash_strings import EMAILS_SUMMARY_FILE
-from dash_strings import KERNEL_TABLE_FILE
-from dash_strings import KERNEL_WATERFALL_FILE
-from dash_strings import KERNELTEST_TAG
-from dash_strings import PERF_BUILDS_FILE
-from dash_strings import TEST_DETAILS_FILE
-from dash_strings import TEST_LANDING_FILE
-from dash_strings import TEST_WATERFALL_FILE
-from dash_strings import UNKNOWN_TIME_STR
-
-
-def _BuildNetbookHTML(
- dash_base_dir, dash_view, tpl_netbook, tpl_board,
- default_category, dash_options):
- """Create table files for all categories of a netbook-board.
-
- Produces a set of pages of test results for a given netbook,
- board and categories with tests executed.
-
- Args:
- dash_base_dir: base of dashboard output files.
- dash_view: data model with all test result details.
- tpl_netbook: netbook (e.g. netbook_MARIO_MP) for these pages.
- tpl_board: board (e.g. x86-mario) for these pages.
- default_category: landing page when switching pages.
- dash_options: config options used for setting landing pages.
- """
- base_dir = os.path.join(dash_base_dir, tpl_netbook, tpl_board)
- if not os.path.exists(base_dir):
- dash_util.MakeChmodDirs(base_dir)
- logging.info('build %s into %s', tpl_netbook, base_dir)
-
- # Build a details page for each category.
- alternate_landings = dash_options.get('alternatelandings', {})
- tpl_board_netbooks = [
- (n, alternate_landings.get(tpl_board, {}).get(n, BVT_TAG))
- for n in dash_view.GetNetbooksWithBoardType(tpl_board)
- if n != tpl_netbook]
- tpl_other_boards = sorted(
- [b
- for b in dash_view.GetNetbookBoardTypes(tpl_netbook)
- if b != tpl_board])
- tpl_last_updated = dash_view.GetFormattedLastUpdated()
-
- tpl_categories = dash_view.GetUICategories(tpl_netbook, tpl_board)
- if not default_category in tpl_categories:
- tpl_categories.append(default_category)
- tpl_categories.sort()
-
- tpl_categories_with_color = []
- for tpl_category in tpl_categories:
- table_builder = TableBuilder(dash_base_dir, tpl_netbook, tpl_board,
- tpl_category)
- total, failed = table_builder.CountTestList()
- if failed:
- label = tpl_category + '(%d/%d)' % (total, failed)
- else:
- label = tpl_category + '(%d)' % total
- if total == 0:
- bg_class = 'white'
- elif failed == 0:
- bg_class = 'success'
- elif total == failed:
- bg_class = 'failure'
- else:
- bg_class = 'warning'
- tpl_categories_with_color.append(
- (tpl_category, table_builder, label, bg_class))
-
- # Produce a test results page for each test category.
- tpl_perf_builds = None
- for tpl_category, table_builder, label, bg_class in tpl_categories_with_color:
- tpl_tables = table_builder.BuildTables()
- if tpl_category == BVT_TAG and tpl_tables:
- tpl_perf_builds = tpl_tables[0]['body']
- dash_util.SaveHTML(
- os.path.join(base_dir, '%s.html' % tpl_category),
- render_to_response(
- os.path.join('tables/details', TEST_DETAILS_FILE),
- locals()).content)
-
- # Produce a performance landing page.
- tpl_perf_available = False
- if 'alerts' in dash_options:
- for alert in dash_options['alerts']:
- if ('platforms' in alert and
- {tpl_board: tpl_netbook} in alert['platforms']):
- tpl_perf_available = True
- dash_util.SaveHTML(
- os.path.join(base_dir, PERF_BUILDS_FILE),
- render_to_response(
- os.path.join('tables/details', PERF_BUILDS_FILE),
- locals()).content)
-
-
-def _GetLandingDetails(dash_view, summary_ranges, netbook, board, category,
- build):
- """Gather the summary details for one build (row).
-
- If the dashboard presented results from one source this
- would not be needed. Since we grab data from test results,
- test attributes and crash summaries this sort of function
- is needed to collect distributed data for one build and category.
-
- Args:
- dash_view: data model with all test result details.
- summary_ranges: limits for data queries in this summary.
- netbook: netbook (e.g. netbook_MARIO_MP) for these pages.
- board: board (e.g. x86-mario) for these pages.
- category: summary producted for bvt now, maybe something else later.
- build: build to use.
-
- Returns:
- Tuple of data for populating one waterfall row (build).
- """
- job_attempted, job_good, passed, total, xpassed, xtotal = (
- dash_view.GetCategorySummary(netbook, board, category, build))
- kernel = summary_ranges.GetKernel(board, netbook, build)
- failed_tests = dash_view.GetCategoryFailedTests(
- netbook, board, category, build)
- if category == BVT_TAG:
- category = None
- crashes, crash_count, crash_category = (
- dash_view.GetCrashes().GetBuildCrashSummary(netbook, board, build,
- category))
- # x86-alex-r18 -> x86-alex
- # x86-generic-full -> x86-generic-full
- release_index = board.rfind('-r')
- if release_index > 0:
- pure_board = board[:release_index]
- else:
- pure_board = board
- return (board, pure_board, netbook, build, job_attempted, job_good, passed,
- total, xpassed, xtotal, kernel, failed_tests, crashes, crash_count,
- crash_category)
-
-
-def BuildLandingSummaries(dash_view, category, tots, branches, summary_ranges):
- """Produces individual table for each board of test summaries per build.
-
- This produces the data for the 'new-style' (waterfall) dashboard summary.
-
- Args:
- dash_view: data model with all test result details.
- category: summary producted for bvt now, maybe something else later.
- tots: boards to be grouped at the top of page.
- branches: boards to be grouped next.
- summary_ranges: limits for data queries in this summary.
-
- Returns:
- A dictionary with the data for the waterfall display.
- """
- platforms = []
- builds = {}
- build_info = BuildInfo()
- results_dict = {}
- releases = set()
- irregular_releases = set()
-
- for board in tots + branches:
- parsed_board, release = dash_view.ParseBoard(board)
- if release:
- releases.add(release)
- else:
- irregular_releases.add(board)
- for build_number in summary_ranges.GetBuildNumbers(board):
- build_results = results_dict.setdefault(build_number, {})
- for netbook in summary_ranges.GetNetbooks(board):
- # Aggregate the test summaries for each platform.
- platform = (parsed_board, netbook)
- if not platform in platforms:
- platforms.append(platform)
- if build_results.get(platform):
- logging.info('Multiple results for %s, %s', build_number, platform)
- continue
- build_results[platform] = _GetLandingDetails(
- dash_view, summary_ranges, netbook, board, category, build_number)
- # Keep track of earliest test job start time for each build.
- time_key = (netbook, board, category, build_number)
- start_time, _, _ = dash_view.GetJobTimesNone(*time_key)
- if not start_time:
- continue
- early_start = builds.setdefault(build_number, (start_time, time_key))
- if start_time < early_start[0]:
- builds[build_number] = (start_time, time_key)
-
- # Include the earliest job date among the platforms to be shown as the
- # overall 'release' (r15) test start date-time.
- organized_results = []
- for build, (start_time, time_key) in sorted(builds.iteritems(), reverse=True,
- key=lambda (k, v): v[0]):
- build_results = []
- for platform in platforms:
- build_results.append(results_dict.get(build, {}).get(platform))
- if time_key:
- formatted_start, _, _ = dash_view.GetFormattedJobTimes(*time_key)
- else:
- formatted_start = None
- if build[0].lower() == 'r':
- # R16-w.x.y format build number.
- build_release = build.split('-')[0][1:]
- else:
- # 0.15.x.y format build number.
- build_release = build.split('.')[1]
- organized_results.append((build, build_release,
- formatted_start, build_results))
-
- return {'platforms': platforms,
- 'irregular': irregular_releases,
- 'releases': sorted(releases, reverse=True),
- 'results': organized_results}
-
-
-def BuildSummaryHTML(base_dir, html_filename, tpl_summary_data, last_updated):
- """Render actual page and save to an html file.
-
- Args:
- base_dir: base where resulting html file goes.
- html_filename: actual filename differs in views.
- tpl_summary_data: this data consumed by the template.
- last_updated: published on output pages.
-
- Render Variables:
- last_updated: date used by the template.
- """
- full_filepath = os.path.join(base_dir, html_filename)
- tpl_last_updated = last_updated
-
- dash_util.SaveHTML(
- full_filepath,
- render_to_response(
- os.path.join('tables/summary', html_filename), locals()).content)
-
-
-def BuildAllTables(dash_base_dir, dash_view, dash_options, summary_limit,
- waterfall_limit):
- """Build all detail pages and a few summary pages as well.
-
- Builds the detail pages for each netbook/board/category and then a
- waterfall-style summary and a few other summary pages.
-
- Args:
- dash_base_dir: base of dashboard output files.
- dash_view: data model with all test result details.
- dash_options: config options used for setting landing pages.
- summary_limit: only show n rows/table on the summary page.
- waterfall_limit: only show n rows on the waterfall summary page.
-
- Render Variables:
- last_updated: date used by the template.
- """
- tpl_last_updated = dash_view.GetFormattedLastUpdated()
- netbooks = dash_options.get('debug_netbook', dash_view.netbooks)
- for netbook in netbooks:
- for board_type in dash_view.GetNetbookBoardTypes(netbook):
- _BuildNetbookHTML(dash_base_dir, dash_view, netbook, board_type,
- BVT_TAG, dash_options)
-
- for summary_file, build_fn, category, limit in (
- (TEST_WATERFALL_FILE, BuildLandingSummaries, BVT_TAG, waterfall_limit),
- (KERNEL_WATERFALL_FILE, BuildLandingSummaries, KERNELTEST_TAG,
- waterfall_limit)):
-
- summary_ranges = SummaryRanges(dash_view, category, limit)
- boards = summary_ranges.GetBoards()
- tots = []
- branches = []
- nonpriority = []
- for board in boards:
- if board in dash_options['priorityboards_tot']:
- tots.append(board)
- elif board in dash_options['priorityboards']:
- branches.append(board)
- else:
- nonpriority.append(board)
- branches += nonpriority
-
- BuildSummaryHTML(
- dash_base_dir,
- summary_file,
- build_fn(dash_view, category, tots, branches, summary_ranges),
- tpl_last_updated)
-
- dash_util.SaveHTML(
- os.path.join(dash_base_dir, TEST_LANDING_FILE),
- render_to_response(
- os.path.join('tables/summary', TEST_LANDING_FILE),
- locals()).content)
- dash_util.SaveHTML(
- os.path.join(dash_base_dir, EMAILS_SUMMARY_FILE),
- render_to_response(
- os.path.join('tables/summary', EMAILS_SUMMARY_FILE),
- locals()).content)
-
-
-if __name__ == '__main__':
- print 'Run %s with --table-generate.' % DASHBOARD_MAIN
diff --git a/site_utils/dashboard/test_dash_view.out b/site_utils/dashboard/test_dash_view.out
deleted file mode 100644
index 88a8ecc..0000000
--- a/site_utils/dashboard/test_dash_view.out
+++ /dev/null
@@ -1,6 +0,0 @@
-This file is the output of test_dash_view.py.
-It is very useful in looking at the data model
-created by dash_view.py which is complicated.
-It's too large for the depot so the full file
-resides here:
-/home/truty/chromeos-test/test_dash_view.out
diff --git a/site_utils/dashboard/test_dash_view.py b/site_utils/dashboard/test_dash_view.py
deleted file mode 100755
index 2f841d8..0000000
--- a/site_utils/dashboard/test_dash_view.py
+++ /dev/null
@@ -1,90 +0,0 @@
-#!/usr/bin/python
-# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Load dash data model and print output to verify model."""
-
-import logging
-import optparse
-import os
-import sys
-
-import dash_common
-import dash_util
-
-settings = 'autotest_lib.frontend.settings'
-os.environ['DJANGO_SETTINGS_MODULE'] = settings
-
-from dash_view import AutotestDashView
-
-
-# String resources.
-from dash_strings import LAST_N_JOBS_LIMIT
-
-
-def parse_args():
- """Support verbose flag."""
- parser = optparse.OptionParser()
- parser.add_option('-d', '--dash-dir',
- help='base dashboard dir [default: %default]',
- dest='dashdir',
- default='/usr/local/autotest/results/dashboard')
- parser.add_option('-f', '--file-name', help='output filename',
- dest='filename', default=None)
- parser.add_option('-j', '--job-limit', help='limit to last n jobs',
- dest='joblimit', default=LAST_N_JOBS_LIMIT)
- parser.add_option('-k', '--keyvals',
- dest='showkeyvals', action='store_true', default=False,
- help='Take time for keyvals')
- parser.add_option('-n', '--noprint',
- dest='verbose', action='store_false', default=True,
- help='Avoid printing data structures')
- parser.add_option('-s', '--show-model',
- dest='showmodel', action='store_true', default=False,
- help='Show data structures')
- options, args = parser.parse_args()
-
- if options.verbose and not options.filename:
- logging.fatal('Must supply --file-name or --noprint.')
- sys.exit(1)
-
- if options.verbose:
- logging.basicConfig(
- level=logging.DEBUG, filename=options.filename, filemode='w')
- else:
- logging.basicConfig(level=logging.WARNING)
- return options, args
-
-
-def main():
- diag = dash_util.DebugTiming()
-
- options, args = parse_args()
-
- dash_base_dir = options.dashdir
- if not os.path.exists(dash_base_dir):
- dash_util.MakeChmodDirs(dash_base_dir)
-
- dash_view = AutotestDashView()
- dash_view.CrashSetup(dash_base_dir)
- dash_view.LoadFromDB(int(options.joblimit))
- if options.showmodel:
- dash_view.ShowDataModel()
- del dash_view
-
- dash_view = AutotestDashView()
- if options.showkeyvals:
- dash_view.CrashSetup(dash_base_dir)
- dash_view.LoadPerfFromDB(int(options.joblimit))
- if options.showmodel:
- dash_view.ShowKeyVals()
-
- if options.filename:
- os.chmod(options.filename, 0644)
-
- del diag
-
-
-if __name__ == '__main__':
- main()
diff --git a/site_utils/dashboard/test_summary.py b/site_utils/dashboard/test_summary.py
deleted file mode 100644
index f92de54..0000000
--- a/site_utils/dashboard/test_summary.py
+++ /dev/null
@@ -1,171 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Retrieve and process the test summary results.json files.
-
-Specifically, this code enables access to information about crashes
-identified during job runs.
-"""
-
-__author__ = ['truty@google.com (Mike Truty)',
- 'dalecurtis@google.com (Dale Curtis)']
-
-import commands
-import json
-import os
-
-import dash_util
-
-# String resources.
-from dash_strings import AUTOTEST_ARCHIVE
-from dash_strings import AUTOTEST_PATH
-from dash_strings import AUTOTEST_SERVER
-from dash_strings import CGI_RETRIEVE_LOGS_CMD
-from dash_strings import GSUTIL_GET_CMD
-from dash_strings import WGET_CMD
-
-LOG_BASE_PATH = '%s/%s/results.json'
-
-
-class TestSummaryInfo(object):
- """Class to enable retrieval of test summary info files."""
-
- def __init__(self, job_cache_dir):
- """Initialize some job status caches.
-
- There are two caches for job result artifacts: an in-memory one and one
- on disk.
- _job_cache_dir: contains the file-based on-disk cache of job results files.
- _job_summary_cache: an in-memory dictionary of job results for test lookups.
-
- Args:
- job_cache_dir: base location for the file-based job result cache.
- """
- self._job_cache_dir = job_cache_dir
- self._job_summary_cache = {}
-
- @staticmethod
- def _GetJsonFromFileOrString(file_or_string, is_file=True):
- """Helper to convert text retrieved to Json.
-
- Args:
- file_or_string: filename or string to consume.
- is_file: flag to inform logic if open is needed.
-
- Returns:
- Json version of the string (file text).
- """
- try:
- if is_file:
- with open(file_or_string) as f:
- summary_json = json.load(f)
- else:
- summary_json = json.loads(file_or_string)
- except ValueError:
- # This ValueError raised when json.load(s) finds improper Json text.
- summary_json = {}
- return summary_json
-
- def _LocalResultFile(self, job_name, base_dir=None, use_json=True):
- """Helper to find and retrieve the results file on a local machine.
-
- The file may be located in a result dir or a cache dir.
-
- Args:
- job_name: a key to finding the job data under autotest results.
- base_dir: used to find the job result file cache. If None, look in
- the dashboard job file cache.
- use_json: flag to suggest if Json-parsing is needed.
-
- Returns:
- If the file is located:
- and use_json=True, then return Json valid version of the file.
- and not use_json=True, then return raw file contents.
- If file not found, return None.
- """
- base_dir = base_dir or self._job_cache_dir
- log_file_path = os.path.abspath(LOG_BASE_PATH % (base_dir, job_name))
- if os.path.isfile(log_file_path):
- if use_json:
- return self._GetJsonFromFileOrString(log_file_path)
- with open(log_file_path) as f:
- return f.read()
- return None
-
- def _RetrieveResultsJson(self, job_name):
- """Helper to retrieve the results.json file from a result server.
-
- The tko/retrieve_logs.cgi script handles finding the results server
- and/or retrieving results from gs using gsutil.
-
- Args:
- job_name: used to locate the job-specific result.json.
- """
- results_base = os.path.join(AUTOTEST_SERVER, CGI_RETRIEVE_LOGS_CMD)
- log_file_path = LOG_BASE_PATH % (results_base, job_name)
- return commands.getoutput('%s %s' % (WGET_CMD, log_file_path))
-
- def _UpdateFileCache(self, job_name):
- """Helper to update a job file cache with a results Json file.
-
- This is complicated by the fact that results files may be located
- on the local machine, a local autotest server or remotely on a
- results server or in Google Storage.
-
- Args:
- job_name: a key to finding the job data under autotest results.
-
- Returns:
- Json valid version of the file content or None.
- """
- summary_text = self._RetrieveResultsJson(job_name)
- cache_path = os.path.abspath(LOG_BASE_PATH % (self._job_cache_dir,
- job_name))
- dash_util.MakeChmodDirs(os.path.dirname(cache_path))
- dash_util.SaveHTML(cache_path, summary_text)
- return self._GetJsonFromFileOrString(summary_text, is_file=False)
-
- def RetrieveTestSummary(self, job_tag, test_name):
- """Retrieves test artifacts from the Autotest server for a given test.
-
- Autotest drops a Json file which contains failed tests, crashes, and log
- file snippets in each job results directory. We use this information to
- reduce wget usage and find crashes for a given job.
-
- Requests are cached to reduce round-trip time to the server which can be
- very substantial.
-
- Extract path to results from tag. Sometimes the test['tag'] is:
-
- <job_name>/<group name>/<host name(s)>
-
- Other times it's just:
-
- <job_name>/<host name>
-
- It depends on how tests were scheduled. Usually, if present, group name
- indicates that the job was spread across multiple hosts.
-
- The results.json is always in sub directory under <job_name>.
-
- Args:
- job_tag: Path under Autotest results to find test result file.
- test_name: Used to find previously cached test results.
-
- Returns:
- Json test artifact if it can be loaded from the Autotest server, None
- otherwise.
- """
- job_name = '/'.join(job_tag.split('/')[0:2])
-
- # Get job summary from in-memory cache, then the actual file or file cache.
- if job_name not in self._job_summary_cache:
- # Now ensure the file cache is updated since job entry not in memory.
- summary_json = self._LocalResultFile(job_name)
- if summary_json is None:
- summary_json = self._UpdateFileCache(job_name)
- self._job_summary_cache[job_name] = summary_json
-
- # Return the results for this test if we have them in the cache.
- return self._job_summary_cache[job_name].get(test_name)
diff --git a/site_utils/downloader.py b/site_utils/downloader.py
deleted file mode 100755
index 85ca92b..0000000
--- a/site_utils/downloader.py
+++ /dev/null
@@ -1,317 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Tool for downloading and processing the latest Buildbot builds.
-
-Downloader is a tool for downloading and processing images for the various board
-types supported by ChromeOS.
-
-All downloading and processing is driven by a board to archive server mapping in
-a specified JSON config file. Boards are processed sequentially.
-
-Downloader is multi-instance friendly. You can spin up as many instances as
-necessary to handle image processing load (which can be substantial). It is not
-recommended to run more than one instance per machine.
-
-Downloader expects the JSON config file to be in the current working directory
-or to be run with --config pointing to the actual config file.
-"""
-
-__author__ = 'dalecurtis@google.com (Dale Curtis)'
-
-import logging
-import optparse
-import os
-import re
-import shutil
-
-from chromeos_test import autotest_util
-from chromeos_test import build_util
-from chromeos_test import common_util
-from chromeos_test import dash_util
-from chromeos_test import dev_server
-from chromeos_test import log_util
-from chromeos_test import test_config
-
-# Autotest imports
-
-import common
-
-from autotest_lib.client.common_lib.cros import dev_server as new_dev_server
-
-
-# Default location of ChromeOS source checkout.
-DEFAULT_CROS_PATH = os.path.join('/usr/local/google/home',
- os.environ['USER'], 'chromeos/chromeos')
-
-
-class Downloader(object):
- """Main class for Downloader. All the magic happens in ProcessBoards()."""
-
- def __init__(self, options, config):
- """Inits Downloader class with options and config data structures.
-
- Args:
- options: Command line options packages as created by ParseOptions().
- config: Dictionary of configuration as loaded from JSON.
- """
- self._options = options
- self._config = config
-
- def ProcessBoards(self):
- """For each board: find latest build version, create components, and upload.
-
- The main processing function for the Downloader class. Given a configuration
- mapping between boards and locations it will:
-
- - Find the latest version of a build for a given board.
- - Determine if the build already exists on Dev Server.
- - Download and extract the build to a staging directory.
- - Convert binary testing image into relevant components.
- - Upload components to Dev Server.
- """
- # Initialize boards listing. If user has specified a board and it's valid,
- # only process that board.
- boards = self._config['boards']
- if self._options.board and self._options.board in boards:
- boards = {self._options.board: boards[self._options.board]}
-
- # Initialize Dev Server utility class.
- dev = dev_server.DevServer(**self._config['dev_server'])
- new_dev = new_dev_server.DevServer()
-
- # Main processing loop. Look for new builds of each board.
- for board in boards:
- # |board| is the same as target in the new nomenclature, i.e.
- # x86-alex-release. this also uses old style; R18, R16, etc.
- board_cfg = boards[board]
- board_cfg.setdefault('archive_path', None)
- board_cfg.setdefault('build_pattern', None)
- board_cfg.setdefault('boto', None)
- board_cfg.setdefault('import_tests', False)
- if not board_cfg.get('archive_server'):
- logging.info('Skipping %s, devserver handles the download.', board)
- continue
-
- # Bind remote_dir and staging_dir here so we can tell if we need to do any
- # cleanup after an exception occurs before remote_dir is set.
- remote_dir = staging_dir = None
- try:
- logging.info('------------[ Processing board %s ]------------', board)
- # Retrieve the latest build version for this board.
- if not self._options.build:
-
- build = build_util.GetLatestBuildbotBuildVersion(
- archive_server=board_cfg['archive_server'], board=board,
- boto=board_cfg['boto'], archive_path=board_cfg['archive_path'],
- build_pattern=board_cfg['build_pattern'])
-
- if not build:
- logging.info('Bad build version returned from server. Skipping.')
- continue
-
- logging.info('Latest build available on Buildbot is %s .', build)
- else:
- build = self._options.build
-
- if board_cfg.get('download_devserver'):
- # Use new dev server download pathway for staging image.
- image = '%s/%s' % (board, build)
- logging.info('Downloading %s using the dev server.', image)
- new_dev.trigger_download(image)
- continue
-
- # Create Dev Server directory for this build and tell other Downloader
- # instances we're working on this build.
- try:
- remote_dir = dev.AcquireLock('/'.join([board, build]))
- except common_util.ChromeOSTestError:
- # Label as info instead of error because this will be the most common
- # end point for the majority of runs.
- logging.info('Refused lock for build. Assuming build has already been'
- ' processed.')
- continue
-
- # Download and extract build to a temporary directory or process the
- # build at the user specified staging directory.
- if not self._options.staging:
- logging.info('Downloading build from %s/%s',
- board_cfg['archive_server'], board)
-
- staging_dir, archive_path = build_util.DownloadAndExtractBuild(
- archive_server=board_cfg['archive_server'],
- archive_path=board_cfg['archive_path'], board=board,
- boto=board_cfg['boto'], build=build)
-
- else:
- staging_dir = self._options.staging
-
- # Do we need to import tests?
- if board_cfg['import_tests'] and not autotest_util.ImportTests(
- hosts=self._config['import_hosts'], staging_dir=staging_dir):
- logging.warning('One or more hosts failed to import tests!')
-
- # Process build and create update.gz and stateful.image.gz
- logging.info('Creating build components under %s', staging_dir)
- build_util.CreateBuildComponents(
- staging_dir=staging_dir, cros_checkout=self._options.cros_checkout)
-
- # Generate N->N AU payload.
- nton_payload_dir = None
- try:
- nton_payload_dir = os.path.join(dev.AU_BASE, build + '_nton')
- common_util.MakedirsExisting(
- os.path.join(staging_dir, nton_payload_dir))
-
- build_util.CreateUpdateZip(
- cros_checkout=self._options.cros_checkout,
- staging_dir=staging_dir, output_dir=nton_payload_dir,
- source_image=build_util.TEST_IMAGE)
- except common_util.ChromeOSTestError, e:
- if nton_payload_dir:
- shutil.rmtree(os.path.join(staging_dir, nton_payload_dir))
- logging.exception(e)
-
- # Generate N-1->N AU payload.
- mton_payload_dir = None
- try:
- # Retrieve N-1 (current LATEST) build from Dev Server.
-
-
- raise NotImplementedException('This code is broken. Do not use.'
- 'If you must use, contact the lab '
- 'team.')
- # ..because the following function call no longer exists
- # previous_build = dev.GetLatestBuildVersion(board)
-
- previous_image = dev.GetImage(board, previous_build, staging_dir)
-
- mton_payload_dir = os.path.join(dev.AU_BASE, previous_build + '_mton')
- common_util.MakedirsExisting(
- os.path.join(staging_dir, mton_payload_dir))
-
- build_util.CreateUpdateZip(
- cros_checkout=self._options.cros_checkout,
- staging_dir=staging_dir, output_dir=mton_payload_dir,
- source_image=previous_image)
- except common_util.ChromeOSTestError, e:
- if mton_payload_dir:
- shutil.rmtree(os.path.join(staging_dir, mton_payload_dir))
- logging.exception(e)
-
- # TODO(dalecurtis): Sync official chromeos_test_image.bins.
-
- # TODO(dalecurtis): Generate <official>->N AU payloads.
-
- # Upload required components into jailed Dev Server.
- logging.info('Uploading build components to Dev Server.')
- dev.UploadBuildComponents(staging_dir=staging_dir, upload_image=True,
- remote_dir=remote_dir)
-
- # Create and upload LATEST file to the Dev Server.
- if not self._options.build:
- dev.UpdateLatestBuild(board=board, build=build)
-
- #TODO(dalecurtis): Disabled, since it's not under active development.
- #appengine_cfg = self._config.get('appengine', {})
- #if appengine_cfg:
- # dash_util.UploadBuild(appengine_cfg, board, build, archive_path)
- else:
- logging.warning('LATEST file not updated because --build was '
- 'specified. Make sure you manually update the LATEST '
- 'file if required.')
- except Exception, e:
- logging.exception(e)
-
- # Release processing lock, which will remove build components directory
- # so future runs can retry.
- if remote_dir:
- try:
- dev.ReleaseLock('/'.join([board, build]))
- except (KeyboardInterrupt, common_util.ChromeOSTestError):
- logging.critical('Failed to clean up Dev Server after failed run on'
- ' build %s.', build)
-
- # If Exception was a ^C, break out of processing loop.
- if isinstance(e, KeyboardInterrupt):
- break
- if not isinstance(e, common_util.ChromeOSTestError):
- raise
- finally:
- # Always cleanup after ourselves. As an automated system with firm
- # inputs, it's trivial to recreate error conditions manually. Where as
- # repeated failures over a long weekend could bring the system down.
- if staging_dir:
- # Remove the staging directory.
- logging.info('Cleaning up staging directory %s', staging_dir)
- cmd = 'sudo rm -rf ' + staging_dir
- msg = 'Failed to clean up staging directory!'
- common_util.RunCommand(cmd=cmd, error_msg=msg)
-
-
-def ParseOptions():
- """Parse command line options. Returns 2-tuple of options and config."""
- # If default config exists, parse it and use values for help screen.
- config = test_config.TestConfig()
-
- # If config is provided parse values to make help screen more useful.
- boards = config.ParseConfigGroups()[0]
-
- parser = optparse.OptionParser('usage: %prog [options]')
-
- parser.add_option('--board', dest='board',
- help='Process only the specified board. Valid boards: %s'
- % boards)
- parser.add_option('--build', dest='build',
- help=('Specify the build version to process. Must be used '
- 'with the --board option. LATEST file will not be '
- 'updated with this option.'))
- parser.add_option('--cros_checkout', dest='cros_checkout',
- default=DEFAULT_CROS_PATH,
- help=('Location of ChromeOS source checkout. Defaults to '
- '"%default".'))
- parser.add_option('--staging', dest='staging',
- help=('Specify a pre-populated staging directory. Must be '
- 'used with the --board and --build options. Useful '
- 'to finish a run that was interrupted or failed.'))
-
- # Add utility/helper class command line options.
- test_config.AddOptions(parser)
- log_util.AddOptions(parser)
-
- options = parser.parse_args()[0]
-
- if options.build and not options.board:
- parser.error('If --build is used, --board must be specified as well.')
-
- if options.staging and not (options.board and options.build):
- parser.error(('If --staging is used, --board and --build must be'
- ' specified as well.'))
-
- # Load correct config file if alternate is specified.
- if options.config != test_config.DEFAULT_CONFIG_FILE:
- config = test_config.TestConfig(options.config)
- boards = config.ParseConfigGroups()[0]
-
- if options.board and not options.board in boards:
- parser.error('Invalid board "%s" specified. Valid boards are: %s'
- % (options.board, boards))
-
- return options, config.GetConfig()
-
-
-def main():
- # Parse options and load config.
- options, config = ParseOptions()
-
- # Setup logger and enable verbose mode if specified.
- log_util.InitializeLogging(options.verbose)
-
- Downloader(options=options, config=config).ProcessBoards()
-
-
-if __name__ == '__main__':
- main()
diff --git a/site_utils/lab_test.py b/site_utils/lab_test.py
deleted file mode 100755
index 9de3c50..0000000
--- a/site_utils/lab_test.py
+++ /dev/null
@@ -1,582 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Script to run an Autotest job on machines in a remote lab.
-
-Takes an image. Verifies the image to be a test image (we need SSH access). Then
-splits the image into update.gz and stateful.tgz components. Finally, uploads
-the components to a Dev Server in the lab.
-
-Once everything is in the necessary places a job is scheduled using the Autotest
-command line tools on /home/build/static and URL returned to the user.
-"""
-
-__author__ = 'dalecurtis@google.com (Dale Curtis)'
-__version__ = 'v1.3'
-
-import json
-import logging
-import optparse
-import os
-import shutil
-import sys
-import tempfile
-
-import chromeos_test_common
-from chromeos_test import autotest_util
-from chromeos_test import build_util
-from chromeos_test import common_util
-from chromeos_test import test_config
-from chromeos_test.colors import Colors
-from chromeos_test.dev_server import DevServer
-
-
-# Autotest directory relative to CrOS root.
-DEFAULT_AUTOTEST_DIR = 'src/third_party/autotest/files'
-
-# Location of default board file.
-DEFAULT_BOARD_FILE = 'src/scripts/.default_board'
-
-# Root of Chrome OS checkout should be up a few directories relative to us.
-DEFAULT_CROS_DIR = chromeos_test_common.CROS_DIR
-
-# Root of the default build directory relative to CrOS root.
-DEFAULT_IMAGE_DIR = 'src/build/images'
-
-# Tag prefix for Dev builds.
-DEV_BUILD_PREFIX = 'dev'
-
-LAB_TEST_CONFIG = os.path.join(chromeos_test_common.CURRENT_DIR,
- 'lab_test.json')
-
-# Path to ChromeOS testing key in CrOS checkout.
-CROS_TEST_KEYS_DIR = 'src/scripts/mod_for_test_scripts/ssh_keys/'
-CROS_TEST_KEY_PRIV = os.path.join(CROS_TEST_KEYS_DIR, 'testing_rsa')
-CROS_TEST_KEY_PUB = os.path.join(CROS_TEST_KEYS_DIR, 'testing_rsa.pub')
-
-# Exit code to use on error.
-ERROR_EXIT_CODE = 1
-
-# URL base for viewing a job.
-JOB_URL_BASE = 'http://cautotest/afe/#tab_id=view_job&object_id='
-
-
-def KerberosExceptionHandler(f):
- """Decorator which provides additional information for Kerberos exceptions."""
-
- def _Wrapped():
- try:
- return f()
- except common_util.ChromeOSTestError, e:
- if 'Kerberos' in e[-1]:
- LogErrorAndExit(
- 'There appears to be a problem with your credentials. Please run'
- ' kinit and try again.')
- else:
- raise
-
- return _Wrapped
-
-
-def FindTest(autotest_dir, test_regex):
- """Uses a test name regex to find the proper control file in Autotest dirs."""
- search_paths = 'client/tests client/site_tests server/tests server/site_tests'
- cmd = ('find %s -maxdepth 2 -type f \\( -name control.* -or -name control \\)'
- '| egrep -v "~$" | egrep "%s"' % (search_paths, test_regex))
- return common_util.RunCommand(cmd=cmd, cwd=autotest_dir, output=True)
-
-
-def FindAutotestDir(options):
- """Determine whether to use cros_workon or emerged Autotests. Returns path."""
- if options.autotest_dir:
- if not os.path.exists(options.autotest_dir):
- LogErrorAndExit('Could not find the specified Autotest directory.')
- else:
- logging.info('As requested, using the specified Autotest directory '
- 'at %s.', Colors.Color(Colors.BOLD_BLUE,
- options.autotest_dir))
- return options.autotest_dir
-
- autotest_dir = os.path.join(options.cros_dir, DEFAULT_AUTOTEST_DIR)
- if options.use_emerged:
- autotest_dir = os.path.join(
- options.cros_dir, 'chroot/build', options.board, 'usr/local/autotest')
- if not os.path.exists(autotest_dir):
- LogErrorAndExit('Could not find pre-installed autotest, you need to '
- 'emerge-%s autotest autotest-tests.', options.board)
- logging.info('As requested, using emerged autotests already installed at '
- '%s.', Colors.Color(Colors.BOLD_BLUE, autotest_dir))
- elif not os.path.exists(autotest_dir):
- LogErrorAndExit('Could not find Autotest, run "cros_workon start autotest" '
- 'and "repo sync" to continue.')
- else:
- logging.info('Detected cros_workon autotests. Using autotests from %s. To '
- 'use emerged autotest, pass --use_emerged.',
- Colors.Color(Colors.BOLD_BLUE, autotest_dir))
- return autotest_dir
-
-
-def VerifyImageAndGetId(cros_dir, image_path, install_shim=False):
- """Verifies image is a test image and returns tuple of version, hash.
-
- Args:
- cros_dir: Location of Chrome OS code base.
- image_path: Path to image to verify and convert.
- install_shim: True to verify an install shim instead of a test image.
-
- Returns:
- Tuple of (build_version, build_hash).
- """
- tempdir = tempfile.mkdtemp()
- build_util.MountImage(cros_dir, tempdir,
- image_file=os.path.basename(image_path),
- image_dir=os.path.dirname(image_path))
- try:
- cmd = 'cat etc/lsb-release | grep CHROMEOS_RELEASE_DESCRIPTION'
- msg = 'Failed to read /etc/lsb-release from mounted image!'
- version = common_util.RunCommand(
- cmd=cmd, cwd=os.path.join(tempdir, build_util.ROOTFS_MOUNT_DIR),
- error_msg=msg, output=True)
- if install_shim:
- cmd = 'file root/.factory_installer'
- msg = ('The specified image is not an install shim! Only install shims '
- 'allowed.')
- else:
- cmd = ('diff root/.ssh/authorized_keys %s'
- % os.path.join(cros_dir, CROS_TEST_KEY_PUB))
- msg = 'The specified image is not a test image! Only test images allowed.'
- common_util.RunCommand(
- cmd=cmd, cwd=os.path.join(tempdir, build_util.ROOTFS_MOUNT_DIR),
- error_msg=msg)
- finally:
- build_util.UnmountImage(cros_dir, tempdir)
- shutil.rmtree(tempdir, ignore_errors=True)
-
- # String looks like '<tag>=<version> (Test Build <hash> ...' After =, we want
- # the first and third elements. TODO(dalecurtis): verify what we're parsing.
- return version.split('=')[1].split(' ')[0:4:3]
-
-
-def ProcessLocalBuild(cros_dir, dev, image_path, force=False):
- """Process a local build. Verifies and converts a test image into updates.
-
- Args:
- cros_dir: Location of Chrome OS code base.
- dev: Instantiated Dev Server Class.
- image_path: Path to test image to verify and convert.
- force: Force creation of updates even if build already exists on server.
-
- Returns:
- Tuple of (build_tag, image_dir, remote_build_dir).
- build_tag: Unique identifier for this build.
- image_dir: Path on local disk
- """
- logging.info('Verifying the specified image is a test image.')
- build_version, build_hash = VerifyImageAndGetId(cros_dir, image_path)
-
- build_tag = '%s-%s-%s' % (os.environ['USER'], build_version, build_hash)
- logging.info(
- 'Processing build %s.', Colors.Color(Colors.BOLD_BLUE, build_tag))
-
- if force:
- logging.info('Forcing upload of new build components due to --force.')
-
- # Prepare the Dev Server for this build.
- remote_build_dir, exists = dev.PrepareDevServer(
- '/'.join([DEV_BUILD_PREFIX, build_tag]), force=force)
-
- image_dir = os.path.dirname(image_path)
- image_file = os.path.basename(image_path)
-
- try:
- # Create update zips if they don't exist.
- if not exists:
- logging.info('Generating update.')
- build_util.CreateUpdateZip(
- cros_dir, image_dir, image_file=image_file)
-
- # Create stateful update zip.
- logging.info('Generating stateful update.')
- build_util.CreateStatefulZip(cros_dir, image_dir, image_file=image_file)
- else:
- logging.info(Colors.Color(
- Colors.BOLD_BLUE, 'Using existing build found on Dev Server.'))
- except:
- if remote_build_dir:
- dev.RemoteCommand('rmdir ' + remote_build_dir)
- raise
-
- return build_tag, image_dir, remote_build_dir, exists
-
-
-def LogErrorAndExit(msg, *args, **kwargs):
- """Simple log error and exit method."""
- logging.error(Colors.Color(Colors.BOLD_RED, msg), *args, **kwargs)
- sys.exit(ERROR_EXIT_CODE)
-
-
-@KerberosExceptionHandler
-def GetPlatformDict():
- """Return a list of Autotest platform labels accessible to current user."""
- platform_dict = autotest_util.GetPlatformDict()
- if not platform_dict:
- LogErrorAndExit('There are no platforms ACL accessible by you. Please'
- ' contact the ChromeOS Autotest team'
- ' (chromeos-lab-infrastructure@google.com).')
- return platform_dict
-
-
-@KerberosExceptionHandler
-def PrintMachineList():
- """Display the output of atest host list."""
- cmd = '%s host list --user $USER' % autotest_util.ATEST_PATH
- msg = 'Failed to retrieve host list from Autotest.'
- print common_util.RunCommand(cmd, error_msg=msg, output=True)
-
-
-def ParseOptions():
- """Parse and verify command line options.
-
- Returns:
- Tuple of options dictionary, relative path to test control file, the path to
- Autotest, and the lab test JSON config.
- """
- parser = optparse.OptionParser(
- 'usage: %prog [options] <test name>\n'
- '\n'
- 'The test name can be a regular expression so long as it only'
- ' matches a single test. For example:\n'
- '\n'
- ' %prog -i test.bin --board x86-generic BootPerfServer')
-
- parser.add_option('--autotest_dir', help='Skip autodetection of autotest and '
- 'use the specified location.')
- parser.add_option('--board', dest='board',
- help=('The board for which you are building autotest. Will '
- 'attempt to read default from <cros_dir>/%s'
- % DEFAULT_BOARD_FILE))
- parser.add_option('--build', dest='build',
- help=('Instead of using a local build, use an official '
- 'build already on the server; e.g. 0.13.507.0 or '
- 'latest to use the most recent build.'))
- parser.add_option('-c', '--cros', dest='cros_dir',
- default=chromeos_test_common.CROS_DIR,
- help=('Location of Chrome OS code base. Defaults to '
- '"%default".'))
- parser.add_option('-v', '--verbose', dest='verbose', action='store_true',
- default=False, help='Enable debugging/verbose output.')
- parser.add_option('-d', '--deps', dest='deps', default=None,
- help='Comma deliminated list of dependencies.')
- parser.add_option('-f', '--force', dest='force', action='store_true',
- default=False,
- help='Force upload even if build already exists on server.')
- parser.add_option('-i', '--image', dest='image_path',
- help=('Path to test image to deploy for testing. If no'
- ' image is specified, the script attempts to use'
- ' <cros_dir>/%s/<board>/latest/%s'
- % (DEFAULT_IMAGE_DIR, build_util.TEST_IMAGE)))
- parser.add_option('--list_machines', dest='list_machines',
- action='store_true',
- help=('Display the list of available machines as well as'
- ' their current status.'))
- parser.add_option('-l', '--list_platforms', dest='list_platforms',
- action='store_true',
- help=('Display the list of valid platforms for use with'
- ' --platforms.'))
- parser.add_option('-m', '--mail', dest='mail',
- help=('A comma seperated list of email addresses to notify'
- ' upon job completion.'))
- parser.add_option('-o', '--override', dest='override', action='store_true',
- default=False,
- help=('Override board and platform safety checks.'
- ' Experienced users only! Please don\'t brick our'
- ' machines :)'))
- parser.add_option('-p', '--platforms', dest='platforms',
- help=('Comma separated list of platforms to use for'
- ' testing. Use the --list_platforms option to see the'
- ' list of valid platforms. Multiple tests on the same'
- ' platform can be run by using the * character; e.g.,'
- ' 2*<platform> would use two machines of type'
- ' <platform>.'))
- parser.add_option('-t', '--tests', dest='tests', action='store_true',
- default=False,
- help=('Package tests with stateful partition. Will cause'
- ' the stateful partition to be reuploaded to the'
- ' server even if it already exists. If tests aren\'t'
- ' packaged, the versions on the Autotest server will'
- ' be used.'))
- parser.add_option('-x', '--priority', dest='priority', default='urgent',
- help='The priority of the job. default: [%default].')
- parser.add_option('--use_emerged', dest='use_emerged', action='store_true',
- default=False,
- help='Force use of emerged autotest packages')
- options, args = parser.parse_args()
-
- if options.verbose:
- logging.getLogger().setLevel(logging.DEBUG)
-
- # Make sure we're outside the chroot.
- if os.path.isfile('/etc/debian_chroot'):
- LogErrorAndExit(
- 'LabTest must be run outside the chroot to access corp resources.')
-
- if options.list_machines:
- parser.print_help()
- print Colors.Color(
- Colors.BOLD_BLUE,
- '\nGenerating list of machines (this may take a few seconds):')
- PrintMachineList()
- sys.exit(0)
-
- if options.list_platforms:
- parser.print_help()
- print Colors.Color(
- Colors.BOLD_BLUE,
- '\nGenerating list of valid platforms (this may take a few seconds):')
- format = '%-40s %-20s'
- print format % ('Platform', 'Dependencies')
- for platform, deps in GetPlatformDict().iteritems():
- print format % (platform, ' '.join(deps))
- sys.exit(0)
-
- logging.info('Verifying command line options.')
-
- if not args:
- LogErrorAndExit('A test name must be specified.')
-
- # Make sure CrOS checkout directory exists.
- if not os.path.exists(options.cros_dir):
- LogErrorAndExit('Could not find Chrome OS checkout, please specify the path'
- ' with -c.')
-
- # Convert paths to abs path.
- for item in ('autotest_dir', 'cros_dir', 'image_path'):
- if getattr(options, item):
- abs_path = os.path.normpath(os.path.join(os.getcwd(),
- getattr(options, item)))
- setattr(options, item, abs_path)
-
- # Attempt to load LabTest config.
- with open(LAB_TEST_CONFIG) as config_file:
- config = json.load(config_file)
-
- # Attempt to determine the default board.
- default_board_file = os.path.join(options.cros_dir, DEFAULT_BOARD_FILE)
- if not options.board:
- logging.info('No board specified, attempting to load the default.')
- if not os.path.isfile(default_board_file):
- LogErrorAndExit('The default board could not be read. Please specify the '
- 'board type with --board.')
- with open(default_board_file, 'r') as f:
- options.board = f.read().strip()
- logging.info('Using default board "%s"',
- Colors.Color(Colors.BOLD_BLUE, options.board))
-
- # Convert boards with multiple names into a single format.
- if options.board in config['preferred_board_fixups']:
- options.board = config['preferred_board_fixups'][options.board]
-
- if not options.platforms:
- if options.board in config['board_platform_map']:
- # If the platform exists in the map, override any further checks.
- options.override = True
- options.platforms = config['board_platform_map'][options.board]
- logging.info(
- 'No platform specified, using the default platform for this board '
- '"%s"', Colors.Color(Colors.BOLD_BLUE, options.platforms))
- else:
- LogErrorAndExit(
- 'An unknown board has been specified, please specify the platform '
- 'type with --platform.')
-
- # Make sure the specified image actually exists...
- if options.image_path:
- if not os.path.isfile(options.image_path):
- LogErrorAndExit('The specified test image does not exist.')
- elif not options.build:
- logging.info('No image specified, attempting to find the latest image.')
- options.image_path = os.path.join(
- options.cros_dir, DEFAULT_IMAGE_DIR, options.board, 'latest',
- build_util.TEST_IMAGE)
- if not os.path.isfile(options.image_path):
- LogErrorAndExit(
- 'No test image specified and the default could not be found.')
- logging.info(
- 'Default image found, using %s',
- Colors.Color(Colors.BOLD_BLUE, options.image_path))
-
- # Figure out the Autotest directory based on command line options.
- autotest_dir = FindAutotestDir(options)
-
- # Identify the desired test case. Limit to only one test for now.
- test_pattern = ' '.join(args)
- try:
- matched_test = FindTest(autotest_dir, test_pattern).strip()
- except common_util.ChromeOSTestError:
- LogErrorAndExit('Cannot find a match for test name "%s"' % test_pattern)
-
- if len(matched_test.split('\n')) > 1:
- logging.error('The given test pattern is ambiguous. Disambiguate by '
- 'passing one of these patterns instead:')
- for test in matched_test.split('\n'):
- logging.error(' ^%s$', test)
- sys.exit(ERROR_EXIT_CODE)
-
- # Verify the requested platforms.
- platform_dict = GetPlatformDict()
-
- # Strip out any multipliers from the platform list.
- platform_split = options.platforms.split(',')
- platform_names = set(p.lstrip('0123456789* ') for p in platform_split)
- bad_platforms = platform_names - set(platform_dict.keys())
- if bad_platforms:
- LogErrorAndExit('The following platforms are invalid: %s',
- ', '.join(bad_platforms))
-
- # Add 1* for any platforms without a count.
- for i in xrange(0, len(platform_split)):
- if not platform_split[i][0].isdigit():
- platform_split[i] = '1*' + platform_split[i]
- options.platforms = ','.join(platform_split)
-
- # Verify specified platforms match the provided board.
- if not options.override and options.board != 'x86-generic':
- # Only allow board, platform pairs we have configured for testing.
- cros_config = test_config.TestConfig(
- os.path.join(chromeos_test_common.CRON_DIR,
- test_config.DEFAULT_CONFIG_FILE))
- valid_platforms = cros_config.ParseConfigGroups(board_re=options.board)[2]
-
- for p in platform_names:
- if not p in valid_platforms:
- LogErrorAndExit('The specified platform (%s) is not valid for the '
- 'specified board (%s). Valid platforms for this board '
- 'are: %s.', p, options.board,
- ', '.join(valid_platforms))
-
- return options, matched_test, autotest_dir, config
-
-
-def main():
- # Setup logging.
- logging.basicConfig(format=' - %(levelname)s: %(message)s')
- logging.getLogger().setLevel(logging.INFO)
-
- print '-' * 80
- print ('LabTest! A script to run Autotest jobs on machines in a remote lab.'
- ' (%s)' % __version__)
- print '-' * 80
-
- options = local_build_dir = remote_build_dir = None
- try:
- # Parse options and find the requested control file.
- options, control_file, autotest_dir, config = ParseOptions()
- start_str = 'Running %s on the following platforms: %s' % (
- Colors.Color(Colors.BOLD_GREEN, control_file),
- Colors.Color(Colors.BOLD_GREEN, options.platforms))
-
- if options.deps:
- start_str += ' with deps: %s' % Colors.Color(Colors.BOLD_GREEN,
- options.deps)
- logging.info(start_str)
- # Load Dev Server configuration.
- dev_config = config['dev_server']
-
- remote_host = dev_config.get('remote_host', None)
-
- # Initialize Dev Server.
- dev = DevServer(
- dev_config['dev_host'], dev_config['dir'], dev_config['user'],
- private_key=os.path.join(options.cros_dir, CROS_TEST_KEY_PRIV),
- remote_host=remote_host)
-
- # Determine if we have any tests to upload.
- tests_to_upload = options.tests
-
- # If the user hasn't specified an official build, process their local build.
- if not options.build:
- build_tag, local_build_dir, remote_build_dir, exists = ProcessLocalBuild(
- options.cros_dir, dev, options.image_path, force=options.force)
- else:
- # Scan the Dev Server for using the partial board, build information we
- # have. Afterward, update the options values with the full ids.
- options.board, options.build = dev.FindDevServerBuild(
- options.board, options.build)
- build_tag = '%s-%s' % (os.environ['USER'], options.build)
-
- logging.info(
- 'Official build requested, using build %s for testing.',
- Colors.Color(Colors.BOLD_GREEN, options.build))
-
- if tests_to_upload:
- # Create a temporary directory to hold Autotest packages.
- local_build_dir = tempfile.mkdtemp()
-
- # Make a copy of the official build so we don't corrupt it.
- remote_build_dir = dev.CloneDevServerBuild(
- options.board, options.build,
- '/'.join([DEV_BUILD_PREFIX, build_tag]), force=options.force)
-
- # Extract test name from path and prepare Autotest packages for upload.
- test_name = os.path.basename(os.path.dirname(control_file))
- if tests_to_upload:
- logging.info('Preparing Autotest packages for upload to Dev Server.')
- build_util.PrepareAutotestPkgs(
- autotest_dir, local_build_dir, test_name=test_name)
-
- # If we've processed a build, upload all build components.
- if remote_build_dir and not options.build and not exists:
- logging.info('Uploading build components to Dev Server.')
- dev.UploadBuildComponents(remote_build_dir, local_build_dir)
- elif tests_to_upload:
- # Otherwise, just upload Autotest packages if there are any.
- logging.info('Uploading Autotest packages to Dev Server.')
- dev.UploadAutotestPackages(remote_build_dir, local_build_dir)
-
- # If official build and no modified tests, use an existing build URL.
- if options.build and not tests_to_upload:
- update_url = dev.GetUpdateUrl(options.board, options.build)
- else:
- # Otherwise determine the update URL for the processed build.
- update_url = dev.GetUpdateUrl(DEV_BUILD_PREFIX, build_tag)
-
- # Hackish, but the only way we have to determine server versus client jobs.
- server = control_file.startswith('server/')
-
- # Special case to fix up job names in the suites directory. These files are
- # all of the format suites/control.<name>.
- if test_name.lower() == 'suites':
- test_name = os.path.basename(control_file).split('.')[-1]
-
- # Now that all components are uploaded, start the Autotest job.
- job_name = '%s_%s' % (build_tag, test_name)
- logging.info('Creating job %s.', Colors.Color(Colors.BOLD_BLUE, job_name))
- job_id = autotest_util.CreateJob(
- name=job_name, control=os.path.join(autotest_dir, control_file),
- platforms=options.platforms, update_url=update_url, server=server,
- mail=options.mail, labels=options.deps, priority=options.priority)
-
- logging.info(
- Colors.Color(Colors.BOLD_GREEN, 'Job created successfully, URL: %s%s'),
- JOB_URL_BASE, job_id)
- except Exception, e:
- if remote_build_dir:
- dev.RemoteCommand('rm -rf ' + remote_build_dir)
-
- if isinstance(e, common_util.ChromeOSTestError):
- logging.error(Colors.Color(Colors.BOLD_RED, e[0]))
- if not options or options.verbose:
- logging.exception(e)
- else:
- raise
- finally:
- # When --build is used, local_build_dir contains only tmp files, so cleanup.
- if options and options.build and local_build_dir:
- common_util.RunCommand('rm -rf ' + local_build_dir)
-
-
-if __name__ == '__main__':
- main()
diff --git a/site_utils/legacy_unittest_suite.py b/site_utils/legacy_unittest_suite.py
deleted file mode 100755
index 86c7c2b..0000000
--- a/site_utils/legacy_unittest_suite.py
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2010 Google Inc. All Rights Reserved.
-
-"""Unit test suite for downloader."""
-
-__author__ = 'dalecurtis@google.com (Dale Curtis)'
-
-import unittest
-
-from chromeos_test import common_util_test
-from chromeos_test import dev_server_test
-from chromeos_test import log_util_test
-from chromeos_test import test_config_test
-
-
-def TestSuite():
- suites = []
-
- suites.append(unittest.TestLoader().loadTestsFromTestCase(
- common_util_test.CommonUtilityTest))
-
- suites.append(unittest.TestLoader().loadTestsFromTestCase(
- dev_server_test.DevServerTest))
-
- suites.append(unittest.TestLoader().loadTestsFromTestCase(
- log_util_test.LogUtilityTest))
-
- suites.append(unittest.TestLoader().loadTestsFromTestCase(
- test_config_test.TestConfigTest))
-
- return unittest.TestSuite(suites)
-
-
-if __name__ == '__main__':
- unittest.TextTestRunner(verbosity=2).run(TestSuite())
diff --git a/site_utils/mass_command.py b/site_utils/mass_command.py
deleted file mode 100755
index a20a25d..0000000
--- a/site_utils/mass_command.py
+++ /dev/null
@@ -1,453 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Executes on all unlocked hosts in Autotest lab in parallel at a given rate.
-
-Used to run a command or script on all hosts, or only those of a given platform,
-in the Autotest lab. Allows a configurable number of commands to be started in
-parallel.
-"""
-
-
-import datetime
-import logging
-import optparse
-import os
-import time
-
-import chromeos_test_common
-from chromeos_test import autotest_util
-from chromeos_test import common_util
-from chromeos_test import mp_log_util
-from chromeos_test import mp_thread_pool as tp
-
-# Default number of hosts to run command/script in parallel.
-DEFAULT_CONCURRENCY = 64
-
-# Default number of hosts to update in parallel.
-DEFAULT_UPDATE_CONCURRENCY = 24
-
-# Default location of ChromeOS checkout.
-DEFAULT_GCLIENT_ROOT = '/usr/local/google/home/${USER}/chromeos'
-
-# Default path for individual host logs. Each host will have it's own file. E.g.
-# <default_log_path>/<host>.log
-DEFAULT_LOG_PATH = ('/tmp/mass_command_logs/%s/'
- % time.strftime('%Y-%m-%d-%H-%M', time.localtime()))
-
-# Default root path on remote device to copy scripts to
-DEFAULT_REMOTE_COPY_PATH = '/tmp/'
-
-# Amount of seconds to wait before declaring an command/script has failed.
-DEFAULT_TIMEOUT = 120
-
-# Amount of seconds to wait before declaring an update has failed.
-DEFAULT_UPDATE_TIMEOUT = 2400
-
-
-def ExecuteTask(failure_desc):
- """Decorator for try/except/log pattern for reporting status and failures.
-
- Args:
- failure_desc: Simple string description of task.
-
- Returns:
- Decorator function to wrap a method call.
- """
-
- def DecoratorFunc(func):
- """Function that takes the user called method as an argument."""
-
- def WrappedFunc(self, *args):
- """Function that wraps and executes user called method.
-
- Args:
- self: Self object of the class method called by decorator.
- args: Arguments to user called method.
-
- Returns:
- True/False if user called method succeeded.
- """
- try:
- output = func(self, *args)
- if output:
- if self.output:
- self.output += '\n' + output
- else:
- self.output = output
- except common_util.ChromeOSTestError:
- if self.logger:
- self.logger.exception('Failed running %s %s.', self.host,
- failure_desc)
- self.result = failure_desc
- return False
- return True
-
- return WrappedFunc
- return DecoratorFunc
-
-
-class HostWorker(object):
- """Responsible for ssh-test, locking, executing, and unlocking a host."""
-
- def __init__(self, host, options):
- """Create instance to perform work on a host.
-
- Args:
- host: IP address of the host to connect to.
- options: Command line options.
- """
- self.host = host
- self.options = options
- self.result = None
- self.output = None
- self.logger = None
-
- def Execute(self, logger=None):
- """Callback method to execute the requested action on the host.
-
- Usual sequence is to test connectivity by SSH-ing to the host, locking
- the host in Autotest, running the command, then unlocking the host.
-
- Args:
- logger: optional logger.
-
- Sets:
- self.result to 'PASS' or failure ['SSH', 'LOCK', 'COPY', 'CMD', 'URL'].
- self.output to standard out of command.
- """
- try:
- if logger:
- # Store logger in self.logger so it is accessible in ExecuteTask.
- self.logger = logger
- logger.info('Executing for host %s', self.host)
-
- if not self.options.skip_ssh:
- if not self.PingHost():
- return
-
- if self.options.lock:
- if not self.LockUnlockHost(True):
- return
-
- # Now that the host may be locked in Autotest the rest of the loop will
- # execute in a try/finally to make sure the host is still unlocked if
- # any of the remaining steps throw an exception.
- try:
- if self.options.url:
- if not self.ImageHost():
- return
- else:
- cmd = self.options.cmd
- if self.options.script:
- cmd = self.options.remote_file
- if not self.CopyToDevice():
- return
- if not self.SSHCmdOnHost(cmd, self.options.extra_args):
- return
- finally:
- if self.options.lock:
- self.LockUnlockHost(False)
-
- self.result = 'PASS'
- self.ProcessResult()
-
- finally:
- # Loggers hold a thread lock which cannot be pickled, so it must be
- # cleared before returning.
- self.logger = None
-
- def ProcessResult(self):
- """Dump the results to the screen and/or log file."""
- if self.logger:
- msg = [self.host, ' finished with ', self.result]
-
- if self.options.echo_output:
- if self.output:
- msg += ['\nStdOut=[\n', self.output, '\n]']
- self.logger.info(''.join(msg))
-
- if not self.options.no_log_files:
- log = open(os.path.join(self.options.log_path, self.host + '.log'), 'w')
- log.write(self.output)
- log.close()
-
- @ExecuteTask('SSH')
- def PingHost(self):
- """Tests if the requested host is reachable over SSH."""
- msg = 'Failed to ssh to host=%s' % self.host
- return common_util.RemoteCommand(self.host, 'root', 'true', error_msg=msg,
- output=True)
-
- @ExecuteTask('CMD')
- def SSHCmdOnHost(self, command, args=None):
- """Executes a command on the target host using an SSH connection.
-
- Args:
- command: Command to run.
- args: Extra arguments to main command to run on the remote host.
-
- Returns:
- String output from the command.
- """
- cmd = '"%s %s"' % (command, args)
- msg = 'Failed to run command=%s' % cmd
- return common_util.RemoteCommand(self.host, 'root', cmd, error_msg=msg,
- output=True)
-
- @ExecuteTask('COPY')
- def CopyToDevice(self):
- """Copies a file (usually a script file) to a host using scp.
-
- Returns:
- String output from the command.
- """
- msg = 'Failed to copy %s to root@%s:%s'% (self.options.script, self.host,
- self.options.remote_file)
- return common_util.RemoteCopy(self.host, 'root', self.options.script,
- self.options.remote_file, error_msg=msg,
- output=True)
-
- @ExecuteTask('URL')
- def ImageHost(self):
- """Uses the image_to_live script to update a host.
-
- Returns:
- String output from the command.
- """
- cmd = ('/usr/local/scripts/alarm %d %s/src/scripts/image_to_live.sh '
- '--update_url %s --remote %s' % (self.options.timeout,
- self.options.gclient,
- self.options.url, self.host))
- return common_util.RunCommand(cmd, output=True)
-
- @ExecuteTask('LOCK')
- def LockUnlockHost(self, lock=True):
- """Locks a host using the atest CLI.
-
- Locking a host tells Autotest that the host shouldn't be scheduled for
- any other tasks. Returns true if the locking process was successful.
-
- Args:
- lock: True=lock the host, False=unlock the host.
-
- Returns:
- String output from the command.
- """
- if lock:
- cmd = '%s host mod -l %s' % (self.options.cli, self.host)
- else:
- cmd = '%s host mod -u %s' % (self.options.cli, self.host)
- return common_util.RunCommand(cmd, output=True)
-
-
-class CommandManager(object):
- """Executes a command on all of the selected remote hosts.
-
- The hosts are selected from Autotest using the parameters supplied on the
- command line.
- """
-
- def __init__(self):
- self.options = self.ParseOptions()
- mp_log_util.InitializeLogging(**vars(self.options))
- if self.options.ip_addr:
- self.host_list = [self.options.ip_addr]
- else:
- self.host_list = autotest_util.GetHostList(self.options.cli,
- self.options.acl,
- self.options.label,
- self.options.user,
- self.options.status)
-
- @staticmethod
- def ParseOptions():
- """Grab the options from the command line."""
-
- parser = optparse.OptionParser(
- 'Used to run a command or script or update on all hosts, or only those '
- 'of a given platform, in the Autotest lab. Allows a configurable '
- 'number of commands to be started in parallel.\n\n'
- '\texample: %prog [options] command\n\n'
- 'Arguments after command are interpreted as arguments to the command.\n'
- '\n\texample: %prog [options] command [cmd_arg_1] [cmd_arg_2]\n\n'
- 'Multiple command can be run by enclosing them in quotation marks.\n\n'
- '\texample: %prog [options] "command1; command2; command2"\n\n'
- 'When using the --script option, additional arguments are interpreted '
- 'as script options and are passed to the script after being copied to '
- 'the remote device.\n\n'
- '\texample: %prog [options] --script /path/to/script.sh '
- '[script_arg_1] [script_arg_2] [script_arg_3]\n\n'
- 'When using the --url option specify the path to the new build. '
- 'Additional arguments are ignored.\n\n'
- '\texample: %prog [options] --url /path/to/build')
-
- # Args for describing the environment of the server machine
- group = optparse.OptionGroup(
- parser, 'Server Configuration', 'Options that specify the layout of '
- 'the machine hosting this script.')
- group.add_option(
- '-g', '--gclient', default=DEFAULT_GCLIENT_ROOT,
- help=('Location of ChromeOS checkout. [default: %default]'))
- parser.add_option_group(group)
-
- # Args for configuring logging.
- group = mp_log_util.AddOptions(parser)
- group.add_option(
- '--log_path', default=DEFAULT_LOG_PATH,
- help=('Where to put individual host log files. [default: %default]'))
- group.add_option(
- '-n', '--no_log_files', default=False, action='store_true',
- help=('Skip writing output to files, instead display results on the '
- 'console window only. [default: %default]'))
- group.add_option(
- '-e', '--echo_output', default=False, action='store_true',
- help=('Write command output to console. [default: %default]'))
- parser.add_option_group(group)
-
- # Args for selecting machines from Autotest
- group = autotest_util.AddOptions(parser)
- group.add_option(
- '-i', '--ip_addr',
- help=('IP address of single machine to run on.'))
- parser.add_option_group(group)
-
- # Args for defining how to run tasks from the server
- group = optparse.OptionGroup(
- parser, 'Execution Options', 'Options that define how commands are '
- 'run on the remote machines.')
- group.add_option(
- '-p', '--parallel', type='int', default=DEFAULT_CONCURRENCY,
- help=('Number of hosts to be run concurrently. '
- '[default: %default].'))
- group.add_option(
- '-t', '--timeout', type='int', default=DEFAULT_TIMEOUT,
- help=('Time to wait before killing the attempt to run command. '
- '[default: %default]'))
- group.add_option(
- '--skip_ssh', default=False, action='store_true',
- help=('Skip SSH check before running on each device. '
- '[default: %default]'))
- group.add_option(
- '-l', '--lock', default=False, action='store_true',
- help='Lock device in Autotest while running. [default: %default]')
- parser.add_option_group(group)
-
- # Args for the action to take on each remote device
- group = optparse.OptionGroup(
- parser, 'Main Options', 'Options that define main action. Selecting '
- 'neither --script nor --url defaults to running a command on the '
- 'hosts.')
- group.add_option(
- '-s', '--script', nargs=2,
- help=('Path to script to copy to host then execute. 2 args are '
- 'required. If the script does not take any args pass an empty '
- 'string \" \"'))
- group.add_option(
- '--url',
- help=('Run image_to_live.sh with provided image URL. Note: Resets '
- 'defaults for --lock=TRUE and --timeout=2400 and --parallel='
- '24.'))
- parser.add_option_group(group)
-
- options, args = parser.parse_args()
-
- options.cmd = None
- options.extra_args = None
- options.remote_file = None
-
- # If script/url was not specified, the remaining args are commands.
- if not options.script and not options.url:
- if not args:
- parser.error('Either script, command, or URL must be selected.')
- else:
- options.cmd, options.extra_args = args[0], ' '.join(args[1:])
-
- # Grab the arguments to the script and setup any extra args.
- if options.script:
- options.script, options.extra_args = options.script[0], options.script[1]
- options.remote_file = os.path.join(DEFAULT_REMOTE_COPY_PATH,
- options.script.split(os.path.sep)[-1])
- else:
- options.remote_file = ''
-
- # For updates reset default lock and timeout.
- if options.url:
- # Only modify these options if they still have their default values. If
- # the user has already overwritten them keep the users values.
- if options.timeout == DEFAULT_TIMEOUT:
- options.timeout = DEFAULT_UPDATE_TIMEOUT
- if options.parallel == DEFAULT_CONCURRENCY:
- options.parallel = DEFAULT_UPDATE_CONCURRENCY
-
- # Create log folder if it doesn't exist.
- if not options.no_log_files and not os.path.exists(options.log_path):
- os.makedirs(options.log_path)
-
- return options
-
-
-def ProcessResults(results, result_type):
- """Dump the results to the screen and/or log file.
-
- Args:
- results: Hosts with the same result type.
- result_type: String description of the result type.
- """
- msg = '%d hosts %s.\n' % (len(results), result_type)
- msg += ', '.join(results)
- mp_log_util.LogWithHeader(msg, width=80, symbol='-')
-
-
-def main():
- """Run commands in parallel on remote hosts."""
- script_start_time = datetime.datetime.now()
- cm = CommandManager()
- if not cm.host_list:
- logging.error('No hosts found.')
- return
- logging.info('Found %d hosts.', len(cm.host_list))
-
- # Create work object for each host retrieved.
- hosts = [HostWorker(host, cm.options) for host in cm.host_list]
-
- # Submit work to pool.
- mp_tp = tp.MultiProcWorkPool(max_threads=cm.options.parallel)
- hosts = mp_tp.ExecuteWorkItems(
- hosts, provide_logger=True,
- logger_init_callback=mp_log_util.InitializeLogging, **vars(cm.options))
-
- # Now that work is done, output results.
- status_strings = {'PASS': 'succeeded',
- 'SSH': 'failed connecting via SSH',
- 'LOCK': 'failed locking in Autotest',
- 'COPY': 'failed copying script',
- 'CMD': 'failed executing command',
- 'URL': 'failed updating image'}
- results = {}
- for key in status_strings:
- results[key] = []
-
- # Divide results by result type for prettier reporting.
- for h in hosts:
- results[h.result].append(h.host)
-
- # Output final results.
- for result, hosts in results.items():
- if hosts:
- ProcessResults(hosts, status_strings[result])
-
- if not cm.options.no_log_files:
- logging.info('Log files located in %s', cm.options.log_path)
-
- # Follow up with some timing info.
- script_runtime = datetime.datetime.now() - script_start_time
- logging.info('Running Time = %d.%d seconds.',
- script_runtime.seconds, script_runtime.microseconds)
-
-
-if __name__ == '__main__':
- main()
diff --git a/site_utils/run_tests.py b/site_utils/run_tests.py
deleted file mode 100755
index f4ef354..0000000
--- a/site_utils/run_tests.py
+++ /dev/null
@@ -1,115 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Tool for running test groups against different ChromeOS boards and platforms.
-
-run_tests allows users to execute test groups against Autotest hosts. Will
-create Autotest jobs given a board, platform, and list of test groups. Test
-groups are configurable via a JSON configuration file.
-
-run_tests will create jobs using a specialized control file which will update
-the targeted hosts to a specific board type and build version before running
-jobs against them.
-"""
-
-__author__ = 'dalecurtis@google.com (Dale Curtis)'
-
-import logging
-import optparse
-
-from chromeos_test import autotest_util
-from chromeos_test import dev_server
-from chromeos_test import log_util
-from chromeos_test import test_config
-import test_scheduler
-
-
-def ParseOptions():
- """Parse command line options. Returns 2-tuple of options and config."""
- # If default config exists, parse it and use values for help screen.
- config = test_config.TestConfig()
-
- # If config is provided parse values to make help screen more useful.
- boards, groups, platforms = config.ParseConfigGroups()
-
- parser = optparse.OptionParser(
- 'usage: %prog --board <BOARD> --platform <PLATFORM> [options]')
- parser.add_option('--board', dest='board',
- help=('Run tests only on the specified board. Valid boards:'
- ' %s' % boards))
- parser.add_option('--build', dest='build',
- help='Specify the build version to process.')
- parser.add_option('--groups', dest='groups',
- help=('Comma separated list of test groups. Valid groups:'
- ' %s' % groups))
- parser.add_option('--platform', dest='platform',
- help=('Run tests on the specified platform. Valid platforms'
- ': %s' % platforms))
-
- # Add utility/helper class command line options.
- test_config.AddOptions(parser)
- autotest_util.AddOptions(parser, cli_only=True)
-
- options = parser.parse_args()[0]
-
- if not options.board or not options.platform:
- parser.error('A board, build, and platform must be provided.')
-
- # Load correct config file if alternate is specified.
- if options.config != test_config.DEFAULT_CONFIG_FILE:
- config = test_config.TestConfig(options.config)
- boards, groups, platforms = config.ParseConfigGroups()
-
- if not options.groups:
- options.groups = config.GetConfig()['default_groups']
- else:
- options.groups = options.groups.split(',')
-
- if not options.board in boards:
- parser.error('Invalid board "%s" specified. Valid boards are: %s'
- % (options.board, boards))
-
- for group in options.groups:
- if not group in groups:
- parser.error('Invalid group "%s" specified. Valid groups are: %s'
- % (group, groups))
-
- if not options.platform in platforms:
- parser.error('Invalid platform "%s" specified. Valid platforms are: %s'
- % (options.platform, platforms))
-
- return options, config.GetConfig()
-
-
-def main():
- options, config = ParseOptions()
-
- # Setup logger and enable verbose mode.
- log_util.InitializeLogging(True)
-
- logging.info('------------[ Processing board %s ]------------', options.board)
-
- # Initialize Dev Server Utility class.
- dev = dev_server.DevServer(**config['dev_server'])
-
- # Get latest version for this board.
- if options.build:
- build = options.build
- else:
- raise NotImplementedException('You must pass in a build with the --build '
- 'flag. Detecting the latest build for a '
- 'board is no longer supported.')
-
- logging.info('Latest build version available on Dev Server is %s.', build)
-
- tr = test_scheduler.TestRunner(
- board=options.board, build=build, cli=options.cli, config=config, dev=dev)
-
- tr.RunTestGroups(groups=options.groups, lock=False, platform=options.platform)
-
-
-if __name__ == '__main__':
- main()
diff --git a/site_utils/system_health/chromeos_test_common.py b/site_utils/system_health/chromeos_test_common.py
deleted file mode 100644
index 3db8ba4..0000000
--- a/site_utils/system_health/chromeos_test_common.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""A common helper that adds chromeos_test libraries to the path.
-
-Also defines:
- chromeos_test_common.CURRENT_DIR: As the current directory.
-"""
-
-import os
-import sys
-
-# Figure out our absolute path so we can simplify configuration.
-CURRENT_DIR = os.path.realpath(os.path.abspath(os.path.join(
- os.getcwd(), os.path.dirname(__file__))))
-sys.path.append(os.path.join(CURRENT_DIR, '../'))
diff --git a/site_utils/system_health/monitor.py b/site_utils/system_health/monitor.py
deleted file mode 100755
index 2d11d11..0000000
--- a/site_utils/system_health/monitor.py
+++ /dev/null
@@ -1,1202 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""System Monitor.
-
-This program monitors the health of Chrome OS devices in the AutoTest testbed.
-
- Classes:
-
- Monitor - The Monitor is responsible for managing the overall process of
- keeping an updated status of each host available to AutoTest.
-
- RemoteWorker - responsible for SSHing to remote hosts to gather resources.
-
- Resource - maintains all of the resources that are monitored, and methods to
- parse their data for consumption by RRDTool.
-
- RRD - maintains all interfaces to RRDTool, including graph definitions, and
- methods to create, update, and graph resources.
-
- TestBed - a global class used to hold configuration data and data collected
- from each remote host. Additionally, formatted data for RRD will be kept
- associated with each host, and some general information about the update
- process of each remote host.
-
-
-Usage:
- The following options are supported:
- --webdir: Systemhealth web directory.
- --url: URL for landing page.
- --datadir: Non-NFS directory for RRD files.
-
- --graph: Create 1, 4, and 24 hour graphs for each host.
- --all_graphs: Create all graphs for each host.
- --html: Build HTML pages for hosts.
- --update: Collect data from hosts.
- --skip_at_status: Don't collect data about hosts from autotest CLI.
- --timout: Seconds to wait for remote commands to complete.
-
- --log_file: Write log messages to specified log file.
- --skip_console: Do not write log messages to the console.
- --verbose: Set the logging level to debug.
-
- --cli: Autotest CLI executable location.
- --acl: Autotest ACL Group to query for host machines.
- --label: Only run on hosts with the specified label.
- --status: Only run on hosts with the specified status.
- --user: Only run on hosts with the specified user.
-
- Arguments should be space separated.
-"""
-
-__author__ = ('kdlucas@gmail.com (Kelly Lucas) & '
- 'pauldean@google.com (Paul Pendlebury)')
-__version__ = '3.10'
-
-
-import cPickle
-import datetime
-import json
-import logging
-import optparse
-import os
-import shutil
-import sys
-import time
-import traceback
-
-import chromeos_test_common
-from chromeos_test import autotest_util
-from chromeos_test import common_util
-from chromeos_test import mp_log_util
-from chromeos_test import mp_thread_pool as tp
-import IPy
-
-
-class RemoteWorker(object):
- """Obtain resource data from remote hosts using monitor_remote_worker.py."""
-
- def __init__(self, hostname, platform, testbed):
- """Inits RemoteWorker with hostname and test configuration.
-
- Args:
- hostname: string, hostname of AutoTest host.
- platform: string, platform of hostname.
- testbed: testbed object for this run.
- """
- self.h = hostname
- self.platform = platform
- self.tb = testbed
-
- # Set up some dictionaries for each host.
- self.host_data = {}
- self.host_data['rrddata'] = {} # Formatted data.
- self.host_data['status'] = False
- self.host_data['time'] = None
- for v in self.tb.version:
- self.host_data[v] = {}
- self.host_data[v]['PTR'] = None
-
- def Run(self, logger):
- """Method called into by thread pool."""
-
- logger.debug('Starting host %s.', self.h)
- updated_html_needed = False
- data_file = os.path.join(self.tb.datadir, 'hosts', self.h, 'data.pkl')
- local_script = os.path.join(chromeos_test_common.CURRENT_DIR,
- 'monitor_remote_worker.py')
- remote_script = '/tmp/monitor_remote_worker.py'
-
- try:
- if self.tb.update:
- if not os.path.isfile(local_script):
- logger.error('Script file %s missing.', local_script)
- return
-
- # Copy script
- try:
- common_util.RemoteCopy(self.h, 'root', local_script, remote_script)
- except common_util.ChromeOSTestError:
- logger.error('Skipping unreachable host %s.', self.h)
- return
- # Run Script
- try:
- output = common_util.RemoteCommand(self.h, 'root', remote_script,
- output=True)
- self.host_data = cPickle.loads(output)
- except common_util.ChromeOSTestError:
- logger.exception('Error running script on host %s.', self.h)
- self.host_data['status'] = 'CollectionError'
- else:
- # If it exists, load saved host_data.
- if os.path.isfile(data_file):
- with open(data_file, 'rb') as in_file:
- self.host_data = cPickle.load(in_file)
-
- advisor = Resource()
- if ((self.tb.update or self.tb.graph) and
- self.host_data['status'] != 'CollectionError'):
- updated_html_needed = self.UpdateRelease(logger)
- advisor.ProcessHostRRD(self.h, self.host_data, self.tb, logger)
- if self.tb.html:
- advisor.BuildHTML(self.h, self.platform, self.host_data, self.tb,
- updated_html_needed)
-
- # Save successful host data so it can be loaded later.
- if self.tb.update and self.host_data['status'] == 'True':
- # rrd data is no longer needed, so don't save it.
- del self.host_data['rrddata']
- self.host_data['rrddata'] = {}
- with open(data_file, 'wb') as out_file:
- cPickle.dump(self.host_data, out_file, cPickle.HIGHEST_PROTOCOL)
-
- # Lots of exception handling happening here. This is the entry point
- # for this thread/process and if we let an exception go unhandled
- # we wouldn't it from the main thread and we would miss any
- # notifications of problems.
- except (KeyboardInterrupt, SystemExit):
- logging.exception('Shutdown requested.')
- sys.exit(1)
- except Exception:
- logging.exception('Unexpected Exception on %s', self.h)
- raise
- logger.debug('Finished host %s.', self.h)
-
- def UpdateRelease(self, logger):
- """Update Release info with most current release versions.
-
- The PTR key points to the most recent released version. This will also
- preserve the last known release version in case the host is down.
-
- Args:
- logger: multiprocess logger
-
- Returns:
- True/False if new HTML files are needed for this host.
- """
- rrd_dir = os.path.join(self.tb.datadir, 'hosts', self.h, 'rrd')
- # Check if the host directory exists, if not create it.
- common_util.MakedirsExisting(rrd_dir)
-
- update_html = False
- for v in self.tb.version:
- update_file = False
- relfile = os.path.join(rrd_dir, v)
- tmpfile = os.path.join(rrd_dir, v + '.tmp')
- if os.path.isfile(relfile):
- try:
- rf = open(relfile, 'r')
- lines = rf.readlines()
- except IOError, e:
- logger.error('Parsing release file %s\n%s', relfile, e)
- finally:
- rf.close()
-
- for line in lines:
- fields = line.split('=')
- # The correct format will have two strings separated by =.
- if len(fields) == 2:
- if fields[0] == 'PTR':
- if self.host_data[v]['PTR']:
- if self.host_data[v]['PTR'] != fields[1]:
- # Most recent version has changed.
- update_file = True
- lines.pop(lines.index(line))
- self.host_data[v][self.tb.time] = (self.host_data[v]['PTR'])
- else:
- # Host is down so use last known value.
- self.host_data[v]['PTR'] = (fields[1].strip())
- else:
- self.host_data[v][fields[0]] = (fields[1].strip())
- elif len(line) > 3:
- # This means the release file has the wrong format, so
- # we'll just write a new one with current values.
- update_file = True
- lines.pop(lines.index(line))
- else:
- # If we get here than it's probably a blank line.
- update_file = True
- lines.pop(lines.index(line))
-
- if update_file:
- update_html = True
- logger.debug('Updating %s', relfile)
- shutil.move(relfile, tmpfile)
- # Put the most recent update in the new file, and make the
- # PTR key to point to it.
- lines.append('%s=%s\n' % (self.tb.time, self.host_data[v]['PTR']))
- lines.append('PTR=%s' % self.host_data[v]['PTR'])
- try:
- rf = open(relfile, 'w')
- for line in lines:
- rf.write(line)
- except IOError, e:
- logger.error('Writing %s\n%s', relfile, e)
- finally:
- rf.close()
- else:
- # Create a new release file, as it does not exist.
- if self.host_data[v]['PTR']:
- update_html = True
- logger.info('Creating new %s', relfile)
- try:
- rf = open(relfile, 'w')
- rf.write('%s=%s\n' % (self.tb.time, self.host_data[v]['PTR']))
- rf.write('PTR=%s' % self.host_data[v]['PTR'])
- except IOError, e:
- logger.error('Writing %s\n%s', relfile, e)
- finally:
- rf.close()
-
- self.host_data[v][self.tb.time] = (self.host_data[v]['PTR'])
- return update_html
-
-
-class TestBed(object):
- """Used to hold all of the global variables."""
-
- def __init__(self, options):
- """Inits TestBed with run options.
-
- Args:
- options: Command line args for this run.
- """
- # Save run start time.
- self.time = int(time.time())
-
- # Setup logging.
- self.options = options
- self.logfile = options.log_file
-
- logger = logging.getLogger()
- mp_log_util.InitializeLogging(logger, **vars(options))
-
- # Warn and exit if SSH is not in the environment.
- if not 'SSH_AGENT_PID' in os.environ:
- logger.error('SSH_AGENT_PID not in environment, ssh commands will fail '
- 'to execute.')
- sys.exit(1)
-
- # Verify RRD is installed where we expect it.
- if not os.path.exists('/usr/bin/rrdtool'):
- logger.error('RRD is not installed to /usr/bin/rrdtool. Run \'sudo '
- 'apt-get install rrdtool\'.')
- sys.exit(1)
-
- # Assign TestBed values used for RRD and HTML pages.
- self.version = ['ec_firmware', 'firmware', 'release']
- self.rrdtimes = ['-1hours', '-4hours', '-24hours', '-1week', '-1month',
- '-1year']
-
- # Make sure directories exist to hold status and data files.
- run_dir = os.path.normpath('/tmp/systemhealth')
- common_util.MakedirsExisting(run_dir)
-
- # Default status files. Used to prevent more than one instance from
- # running at the same time.
- self.update_runfile = os.path.join(run_dir, 'update.running')
- self.graph_runfile = os.path.join(run_dir, 'graph.running')
-
- # Requested run actions.
- self.graph = options.graph
- self.all_graphs = options.all_graphs
- self.html = options.html
- self.update = options.update
- self.timeout = options.timeout
- self.skip_at_status = options.skip_at_status
-
- # Machine setup.
- self.webdir = options.webdir
- self.url = options.url
- self.datadir = options.datadir
-
- # Output some debug info.
- self.run_description = str(os.getpid()) + ':'
- if self.update:
- self.run_description += ' Update'
- if self.graph:
- self.run_description += ' Graph'
- if self.all_graphs:
- self.run_description += '_All'
- if self.html:
- self.run_description += ' HTML'
- if not self.skip_at_status:
- self.run_description += ' Status'
- mp_log_util.LogWithHeader('Start ' + self.run_description, logger)
-
-
-class Monitor(object):
- """Main class used to manage the monitoring of remote hosts.
-
- This class is used to determine the current status of hosts in the AutoTest
- testbed. AutoTest will be queried to populate self.rhosts. It will populate
- a list of RemoteWorkes and submit that list to MultiProcWorkPool to query
- each host to gather resource data.
- """
-
- def __init__(self, testbed, options):
- """Monitor will use config data from TestBed."""
- self.tb = testbed
- self.options = options
- self.mp_wp = tp.MultiProcWorkPool()
- self.afe_hosts = autotest_util.GetHostData(self.tb.options.cli,
- self.tb.options.acl,
- self.tb.options.label,
- self.tb.options.user,
- self.tb.options.status)
- self.host_status = []
-
- def UpdateStatus(self):
- """Update data from all monitored hosts."""
-
- # Don't attempt work when no hosts are known.
- if not self.afe_hosts:
- return
-
- # Record known host status from Autotest
- if not self.options.skip_at_status:
- self.RecordAutotestHostStatus(self.afe_hosts)
-
- # Create instance of RemoteWorker class for every host from atest.
- self.host_status = [RemoteWorker(host, self.afe_hosts[host]['platform'],
- self.tb) for host in self.afe_hosts.keys()]
-
- # Submit RemoteWorker items to thread pool.
- self.host_status = self.mp_wp.ExecuteWorkItems(
- self.host_status, 'Run', provide_logger=True,
- logger_init_callback=mp_log_util.InitializeLogging,
- **vars(self.options))
-
- loglevel = logging.getLogger().getEffectiveLevel()
- if loglevel == logging.DEBUG:
- for worker in self.host_status:
- logging.debug('%s status is %s/%s', worker.h,
- worker.host_data['status'],
- self.afe_hosts[worker.h]['status'])
-
- def RecordAutotestHostStatus(self, hosts):
- """Record Autotest status of all hosts in rrd files.
-
- Args:
- hosts: Dictionary of host information from autotest cli.
- """
-
- # Maps a host status string to an index in an array.
- status_key = {'Repairing': 0, 'Verifying': 1, 'Repair_Failed': 2,
- 'Running': 3, 'Cleaning': 4, 'Ready': 5, 'Pending': 6}
-
- # lab_status holds the lab data in the format rrd needs. The special
- # netbook_ALL platform is the sum of all the platforms.
- lab_status = {'netbook_ALL': [0] * len(status_key)}
-
- # Loop through all the hosts recording their status in lab_status
- for host in hosts:
- status = hosts[host]['status'].replace(' ', '_')
- platform = hosts[host]['platform']
-
- if platform not in lab_status:
- lab_status[platform] = [0] * len(status_key)
- if status in status_key:
- lab_status[platform][status_key[status]] += 1
- lab_status['netbook_ALL'][status_key[status]] += 1
- else:
- logging.error('Status=%s not a known status of %s', status, status_key)
-
- Resource().ProcessAutotestRRD(lab_status, self.tb, logging.getLogger())
-
- # Save data for later analysis in a pickled data file.
- for platform in lab_status:
- data_folder = os.path.join(self.tb.datadir, 'hosts', platform)
- common_util.MakedirsExisting(data_folder)
-
- data_file = os.path.join(data_folder, 'utilization.pkl')
- platform_data = {}
- if os.path.isfile(data_file):
- with open(data_file, 'rb') as in_file:
- platform_data = cPickle.load(in_file)
-
- date_entry = datetime.datetime.strftime(datetime.datetime.now(),
- '%Y_%m_%d_%H_%M_%S')
- platform_data[date_entry] = lab_status[platform]
- with open(data_file, 'wb') as out_file:
- cPickle.dump(platform_data, out_file, cPickle.HIGHEST_PROTOCOL)
-
- @staticmethod
- def ValidIP(address):
- """Verify address is a valid IP address.
-
- Args:
- address: string.
- Returns:
- boolean: True = valid IP address, False = not valid IP address.
- """
- octets = address.split('.')
- if len(octets) != 4:
- return False
- for octet in octets:
- if not 0 <= int(octet) <= 255:
- return False
- return True
-
- def SortAFEHosts(self, afelist):
- """Sort AFE host list by IP address.
-
- Args:
- afelist: list of AFE host objects.
- Returns:
- newlist: list of sorted AFE host objects.
- """
- iplist = []
- hostlist = []
-
- for h in afelist:
- if self.ValidIP(h):
- iplist.append(h)
- else:
- hostlist.append(h)
-
- templist = [(IPy.IP(h).int(), h) for h in iplist]
- templist.sort()
- newlist = [h[1] for h in templist]
- hostlist.sort()
- newlist.extend(hostlist)
-
- return newlist
-
- def BuildLandingPage(self):
- """Build the initial HTML landing page with links to all hosts."""
- logging.debug('Building Landing Page')
- sorted_hosts = []
- downhosts = 0
- down_repair = 0
- down_running = 0
- down_ready = 0
- down_other = 0
-
- readyhosts = 0
- ready_repair = 0
- ready_running = 0
- ready_ready = 0
- ready_other = 0
-
- scripthosts = 0
- script_repair = 0
- script_running = 0
- script_ready = 0
- script_other = 0
-
- hostlist = self.afe_hosts.keys()
- sorted_ip = self.SortAFEHosts(hostlist)
-
- # Create a dictionary to easily map host name to host result.
- host_results = {}
- for host in self.host_status:
- host_results[host.h] = host
-
- # Put host that are down first
- for h in sorted_ip:
- insert_offset = 0
- # Up hosts.
- if host_results[h].host_data['status'] == 'True':
- readyhosts += 1
- insert_offset += downhosts + scripthosts
- if self.afe_hosts[h]['status'] == 'Repair':
- insert_offset += ready_repair
- ready_repair += 1
- self.afe_hosts[h]['color'] = '#96BAC6'
- self.afe_hosts[h]['status_string'] = 'Repair'
- elif self.afe_hosts[h]['status'] == 'Running':
- insert_offset += ready_repair + ready_running
- ready_running += 1
- self.afe_hosts[h]['color'] = '#BBD9EE'
- self.afe_hosts[h]['status_string'] = 'Running'
- elif self.afe_hosts[h]['status'] == 'Ready':
- insert_offset += ready_repair + ready_running + ready_ready
- ready_ready += 1
- self.afe_hosts[h]['color'] = '#FFFFFF'
- self.afe_hosts[h]['status_string'] = 'Ready'
- else:
- insert_offset += (ready_repair + ready_running + ready_ready +
- ready_other)
- ready_other += 1
- self.afe_hosts[h]['color'] = '#788D9A'
- status_str = self.afe_hosts[h]['status']
- self.afe_hosts[h]['status_string'] = status_str
- # Up hosts with python problems.
- elif host_results[h].host_data['status'] == 'CollectionError':
- scripthosts += 1
- insert_offset += downhosts
- if self.afe_hosts[h]['status'] == 'Repair':
- insert_offset += script_repair
- script_repair += 1
- self.afe_hosts[h]['color'] = '#245403'
- self.afe_hosts[h]['status_string'] = 'ScriptError/Repair'
- elif self.afe_hosts[h]['status'] == 'Running':
- insert_offset += script_repair + script_running
- script_running += 1
- self.afe_hosts[h]['color'] = '#406331'
- self.afe_hosts[h]['status_string'] = 'ScriptError/Running'
- elif self.afe_hosts[h]['status'] == 'Ready':
- insert_offset += (script_repair + script_running + script_ready)
- script_ready += 1
- self.afe_hosts[h]['color'] = '#5E924E'
- self.afe_hosts[h]['status_string'] = 'ScriptError/Ready'
- else:
- insert_offset += (script_repair + script_running + script_ready +
- script_other)
- script_other += 1
- self.afe_hosts[h]['color'] = '#183503'
- status_str = 'ScriptError/' + self.afe_hosts[h]['status']
- self.afe_hosts[h]['status_string'] = status_str
- # Down hosts.
- else:
- downhosts += 1
- if self.afe_hosts[h]['status'] == 'Repair':
- insert_offset += down_repair
- down_repair += 1
- self.afe_hosts[h]['color'] = '#867146'
- self.afe_hosts[h]['status_string'] = 'Down/Repair'
- elif self.afe_hosts[h]['status'] == 'Running':
- insert_offset += down_repair + down_running
- down_running += 1
- self.afe_hosts[h]['color'] = '#E5DCBD'
- self.afe_hosts[h]['status_string'] = 'Down/Running'
- elif self.afe_hosts[h]['status'] == 'Ready':
- insert_offset += down_repair + down_running + down_ready
- down_ready += 1
- self.afe_hosts[h]['color'] = '#D6C085'
- self.afe_hosts[h]['status_string'] = 'Down/Ready'
- else:
- insert_offset += (down_repair + down_running + down_ready +
- down_other)
- down_other += 1
- self.afe_hosts[h]['color'] = '#4F4126'
- status_str = 'Down/' + self.afe_hosts[h]['status']
- self.afe_hosts[h]['status_string'] = status_str
- sorted_hosts.insert(insert_offset, h)
-
- # If we didn't connect to the host this run, load data from
- # the last successful run.
- if host_results[h].host_data['status'] != 'True':
- data_file = os.path.join(self.tb.datadir, 'hosts', h, 'data.pkl')
- if os.path.isfile(data_file):
- with open(data_file, 'rb') as in_file:
- host_results[h].host_data = cPickle.load(in_file)
-
- # Create symlink to the log file if it does not exist.
- log_filename = os.path.join(self.tb.webdir, 'monitor.log')
- if not os.path.isfile(log_filename):
- try:
- os.symlink(self.tb.logfile, log_filename)
- except OSError, e:
- logging.error('Linking to logfile\n%s', e)
- land_page_file = os.path.join(self.tb.webdir, 'index.html')
- # The temp file is used so that there will always be viewable html page
- # when the new page is being built.
- land_page_temp = os.path.join(self.tb.webdir, 'temp.html')
- f = open(land_page_temp, 'w')
- f.write('<HTML><HEAD>')
- f.write('<LINK REL="stylesheet" TYPE="text/css" HREF="table.css">')
- f.write('<TITLE>AutoTest System Health Check</TITLE></HEAD>')
- f.write('<BODY>')
- f.write('<img src="chrome.png" style="float:left;"/>')
- f.write('<table style="float: right">')
- f.write(('<TR><TD><a href=%s>%s</a><TD>Hosts<TD>Ready<TD>Repair<TD>'
- 'Running<TD>Other') % ('monitor.log', 'Log File'))
- f.write('<TR><TD>Total')
- f.write('<TD>%d<TD>%d<TD>%d<TD>%d<TD>%d' % (
- downhosts + readyhosts + scripthosts,
- down_ready + ready_ready + script_ready,
- down_repair + ready_repair + script_repair,
- down_running + ready_running + script_running,
- down_other + ready_other + script_other))
- f.write('<TR><TD>Inaccessible')
- f.write('<TD>%d<TD>%d<TD>%d<TD>%d<TD>%d' % (downhosts, down_ready,
- down_repair, down_running,
- down_other))
- f.write('<TR><TD>Script Error')
- f.write('<TD>%d<TD>%d<TD>%d<TD>%d<TD>%d' % (scripthosts, script_ready,
- script_repair, script_running,
- script_other))
- f.write('<TR><TD>Accessible')
- f.write('<TD>%d<TD>%d<TD>%d<TD>%d<TD>%d' % (readyhosts, ready_ready,
- ready_repair, ready_running,
- ready_other))
- f.write('</table>')
- f.write('<center><H1>CAUTOTEST Testbed</H1>')
- f.write('<H2>System Health</H2>')
- plat_graph = os.path.join(self.tb.url, 'hosts', 'netbook_ALL',
- 'utilization-24hours.png')
- f.write('<BR><BR><img src=%s ><BR><BR>' % plat_graph)
- f.write('<table>')
- f.write('<CAPTION>Hosts last updated: %s</CAPTION>' % time.strftime(
- '%d %b %Y - %I:%M:%S %p %Z', time.localtime()))
- f.write('<TR><TH>Hostname<TH>Status<TH>Labels<TH>Last Update')
- f.write('<TH>Release<TH>Health</TR>')
- for h in sorted_hosts:
- link_dir = 'hosts/' + h
- web_dir = os.path.join(self.tb.webdir, 'hosts', h)
- common_util.MakedirsExisting(web_dir, 0755)
- fqn = 'http://cautotest.corp.google.com/'
- view_host = 'afe/#tab_id=view_host&object_id=%s' % h
- hlink = fqn + view_host
- f.write('<tr bgcolor=%s><th>' % self.afe_hosts[h]['color'])
- f.write('<a href=%s>%s</a></th>' % (hlink, h))
- f.write('<td><em>%s</em>' % self.afe_hosts[h]['status_string'])
- f.write('<td>')
- f.write('<em><b>%s</b></em><br>' % self.afe_hosts[h]['platform'])
- for label in self.afe_hosts[h]['labels']:
- f.write('%s<br>' % label)
- f.write('<td>%s' % host_results[h].host_data['time'])
- if host_results[h].host_data['release']['PTR']:
- f.write('<td>%s' % host_results[h].host_data['release']['PTR'])
- else:
- f.write('<td>Unknown')
- index_file = os.path.join(web_dir, 'index.html')
- if os.path.isfile(index_file):
- f.write('<td><a href=%s' % self.tb.url)
- f.write('%s/index.html target="_blank">' % link_dir)
- f.write('health</a></td>')
- else:
- f.write('<td>None</td>')
- f.write('</table><p>\n</center>\n</BODY></HTML>')
- f.close()
- shutil.copyfile(land_page_temp, land_page_file)
- os.chmod(land_page_file, 0644)
-
-
-class Resource(object):
- """Contains structures and methods to collect health data on hosts.
-
- For each resource in self.resources, there must also be a corresponding
- method to format the data into what RRDTool expects.
- """
-
- def __init__(self):
- self.resources = [
- 'battery',
- 'boot',
- 'cpu',
- 'load',
- 'memory',
- 'network',
- 'power',
- 'temp',
- 'uptime'
- ]
- self.fs = [
- 'rootfsA_space',
- 'rootfsA_inodes',
- 'rootfsA_stats',
- 'rootfsB_space',
- 'rootfsB_inodes',
- 'rootfsB_stats',
- 'stateful_space',
- 'stateful_inodes',
- 'stateful_stats'
- ]
- self.resources += self.fs
-
- @staticmethod
- def ProcessAutotestRRD(hosts, testbed, logger):
- """Process formatted data into RRD files for each host in hosts.
-
- Args:
- hosts: dictionary of platforms and their data for rrd.
- testbed: configuration data for this run.
- logger: logger for this process/thread.
- """
- for platform in hosts:
- rrd_dir = os.path.join(testbed.datadir, 'hosts', platform, 'rrd')
- web_dir = os.path.join(testbed.webdir, 'hosts', platform)
-
- common_util.MakedirsExisting(rrd_dir)
- common_util.MakedirsExisting(web_dir, 0755)
-
- rrd_list = []
- for v in hosts[platform]:
- rrd_list += [str(v)]
-
- rrd_dict = {'rrddata': {'utilization': rrd_list}}
- rrd = RRD('utilization', platform, rrd_dir, web_dir, testbed)
- if not os.path.exists(rrd.rrdfile):
- rrd.Create(logger, 600)
- rrd.Update(rrd_dict, logger)
- rrd.Graph(rrd_dict, logger, False)
-
- def ProcessHostRRD(self, hostname, hostdata, testbed, logger):
- """Process formatted data into RRD files for host hostname.
-
- Args:
- hostname: string, hostname of AutoTest host.
- hostdata: raw data from the host.
- testbed: configuration data for this run.
- logger: logger for this process/thread.
- """
- rrd_dir = os.path.join(testbed.datadir, 'hosts', hostname, 'rrd')
- web_dir = os.path.join(testbed.webdir, 'hosts', hostname)
-
- common_util.MakedirsExisting(rrd_dir)
- common_util.MakedirsExisting(web_dir, 0755)
-
- for r in self.resources:
- dk = None # datakey only needs to be set if it's a file system.
- if r in self.fs:
- if '_space' in r:
- dk = 'fs_space'
- elif '_inode' in r:
- dk = 'fs_inode'
- elif '_stat' in r:
- dk = 'fs_stat'
-
- rrd = RRD(r, hostname, rrd_dir, web_dir, testbed, dk)
- if not os.path.exists(rrd.rrdfile):
- rrd.Create(logger)
- if testbed.update == True:
- logger.debug('Updating %s for host %s', r, hostname)
- rrd.Update(hostdata, logger)
- if testbed.graph:
- logger.debug('Building %s graphs for %s', r, hostname)
- rrd.Graph(hostdata, logger)
-
- def BuildHTML(self, hostname, platform, hostdata, testbed,
- update_needed=False):
- """Create HTML pages for to display the graphs.
-
- Args:
- hostname: string, hostname of AutoTest host.
- platform: string, platform of hostname.
- hostdata: raw data from the host.
- testbed: configuration data for this run.
- update_needed: new html needed, existing has wrong info.
- """
- web_dir = os.path.join(testbed.webdir, 'hosts', hostname)
- plat_dir = os.path.join(testbed.url, 'hosts', platform)
- index_file = os.path.join(web_dir, 'index.html')
-
- # If the index file exists, and the release info hasn't changed, skip.
- if os.path.isfile(index_file) and not update_needed:
- return
-
- mainindex = testbed.url + 'index.html'
- resource_list = []
- for r in self.resources:
- resource_list.append(r)
- resource_list.sort()
-
- html_file = {}
- for t in testbed.rrdtimes:
- html_file[t] = hostname + t + '.html'
- pathname = {}
- for name in html_file:
- pathname[name] = os.path.join(web_dir, html_file[name])
-
- # Create directory for html/graphs.
- common_util.MakedirsExisting(web_dir, 0755)
-
- # Create HTML files for each time period we are graphing.
- for path in pathname:
- f = open(pathname[path], 'w')
- f.write('<HTML><HEAD>')
- f.write('<center><TITLE>%s System Health</TITLE></HEAD>' % hostname)
- f.write('<BODY><H1>%s System Health</H1>' % hostname)
- for v in testbed.version:
- f.write('<H4>%s: %s</H4>' % (v, hostdata[v]['PTR']))
- for t in testbed.rrdtimes:
- f.write('<a href="%s">%s</a> <b>|</b>' % (html_file[t], t))
- f.write('<a href="%s">SystemHealth Home</a>' % mainindex)
- f.write('<p><HR>')
- plat_graph = os.path.join(plat_dir, 'utilization' + path + '.png')
- f.write('<img src=%s ><BR><BR>' % plat_graph)
- f.write('<table border=1 bgcolor=#EEEEEE>')
- newrow = True
- for r in resource_list:
- if newrow:
- f.write('<tr>')
- f.write('<td>%s<br><a href=%s.html>' % (r, r))
- f.write('<img src=%s%s.png width=475 height=250></a></td>' % (r, path))
- if newrow:
- newrow = False
- else:
- f.write('</tr>\n')
- newrow = True
- f.write('</table><p>\n')
- f.write('</center>\n')
- f.write('<H5>Last Update: %s</H5>' % hostdata['time'])
- f.write('</BODY></HTML>')
- f.close()
- os.chmod(pathname[path], 0644)
- # Set default landing page to 24-hour graphs
- if not os.path.isfile(index_file):
- os.symlink(pathname[testbed.rrdtimes[2]], index_file)
-
- # Create HTML files for each resource for all time periods.
- for r in resource_list:
- rrdfile = os.path.join(web_dir, r + '.html')
- f = open(rrdfile, 'w')
- f.write('<HTML><HEAD>')
- f.write('<center><TITLE>%s %s Resources</TITLE></HEAD>' % (hostname, r))
- f.write('<BODY><H1>%s %s Resources</H1>' % (hostname, r))
- for v in testbed.version:
- f.write('<H4>%s: %s</H4>' % (v, hostdata[v]['PTR']))
- f.write('<table border=5 bgcolor=#B5B5B5>')
- f.write('<tr>')
- for t in testbed.rrdtimes:
- f.write('<td><a href="#%s"><b>%s</b></a>' % (t, t))
- f.write('</table>')
- f.write('<HR>')
- f.write('<table border=1 bgcolor=#EEEEEE>')
- for t in testbed.rrdtimes:
- f.write('<tr><td><a name="%s"><img src=%s%s.png>' % (t, r, t))
- f.write('</a></td></tr>\n')
- f.write('</table><p>\n')
- f.write('</center>\n')
- f.write('<H5>Last Update: %s</H5>' % hostdata['time'])
- f.write('</BODY></HTML>')
- f.close()
- os.chmod(rrdfile, 0644)
-
-
-class RRD(object):
- """The class to create and update RRD data stores and graph them.
-
- This class should be used to access all of the functions of RRDTool. It will
- create the data files, update them, and create graphs/charts based on that
- data. Datakey is needed when we are using the same data definitions for many
- items of the same type, like file systems.
- """
-
- def __init__(self, rrdname, hostname, rrd_dir, web_dir, tb, datakey=None):
- """Inits RRD class.
-
- Args:
- rrdname: string, item name(should match key from Resources)
- hostname: string, hostname of the machine.
- rrd_dir: string, directory for rrd files.
- web_dir: string, directory for generated graphs.
- tb: testbase object for this run.
- datakey: string, overrides which data definition to use.
- """
- self.tb = tb
- self.rrdtool = '/usr/bin/rrdtool'
- self.rrd_dir = rrd_dir
- self.web_dir = web_dir
- self.rrdname = rrdname
- self.hostname = hostname
- rrd_filename = rrdname + '.rrd'
- self.rrdfile = os.path.join(self.rrd_dir, rrd_filename)
- file_system = 'Unknown'
-
- if not datakey:
- datakey = rrdname
- else:
- fields = rrdname.split('_')
- if fields[0]:
- file_system = fields[0]
-
- self.dd = json.load(open(os.path.join(sys.path[0], 'rrd.json')))[datakey]
- self.dd['title'] %= {'host': self.hostname, 'file_system': file_system}
-
- def Create(self, logger, step=600):
- """Create an empty RRD file.
-
- Args:
- logger: Multiprocess logger.
- step: Default rrdtool step.
-
- Returns:
- boolean: True = Success, False = failure.
- """
-
- stime = int(time.time()) - 5 * 86400
- rrd_suffix = ['RRA:AVERAGE:0.5:1:576', 'RRA:AVERAGE:0.5:6:672',
- 'RRA:AVERAGE:0.5:24:732', 'RRA:AVERAGE:0.5:144:1460']
-
- rrd_cmd = [self.rrdtool, 'create', self.rrdfile, '--start', str(stime),
- '--step', str(step)]
- for ds in self.dd['items']:
- ds_str = 'DS:%s:%s:%s:%s:%s' % (ds, self.dd['type'], self.dd['heartbeat'],
- self.dd['min'], self.dd['max'])
- rrd_cmd.append(ds_str)
- rrd_cmd += rrd_suffix
- # Convert the rrd_cmd to a string with space separated commands.
- exec_str = ' '.join(rrd_cmd)
- try:
- common_util.RunCommand(exec_str)
- except common_util.ChromeOSTestError:
- logger.error('Executing: "%s".', exec_str)
- return False
- return True
-
- def Update(self, hostdata, logger):
- """Update an existing RRD file.
-
- Args:
- hostdata: dictionary of raw data from this host.
- logger: logger for this process/thread
-
- Returns:
- boolean: True = Success, False = errors.
- """
- if self.rrdname in hostdata['rrddata']:
- data_count = len(hostdata['rrddata'][self.rrdname])
- if data_count == 0:
- logger.debug('Key "%s" empty in hostdata for host %s.', self.rrdname,
- self.hostname)
- return False
-
- if data_count < 2:
- data = 'N:' + hostdata['rrddata'][self.rrdname][0]
- else:
- data = 'N:' + ':'.join(hostdata['rrddata'][self.rrdname])
- rrd_cmd = [self.rrdtool, 'update', self.rrdfile, data]
- exec_str = ' '.join(rrd_cmd)
- try:
- common_util.RunCommand(exec_str)
- except common_util.ChromeOSTestError:
- logger.error('Executing: "%s".', exec_str)
- return False
-
- return True
- else:
- logger.debug('Key "%s" not found in hostdata for host %s.', self.rrdname,
- self.hostname)
- return False
-
- def Graph(self, hostdata, logger, include_updates=True, file_prefix=''):
- """Create a graph of a tracked resource.
-
- Args:
- hostdata: Dictionary of raw data from this host.
- logger: Logger for this process/thread.
- include_updates: Include firmware update history in graphs.
- file_prefix: String to append to front of graph file names.
- """
- width = '850'
- height = '300'
- end = 'now'
- rcolor = {'release': '#9966FF', 'firmware': '#990033',
- 'ec_firmware': '#009933'}
-
- if self.tb.all_graphs:
- rrdtimes = self.tb.rrdtimes
- else:
- rrdtimes = self.tb.rrdtimes[:3]
-
- for rrdtime in rrdtimes:
- png_filename = file_prefix + self.rrdname + rrdtime + '.png'
- png_file = os.path.join(self.web_dir, png_filename)
-
- title = self.dd['title'] + ' ' + rrdtime + '"'
-
- rrd_cmd = [self.rrdtool, 'graph', png_file, '--imgformat PNG', '-s',
- rrdtime, '--end', end, '--width', width, '--height', height,
- '--vertical-label', self.dd['units'], '--title', title]
-
- for ds in self.dd['items']:
- rrd_cmd.append('DEF:%s=%s:%s:AVERAGE' % (ds, self.rrdfile, ds))
- rrd_cmd += self.dd['graph']
- if include_updates:
- rrd_cmd.append('COMMENT:"Release History \\s"')
- rrd_cmd.append('COMMENT:"=============== \\n"')
- for v in self.tb.version:
- sorted_items = []
- for k in hostdata[v]:
- if k != 'PTR':
- sorted_items.append(k)
- sorted_items.sort()
- for i in sorted_items:
- # Get a date/time string to display, localtime requires
- # a float, so convert i to float.
- fw_datetime = time.strftime('%D %H\\:%M', time.localtime(float(i)))
- # Need to escape any ':' for RRDTool.
- filter_val = (hostdata[v][i].replace(':', '\\:'))
- if not self.tb.all_graphs:
- # Insert Veritical Lines for release and firmware updates.
- vrule = 'VRULE:%s%s:"%s %s=%s \\n"' % (i, rcolor[v], fw_datetime,
- v, filter_val)
- else:
- # On Week + graphs, only insert release comment. There are too
- # many vertical lines on the longer graphs to make see anything
- # else.
- vrule = 'COMMENT:"%s %s=%s \\n"' % (fw_datetime, v, filter_val)
- rrd_cmd.append(vrule)
-
- exec_str = ' '.join(rrd_cmd)
- try:
- common_util.RunCommand(exec_str)
- except common_util.ChromeOSTestError:
- logger.error('Executing: "%s".', exec_str)
- if os.path.isfile(png_file):
- os.chmod(png_file, 0644)
-
-
-def ParseArgs():
- """Parse all command line options."""
- homedir = os.environ['HOME']
- datadir = os.path.normpath('/usr/local/google/%s/systemhealth' % homedir)
- systemhealth_webdir = os.path.join(homedir, 'www', 'systemhealth')
- logfile = os.path.join(systemhealth_webdir, 'monitor.log')
- defaul_url = 'http://www/~%s/systemhealth/' % os.environ['USER']
-
- parser = optparse.OptionParser(version=__version__)
-
- # Args for describing the environment of the server machine.
- group = optparse.OptionGroup(
- parser, title='Server Configuration',
- description=('Options specifying the layout of this machine.'))
- group.add_option(
- '-w', '--webdir',
- help='Systemhealth web directory [default: %default]',
- default=systemhealth_webdir,
- dest='webdir')
- group.add_option(
- '-u', '--url',
- help='URL for landing page [default: %default]',
- default=defaul_url,
- dest='url')
- group.add_option(
- '-d', '--datadir',
- help='Non-NFS directory for RRD. [default: %default]',
- default=datadir,
- dest='datadir')
- parser.add_option_group(group)
-
- # Args for describing logging.
- mp_log_util.AddOptions(parser)
-
- # Args for selecting hosts from Autotest.
- autotest_util.AddOptions(parser)
-
- # Args for describing what work to perform.
- group = optparse.OptionGroup(
- parser, title='Run Configuration',
- description=('Options specifying what actions the script will perform.'))
- group.add_option(
- '--graph',
- help=('Create 1, 4, & 24 hour graphs for each host [default: %default]'),
- default=False, action='store_true', dest='graph')
- group.add_option(
- '--all_graphs',
- help='Create all graphs for each host [default: %default]',
- default=False, action='store_true', dest='all_graphs')
- group.add_option(
- '--html',
- help='Build HTML pages for hosts [default: %default]',
- default=False, action='store_true', dest='html')
- group.add_option(
- '--update',
- help='Collect data from hosts [default: %default]',
- default=False, action='store_true', dest='update')
- group.add_option(
- '--timout',
- help=('Timeout for remote commands to complete [default: %default]'),
- default=30, dest='timeout')
- group.add_option(
- '--skip_at_status',
- help=('Record the host status in autotest [default: %default]'),
- default=False, action='store_true', dest='skip_at_status')
-
- parser.add_option_group(group)
-
- options = parser.parse_args()[0]
-
- if not options.log_file:
- options.log_file = logfile
-
- if options.all_graphs:
- options.graph = True
-
- if not (options.graph or options.html or options.update):
- parser.error('Must specify at least one of the --graph, --html, or '
- '--update options.')
-
- # Create required directories if they don't exist.
- common_util.MakedirsExisting(options.datadir)
- common_util.MakedirsExisting(options.webdir, 0755)
- common_util.MakedirsExisting(os.path.join(options.webdir, 'hosts'), 0755)
-
- return options
-
-
-def CheckRun(action, tb):
- """Check the run status of monitor.py, and add/remove run files.
-
- This function will ensure we only running one program with either the graph
- or update option.
- Args:
- action: string, indicates if monitor.py is starting or stopping.
- tb: options for this run.
- """
- if action == 'start':
- if tb.update == True:
- if os.path.isfile(tb.update_runfile):
- logging.info('Exiting, already running with update option')
- sys.exit(1)
- else:
- try:
- open(tb.update_runfile, 'w').close()
- except IOError, e:
- logging.error('Opening %s\n%s', tb.update_runfile, e)
- if tb.graph:
- if os.path.isfile(tb.graph_runfile):
- logging.info('Exiting, already running with graph option')
- sys.exit(1)
- else:
- try:
- open(tb.graph_runfile, 'w').close()
- except IOError, e:
- logging.error('Opening %s\n%s', tb.graph_runfile, e)
- elif action == 'stop':
- if tb.update == True:
- if os.path.isfile(tb.update_runfile):
- try:
- os.remove(tb.update_runfile)
- except IOError, e:
- logging.error('Removing %s\n%s', tb.update_runfile, e)
- if tb.graph:
- if os.path.isfile(tb.graph_runfile):
- try:
- os.remove(tb.graph_runfile)
- except IOError, e:
- logging.error('Removing %s\n%s', tb.graph_runfile, e)
- else:
- logging.error('Unknown option passed to CheckRun(): %s', action)
- sys.exit(1)
-
-
-def main():
- start_time = time.time()
- options = ParseArgs()
-
- test_bed = TestBed(options)
- CheckRun('start', test_bed)
- try:
- sysmon = Monitor(test_bed, options)
- if not sysmon.afe_hosts:
- logging.error('No hosts found, nothing to do, exiting.')
- sys.exit(1)
- sysmon.UpdateStatus()
- if test_bed.update:
- sysmon.BuildLandingPage()
-
- runtime = time.time() - start_time
- msg = 'End [ %s ] Runtime %d seconds' % (test_bed.run_description, runtime)
- mp_log_util.LogWithHeader(msg, symbol='-')
-
- except (KeyboardInterrupt, SystemExit):
- logging.error('Shutdown requested.')
- sys.exit(1)
- except Exception, e:
- logging.error('Exception: %s\n%s', e, traceback.format_exc())
- raise
- finally:
- CheckRun('stop', test_bed)
- os.chmod(options.log_file, 0755)
-
-if __name__ == '__main__':
- main()
diff --git a/site_utils/system_health/monitor_remote_worker.py b/site_utils/system_health/monitor_remote_worker.py
deleted file mode 100755
index 89e23db..0000000
--- a/site_utils/system_health/monitor_remote_worker.py
+++ /dev/null
@@ -1,473 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Monitor Remote Worker.
-
-This program gathers statistics from a Chromium device.
-
- Classes:
-
- HostWorker - responsible for gathering host resources.
-
- Resource - maintains all of the resources that are monitored, and methods to
- parse their data for consumption by RRDTool.
-
-"""
-
-__author__ = ('kdlucas@gmail.com (Kelly Lucas) & '
- 'pauldean@google.com (Paul Pendlebury)')
-__version__ = '1.00'
-
-
-import cPickle
-import logging
-import subprocess
-import sys
-import time
-import traceback
-
-
-class HostWorker(object):
- """Obtain host resource data."""
-
- def __init__(self):
- """Inits HostWorker."""
- self.logger = logging.getLogger()
- self.version = ['ec_firmware', 'firmware', 'release']
-
- # Set up some data dictionaries
- self.host_data = {}
- self.host_data['data'] = {} # Raw data from hosts.
- self.host_data['rrddata'] = {} # Formatted data.
- self.host_data['status'] = 'True'
- self.host_data['time'] = None
-
- for v in self.version:
- self.host_data[v] = {}
- self.host_data[v]['PTR'] = None
-
- def Run(self):
- """Main class method to gather host resources."""
-
- try:
- self.host_data['time'] = time.strftime('%d%b%Y %H:%M:%S',
- time.localtime())
- self.ReadRelease()
- self.ReadFirmware()
- self.ReadResources()
- except (KeyboardInterrupt, SystemExit):
- self.logger.exception('Shutdown requested.')
- sys.exit(1)
- except Exception:
- self.logger.exception('Unexpected Exception.')
- raise
-
- def ReadRelease(self):
- """Get the Chrome OS Release version.
-
- The PTR key in host_data['release'] will mark the current version.
- """
- # Use grep to find the one line in the file we are after.
- cmd = ('grep CHROMEOS_RELEASE_DESCRIPTION /etc/lsb-release')
- output = ExecuteCommand(cmd)
- if output[0] == 0:
- if 'CHROMEOS_RELEASE_DESCRIPTION' in output[1]:
- release = output[1].split('=')
- self.host_data['release']['PTR'] = release[1].strip()
-
- def ReadFirmware(self):
- """Get the Firmware versions.
-
- The PTR key in host_data['ec_firmware'] and host_data['firmware'] will
- mark the current versions.
- """
- # Use grep to return the two segments of the string we are after.
- # Message 'Unable to auto-detect platform. Limited functionality only.'
- # is showing up on StandardError, so redirect that to /dev/null.
- cmd = ('/usr/sbin/mosys -k smbios info bios 2>/dev/null | '
- 'grep -o \'[ec_]*version=\\"[^ ]*\\"\'')
- output = ExecuteCommand(cmd)
- if output[0] == 0:
- lines = output[1].split()
- for item in lines:
- if 'ec_version' in item:
- fields = item.split('=')
- # We must sanitize the string for RRDTool.
- val = fields[1].strip('\n" ')
- self.host_data['ec_firmware']['PTR'] = val
- elif 'version' in item:
- fields = item.split('=')
- val = fields[1].strip('\n" ')
- self.host_data['firmware']['PTR'] = val
-
- def ReadResources(self):
- """Get resources that we are monitoring on the host.
-
- Combine all the individual commands to execute into one large command
- so only one SSH connection to the host is required instead of an
- individual connection for each command in advisor.resources.
- """
- advisor = Resource()
- # Get the individual commands from the Resource class.
- cmds = advisor.GetCommands(self.logger)
- for r in advisor.resources:
- output = ExecuteCommand(cmds[r])
- if output[0] == 0:
- self.host_data['data'][r] = output[1]
- advisor.FormatData(self.host_data, self.logger)
-
-
-class Resource(object):
- """Contains structures and methods to collect health data on hosts.
-
- For each resource in self.resources, there must also be a corresponding
- method to format the data into what RRDTool expects.
- """
-
- def __init__(self):
- self.files = {'battery': '/proc/acpi/battery/BAT?/state',
- 'boot': ('/tmp/firmware-boot-time'
- ' /tmp/uptime-login-prompt-ready'),
- 'cpu': '/proc/stat',
- 'load': '/proc/loadavg',
- 'memory': '/proc/meminfo',
- 'network': '/proc/net/dev',
- 'power': '/proc/acpi/processor/CPU0/throttling',
- 'temp': '/proc/acpi/thermal_zone/*/temperature',
- 'uptime': '/proc/uptime',
- }
- self.fs = {'rootfsA_space': '/',
- 'rootfsA_inodes': '/',
- 'rootfsA_stats': 'sda2 ',
- 'rootfsB_space': '/',
- 'rootfsB_inodes': '/',
- 'rootfsB_stats': 'sda5 ',
- 'stateful_space': '/mnt/stateful_partition',
- 'stateful_inodes': '/mnt/stateful_partition',
- 'stateful_stats': 'sda1 ',
- }
- self.resources = []
- for k in self.files:
- self.resources.append(k)
- for k in self.fs:
- self.resources.append(k)
-
- def FormatData(self, hostdata, logger):
- """Convert collected data into the correct format for RRDTool.
-
- Args:
- hostdata: raw data from the host.
- logger: logger for this process/thread.
- """
-
- parse_method = {'battery': self.ParseBattery,
- 'boot': self.ParseBoot,
- 'fs': self.ParseFS,
- 'diskstats': self.ParseDiskStats,
- 'cpu': self.ParseStat,
- 'load': self.ParseLoadAvg,
- 'memory': self.ParseMemInfo,
- 'network': self.ParseNetDev,
- 'power': self.ParsePower,
- 'temp': self.ParseTemp,
- 'uptime': self.ParseUpTime,
- }
-
- # method_key is used here because multiple resource keys will use the
- # same method to parse them.
- for key in hostdata['data']:
- method_key = key
- if key in self.fs:
- if '_space' in key:
- method_key = 'fs'
- elif '_inode' in key:
- method_key = 'fs'
- elif '_stats' in key:
- method_key = 'diskstats'
- else:
- logger.error('Invalid key "%s".', key)
- if method_key in parse_method:
- if hostdata['data'][key]:
- parse_method[method_key](key, hostdata)
- else:
- logger.debug('%s missing from hostdata.', key)
- else:
- logger.error('No method to parse resource %s', key)
-
- @staticmethod
- def ParseBattery(k, hostdata):
- """Convert /proc/acpi/battery/BAT0/state to a list of strings.
-
- Args:
- k: string, resource key.
- hostdata: dictionary of raw data from this host.
- We only care about the values corresponding to the rrdkeys, so the other
- values will be discarded.
- """
- rrdkeys = ['charging state', 'present rate', 'remaining capacity']
- hostdata['rrddata'][k] = []
- statlist = hostdata['data'][k].split('\n')
- for stat in statlist:
- for key in rrdkeys:
- if key in stat:
- stats = stat.split(':')
- temp = stats[1].split()
- if key == 'charging state':
- if temp[0] == 'discharging':
- hostdata['rrddata'][k].append('0')
- else:
- hostdata['rrddata'][k].append('1')
- else:
- hostdata['rrddata'][k].append(temp[0])
-
- @staticmethod
- def ParseBoot(k, hostdata):
- """Parse /tmp/uptime-login-prompt-ready for boot time.
-
- Args:
- k: string, resource key.
- hostdata: dictionary of raw data from this host.
- We only want the first and 2nd values from the raw data.
- """
- fields = []
- hostdata['rrddata'][k] = []
- lines = hostdata['data'][k].split('\n')
- for line in lines:
- if not '==>' in line:
- fields.extend(line.split())
- hostdata['rrddata'][k] = fields[0:2]
-
- @staticmethod
- def ParseFS(k, hostdata):
- """Convert file system space and inode readings to a list of strings.
-
- Args:
- k: string, resource key.
- hostdata: dictionary of raw data from this host.
- """
- hostdata['rrddata'][k] = []
- lines = hostdata['data'][k].split('\n')
- for line in lines:
- if not line.startswith('Filesystem'):
- fields = line.split()
- if len(fields) > 4:
- hostdata['rrddata'][k].append(fields[2])
- hostdata['rrddata'][k].append(fields[3])
-
- @staticmethod
- def ParseDiskStats(k, hostdata):
- """Parse read and write sectors from /proc/diskstats to list of strings.
-
- Args:
- k: string, resource key.
- hostdata: dictionary of raw data from this host.
- """
- hostdata['rrddata'][k] = []
- fields = hostdata['data'][k].split()
- if len(fields) > 9:
- hostdata['rrddata'][k].append(fields[5])
- hostdata['rrddata'][k].append(fields[9])
-
- @staticmethod
- def ParseStat(k, hostdata):
- """Convert /proc/stat to lists for CPU usage.
-
- Args:
- k: string, resource key.
- hostdata: dictionary of raw data from this host.
- """
- lines = hostdata['data'][k].split('\n')
- for line in lines:
- if 'cpu ' in line:
- vals = line.split()
- hostdata['rrddata'][k] = vals[1:5]
-
- @staticmethod
- def ParseLoadAvg(k, hostdata):
- """Convert /proc/loadavg to a list of strings to monitor.
-
- Args:
- k: string, resource key.
- hostdata: dictionary of raw data from this host.
- Process ID is discarded, as it's not needed.
- """
- statlist = hostdata['data'][k].split()
- hostdata['rrddata'][k] = statlist[0:3]
-
- @staticmethod
- def ParseMemInfo(k, hostdata):
- """Convert specified fields in /proc/meminfo to a list of strings.
-
- Args:
- k: string, resource key.
- hostdata: dictionary of raw data from this host.
- """
- hostdata['rrddata'][k] = []
- mem_keys = ['MemTotal', 'MemFree', 'Buffers', 'Cached', 'SwapTotal',
- 'SwapFree']
- lines = hostdata['data'][k].split('\n')
- for line in lines:
- for key in mem_keys:
- if key in line:
- if not 'SwapCached' in line:
- fields = line.split()
- hostdata['rrddata'][k].append(fields[1])
-
- @staticmethod
- def ParseNetDev(k, hostdata):
- """Convert /proc/net/dev to a list of strings of rec and xmit values.
-
- Args:
- k: string, resource key.
- hostdata: dictionary of raw data from this host.
- """
- net_keys = ['eth0', 'wlan0']
- rrdlist = ['0', '0', '0', '0']
- lines = hostdata['data'][k].split('\n')
- for key in net_keys:
- for line in lines:
- if key in line:
- # The following routine will ensure that the values are
- # placed in the correct order in case there is an expected
- # interface is not present.
- index = net_keys.index(key)
- if index:
- index *= 2
- data = line.split(':')[1]
- fields = data.split()
- rrdlist[index] = fields[0]
- rrdlist[index + 1] = fields[8]
-
- hostdata['rrddata'][k] = rrdlist
-
- @staticmethod
- def ParsePower(k, hostdata):
- """Convert /proc/acpi/processor/CPU0/throttling to power percentage.
-
- Args:
- k: string, resource key.
- hostdata: dictionary of raw data from this host.
- """
- hostdata['rrddata'][k] = []
- lines = hostdata['data'][k].split('\n')
- for line in lines:
- line = line.strip()
- if line.startswith('*'):
- fields = line.split(':')
-
- if 'fields' in locals():
- if len(fields) > 1:
- percent = fields[1].strip('%')
- percent = percent.strip()
- hostdata['rrddata'][k].append(percent)
-
- @staticmethod
- def ParseTemp(k, hostdata):
- """Convert temperature readings to a list of strings.
-
- Args:
- k: string, resource key.
- hostdata: dictionary of raw data from this host.
- """
- hostdata['rrddata'][k] = []
- statlist = hostdata['data'][k].split()
- if len(statlist) > 1:
- hostdata['rrddata'][k].append(statlist[1])
-
- @staticmethod
- def ParseUpTime(k, hostdata):
- """Convert /proc/uptime to a list of strings.
-
- Args:
- k: string, resource key.
- hostdata: dictionary of raw data from this host.
- Returns:
- list of strings.
- """
-
- hostdata['rrddata'][k] = hostdata['data'][k].split()
-
- def GetCommands(self, logger):
- """Routine for gathering data from files and file systems.
-
- Args:
- logger: multiprocess logger.
-
- Returns:
- dictionary of commands to run on hosts.
- """
-
- command = {}
-
- for r in self.resources:
- if r in self.files:
- if r == 'boot':
- command[r] = 'head %s' % self.files[r]
- else:
- command[r] = 'cat %s' % self.files[r]
- elif r in self.fs:
- if '_space' in r:
- command[r] = 'df -lP %s' % self.fs[r]
- elif '_inode' in r:
- command[r] = 'df -iP %s' % self.fs[r]
- elif '_stat' in r:
- command[r] = 'cat /proc/diskstats | grep %s' % self.fs[r]
- else:
- logger.error('Invalid key "%s".', r)
- return command
-
-
-def ExecuteCommand(cmd):
- """Execute a command.
-
- Args:
- cmd: command string to run
-
- Returns:
- tuple(command return code, standard out, standard error)
-
- Note: If the command throws an OSError or ValueError the return code will
- be -1 and standard out will have the exception traceback.
- """
- try:
- proc = subprocess.Popen(cmd, shell=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- stdout, stderr = proc.communicate()
- except OSError, e:
- logging.exception('OSError on cmd=%s.', cmd)
- return (-1, traceback.format_exc(), str(e))
- except ValueError, e:
- logging.exception('ValueError on cmd=%s.', cmd)
- return (-1, traceback.format_exc(), str(e))
-
- return (proc.returncode, stdout, stderr)
-
-
-def main():
- """Gather host information and return data to monitor on the server."""
- logging.basicConfig(level=logging.INFO, strem=sys.stderr)
-
- try:
- worker = HostWorker()
- worker.Run()
-
- # Remove the raw data, leave the formatted data.
- del worker.host_data['data']
-
- # Serialize to Stdout, Monitory.py will read this into host_data[].
- print cPickle.dumps(worker.host_data)
-
- except (KeyboardInterrupt, SystemExit):
- logging.exception('Shutdown requested.')
- sys.exit(1)
- except Exception, e:
- logging.exception('Exception: %s\n%s', e, traceback.format_exc())
- raise
-
-
-if __name__ == '__main__':
- main()
diff --git a/site_utils/system_health/rrd.json b/site_utils/system_health/rrd.json
deleted file mode 100644
index 19cb655..0000000
--- a/site_utils/system_health/rrd.json
+++ /dev/null
@@ -1,486 +0,0 @@
-{
- "battery": {
- "heartbeat": "1200",
- "min": "0",
- "max": "U",
- "title": "\"%(host)s Battery Status",
- "type": "GAUGE",
- "units": "\"Mili Amps\"",
- "items": ["State", "Rate", "Capacity"],
- "graph": [
- "-l 0 -r",
- "CDEF:bat=State,1,LT,50,UNKN,IF",
- "CDEF:ac=State,1,LT,UNKN,50,IF",
- "CDEF:RateD=State,1,LT,Rate,UNKN,IF",
- "CDEF:RateC=State,1,LT,UNKN,Rate,IF",
- "CDEF:bg=Capacity,UN,0,Capacity,IF,0,GT,INF,UNKN,IF",
- "AREA:bg#DDDDDD:",
- "AREA:Capacity#99CCFF:\"Capacity \"",
- "LINE1:Capacity#3399FF:",
- "VDEF:max1=Capacity,MAXIMUM",
- "VDEF:min1=Capacity,MINIMUM",
- "VDEF:avg1=Capacity,AVERAGE",
- "GPRINT:max1:\"Max %6.3lf%s\"",
- "GPRINT:min1:\"Min %6.3lf%s\"",
- "GPRINT:avg1:\" Avg %6.3lf%s\"",
- "AREA:bat#CC0000:\"Battery \\n\"",
- "LINE2:RateD#990033:\"Discharge Rate\"",
- "VDEF:max2=RateD,MAXIMUM",
- "VDEF:min2=RateD,MINIMUM",
- "VDEF:avg2=RateD,AVERAGE",
- "GPRINT:max2:\"Max %6.3lf%s\"",
- "GPRINT:min2:\"Min %6.3lf%s\"",
- "GPRINT:avg2:\"Avg %6.3lf%s\"",
- "AREA:ac#33FF66:\"AC Connected \\n\"",
- "LINE2:RateC#009966:\"Charge Rate \"",
- "VDEF:max3=RateC,MAXIMUM",
- "VDEF:min3=RateC,MINIMUM",
- "VDEF:avg3=RateC,AVERAGE",
- "GPRINT:max3:\"Max %6.3lf%s\"",
- "GPRINT:min3:\"Min %6.3lf%s\"",
- "GPRINT:avg3:\" Avg %6.3lf%s\\n\""
- ]
- },
-
- "boot": {
- "heartbeat": "1200",
- "min": "0",
- "max": "U",
- "title": "\"%(host)s Boot time to Login Prompt",
- "type": "GAUGE",
- "units": "\"Seconds\"",
- "items": ["firmware", "ready"],
- "graph": [
- "-l 0 -u 30 -r",
- "CDEF:total=firmware,ready,+",
- "CDEF:bg=total,UN,0,total,IF,0,GT,UNKN,INF,IF",
- "AREA:bg#DDDDDD:",
- "AREA:firmware#26466D:\"Firmware \"",
- "LINE1:firmware#660000:",
- "VDEF:maxF=firmware,MAXIMUM",
- "VDEF:minF=firmware,MINIMUM",
- "VDEF:avgF=firmware,AVERAGE",
- "GPRINT:minF:\"Min %2.1lf\"",
- "GPRINT:maxF:\"Max %2.1lf\"",
- "GPRINT:avgF:\"Avg %2.1lf Seconds \\n\"",
- "AREA:ready#0BB5FF:\"Login Prompt\":STACK",
- "LINE1:firmware#660000:",
- "VDEF:maxR=ready,MAXIMUM",
- "VDEF:minR=ready,MINIMUM",
- "VDEF:avgR=ready,AVERAGE",
- "GPRINT:minR:\"Min %2.1lf\"",
- "GPRINT:maxR:\"Max %2.1lf\"",
- "GPRINT:avgR:\"Avg %2.1lf Seconds \\n\"",
- "VDEF:maxT=total,MAXIMUM",
- "VDEF:minT=total,MINIMUM",
- "VDEF:avgT=total,AVERAGE",
- "GPRINT:minT:\"Total Min %2.1lf\"",
- "GPRINT:maxT:\"Max %2.1lf\"",
- "GPRINT:avgT:\"Avg %2.1lf Seconds \\n\"",
- "HRULE:15#FF0000",
- "HRULE:10#FFA500"
- ]
- },
-
- "cpu": {
- "heartbeat": "1200",
- "min": "0",
- "max": "U",
- "title": "\"%(host)s CPU Usage",
- "type": "DERIVE",
- "units": "\"jiffies\"",
- "items": ["user", "nice", "system", "idle"],
- "graph": [
- "-l 0 -r -u 99.99",
- "CDEF:l=user,0.1,0.1,IF",
- "CDEF:bg=user,UN,0,user,IF,0,GT,UNKN,INF,IF",
- "AREA:bg#DDDDDD:",
- "CDEF:tj=user,nice,+,system,+,idle,+",
- "CDEF:usr=100,user,*,tj,/",
- "CDEF:nic=100,nice,*,tj,/",
- "CDEF:sys=100,system,*,tj,/",
- "CDEF:idl=100,idle,*,tj,/",
- "CDEF:tot=100,tj,*,tj,/",
- "AREA:nic#0040A2:\"Nice \"",
- "VDEF:maxN=nic,MAXIMUM",
- "VDEF:minN=nic,MINIMUM",
- "VDEF:avgN=nic,AVERAGE",
- "GPRINT:maxN:\"Max %6.2lf%s\"",
- "GPRINT:minN:\"Min %6.2lf%s\"",
- "GPRINT:avgN:\"Avg %6.2lf%s \\n\"",
- "AREA:sys#3399FF:System:STACK",
- "LINE2:l#70A5AC::STACK",
- "VDEF:maxS=sys,MAXIMUM",
- "VDEF:minS=sys,MINIMUM",
- "VDEF:avgS=sys,AVERAGE",
- "GPRINT:maxS:\"Max %6.2lf%s\"",
- "GPRINT:minS:\"Min %6.2lf%s\"",
- "GPRINT:avgS:\"Avg %6.2lf%s \\n\"",
- "AREA:usr#B0F5EC:\"User \":STACK",
- "LINE2:l#90C5CC::STACK",
- "VDEF:maxU=usr,MAXIMUM",
- "VDEF:minU=usr,MINIMUM",
- "VDEF:avgU=usr,AVERAGE",
- "GPRINT:maxU:\"Max %6.2lf%s\"",
- "GPRINT:minU:\"Min %6.2lf%s\"",
- "GPRINT:avgU:\"Avg %6.2lf%s \\n\"",
- "AREA:idl#EEFFFF:\"Idle \":STACK",
- "VDEF:maxI=idl,MAXIMUM",
- "VDEF:minI=idl,MINIMUM",
- "VDEF:avgI=idl,AVERAGE",
- "GPRINT:maxI:\"Max %6.2lf%s\"",
- "GPRINT:minI:\"Min %6.2lf%s\"",
- "GPRINT:avgI:\"Avg %6.2lf%s \\n\""
- ]
- },
-
- "fs_inode": {
- "heartbeat": "1200",
- "min": "0",
- "max": "U",
- "title": "\"%(host)s %(file_system)s File System Inodes",
- "type": "GAUGE",
- "units": "\"Quantity\"",
- "items": ["Used", "Free"],
- "graph": [
- "-l 0 -r",
- "CDEF:bg=Used,UN,0,Used,IF,0,GT,UNKN,INF,IF",
- "AREA:bg#DDDDDD",
- "CDEF:inodes=Used,Free,+",
- "VDEF:inodesTotal=inodes,LAST",
- "GPRINT:inodesTotal:\"Total %6.2lf %s\\n\"",
- "AREA:Used#000066:\"Used\"",
- "VDEF:usedLast=Used,LAST",
- "GPRINT:usedLast:\"%6.2lf %s\"",
- "CDEF:usedPct=Used,100,*,inodes,/",
- "VDEF:pctUsed=usedPct,LAST",
- "GPRINT:pctUsed:\"%6.2lf%%\\n\"",
- "AREA:Free#3399FF:\"Free\":STACK",
- "VDEF:freeLast=Free,LAST",
- "GPRINT:freeLast:\"%6.2lf %s\"",
- "CDEF:freePct=100,usedPct,-",
- "VDEF:pctFree=freePct,LAST",
- "GPRINT:pctFree:\"%6.2lf%%\\n\""
- ]
- },
-
- "fs_space": {
- "heartbeat": "1200",
- "min": "0",
- "max": "U",
- "title": "\"%(host)s %(file_system)s File System Space",
- "type": "GAUGE",
- "units": "\"Bytes\"",
- "items": ["Used", "Free"],
- "graph": [
- "-l 0 -r",
- "CDEF:bg=Used,UN,0,Used,IF,0,GT,UNKN,INF,IF",
- "AREA:bg#DDDDDD",
- "CDEF:UsedB=Used,1024,*",
- "CDEF:FreeB=Free,1024,*",
- "CDEF:fs=UsedB,FreeB,+",
- "VDEF:fsTotal=fs,LAST",
- "GPRINT:fsTotal:\"Total %6.2lf %sB\\n\"",
- "AREA:UsedB#003399:\"Used\"",
- "VDEF:usedLast=UsedB,LAST",
- "GPRINT:usedLast:\"%6.2lf %sB\"",
- "CDEF:usedPct=UsedB,100,*,fs,/",
- "VDEF:pctUsed=usedPct,LAST",
- "GPRINT:pctUsed:\"%6.2lf%%\\n\"",
- "AREA:FreeB#6699CC:\"Free\":STACK",
- "VDEF:freeLast=FreeB,LAST",
- "GPRINT:freeLast:\"%6.2lf %sB\"",
- "CDEF:freePct=100,usedPct,-",
- "VDEF:pctFree=freePct,LAST",
- "GPRINT:pctFree:\"%6.2lf%%\\n\""
- ]
- },
-
- "fs_stat": {
- "heartbeat": "1200",
- "min": "U",
- "max": "U",
- "title": "\"%(host)s %(file_system)s File System Activity",
- "type": "DERIVE",
- "units": "\"Bytes\"",
- "items": ["Reads", "Writes"],
- "graph": [
- "-r",
- "CDEF:bWrites=Writes,-512,*",
- "CDEF:bReads=Reads,512,*",
- "AREA:bWrites#990000:\"Bytes Written\\n\"",
- "AREA:bReads#0066CC:\"Bytes Read\"",
- "HRULE:0#000000"
- ]
- },
-
- "load": {
- "heartbeat": "1200",
- "min": "0",
- "max": "100",
- "title": "\"%(host)s Load Levels",
- "type": "GAUGE",
- "units": "\"proc/min\"",
- "items": ["load_1", "load_5", "load_15"],
- "graph": [
- "-r",
- "CDEF:bg=load_1,UN,0,load_1,IF,0,GT,UNKN,INF,IF",
- "AREA:bg#DDDDDD:",
- "CDEF:bi=load_1,UN,0,load_1,IF,0,GT,INF,UNKN,IF",
- "AREA:bi#FEFEED:",
- "HRULE:1.0#44B5FF",
- "AREA:load_15#99FFCC:\"Last 15 min\"",
- "VDEF:max3=load_15,MAXIMUM",
- "VDEF:min3=load_15,MINIMUM",
- "VDEF:avg3=load_15,AVERAGE",
- "GPRINT:max3:\"Max %6.2lf\"",
- "GPRINT:min3:\"Min %6.2lf\"",
- "GPRINT:avg3:\"Avg %6.2lf\\n\"",
- "LINE2:load_5#3399FF:\"Last 5 min \"",
- "VDEF:max2=load_5,MAXIMUM",
- "VDEF:min2=load_5,MINIMUM",
- "VDEF:avg2=load_5,AVERAGE",
- "GPRINT:max2:\"Max %6.2lf\"",
- "GPRINT:min2:\"Min %6.2lf\"",
- "GPRINT:avg2:\"Avg %6.2lf\\n\"",
- "LINE2:load_1#993366:\"Last 1 min \"",
- "VDEF:max1=load_1,MAXIMUM",
- "VDEF:min1=load_1,MINIMUM",
- "VDEF:avg1=load_1,AVERAGE",
- "GPRINT:max1:\"Max %6.2lf\"",
- "GPRINT:min1:\"Min %6.2lf\"",
- "GPRINT:avg1:\"Avg %6.2lf\\n\""
- ]
- },
-
- "memory": {
- "heartbeat": "1200",
- "min": "0",
- "max": "10000000",
- "title": "\"%(host)s Memory Usage",
- "type": "GAUGE",
- "units": "\"bytes\"",
- "items": ["MemTotal", "MemFree", "Buffers", "Cached", "SwapTotal",
- "SwapFree"],
- "graph": [
- "-r",
- "CDEF:bg=MemTotal,UN,0,MemTotal,IF,0,GT,UNKN,INF,IF",
- "AREA:bg#DDDDDD:",
- "CDEF:sum=MemTotal,1024,*",
- "CDEF:free=MemFree,1024,*",
- "CDEF:buff=Buffers,1024,*",
- "CDEF:buffP=buff,100,*,sum,/",
- "CDEF:cache=Cached,1024,*",
- "CDEF:user=MemTotal,MemFree,Cached,+,Buffers,+,-,1024,*",
- "CDEF:l=user,1,1,IF",
- "AREA:user#003366:\"User \"",
- "LINE2:l#AC1300::STACK",
- "VDEF:maxUser=user,MAXIMUM",
- "VDEF:minUser=user,MINIMUM",
- "VDEF:avgUser=user,AVERAGE",
- "VDEF:curUser=user,LAST",
- "GPRINT:curUser:\"Last %6.2lf %s\"",
- "GPRINT:avgUser:\"Avg %6.2lf %s\"",
- "GPRINT:maxUser:\"Max %6.2lf %s\"",
- "GPRINT:minUser:\"Min %6.2lf %s\\n\"",
- "AREA:cache#336699:\"Cached \":STACK",
- "LINE2:l#DF7900::STACK",
- "VDEF:maxCache=cache,MAXIMUM",
- "VDEF:minCache=cache,MINIMUM",
- "VDEF:avgCache=cache,AVERAGE",
- "VDEF:curCache=cache,LAST",
- "GPRINT:curCache:\"Last %6.2lf %s\"",
- "GPRINT:avgCache:\"Avg %6.2lf %s\"",
- "GPRINT:maxCache:\"Max %6.2lf %s\"",
- "GPRINT:minCache:\"Min %6.2lf %s\\n\"",
- "AREA:buff#99CCFF:\"Buffers\":STACK",
- "LINE2:l#DFAC00::STACK",
- "VDEF:maxBuff=buff,MAXIMUM",
- "VDEF:minBuff=buff,MINIMUM",
- "VDEF:avgBuff=buff,AVERAGE",
- "VDEF:curBuff=buff,LAST",
- "GPRINT:curBuff:\"Last %6.2lf %s\"",
- "GPRINT:avgBuff:\"Avg %6.2lf %s\"",
- "GPRINT:maxBuff:\"Max %6.2lf %s\"",
- "GPRINT:minBuff:\"Min %6.2lf %s\\n\"",
- "AREA:free#CCFFCC:\"Unused \":STACK",
- "VDEF:maxFree=free,MAXIMUM",
- "VDEF:minFree=free,MINIMUM",
- "VDEF:avgFree=free,AVERAGE",
- "VDEF:curFree=free,LAST",
- "GPRINT:curFree:\"Last %6.2lf %s\"",
- "GPRINT:avgFree:\"Avg %6.2lf %s\"",
- "GPRINT:maxFree:\"Max %6.2lf %s\"",
- "GPRINT:minFree:\"Min %6.2lf %s\\n\""
- ]
- },
-
- "network": {
- "heartbeat": "1200",
- "min": "0",
- "max": "12500000",
- "title": "\"%(host)s Network Traffic",
- "type": "DERIVE",
- "units": "\"bytes/s\"",
- "items": ["r_eth0", "x_eth0", "r_wlan0", "x_wlan0"],
- "graph": [
- "-r",
- "VDEF:max1=r_eth0,MAXIMUM",
- "CDEF:eoff=r_eth0,UN,0,r_eth0,IF,0,GT,UNKN,0,IF",
- "CDEF:eon=0,r_eth0,UN,0,r_eth0,IF,0,GT,max1,50,/,UNKN,IF,-",
- "CDEF:bi=r_eth0,UN,0,r_eth0,IF,0,GT,INF,UNKN,IF",
- "CDEF:bg=r_eth0,UN,0,r_eth0,IF,0,GT,UNKN,INF,IF",
- "AREA:bi#DDDDDD:",
- "AREA:r_eth0#000066:\"Eth0 In \"",
- "LINE1:r_eth0#0000CC:",
- "VDEF:min1=r_eth0,MINIMUM",
- "VDEF:avg1=r_eth0,AVERAGE",
- "VDEF:tot1=r_eth0,TOTAL",
- "GPRINT:max1:\"Max %6.2lf%s\"",
- "GPRINT:min1:\"Min %6.2lf%s\"",
- "GPRINT:avg1:\"Avg %6.2lf%s\"",
- "GPRINT:tot1:\"Sum %6.2lf%s\\n\"",
- "CDEF:xmit0=x_eth0,-1,*",
- "AREA:xmit0#990033:\"Eth0 Out\"",
- "VDEF:max2=x_eth0,MAXIMUM",
- "VDEF:min2=x_eth0,MINIMUM",
- "VDEF:avg2=x_eth0,AVERAGE",
- "VDEF:tot2=x_eth0,TOTAL",
- "GPRINT:max2:\"Max %6.2lf%s\"",
- "GPRINT:min2:\"Min %6.2lf%s\"",
- "GPRINT:avg2:\"Avg %6.2lf%s\"",
- "GPRINT:tot2:\"Sum %6.2lf%s\\n\"",
- "AREA:bg#DDDDDD:",
- "LINE3:eoff#000000:\"Eth0 Offline \\n\"",
- "LINE3:eon#00CC66:\"Eth0 Online \\n\"",
- "AREA:r_wlan0#6699CC:\"Wlan0 In \"",
- "VDEF:min3=r_wlan0,MINIMUM",
- "VDEF:max3=r_wlan0,MAXIMUM",
- "VDEF:avg3=r_wlan0,AVERAGE",
- "VDEF:tot3=r_wlan0,TOTAL",
- "GPRINT:max3:\"Max %6.2lf%s\"",
- "GPRINT:min3:\"Min %6.2lf%s\"",
- "GPRINT:avg3:\"Avg %6.2lf%s\"",
- "GPRINT:tot3:\"Sum %6.2lf%s\\n\"",
- "CDEF:xmit1=x_wlan0,-1,*",
- "AREA:xmit1#FF6666:\"Wlan0 Out\"",
- "VDEF:max4=x_wlan0,MAXIMUM",
- "VDEF:min4=x_wlan0,MINIMUM",
- "VDEF:avg4=x_wlan0,AVERAGE",
- "VDEF:tot4=x_wlan0,TOTAL",
- "GPRINT:max4:\"Max %6.2lf%s\"",
- "GPRINT:min4:\"Min %6.2lf%s\"",
- "GPRINT:avg4:\"Avg %6.2lf%s\"",
- "GPRINT:tot4:\"Sum %6.2lf%s\\n\""
- ]
- },
-
- "power": {
- "heartbeat": "1200",
- "min": "0",
- "max": "100",
- "title": "\"%(host)s Power State",
- "type": "GAUGE",
- "units": "\"Percentage\"",
- "items": ["state"],
- "graph": [
- "-l 0 -r",
- "CDEF:bg=state,UN,0,state,IF,0,GT,UNKN,INF,IF",
- "AREA:bg#DDDDDD:",
- "VDEF:pstate=state,LAST",
- "AREA:pstate#CC3333:\"Power Setting \"",
- "VDEF:pstateMax=state,MAXIMUM",
- "VDEF:pstateMin=state,MINIMUM",
- "VDEF:pstateAvg=state,AVERAGE",
- "GPRINT:pstateMax:\"Max %6.2lf%s%%\"",
- "GPRINT:pstateMin:\"Min %6.2lf%s%%\"",
- "GPRINT:pstateAvg:\"Avg %6.2lf%s%%\\n\""
- ]
- },
-
- "temp": {
- "heartbeat": "1200",
- "min": "0",
- "max": "100",
- "title": "\"%(host)s Temperature Readings",
- "type": "GAUGE",
- "units": "\"Celsius\"",
- "items": ["cpu"],
- "graph": [
- "-l 20 -r",
- "CDEF:bg=cpu,UN,0,cpu,IF,0,GT,UNKN,INF,IF",
- "AREA:bg#DDDDDD:",
- "CDEF:cool=cpu,40,LE,cpu,UNKN,IF",
- "CDEF:warm=cpu,40,60,LIMIT",
- "CDEF:hot=cpu,60,GE,cpu,UNKN,IF",
- "AREA:cool#B0F5EC:\"Cool \"",
- "AREA:warm#FFCC00:\"Warm \"",
- "AREA:hot#CC3300:\"Hot \\n\"",
- "VDEF:maxC=cpu,MAXIMUM",
- "VDEF:minC=cpu,MINIMUM",
- "VDEF:avgC=cpu,AVERAGE",
- "GPRINT:minC:\"Min %2.1lf\"",
- "GPRINT:maxC:\"Max %2.1lf\"",
- "GPRINT:avgC:\"Avg %2.1lf Celsius \\n\"",
- "LINE1:cpu#660000:",
- "HRULE:60#FF0000",
- "HRULE:20#FFA500"
- ]
- },
-
- "uptime": {
- "heartbeat": "1200",
- "min": "0",
- "max": "U",
- "title": "\"%(host)s Uptime Readings",
- "type": "GAUGE",
- "units": "\"hours\"",
- "items": ["uptime", "idletime"],
- "graph": [
- "-r",
- "CDEF:bg=uptime,UN,0,uptime,IF,0,GT,UNKN,INF,IF",
- "AREA:bg#DDDDDD:",
- "CDEF:upHours=uptime,3600,/",
- "CDEF:idleHours=idletime,3600,/",
- "AREA:upHours#99CC99:\"Uptime \"",
- "GPRINT:upHours:MIN:\"Min %8.2lf\"",
- "GPRINT:upHours:MAX:\"Max %8.2lf\"",
- "GPRINT:upHours:AVERAGE:\"Avg %8.2lf\"",
- "GPRINT:upHours:LAST:\"Last %8.2lf\\n\"",
- "LINE2:idleHours#333333:\"Idletime\"",
- "GPRINT:idleHours:MIN:\"Min %8.2lf\"",
- "GPRINT:idleHours:MAX:\"Max %8.2lf\"",
- "GPRINT:idleHours:AVERAGE:\"Avg %8.2lf\"",
- "GPRINT:idleHours:LAST:\"Last %8.2lf\\n\""
- ]
- },
-
- "utilization": {
- "heartbeat": "1200",
- "min": "0",
- "max": "U",
- "title": "\"%(host)s Utilization",
- "type": "GAUGE",
- "units": "\"Host Count\"",
- "items": ["Repairing", "Verifying", "Repair_Failed", "Running",
- "Cleaning", "Ready", "Pending"],
- "graph": [
- "LINE2:Repair_Failed#FF0000:\"Repair_Failed\"",
- "LINE2:Repairing#FF9900:\"Repairing\"",
- "LINE2:Cleaning#99FF00:\"Cleaning\"",
- "LINE2:Verifying#006600:\"Verifying\"",
- "LINE2:Pending#CC00FF:\"Pending\"",
- "LINE2:Ready#00FFFF:\"Ready\"",
- "LINE2:Running#0000FF:\"Running\"",
- "COMMENT:\"\\\\n\"",
- "GPRINT:Repair_Failed:LAST:\"Repair Failed\\:%3.0lf\"",
- "GPRINT:Repairing:LAST:\"Repairing\\:%3.0lf\"",
- "GPRINT:Cleaning:LAST:\"Cleaning\\:%3.0lf\"",
- "GPRINT:Verifying:LAST:\"Verifying\\:%3.0lf\"",
- "GPRINT:Pending:LAST:\"Pending\\:%3.0lf\"",
- "GPRINT:Ready:LAST:\"Ready\\:%3.0lf\"",
- "GPRINT:Running:LAST:\"Running\\:%3.0lf\""
- ]
- }
-}
-
diff --git a/site_utils/test_scheduler.py b/site_utils/test_scheduler.py
deleted file mode 100755
index abda3ff..0000000
--- a/site_utils/test_scheduler.py
+++ /dev/null
@@ -1,353 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Tool for scheduling BVT and full suite testing of Chrome OS images.
-
-Test Scheduler is a tool for scheduling the testing of Chrome OS images across
-multiple boards and platforms. All testing is driven through a board to platform
-mapping specified in a JSON config file.
-
-For each board, platform tuple the bvt group is scheduled. Once the bvt has
-completed and passed, all groups from 'default_full_groups' are scheduled.
-
-Test Scheduler expects the JSON config file to be in the current working
-directory or to be run with --config pointing to the actual config file.
-"""
-
-__author__ = 'dalecurtis@google.com (Dale Curtis)'
-
-import logging
-import optparse
-import os
-import re
-import tempfile
-
-from chromeos_test import autotest_util
-from chromeos_test import common_util
-from chromeos_test import dash_util
-from chromeos_test import dev_server
-from chromeos_test import log_util
-from chromeos_test import test_config
-
-# Autotest imports
-
-import common
-
-from autotest_lib.client.common_lib.cros import dev_server as new_dev_server
-
-
-# RegEx for extracting versions from build strings.
-_3_TUPLE_VERSION_RE = re.compile('R\d+-(\d+\.\d+\.\d+)')
-_4_TUPLE_VERSION_RE = re.compile('(\d+\.\d+\.\d+\.\d+)+-')
-
-
-def _ParseVersion(build):
- """Extract version from build string. Parses x.x.x.x* and Ryy-x.x.x* forms."""
- match = _3_TUPLE_VERSION_RE.match(build)
- if not match:
- match = _4_TUPLE_VERSION_RE.match(build)
-
- # Will generate an exception if no match was found.
- return match.group(1)
-
-
-class TestRunner(object):
- """Helper class for scheduling jobs from tests and groups."""
-
- def __init__(self, board, build, cli, config, dev, new_dev, upload=False):
- """Initializes class variables.
-
- Args:
- board: Board name for this build; e.g., x86-generic-rel
- build: Full build string to look for; e.g., 0.8.61.0-r1cf43296-b269
- cli: Path to Autotest CLI.
- config: Dictionary of configuration as loaded from JSON.
- dev: An initialized DevServer() instance.
- new_dev: new dev_server interface under client/common_lib/cros.
- upload: Whether to upload created job information to appengine.
- """
- self._board = board
- self._build = build
- self._config = config
- self._cli = cli
- self._dev = dev
- self._new_dev = new_dev
- self._upload = upload
-
- def RunTest(self, job_name, platform, test, build=None, control_mods=None):
- """Given a test dictionary: retrieves control file and creates jobs.
-
- Test dictionary format is as follows:
-
- {'name': '', 'control': '', 'count': ##, 'labels': [...], 'sync': <T/F>}
-
- Optional keys are count, labels, and sync. If not specified they will be set
- to default values of 1, None, and False respectively.
-
- Jobs are created with the name <board>-<build>_<name>.
-
- Args:
- job_name: Name of job to create.
- platform: Platform to schedule job for.
- test: Test config dictionary.
- build: Build to use, if different than the one used to initialize class.
- control_mods: List of functions to call for control file preprocessing.
- Each function will be passed the contents of the control file.
-
- Raises:
- common_util.ChromeOSTestError: If any steps fail.
- """
- # Initialize defaults for optional keys. Avoids tedious, if <key> in <test>
- default = {'count': 1, 'labels': None, 'sync': None}
- default.update(test)
- test = default
-
- if test['sync']:
- test['sync'] = test['count']
-
- if not build:
- build = self._build
-
- # Pull control file from Dev Server.
- try:
- # Use new style for TOT boards.
- if 'release' in self._board:
- image = '%s/%s' % (self._board, build)
- # Make sure the latest board is already staged. This will hang until
- # the image is properly staged or return immediately if it is already
- # staged. This will have little impact on the rest of this process and
- # ensures we properly launch tests while straddling the old and the new
- # styles.
- self._new_dev.trigger_download(image)
- control_file_data = self._new_dev.get_control_file(image,
- test['control'])
- if 'Unknown control path' in control_file_data:
- raise common_util.ChromeOSTestError(
- 'Control file %s not yet staged, skipping' % test['control'])
- else:
- control_file_data = self._dev.GetControlFile(self._board, build,
- test['control'])
- except (new_dev_server.DevServerException, common_util.ChromeOSTestError):
- logging.error('Missing %s for %s on %s.', test['control'], job_name,
- platform)
- raise
-
- # If there's any preprocessing to be done call it now.
- if control_mods:
- for mod in control_mods:
- control_file_data = mod(control_file_data)
-
- # Create temporary file and write control file contents to it.
- temp_fd, temp_fn = tempfile.mkstemp()
- os.write(temp_fd, control_file_data)
- os.close(temp_fd)
-
- # Create Autotest job using control file and image parameter.
- try:
- # Inflate the priority of BVT runs.
- if job_name.endswith('_bvt'):
- priority = 'urgent'
- else:
- priority = 'medium'
-
- # Add pool:suites to all jobs to avoid using the BVT machines with the
- # same platform label.
- if test['labels'] is None:
- test['labels'] = ['pool:suites']
- else:
- test['labels'].append('pool:suites')
-
- job_id = autotest_util.CreateJob(
- name=job_name, control=temp_fn,
- platforms='%d*%s' % (test['count'], platform), labels=test['labels'],
- sync=test['sync'],
- update_url=self._dev.GetUpdateUrl(self._board, build),
- cli=self._cli, priority=priority)
- finally:
- # Cleanup temporary control file. Autotest doesn't need it anymore.
- os.unlink(temp_fn)
-
- #TODO(dalecurtis): Disabled, since it's not under active development.
- #try:
- # appengine_cfg = self._config.get('appengine', {})
- # if self._upload and appengine_cfg:
- # dash_util.UploadJob(appengine_cfg, job_id)
- #except common_util.ChromeOSTestError:
- # logging.warning('Failed to upload job to AppEngine.')
-
- def RunTestGroups(self, groups, platform, lock=True):
- """Given a list of test groups, creates Autotest jobs for associated tests.
-
- Given a list of test groups, map each into the "groups" dictionary from the
- JSON configuration file and launch associated tests. If lock is specified it
- will attempt to acquire a dev server lock for each group before starting. If
- a lock can't be obtained, the group won't be started.
-
- Args:
- groups: List of group names to run tests for. See test config for valid
- group names.
- platform: Platform label to look for. See test config for valid platforms.
- lock: Attempt to acquire lock before running tests?
- """
- for group in groups:
- if not group in self._config['groups']:
- logging.warning('Skipping unknown group "%s".', group)
- continue
-
- # Start tests for the given group.
- for test in self._config['groups'][group]:
- has_lock = False
- try:
- job_name = '%s-%s_%s' % (self._board, self._build, test['name'])
-
- # Attempt to acquire lock for test.
- if lock:
- tag = '%s/%s/%s_%s_%s' % (self._board, self._build, platform,
- group, test['name'])
- try:
- self._dev.AcquireLock(tag)
- has_lock = True
- except common_util.ChromeOSTestError, e:
- logging.debug('Refused lock for test "%s" from group "%s".'
- ' Assuming it has already been started.',
- test['name'], group)
- continue
-
- self.RunTest(platform=platform, test=test, job_name=job_name)
- logging.info('Successfully created job "%s".', job_name)
- except common_util.ChromeOSTestError, e:
- logging.exception(e)
- logging.error('Failed to schedule test "%s" from group "%s".',
- test['name'], group)
-
- # We failed, so release lock and let next run pick this test up.
- if has_lock:
- self._dev.ReleaseLock(tag)
-
- def RunAutoupdateTests(self, platform):
- # Process the autoupdate targets.
- for target in self._dev.ListAutoupdateTargets(self._board, self._build):
- has_lock = False
- try:
- # Tell other instances of the scheduler we're processing this target.
- tag = '%s/%s/%s_%s' % (self._board, self._build, platform['platform'],
- target)
- try:
- self._dev.AcquireLock(tag)
- has_lock = True
- except common_util.ChromeOSTestError, e:
- logging.debug('Refused lock for autoupdate target "%s". Assuming'
- ' it has already been started.', target)
- continue
-
- # Split target into base build and convenience label.
- base_build, label = target.split('_')
-
- # Setup preprocessing function to insert the correct update URL into
- # the control file.
- control_preprocess_fn = lambda x: x % {'update_url': '%s/%s/%s' % (
- self._dev.GetUpdateUrl(
- self._board, self._build), self._dev.AU_BASE, target)}
-
- # E.g., x86-mario-r14-0.14.734.0_to_0.14.734.0-a1-b123_nton_au
- job_name = '%s-%s_to_%s_%s_au' % (
- self._board, _ParseVersion(base_build), self._build, label)
-
- self.RunTest(
- platform=platform['platform'],
- test=self._config['groups']['autoupdate'][0], job_name=job_name,
- build=base_build,
- control_mods=[control_preprocess_fn])
- logging.info('Successfully created job "%s".', job_name)
- except common_util.ChromeOSTestError, e:
- logging.exception(e)
- logging.error('Failed to schedule autoupdate target "%s".', target)
-
- # We failed, so release lock and let next run pick this target up.
- if has_lock:
- self._dev.ReleaseLock(tag)
-
-
-def ParseOptions():
- """Parse command line options. Returns 2-tuple of options and config."""
- parser = optparse.OptionParser('usage: %prog [options]')
-
- # Add utility/helper class command line options.
- test_config.AddOptions(parser)
- log_util.AddOptions(parser)
- autotest_util.AddOptions(parser, cli_only=True)
-
- options = parser.parse_args()[0]
- config = test_config.TestConfig(options.config)
-
- return options, config.GetConfig()
-
-
-def main():
- options, config = ParseOptions()
-
- # Setup logger and enable verbose mode if specified.
- log_util.InitializeLogging(options.verbose)
-
- # Initialize Dev Server Utility class.
- dev = dev_server.DevServer(**config['dev_server'])
-
- # Main processing loop. Look for new builds of each board.
- for board in config['boards']:
- for platform in config['boards'][board]['platforms']:
- logging.info('----[ Processing board %s, platform %s ]----',
- board, platform['platform'])
- try:
- new_dev = new_dev_server.DevServer()
- # The variable board is akin to target in the new nomenclature. This is
- # the old style and the new style clashing.
- # TODO(scottz): remove kludge once we move to suite scheduler.
- for milestone in ['r19', 'r20']:
- try:
- build = new_dev.get_latest_build(board, milestone=milestone)
- except new_dev_server.DevServerException:
- continue
- # Leave just in case we do get an empty response from the server
- # but we shouldn't.
- if not build:
- continue
- test_runner = TestRunner(
- board=board, build=build, cli=options.cli, config=config,
- dev=dev, new_dev=new_dev, upload=True)
-
- # Determine which groups to run.
- full_groups = []
- if 'groups' in platform:
- full_groups += platform['groups']
- else:
- # Add default groups to the job since 'groups' was not defined.
- # if test_suite is set to True use 'default_tot_groups' from the
- # json configuration, otherwise use 'default_groups.'
- if platform.get('test_suite'):
- full_groups += config['default_tot_groups']
- else:
- full_groups += config['default_groups']
-
- if 'extra_groups' in platform:
- full_groups += platform['extra_groups']
-
- test_runner.RunTestGroups(
- groups=full_groups, platform=platform['platform'])
-
- # Skip platforms which are not marked for AU testing.
- if not platform.get('au_test', False):
- continue
-
- # Process AU targets.
- test_runner.RunAutoupdateTests(platform)
- except (new_dev_server.DevServerException,
- common_util.ChromeOSTestError) as e:
- logging.exception(e)
- logging.warning('Exception encountered during processing. Skipping.')
-
-
-if __name__ == '__main__':
- main()