site_utils/lab_inventory.py

#!/usr/bin/env python
# Copyright 2015 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Create e-mail reports of the Lab's DUT inventory.

Gathers a list of all DUTs of interest in the Lab, segregated by
board and pool, and determines whether each DUT is working or
broken.  Then, send one or more e-mail reports summarizing the
status to e-mail addresses provided on the command line.

usage:  lab_inventory.py [ options ] [ board ... ]

Options:
--duration / -d <hours>
    How far back in time to search job history to determine DUT
    status.

--board-notify <address>[,<address>]
    Send the "board status" e-mail to all the specified e-mail
    addresses.

--pool-notify <address>[,<address>]
    Send the "pool status" e-mail to all the specified e-mail
    addresses.

--recommend <number>
    When generating the "board status" e-mail, included a list of
    <number> specific DUTs to be recommended for repair.

--repair-loops
    Scan the inventory for DUTs stuck in repair loops, and report them
    via a Monarch presence metric.

--logdir <directory>
    Log progress and actions in a file under this directory.  Text
    of any e-mail sent will also be logged in a timestamped file in
    this directory.

--debug
    Suppress all logging, metrics reporting, and sending e-mail.
    Instead, write the output that would be generated onto stdout.

<board> arguments:
    With no arguments, gathers the status for all boards in the lab.
    With one or more named boards on the command line, restricts
    reporting to just those boards.

"""


import argparse
import collections
import logging
import logging.handlers
import os
import re
import sys
import time

import common
from autotest_lib.client.bin import utils
from autotest_lib.client.common_lib import time_utils
from autotest_lib.server import site_utils
from autotest_lib.server.cros.dynamic_suite import frontend_wrappers
from autotest_lib.server.hosts import servo_host
from autotest_lib.server.lib import status_history
from autotest_lib.site_utils import gmail_lib
from autotest_lib.site_utils.suite_scheduler import constants
from autotest_lib.utils import labellib
from chromite.lib import metrics


CRITICAL_POOLS = constants.Pools.CRITICAL_POOLS
SPARE_POOL = constants.Pools.SPARE_POOL
MANAGED_POOLS = constants.Pools.MANAGED_POOLS

# _EXCLUDED_LABELS - A set of labels that disqualify a DUT from
#     monitoring by this script.  Currently, we're excluding these:
#   + 'adb' - We're not ready to monitor Android or Brillo hosts.
#   + 'board:guado_moblab' - These are maintained by a separate
#     process that doesn't use this script.

_EXCLUDED_LABELS = {'adb', 'board:guado_moblab'}

# _DEFAULT_DURATION:
#     Default value used for the --duration command line option.
#     Specifies how far back in time to search in order to determine
#     DUT status.

_DEFAULT_DURATION = 24

# _LOGDIR:
#     Relative path used in the calculation of the default setting for
#     the --logdir option.  The full path is relative to the root of the
#     autotest directory, as determined from sys.argv[0].
# _LOGFILE:
#     Basename of a file to which general log information will be
#     written.
# _LOG_FORMAT:
#     Format string for log messages.

_LOGDIR = os.path.join('logs', 'dut-data')
_LOGFILE = 'lab-inventory.log'
_LOG_FORMAT = '%(asctime)s | %(levelname)-10s | %(message)s'

# Pattern describing location-based host names in the Chrome OS test
# labs.  Each DUT hostname designates the DUT's location:
#   * A lab (room) that's physically separated from other labs
#     (i.e. there's a door).
#   * A row (or aisle) of DUTs within the lab.
#   * A vertical rack of shelves on the row.
#   * A specific host on one shelf of the rack.

_HOSTNAME_PATTERN = re.compile(
        r'(chromeos\d+)-row(\d+)-rack(\d+)-host(\d+)')

# Default entry for managed pools.

_MANAGED_POOL_DEFAULT = 'all_pools'

# _REPAIR_LOOP_THRESHOLD:
#    The number of repeated Repair tasks that must be seen to declare
#    that a DUT is stuck in a repair loop.

_REPAIR_LOOP_THRESHOLD = 4


class _CachedHostJobHistories(object):
    """Maintains a set of `HostJobHistory` objects for a pool.

    The collected history objects are nominally all part of a single
    scheduling pool of DUTs.  The collection maintains a list of
    working DUTs, a list of broken DUTs, and a list of all DUTs.

    Performance note:  Certain methods in this class are potentially
    expensive:
      * `get_working()`
      * `get_working_list()`
      * `get_broken()`
      * `get_broken_list()`
      * `get_idle()`
      * `get_idle_list()`
    The first time any one of these methods is called, it causes
    multiple RPC calls with a relatively expensive set of database
    queries.  However, the results of the queries are cached in the
    individual `HostJobHistory` objects, so only the first call
    actually pays the full cost.

    Additionally, `get_working_list()`, `get_broken_list()` and
    `get_idle_list()` cache their return values to avoid recalculating
    lists at every call; this caching is separate from the caching of RPC
    results described above.

    This class is deliberately constructed to delay the RPC cost
    until the accessor methods are called (rather than to query in
    `record_host()`) so that it's possible to construct a complete
    `_LabInventory` without making the expensive queries at creation
    time.  `_populate_board_counts()`, below, assumes this behavior.

    """

    def __init__(self):
        self._histories = []
        self._working_list = None
        self._broken_list = None
        self._idle_list = None


    def record_host(self, host_history):
        """Add one `HostJobHistory` object to the collection.

        @param host_history The `HostJobHistory` object to be
                            remembered.

        """
        self._working_list = None
        self._broken_list = None
        self._idle_list = None
        self._histories.append(host_history)


    def get_working_list(self):
        """Return a list of all working DUTs in the pool.

        Filter `self._histories` for histories where the last
        diagnosis is `WORKING`.

        Cache the result so that we only cacluate it once.

        @return A list of HostJobHistory objects.

        """
        if self._working_list is None:
            self._working_list = [h for h in self._histories
                    if h.last_diagnosis()[0] == status_history.WORKING]
        return self._working_list


    def get_working(self):
        """Return the number of working DUTs in the pool."""
        return len(self.get_working_list())


    def get_broken_list(self):
        """Return a list of all broken DUTs in the pool.

        Filter `self._histories` for histories where the last
        diagnosis is `BROKEN`.

        Cache the result so that we only cacluate it once.

        @return A list of HostJobHistory objects.

        """
        if self._broken_list is None:
            self._broken_list = [h for h in self._histories
                    if h.last_diagnosis()[0] == status_history.BROKEN]
        return self._broken_list


    def get_broken(self):
        """Return the number of broken DUTs in the pool."""
        return len(self.get_broken_list())


    def get_idle_list(self):
        """Return a list of all idle DUTs in the pool.

        Filter `self._histories` for histories where the last
        diagnosis is `UNUSED` or `UNKNOWN`.

        Cache the result so that we only cacluate it once.

        @return A list of HostJobHistory objects.

        """
        idle_list = [status_history.UNUSED, status_history.UNKNOWN]
        if self._idle_list is None:
            self._idle_list = [h for h in self._histories
                    if h.last_diagnosis()[0] in idle_list]
        return self._idle_list


    def get_idle(self):
        """Return the number of idle DUTs in the pool."""
        return len(self.get_idle_list())


    def get_total(self):
        """Return the total number of DUTs in the pool."""
        return len(self._histories)


class _ManagedPoolsHostJobHistories(object):
    """Maintains a set of `HostJobHistory`s per managed pool.

    The collection maintains a count of working DUTs, a count of broken DUTs,
    and a total count.  The counts can be obtained either for a single pool, or
    as a total across all pools.

    DUTs in the collection must be assigned to one of the pools in
    `_MANAGED_POOLS`.

    The `get_working()` and `get_broken()` methods rely on the
    methods of the same name in _CachedHostJobHistories, so the performance
    note in _CachedHostJobHistories applies here as well.

    """

    def __init__(self):
        self._histories_by_pool = {
            pool: _CachedHostJobHistories() for pool in MANAGED_POOLS
        }

    def record_host(self, host_history):
        """Add one `HostJobHistory` object to the collection.

        @param host_history The `HostJobHistory` object to be
                            remembered.

        """
        pool = host_history.host_pool
        self._histories_by_pool[pool].record_host(host_history)


    def _count_pool(self, get_pool_count, pool=None):
        """Internal helper to count hosts in a given pool.

        The `get_pool_count` parameter is a function to calculate
        the exact count of interest for the pool.

        @param get_pool_count  Function to return a count from a
                               _PoolCount object.
        @param pool            The pool to be counted.  If `None`,
                               return the total across all pools.

        """
        if pool is None:
            return sum([get_pool_count(cached_history) for cached_history in
                        self._histories_by_pool.values()])
        else:
            return get_pool_count(self._histories_by_pool[pool])


    def get_working_list(self):
        """Return a list of all working DUTs (across all pools).

        Go through all HostJobHistory objects across all pools, selecting the
        ones where the last diagnosis is `WORKING`.

        @return A list of HostJobHistory objects.

        """
        l = []
        for p in self._histories_by_pool.values():
            l.extend(p.get_working_list())
        return l


    def get_working(self, pool=None):
        """Return the number of working DUTs in a pool.

        @param pool  The pool to be counted.  If `None`, return the
                     total across all pools.

        @return The total number of working DUTs in the selected
                pool(s).
        """
        return self._count_pool(_CachedHostJobHistories.get_working, pool)


    def get_broken_list(self):
        """Return a list of all broken DUTs (across all pools).

        Go through all HostJobHistory objects in the across all pools,
        selecting the ones where the last diagnosis is `BROKEN`.

        @return A list of HostJobHistory objects.

        """
        l = []
        for p in self._histories_by_pool.values():
            l.extend(p.get_broken_list())
        return l


    def get_broken(self, pool=None):
        """Return the number of broken DUTs in a pool.

        @param pool  The pool to be counted.  If `None`, return the
                     total across all pools.

        @return The total number of broken DUTs in the selected pool(s).
        """
        return self._count_pool(_CachedHostJobHistories.get_broken, pool)


    def get_idle_list(self, pool=None):
        """Return a list of all idle DUTs in the given pool.

        Go through all HostJobHistory objects in the given pool, selecting the
        ones where the last diagnosis is `UNUSED` or `UNKNOWN`.

        @param pool: The pool to be counted. If `None`, return the total list
                     across all pools.

        @return A list of HostJobHistory objects.

        """
        if pool is None:
            l = []
            for p in self._histories_by_pool.values():
                l.extend(p.get_idle_list())
            return l
        else:
            return _CachedHostJobHistories.get_idle_list(
                    self._histories_by_pool[pool])


    def get_idle(self, pool=None):
        """Return the number of idle DUTs in a pool.

        @param pool: The pool to be counted. If `None`, return the total
                     across all pools.

        @return The total number of idle DUTs in the selected pool(s).
        """
        return self._count_pool(_CachedHostJobHistories.get_idle, pool)


    def get_spares_buffer(self):
        """Return the the nominal number of working spares.

        Calculates and returns how many working spares there would
        be in the spares pool if all broken DUTs were in the spares
        pool.  This number may be negative, indicating a shortfall
        in the critical pools.

        @return The total number DUTs in the spares pool, less the total
                number of broken DUTs in all pools.
        """
        return self.get_total(SPARE_POOL) - self.get_broken()


    def get_total(self, pool=None):
        """Return the total number of DUTs in a pool.

        @param pool  The pool to be counted.  If `None`, return the
                     total across all pools.

        @return The total number of DUTs in the selected pool(s).
        """
        return self._count_pool(_CachedHostJobHistories.get_total, pool)


class _LabInventory(object):
    """Collection of `HostJobHistory` objects for the Lab's inventory.

    Important attributes:
      by_board: A dict mapping board to ManagedPoolsHostJobHistories

    """

    @staticmethod
    def _eligible_host(afehost):
        """Return whether this host is eligible for monitoring.

        A host is eligible if it's in exactly one pool and it has no
        labels from the `_EXCLUDED_LABELS` set.

        @param afehost  The host to be tested for eligibility.
        """
        pools = [l for l in afehost.labels
                     if l.startswith(constants.Labels.POOL_PREFIX)]
        excluded = _EXCLUDED_LABELS.intersection(afehost.labels)
        return len(pools) == 1 and not excluded


    @classmethod
    def create_inventory(cls, afe, start_time, end_time, boardlist=[]):
        """Return a Lab inventory with specified parameters.

        By default, gathers inventory from `HostJobHistory` objects
        for all DUTs in the `MANAGED_POOLS` list.  If `boardlist`
        is supplied, the inventory will be restricted to only the
        given boards.

        @param afe         AFE object for constructing the
                           `HostJobHistory` objects.
        @param start_time  Start time for the `HostJobHistory`
                           objects.
        @param end_time    End time for the `HostJobHistory`
                           objects.
        @param boardlist   List of boards to include.  If empty,
                           include all available boards.
        @return A `_LabInventory` object for the specified boards.

        """
        label_list = [constants.Labels.POOL_PREFIX + l
                          for l in MANAGED_POOLS]
        afehosts = afe.get_hosts(labels__name__in=label_list)
        if boardlist:
            # We're deliberately not checking host eligibility in this
            # code path.  This is a debug path, not used in production;
            # it may be useful to include ineligible hosts here.
            boardhosts = []
            for board in boardlist:
                board_label = constants.Labels.BOARD_PREFIX + board
                host_list = [h for h in afehosts
                                  if board_label in h.labels]
                boardhosts.extend(host_list)
            afehosts = boardhosts
        else:
            afehosts = [h for h in afehosts if cls._eligible_host(h)]
        create = lambda host: (
                status_history.HostJobHistory(afe, host,
                                              start_time, end_time))
        return cls([create(host) for host in afehosts])


    def __init__(self, histories):
        # N.B. The query that finds our hosts is restricted to those
        # with a valid pool: label, but doesn't check for a valid
        # board: label.  In some (insufficiently) rare cases, the
        # AFE hosts table has been known to (incorrectly) have DUTs
        # with a pool: but no board: label.  We explicitly exclude
        # those here.
        histories = [h for h in histories
                     if h.host_board is not None]
        self.histories = histories
        self._dut_count = len(histories)
        self._managed_boards = {}
        self._managed_models = {}
        self.by_board = self._classify_by_label_type('board')
        self.by_model = self._classify_by_label_type('model')


    def _classify_by_label_type(self, label_key):
        """Classify histories by labels with the given key.

        @returns a dict mapping labels with the given key to
        _ManagedPoolsHostJobHistories for DUTs with that label.
        """
        classified = collections.defaultdict(_ManagedPoolsHostJobHistories)
        for h in self.histories:
            labels = labellib.LabelsMapping(h.host.labels)
            if label_key in labels:
                classified[labels[label_key]].record_host(h)
        return dict(classified)


    def get_managed_boards(self, pool=_MANAGED_POOL_DEFAULT):
        """Return the set of "managed" boards.

        @param pool: The specified pool for managed boards.
        @return A set of all the boards that have both spare and
                non-spare pools, unless the pool is specified,
                then the set of boards in that pool.
        """
        if self._managed_boards.get(pool) is None:
            self._managed_boards[pool] = set()
            for board, counts in self.by_board.iteritems():
                if self._is_managed(pool, counts):
                    self._managed_boards[pool].add(board)
        return self._managed_boards[pool]


    def get_managed_models(self, pool=_MANAGED_POOL_DEFAULT):
        """Return the set of "managed" models.

        @param pool: The specified pool for managed models.
        @return A set of all the models that have both spare and
                non-spare pools, unless the pool is specified,
                then the set of models in that pool.
        """
        if self._managed_models.get(pool) is None:
            self._managed_models[pool] = set()
            for board, counts in self.by_model.iteritems():
                if self._is_managed(pool, counts):
                    self._managed_models[pool].add(board)
        return self._managed_models[pool]


    def _is_managed(self, pool, histories):
        """Deterime if the given histories contain DUTs to be managed for pool.

        Operationally, saying a board is "managed" means that the
        board will be included in the "board" and "repair
        recommendations" reports.  That is, if there are failures in
        the board's inventory then lab techs will be asked to fix
        them without a separate ticket.

        For purposes of implementation, a board is "managed" if it
        has DUTs in both the spare and a non-spare (i.e. critical)
        pool.

        """
        # Get the counts for all pools, otherwise get it for the
        # specified pool.
        if pool == _MANAGED_POOL_DEFAULT:
            spares = histories.get_total(SPARE_POOL)
            total = histories.get_total()
            return spares != 0 and spares != total
        else:
            return histories.get_total(pool) != 0


    def get_num_duts(self):
        """Return the total number of DUTs in the inventory."""
        return self._dut_count


    def get_num_boards(self):
        """Return the total number of boards in the inventory."""
        return len(self.by_board)


    def get_num_models(self):
        """Return the total number of models in the inventory."""
        return len(self.by_model)


def _sort_by_location(inventory_list):
    """Return a list of DUTs, organized by location.

    Take the given list of `HostJobHistory` objects, separate it
    into a list per lab, and sort each lab's list by location.  The
    order of sorting within a lab is
      * By row number within the lab,
      * then by rack number within the row,
      * then by host shelf number within the rack.

    Return a list of the sorted lists.

    Implementation note: host locations are sorted by converting
    each location into a base 100 number.  If row, rack or
    host numbers exceed the range [0..99], then sorting will
    break down.

    @return A list of sorted lists of DUTs.

    """
    BASE = 100
    lab_lists = {}
    for history in inventory_list:
        location = _HOSTNAME_PATTERN.match(history.host.hostname)
        if location:
            lab = location.group(1)
            key = 0
            for idx in location.group(2, 3, 4):
                key = BASE * key + int(idx)
            lab_lists.setdefault(lab, []).append((key, history))
    return_list = []
    for dut_list in lab_lists.values():
        dut_list.sort(key=lambda t: t[0])
        return_list.append([t[1] for t in dut_list])
    return return_list


def _score_repair_set(buffer_counts, repair_list):
    """Return a numeric score rating a set of DUTs to be repaired.

    `buffer_counts` is a dictionary mapping board names to the
    size of the board's spares buffer.

    `repair_list` is a list of DUTs to be repaired.

    This function calculates the new set of buffer counts that would
    result from the proposed repairs, and scores the new set using
    two numbers:
      * Worst case buffer count for any board (higher is better).
        This is the more siginficant number for comparison.
      * Number of boards at the worst case (lower is better).  This
        is the less significant number.

    Implementation note:  The score could fail to reflect the
    intended criteria if there are more than 1000 boards in the
    inventory.

    @param spare_counts A dictionary mapping boards to buffer counts.
    @param repair_list  A list of boards to be repaired.
    @return A numeric score.

    """
    # Go through `buffer_counts`, and create a list of new counts
    # that records the buffer count for each board after repair.
    # The new list of counts discards the board names, as they don't
    # contribute to the final score.
    _NBOARDS = 1000
    repair_inventory = _LabInventory(repair_list)
    new_counts = []
    for b, c in buffer_counts.items():
        if b in repair_inventory.by_board:
            newcount = repair_inventory.by_board[b].get_total()
        else:
            newcount = 0
        new_counts.append(c + newcount)
    # Go through the new list of counts.  Find the worst available
    # spares count, and count how many times that worst case occurs.
    worst_count = new_counts[0]
    num_worst = 1
    for c in new_counts[1:]:
        if c == worst_count:
            num_worst += 1
        elif c < worst_count:
            worst_count = c
            num_worst = 1
    # Return the calculated score
    return _NBOARDS * worst_count - num_worst


def _generate_repair_recommendation(inventory, num_recommend):
    """Return a summary of selected DUTs needing repair.

    Returns a message recommending a list of broken DUTs to be
    repaired.  The list of DUTs is selected based on these
    criteria:
      * No more than `num_recommend` DUTs will be listed.
      * All DUTs must be in the same lab.
      * DUTs should be selected for some degree of physical
        proximity.
      * DUTs for boards with a low spares buffer are more important
        than DUTs with larger buffers.

    The algorithm used will guarantee that at least one DUT from a
    board with the smallest spares buffer will be recommended.  If
    the worst spares buffer number is shared by more than one board,
    the algorithm will tend to prefer repair sets that include more
    of those boards over sets that cover fewer boards.

    @param inventory      Inventory for generating recommendations.
    @param num_recommend  Number of DUTs to recommend for repair.

    """
    logging.debug('Creating DUT repair recommendations')
    board_buffer_counts = {}
    broken_list = []
    for board in inventory.get_managed_boards():
        logging.debug('Listing failed DUTs for %s', board)
        counts = inventory.by_board[board]
        if counts.get_broken() != 0:
            board_buffer_counts[board] = counts.get_spares_buffer()
            broken_list.extend(counts.get_broken_list())
    # N.B. The logic inside this loop may seem complicated, but
    # simplification is hard:
    #   * Calculating an initial recommendation outside of
    #     the loop likely would make things more complicated,
    #     not less.
    #   * It's necessary to calculate an initial lab slice once per
    #     lab _before_ the while loop, in case the number of broken
    #     DUTs in a lab is less than `num_recommend`.
    recommendation = None
    best_score = None
    for lab_duts in _sort_by_location(broken_list):
        start = 0
        end = num_recommend
        lab_slice = lab_duts[start : end]
        lab_score = _score_repair_set(board_buffer_counts,
                                      lab_slice)
        while end < len(lab_duts):
            start += 1
            end += 1
            new_slice = lab_duts[start : end]
            new_score = _score_repair_set(board_buffer_counts,
                                          new_slice)
            if new_score > lab_score:
                lab_slice = new_slice
                lab_score = new_score
        if recommendation is None or lab_score > best_score:
            recommendation = lab_slice
            best_score = lab_score
    # N.B. The trailing space here is manadatory:  Without it, Gmail
    # will parse the URL wrong.  Don't ask.  If you simply _must_
    # know more, go try it yourself...
    line_fmt = '%-30s %-16s %-6s\n    %s '
    message = ['Repair recommendations:\n',
               line_fmt % ( 'Hostname', 'Board', 'Servo?', 'Logs URL')]
    for h in recommendation:
        servo_name = servo_host.make_servo_hostname(h.host.hostname)
        servo_present = utils.host_is_in_lab_zone(servo_name)
        _, event = h.last_diagnosis()
        line = line_fmt % (
                h.host.hostname, h.host_board,
                'Yes' if servo_present else 'No', event.job_url)
        message.append(line)
    return '\n'.join(message)


def _generate_board_inventory_message(inventory):
    """Generate the "board inventory" e-mail message.

    The board inventory is a list by board summarizing the number
    of working and broken DUTs, and the total shortfall or surplus
    of working devices relative to the minimum critical pool
    requirement.

    The report omits boards with no DUTs in the spare pool or with
    no DUTs in a critical pool.

    N.B. For sample output text formattted as users can expect to
    see it in e-mail and log files, refer to the unit tests.

    @param inventory  _LabInventory object with the inventory to
                      be reported on.
    @return String with the inventory message to be sent.

    """
    logging.debug('Creating board inventory')
    nworking = 0
    nbroken = 0
    nidle = 0
    nbroken_boards = 0
    ntotal_boards = 0
    summaries = []
    for board in inventory.get_managed_boards():
        counts = inventory.by_board[board]
        logging.debug('Counting %2d DUTS for board %s',
                      counts.get_total(), board)
        # Summary elements laid out in the same order as the text
        # headers:
        #     Board Avail   Bad  Idle  Good  Spare Total
        #      e[0]  e[1]  e[2]  e[3]  e[4]  e[5]  e[6]
        element = (board,
                   counts.get_spares_buffer(),
                   counts.get_broken(),
                   counts.get_idle(),
                   counts.get_working(),
                   counts.get_total(SPARE_POOL),
                   counts.get_total())
        if element[2]:
            summaries.append(element)
            nbroken_boards += 1
        ntotal_boards += 1
        nbroken += element[2]
        nidle += element[3]
        nworking += element[4]
    ntotal = nworking + nbroken + nidle
    summaries = sorted(summaries, key=lambda e: (e[1], -e[2]))
    broken_percent = int(round(100.0 * nbroken / ntotal))
    idle_percent = int(round(100.0 * nidle / ntotal))
    working_percent = 100 - broken_percent - idle_percent
    message = ['Summary of DUTs in inventory:',
               '%10s %10s %10s %6s' % ('Bad', 'Idle', 'Good', 'Total'),
               '%5d %3d%% %5d %3d%% %5d %3d%% %6d' % (
                   nbroken, broken_percent,
                   nidle, idle_percent,
                   nworking, working_percent,
                   ntotal),
               '',
               'Boards with failures: %d' % nbroken_boards,
               'Boards in inventory:  %d' % ntotal_boards,
               '', '',
               'Full board inventory:\n',
               '%-22s %5s %5s %5s %5s %5s %5s' % (
                   'Board', 'Avail', 'Bad', 'Idle', 'Good',
                   'Spare', 'Total')]
    message.extend(
            ['%-22s %5d %5d %5d %5d %5d %5d' % e for e in summaries])
    return '\n'.join(message)


_POOL_INVENTORY_HEADER = '''\
Notice to Infrastructure deputies:  All boards shown below are at
less than full strength, please take action to resolve the issues.
Once you're satisified that failures won't recur, failed DUTs can
be replaced with spares by running `balance_pool`.  Detailed
instructions can be found here:
    http://go/cros-manage-duts
'''


def _generate_pool_inventory_message(inventory):
    """Generate the "pool inventory" e-mail message.

    The pool inventory is a list by pool and board summarizing the
    number of working and broken DUTs in the pool.  Only boards with
    at least one broken DUT are included in the list.

    N.B. For sample output text formattted as users can expect to
    see it in e-mail and log files, refer to the unit tests.

    @param inventory  _LabInventory object with the inventory to
                      be reported on.
    @return String with the inventory message to be sent.

    """
    logging.debug('Creating pool inventory')
    message = [_POOL_INVENTORY_HEADER]
    newline = ''
    for pool in CRITICAL_POOLS:
        message.append(
            '%sStatus for pool:%s, by board:' % (newline, pool))
        message.append(
            '%-20s   %5s %5s %5s %5s' % (
                'Board', 'Bad', 'Idle', 'Good', 'Total'))
        data_list = []
        for board, counts in inventory.by_board.iteritems():
            logging.debug('Counting %2d DUTs for %s, %s',
                          counts.get_total(pool), board, pool)
            broken = counts.get_broken(pool)
            idle = counts.get_idle(pool)
            # boards at full strength are not reported
            if broken == 0 and idle == 0:
                continue
            working = counts.get_working(pool)
            total = counts.get_total(pool)
            data_list.append((board, broken, idle, working, total))
        if data_list:
            data_list = sorted(data_list, key=lambda d: -d[1])
            message.extend(
                ['%-20s   %5d %5d %5d %5d' % t for t in data_list])
        else:
            message.append('(All boards at full strength)')
        newline = '\n'
    return '\n'.join(message)


_IDLE_INVENTORY_HEADER = '''\
Notice to Infrastructure deputies:  The hosts shown below haven't
run any jobs for at least 24 hours. Please check each host; locked
hosts should normally be unlocked; stuck jobs should normally be
aborted.
'''


def _generate_idle_inventory_message(inventory):
    """Generate the "idle inventory" e-mail message.

    The idle inventory is a host list with corresponding pool and board,
    where the hosts are idle (`UNKWOWN` or `UNUSED`).

    N.B. For sample output text format as users can expect to
    see it in e-mail and log files, refer to the unit tests.

    @param inventory  _LabInventory object with the inventory to
                      be reported on.
    @return String with the inventory message to be sent.

    """
    logging.debug('Creating idle inventory')
    message = [_IDLE_INVENTORY_HEADER]
    message.append('Idle Host List:')
    message.append('%-30s %-20s %s' % ('Hostname', 'Board', 'Pool'))
    data_list = []
    for pool in MANAGED_POOLS:
        for board, counts in inventory.by_board.iteritems():
            logging.debug('Counting %2d DUTs for %s, %s',
                          counts.get_total(pool), board, pool)
            data_list.extend([(dut.host.hostname, board, pool)
                                  for dut in counts.get_idle_list(pool)])
    if data_list:
        message.extend(['%-30s %-20s %s' % t for t in data_list])
    else:
        message.append('(No idle DUTs)')
    return '\n'.join(message)


def _send_email(arguments, tag, subject, recipients, body):
    """Send an inventory e-mail message.

    The message is logged in the selected log directory using `tag`
    for the file name.

    If the --debug option was requested, the message is neither
    logged nor sent, but merely printed on stdout.

    @param arguments   Parsed command-line options.
    @param tag         Tag identifying the inventory for logging
                       purposes.
    @param subject     E-mail Subject: header line.
    @param recipients  E-mail addresses for the To: header line.
    @param body        E-mail message body.

    """
    logging.debug('Generating email: "%s"', subject)
    all_recipients = ', '.join(recipients)
    report_body = '\n'.join([
            'To: %s' % all_recipients,
            'Subject: %s' % subject,
            '', body, ''])
    if arguments.debug:
        print report_body
    else:
        filename = os.path.join(arguments.logdir, tag)
        try:
            report_file = open(filename, 'w')
            report_file.write(report_body)
            report_file.close()
        except EnvironmentError as e:
            logging.error('Failed to write %s:  %s', filename, e)
        try:
            gmail_lib.send_email(all_recipients, subject, body)
        except Exception as e:
            logging.error('Failed to send e-mail to %s:  %s',
                          all_recipients, e)


def _populate_board_counts(inventory):
    """Gather board counts while providing interactive feedback.

    Gathering the status of all individual DUTs in the lab can take
    considerable time (~30 minutes at the time of this writing).

    Normally, we pay that cost by querying as we go.  However, with
    the `--debug` option, we expect a human being to be watching the
    progress in real time.  So, we force the first (expensive)
    queries to happen up front, and provide simple ASCII output
    (without using logging) to show a progress bar and results.

    @param inventory  _LabInventory object with the inventory to
                      be gathered.

    """
    n = 0
    total_broken = 0
    for counts in inventory.by_board.itervalues():
        n += 1
        if n % 10 == 5:
            c = '+'
        elif n % 10 == 0:
            c = '%d' % ((n / 10) % 10)
        else:
            c = '.'
        sys.stdout.write(c)
        sys.stdout.flush()
        # This next call is where all the time goes - it forces all
        # of a board's HostJobHistory objects to query the database
        # and cache their results.
        total_broken += counts.get_broken()
    sys.stdout.write('\n')
    sys.stdout.write('Found %d broken DUTs\n' % total_broken)


def _perform_board_inventory(arguments, inventory, timestamp):
    """Perform the board inventory report.

    The board inventory report consists of the following:
      * A list of DUTs that are recommended to be repaired.
        This list is optional, and only appears if the `--recommend`
        option is present.
      * A list of all boards that have failed DUTs, with counts
        of working, broken, and spare DUTs, among others.

    @param arguments  Command-line arguments as returned by
                      `ArgumentParser`
    @param inventory  _LabInventory object with the inventory to
                      be reported.
    @param timestamp  A string used to identify this run's timestamp
                      in logs and email output.
    """
    if arguments.recommend:
        recommend_message = _generate_repair_recommendation(
                inventory, arguments.recommend) + '\n\n\n'
    else:
        recommend_message = ''
    board_message = _generate_board_inventory_message(inventory)
    _send_email(arguments,
                'boards-%s.txt' % timestamp,
                'DUT board inventory %s' % timestamp,
                arguments.board_notify,
                recommend_message + board_message)


def _perform_pool_inventory(arguments, inventory, timestamp):
    """Perform the pool inventory report.

    The pool inventory report consists of the following:
      * A list of all critical pools that have failed DUTs, with counts
        of working, broken, and idle DUTs.
      * A list of all idle DUTs by hostname including the board and
        pool.

    @param arguments  Command-line arguments as returned by
                      `ArgumentParser`
    @param inventory  _LabInventory object with the inventory to be
                      reported.
    @param timestamp  A string used to identify this run's timestamp in
                      logs and email output.
    """
    pool_message = _generate_pool_inventory_message(inventory)
    idle_message = _generate_idle_inventory_message(inventory)
    _send_email(arguments,
                'pools-%s.txt' % timestamp,
                'DUT pool inventory %s' % timestamp,
                arguments.pool_notify,
                pool_message + '\n\n\n' + idle_message)


def _dut_in_repair_loop(history):
    """Return whether a DUT's history indicates a repair loop.

    A DUT is considered looping if it runs no tests, and no tasks pass
    other than repair tasks.

    @param history  An instance of `status_history.HostJobHistory` to be
                    scanned for a repair loop.  The caller guarantees
                    that this history corresponds to a working DUT.
    @returns  Return a true value if the DUT's most recent history
              indicates a repair loop.
    """
    # Our caller passes only histories for working DUTs; that means
    # we've already paid the cost of fetching the diagnosis task, and
    # we know that the task was successful.  The diagnosis task will be
    # one of the tasks we must scan to find a loop, so if the task isn't
    # a repair task, then our history includes a successful non-repair
    # task, and we're not looping.
    #
    # The for loop below  is very expensive, because it must fetch the
    # full history, regardless of how many tasks we examine.  At the
    # time of this writing, this check against the diagnosis task
    # reduces the cost of finding loops in the full inventory from hours
    # to minutes.
    if history.last_diagnosis()[1].name != 'Repair':
        return False
    repair_ok_count = 0
    for task in history:
        if not task.is_special:
            # This is a test, so we're not looping.
            return False
        if task.diagnosis == status_history.BROKEN:
            # Failed a repair, so we're not looping.
            return False
        if (task.diagnosis == status_history.WORKING
                and task.name != 'Repair'):
            # Non-repair task succeeded, so we're not looping.
            return False
        # At this point, we have either a failed non-repair task, or
        # a successful repair.
        if task.name == 'Repair':
            repair_ok_count += 1
            if repair_ok_count >= _REPAIR_LOOP_THRESHOLD:
                return True


def _perform_repair_loop_report(arguments, inventory):
    """Scan the inventory for DUTs stuck in a repair loop.

    This routine walks through the given inventory looking for DUTs
    where the most recent history shows that the DUT is regularly
    passing repair tasks, but has not run any tests.

    @param arguments  Command-line arguments as returned by
                      `ArgumentParser`
    @param inventory  _LabInventory object with the inventory to be
                      reported.
    """
    loop_presence = metrics.BooleanMetric(
        'chromeos/autotest/inventory/repair_loops',
        'DUTs stuck in repair loops')
    logging.info('Scanning for DUTs in repair loops.')
    for counts in inventory.by_board.itervalues():
        for history in counts.get_working_list():
            # Managed DUTs with names that don't match
            # _HOSTNAME_PATTERN shouldn't be possible.  However, we
            # don't want arbitrary strings being attached to the
            # 'dut_hostname' field, so for safety, we exclude all
            # anomalies.
            if not _HOSTNAME_PATTERN.match(history.hostname):
                continue
            if _dut_in_repair_loop(history):
                fields = {'dut_hostname': history.hostname,
                          'board': history.host_board,
                          'pool': history.host_pool}
                logging.info('Looping DUT: %(dut_hostname)s, '
                             'board: %(board)s, pool: %(pool)s',
                             fields)
                loop_presence.set(True, fields=fields)


def _log_startup(arguments, startup_time):
    """Log the start of this inventory run.

    Print various log messages indicating the start of the run.  Return
    a string based on `startup_time` that will be used to identify this
    run in log files and e-mail messages.

    @param startup_time   A UNIX timestamp marking the moment when
                          this inventory run began.
    @returns  A timestamp string that will be used to identify this run
              in logs and email output.
    """
    timestamp = time.strftime('%Y-%m-%d.%H',
                              time.localtime(startup_time))
    logging.debug('Starting lab inventory for %s', timestamp)
    if arguments.board_notify:
        if arguments.recommend:
            logging.debug('Will include repair recommendations')
        logging.debug('Will include board inventory')
    if arguments.pool_notify:
        logging.debug('Will include pool inventory')
    return timestamp


def _create_inventory(arguments, end_time):
    """Create the `_LabInventory` instance to use for reporting.

    @param end_time   A UNIX timestamp for the end of the time range
                      to be searched in this inventory run.
    """
    start_time = end_time - arguments.duration * 60 * 60
    afe = frontend_wrappers.RetryingAFE(server=None)
    inventory = _LabInventory.create_inventory(
            afe, start_time, end_time, arguments.boardnames)
    logging.info('Found %d hosts across %d boards',
                     inventory.get_num_duts(),
                     inventory.get_num_boards())
    return inventory


def _perform_inventory_reports(arguments):
    """Perform all inventory checks requested on the command line.

    Create the initial inventory and run through the inventory reports
    as called for by the parsed command-line arguments.

    @param arguments  Command-line arguments as returned by
                      `ArgumentParser`.
    """
    startup_time = time.time()
    timestamp = _log_startup(arguments, startup_time)
    inventory = _create_inventory(arguments, startup_time)
    if arguments.debug:
        _populate_board_counts(inventory)
    if arguments.board_notify:
        _perform_board_inventory(arguments, inventory, timestamp)
    if arguments.pool_notify:
        _perform_pool_inventory(arguments, inventory, timestamp)
    if arguments.repair_loops:
        _perform_repair_loop_report(arguments, inventory)


def _separate_email_addresses(address_list):
    """Parse a list of comma-separated lists of e-mail addresses.

    @param address_list  A list of strings containing comma
                         separate e-mail addresses.
    @return A list of the individual e-mail addresses.

    """
    newlist = []
    for arg in address_list:
        newlist.extend([email.strip() for email in arg.split(',')])
    return newlist


def _verify_arguments(arguments):
    """Validate command-line arguments.

    Join comma separated e-mail addresses for `--board-notify` and
    `--pool-notify` in separate option arguments into a single list.

    For non-debug uses, require that notification be requested for
    at least one report.  For debug, if notification isn't specified,
    treat it as "run all the reports."

    The return value indicates success or failure; in the case of
    failure, we also write an error message to stderr.

    @param arguments  Command-line arguments as returned by
                      `ArgumentParser`
    @return True if the arguments are semantically good, or False
            if the arguments don't meet requirements.

    """
    arguments.board_notify = _separate_email_addresses(
            arguments.board_notify)
    arguments.pool_notify = _separate_email_addresses(
            arguments.pool_notify)
    if not arguments.board_notify and not arguments.pool_notify:
        if not arguments.debug:
            sys.stderr.write('Must specify at least one of '
                             '--board-notify or --pool-notify\n')
            return False
        else:
            # We want to run all the reports.  An empty notify list
            # will cause a report to be skipped, so make sure the
            # lists are non-empty.
            arguments.board_notify = ['']
            arguments.pool_notify = ['']
    return True


def _get_default_logdir(script):
    """Get the default directory for the `--logdir` option.

    The default log directory is based on the parent directory
    containing this script.

    @param script  Path to this script file.
    @return A path to a directory.

    """
    basedir = os.path.dirname(os.path.abspath(script))
    basedir = os.path.dirname(basedir)
    return os.path.join(basedir, _LOGDIR)


def _parse_command(argv):
    """Parse the command line arguments.

    Create an argument parser for this command's syntax, parse the
    command line, and return the result of the ArgumentParser
    parse_args() method.

    @param argv Standard command line argument vector; argv[0] is
                assumed to be the command name.
    @return Result returned by ArgumentParser.parse_args().

    """
    parser = argparse.ArgumentParser(
            prog=argv[0],
            description='Gather and report lab inventory statistics')
    parser.add_argument('-d', '--duration', type=int,
                        default=_DEFAULT_DURATION, metavar='HOURS',
                        help='number of hours back to search for status'
                             ' (default: %d)' % _DEFAULT_DURATION)
    parser.add_argument('--board-notify', action='append',
                        default=[], metavar='ADDRESS',
                        help='Generate board inventory message, '
                        'and send it to the given e-mail address(es)')
    parser.add_argument('--pool-notify', action='append',
                        default=[], metavar='ADDRESS',
                        help='Generate pool inventory message, '
                             'and send it to the given address(es)')
    parser.add_argument('-r', '--recommend', type=int, default=None,
                        help=('Specify how many DUTs should be '
                              'recommended for repair (default: no '
                              'recommendation)'))
    parser.add_argument('--repair-loops', action='store_true',
                        help='Check for devices stuck in repair loops.')
    parser.add_argument('--debug', action='store_true',
                        help='Print e-mail messages on stdout '
                             'without sending them.')
    parser.add_argument('--logdir', default=_get_default_logdir(argv[0]),
                        help='Directory where logs will be written.')
    parser.add_argument('boardnames', nargs='*',
                        metavar='BOARD',
                        help='names of boards to report on '
                             '(default: all boards)')
    arguments = parser.parse_args(argv[1:])
    if not _verify_arguments(arguments):
        return None
    return arguments


def _configure_logging(arguments):
    """Configure the `logging` module for our needs.

    How we log depends on whether the `--debug` option was provided on
    the command line.
      * Without the option, we configure the logging to capture all
        potentially relevant events in a log file.  The log file is
        configured to rotate once a week on Friday evening, preserving
        ~3 months worth of history.
      * With the option, we expect stdout to contain other
        human-readable output (including the contents of the e-mail
        messages), so we restrict the output to INFO level.

    For convenience, when `--debug` is on, the logging format has
    no adornments, so that a call like `logging.info(msg)` simply writes
    `msg` to stdout, plus a trailing newline.

    @param arguments  Command-line arguments as returned by
                      `ArgumentParser`
    """
    root_logger = logging.getLogger()
    if arguments.debug:
        root_logger.setLevel(logging.INFO)
        handler = logging.StreamHandler(sys.stdout)
        handler.setFormatter(logging.Formatter())
    else:
        if not os.path.exists(arguments.logdir):
            os.mkdir(arguments.logdir)
        root_logger.setLevel(logging.DEBUG)
        logfile = os.path.join(arguments.logdir, _LOGFILE)
        handler = logging.handlers.TimedRotatingFileHandler(
                logfile, when='W4', backupCount=13)
        formatter = logging.Formatter(_LOG_FORMAT,
                                      time_utils.TIME_FMT)
        handler.setFormatter(formatter)
    # TODO(jrbarnette) This is gross.  Importing client.bin.utils
    # implicitly imported logging_config, which calls
    # logging.basicConfig() *at module level*.  That gives us an
    # extra logging handler that we don't want.  So, clear out all
    # the handlers here.
    for h in root_logger.handlers:
        root_logger.removeHandler(h)
    root_logger.addHandler(handler)


def main(argv):
    """Standard main routine.

    @param argv  Command line arguments, including `sys.argv[0]`.
    """
    arguments = _parse_command(argv)
    if not arguments:
        sys.exit(1)
    _configure_logging(arguments)
    try:
        if not arguments.debug:
            with site_utils.SetupTsMonGlobalState(
                    'repair_loops', short_lived=True, auto_flush=False):
                _perform_inventory_reports(arguments)
        else:
            _perform_inventory_reports(arguments)
    except KeyboardInterrupt:
        pass
    except EnvironmentError as e:
        logging.exception('Unexpected OS error: %s', e)
    except Exception as e:
        logging.exception('Unexpected exception: %s', e)


def get_inventory(afe):
    end_time = int(time.time())
    start_time = end_time - 24 * 60 * 60
    return _LabInventory.create_inventory(afe, start_time, end_time)


def get_managed_boards(afe):
    return get_inventory(afe).get_managed_boards()


if __name__ == '__main__':
    main(sys.argv)