site_utils/dut_mon.py

#!/usr/bin/env python
# Copyright 2018 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""A simple service to monitor DUT statuses from master db/afe."""
import collections
import logging
import sys
import time

import common
from autotest_lib.server import constants
from autotest_lib.server import frontend
from chromite.lib import metrics
from chromite.lib import ts_mon_config

from infra_libs import ts_mon


DutCountBucket = collections.namedtuple('DutCountBucket',
                                        ['board',
                                         'model',
                                         'pool',
                                         'is_locked',
                                         'status']
                                        )


def _get_bucket_for_host(host):
    """Determine the counter bucket for |host|.

    Args:
        host: A Host object as returned by afe.

    Returns:
        A DutCountBucket instance describing the bucket for this host.
    """
    board = _get_unique_label(host.labels, constants.Labels.BOARD_PREFIX)
    model = _get_unique_label(host.labels, constants.Labels.MODEL_PREFIX)
    pool = _get_unique_label(host.labels, constants.Labels.POOL_PREFIX)
    if pool in constants.Pools.MANAGED_POOLS:
        pool = 'managed:' + pool
    status = host.status or '[None]'
    is_locked = host.locked
    return DutCountBucket(board, model, pool, is_locked, status)


def _get_unique_label(labels, prefix):
    """Return the labels for a given prefix, with prefix stripped.

    If prefixed label does not occur, return '[None]'
    If prefixed label occurs multiply, return '[Multiple]'

    _get_unique_label(['foo:1', 'foo:2', 'bar1'], 'foo:') -> '[Multiple]'

    _get_unique_label(['foo:1', 'bar2', 'baz3'], 'foo:') -> '1'

    _get_prefixed_labels(['bar1', 'baz1'], 'foo:') -> '[None]'
    """
    ls = [l[len(prefix):] for l in labels if l.startswith(prefix)]
    if not ls:
        return '[None]'
    elif len(ls) == 1:
        return ls[0]
    else:
        return '[Multiple]'


def main(argv):
    """Entry point for dut_mon."""
    logging.getLogger().setLevel(logging.INFO)

    with ts_mon_config.SetupTsMonGlobalState('dut_mon', indirect=True):
        afe = frontend.AFE()
        counters = collections.defaultdict(lambda: 0)

        field_spec = [ts_mon.StringField('board'),
                      ts_mon.StringField('model'),
                      ts_mon.StringField('pool'),
                      ts_mon.BooleanField('is_locked'),
                      ts_mon.StringField('status'),
                      ]
        dut_count = metrics.Gauge('chromeos/autotest/dut_mon/dut_count',
                                  description='The number of duts in a given '
                                              'state and bucket.',
                                  field_spec=field_spec)
        tick_count = metrics.Counter('chromeos/autotest/dut_mon/tick',
                                     description='Tick counter of dut_mon.')

        while True:
            # Note: We reset all counters to zero in each loop rather than
            # creating a new defaultdict, because we want to ensure that any
            # gauges that were previously set to a nonzero value by this process
            # get set back to zero if necessary.
            for k in counters:
                counters[k] = 0

            logging.info('Fetching all hosts.')
            hosts = afe.get_hosts()
            logging.info('Fetched %s hosts.', len(hosts))
            for host in hosts:
                fields = _get_bucket_for_host(host)
                counters[fields] += 1

            for field, value in counters.iteritems():
                logging.info('%s %s', field, value)
                dut_count.set(value, fields=field.__dict__)

            tick_count.increment()
            logging.info('Sleeping for 2 minutes.')
            time.sleep(120)


if __name__ == '__main__':
    main(sys.argv)