frontend/afe/doctests/001_rpc_test.txt

# setup (you can ignore this)
# ###########################

# a bit of setup to allow overriding rpc_interace with an RPC proxy
# (to use RPC, we would say
#   import rpc_client_lib
#   rpc_interface = rpc_client_lib.get_proxy(
#                             'http://hostname:8000/afe/server/noauth/rpc/')
# )
>>> if 'rpc_interface' not in globals():
...   from frontend.afe import rpc_interface, models
...   from frontend import thread_local
...   # set up a user for us to "login" as
...   user = models.User(login='debug_user')
...   user.access_level = 100
...   user.save()
...   thread_local.set_user(user)
...

# get directory of this test file; we'll need it later
>>> import common
>>> from autotest_lib.frontend.afe import test
>>> import os
>>> test_path = os.path.join(os.path.dirname(test.__file__),
...                                          'doctests')
>>> test_path = os.path.abspath(test_path)

# disable logging
>>> from autotest_lib.client.common_lib import debug
>>> debug.get_logger().setLevel(100)

# basic interface test
######################

# echo a comment
>>> rpc_interface.echo('test string to echo')
'test string to echo'

# basic object management
# #######################

# create a label
>>> rpc_interface.add_label(name='test_label')
1

# we can modify the label by referencing its ID...
>>> rpc_interface.modify_label(1, kernel_config='/my/kernel/config')

# ...or by referencing it's name
>>> rpc_interface.modify_label('test_label', platform=True)

# we use get_labels to retrieve object data
>>> data = rpc_interface.get_labels(name='test_label')
>>> data == [{'id': 1,
...           'name': 'test_label',
...           'platform': 1,
...           'kernel_config': '/my/kernel/config',
...	      'only_if_needed' : False,
...           'invalid': 0}]
True

# get_labels return multiple matches as lists of dictionaries
>>> rpc_interface.add_label(name='label1', platform=False)
2
>>> rpc_interface.add_label(name='label2', platform=True)
3
>>> rpc_interface.add_label(name='label3', platform=False)
4
>>> data = rpc_interface.get_labels(platform=False)
>>> data == [{'id': 2, 'name': 'label1', 'platform': 0, 'kernel_config': '',
...           'only_if_needed': False, 'invalid': 0},
...          {'id': 4, 'name': 'label3', 'platform': 0, 'kernel_config': '',
...           'only_if_needed': False, 'invalid': 0}]
True

# delete_label takes an ID or a name as well
>>> rpc_interface.delete_label(3)
>>> rpc_interface.get_labels(name='label2')
[]
>>> rpc_interface.delete_label('test_label')
>>> rpc_interface.delete_label('label1')
>>> rpc_interface.delete_label('label3')
>>> rpc_interface.get_labels()
[]

# all the add*, modify*, delete*, and get* methods work the same way
# hosts...
>>> rpc_interface.add_host(hostname='ipaj1', locked=True)
1
>>> rpc_interface.modify_host('ipaj1', status='Hello')
>>> data = rpc_interface.get_hosts()

# delete the lock_time field, since that can't be reliably checked
>>> del data[0]['lock_time']
>>> data == [{'id': 1,
...           'hostname': 'ipaj1',
...           'locked': 1,
...           'synch_id': None,
...           'status': 'Hello',
...           'labels': [],
...           'platform': None,
...           'invalid': 0,
...           'protection': 'No protection',
...           'locked_by': 'debug_user',
...           'dirty': True}]
True
>>> rpc_interface.delete_host('ipaj1')
>>> rpc_interface.get_hosts() == []
True

# tests...
>>> rpc_interface.add_test(name='sleeptest', test_type='Client', author='Test',
...                        description='Sleep Test', test_time=1,
...                        test_category='Functional',
...                        test_class='Kernel', path='sleeptest')
1
>>> rpc_interface.modify_test('sleeptest', path='/my/path')
>>> data = rpc_interface.get_tests()
>>> data == [{'id': 1,
...           'name': 'sleeptest',
...           'author': 'Test',
...           'description': 'Sleep Test',
...           'dependencies': '',
...           'experimental': 1,
...           'sync_count': 1,
...           'test_type': 'Client',
...           'test_class': 'Kernel',
...           'test_time': 'SHORT',
...           'run_verify': 1,
...           'test_category': 'Functional',
...           'path': '/my/path'}]
True
>>> rpc_interface.delete_test('sleeptest')
>>> rpc_interface.get_tests() == []
True

# profilers...
>>> rpc_interface.add_profiler(name='oprofile')
1
>>> rpc_interface.modify_profiler('oprofile', description='Oh profile!')
>>> data = rpc_interface.get_profilers()
>>> data == [{'id': 1,
...           'name': 'oprofile',
...           'description': 'Oh profile!'}]
True
>>> rpc_interface.delete_profiler('oprofile')
>>> rpc_interface.get_profilers() == []
True


# users...
>>> rpc_interface.add_user(login='showard')
2
>>> rpc_interface.modify_user('showard', access_level=1)
>>> data = rpc_interface.get_users(login='showard')
>>> data == [{'id': 2,
...           'login': 'showard',
...           'access_level': 1,
...           'reboot_before': 'If dirty',
...           'reboot_after': 'Always',
...           'show_experimental': False}]
True
>>> rpc_interface.delete_user('showard')
>>> rpc_interface.get_users(login='showard') == []
True

# acl groups...
# 1 ACL group already exists, named "Everyone" (ID 1)
>>> rpc_interface.add_acl_group(name='my_group')
2
>>> rpc_interface.modify_acl_group('my_group', description='my new acl group')
>>> data = rpc_interface.get_acl_groups(name='my_group')
>>> data == [{'id': 2,
...           'name': 'my_group',
...           'description': 'my new acl group',
...           'users': ['debug_user'],
...           'hosts': []}]
True
>>> rpc_interface.delete_acl_group('my_group')
>>> data = rpc_interface.get_acl_groups()
>>> data == [{'id': 1,
...           'name': 'Everyone',
...           'description': '',
...           'users': ['debug_user'],
...           'hosts': []}]
True


# managing many-to-many relationships
# ###################################

# first, create some hosts and labels to play around with
>>> rpc_interface.add_host(hostname='host1')
2
>>> rpc_interface.add_host(hostname='host2')
3
>>> rpc_interface.add_label(name='label1')
2
>>> rpc_interface.add_label(name='label2', platform=True)
3

# add hosts to labels
>>> rpc_interface.host_add_labels('host1', ['label1'])
>>> rpc_interface.host_add_labels('host2', ['label1', 'label2'])

# check labels for hosts
>>> data = rpc_interface.get_hosts(hostname='host1')
>>> data[0]['labels']
['label1']
>>> data = rpc_interface.get_hosts(hostname='host2')
>>> data[0]['labels']
['label1', 'label2']
>>> data[0]['platform']
'label2'

# check host lists for labels -- use double underscore to specify fields of
# related objects
>>> data = rpc_interface.get_hosts(labels__name='label1')
>>> [host['hostname'] for host in data]
['host1', 'host2']
>>> data = rpc_interface.get_hosts(labels__name='label2')
>>> [host['hostname'] for host in data]
['host2']

# remove a host from a label
>>> rpc_interface.host_remove_labels('host2', ['label2'])
>>> data = rpc_interface.get_hosts(hostname='host1')
>>> data[0]['labels']
['label1']
>>> rpc_interface.get_hosts(labels__name='label2')
[]

# Cleanup
>>> rpc_interface.host_remove_labels('host2', ['label1'])
>>> rpc_interface.host_remove_labels('host1', ['label1'])


# Other interface for new CLI
# add hosts to labels
>>> rpc_interface.label_add_hosts('label1', ['host1'])
>>> rpc_interface.label_add_hosts('label2', ['host1', 'host2'])

# check labels for hosts
>>> data = rpc_interface.get_hosts(hostname='host1')
>>> data[0]['labels']
['label1', 'label2']
>>> data = rpc_interface.get_hosts(hostname='host2')
>>> data[0]['labels']
['label2']
>>> data[0]['platform']
'label2'

# check host lists for labels -- use double underscore to specify fields of
# related objects
>>> data = rpc_interface.get_hosts(labels__name='label1')
>>> [host['hostname'] for host in data]
['host1']
>>> data = rpc_interface.get_hosts(labels__name='label2')
>>> [host['hostname'] for host in data]
['host1', 'host2']

# remove a host from a label
>>> rpc_interface.label_remove_hosts('label2', ['host2'])
>>> data = rpc_interface.get_hosts(hostname='host1')
>>> data[0]['labels']
['label1', 'label2']
>>> data = rpc_interface.get_hosts(labels__name='label2')
>>> [host['hostname'] for host in data]
['host1']

# Remove multiple hosts from a label
>>> rpc_interface.label_add_hosts('label2', ['host2'])
>>> data = rpc_interface.get_hosts(labels__name='label2')
>>> [host['hostname'] for host in data]
['host1', 'host2']
>>> rpc_interface.label_remove_hosts('label2', ['host2', 'host1'])
>>> rpc_interface.get_hosts(labels__name='label2')
[]


# ACL group relationships work similarly
# note that all users are a member of 'Everyone' by default, and that hosts are
# automatically made a member of 'Everyone' only when they are a member of no
# other group
>>> data = rpc_interface.get_acl_groups(hosts__hostname='host1')
>>> [acl_group['name'] for acl_group in data]
['Everyone']

>>> rpc_interface.add_user(login='showard', access_level=0)
2
>>> rpc_interface.add_acl_group(name='my_group')
2

>>> rpc_interface.acl_group_add_users('my_group', ['showard'])
>>> rpc_interface.acl_group_add_hosts('my_group', ['host1'])
>>> data = rpc_interface.get_acl_groups(name='my_group')
>>> data[0]['users']
['debug_user', 'showard']
>>> data[0]['hosts']
['host1']
>>> data = rpc_interface.get_acl_groups(users__login='showard')
>>> [acl_group['name'] for acl_group in data]
['Everyone', 'my_group']

# note host has been automatically removed from 'Everyone'
>>> data = rpc_interface.get_acl_groups(hosts__hostname='host1')
>>> [acl_group['name'] for acl_group in data]
['my_group']

>>> rpc_interface.acl_group_remove_users('my_group', ['showard'])
>>> rpc_interface.acl_group_remove_hosts('my_group', ['host1'])
>>> data = rpc_interface.get_acl_groups(name='my_group')
>>> data[0]['users'], data[0]['hosts']
(['debug_user'], [])
>>> data = rpc_interface.get_acl_groups(users__login='showard')
>>> [acl_group['name'] for acl_group in data]
['Everyone']

# note host has been automatically added back to 'Everyone'
>>> data = rpc_interface.get_acl_groups(hosts__hostname='host1')
>>> [acl_group['name'] for acl_group in data]
['Everyone']


# job management
# ############

# note that job functions require job IDs to identify jobs, since job names are
# not unique

# add some entries to play with
>>> rpc_interface.add_label(name='my_label', kernel_config='my_kernel_config')
5
>>> test_control_path = os.path.join(test_path, 'test.control')
>>> rpc_interface.add_test(name='sleeptest', test_type='Client', author='Test',
...                        test_category='Test',
...                        test_class='Kernel', path=test_control_path)
1
>>> test_control_path = os.path.join(test_path, 'test.control.2')
>>> rpc_interface.add_test(name='my_test', test_type='Client', author='Test',
...                        test_category='Test',
...                        test_class='Kernel', path=test_control_path)
2
>>> rpc_interface.add_host(hostname='my_label_host1')
4
>>> rpc_interface.add_host(hostname='my_label_host2')
5
>>> rpc_interface.label_add_hosts(id='my_label', hosts=['my_label_host1', 'my_label_host2'])

# generate a control file
>>> cf_info = rpc_interface.generate_control_file(
...     tests=['sleeptest', 'my_test'],
...     kernel=' 2.6.18, 2.6.22 \t2.6.26,2.6.27  \t',
...     label='my_label')
>>> print cf_info['control_file'] #doctest: +NORMALIZE_WHITESPACE
kernel_list = ['2.6.18', '2.6.22', '2.6.26', '2.6.27']
def step_init():
    for kernel_version in kernel_list:
        job.next_step(boot_kernel, kernel_version)
        job.next_step(step_test, kernel_version)
def boot_kernel(kernel_version):
    testkernel = job.kernel(kernel_version)
    testkernel.config('my_kernel_config')
    testkernel.install()
    testkernel.boot(args='')
def step_test(kernel_version):
    global kernel
    kernel = kernel_version  # Set the global in case anyone is using it.
    if len(kernel_list) > 1:
        job.set_test_tag_prefix(kernel_version)  # Separate output by kernel.
    job.next_step('step0')
    job.next_step('step1')
def step0():
    job.run_test('testname')
def step1():
    job.run_test('testname')
>>> cf_info['is_server'], cf_info['synch_count'], cf_info['dependencies']
(False, 1, [])

# create a job to run on host1, host2, and any two machines in my_label
>>> rpc_interface.create_job(name='my_job',
...                          priority='Low',
...                          control_file=cf_info['control_file'],
...                          control_type='Client',
...                          hosts=['host1', 'host2'],
...                          meta_hosts=['my_label', 'my_label'])
1

# get job info - this does not include status info for particular hosts
>>> data = rpc_interface.get_jobs()
>>> data = data[0]
>>> data['id'], data['owner'], data['name'], data['priority']
(1, 'debug_user', 'my_job', 'Low')
>>> data['control_file'] == cf_info['control_file']
True
>>> data['control_type']
'Client'

>>> data['created_on'] #doctest: +ELLIPSIS
'200...'

# get_num_jobs - useful when dealing with large numbers of jobs
>>> rpc_interface.get_num_jobs(name='my_job')
1

# check host queue entries for a job
>>> data = rpc_interface.get_host_queue_entries(job=1)
>>> len(data)
4

# get rid of created_on, it's nondeterministic
>>> data[0]['job']['created_on'] = data[2]['job']['created_on'] = None

# get_host_queue_entries returns full info about the job within each queue entry
>>> job = data[0]['job']
>>> job == {'control_file': cf_info['control_file'], # the control file we used
...         'control_type': 'Client',
...         'created_on': None,
...         'id': 1,
...         'name': 'my_job',
...         'owner': 'debug_user',
...         'priority': 'Low',
...         'synch_count': 1,
...         'timeout': 72,
...         'run_verify': 1,
...         'email_list': '',
...         'reboot_before': 'If dirty',
...         'reboot_after': 'Always'}
True

# get_host_queue_entries returns a lot of data, so let's only check a couple
>>> data[0] == (
... {'active': 0,
...  'complete': 0,
...  'host': {'hostname': 'host1', # full host info here
...           'id': 2,
...           'invalid': 0,
...           'locked': 0,
...           'status': 'Ready',
...           'synch_id': None,
...           'protection': 'No protection',
...           'locked_by': None,
...           'lock_time': None,
...           'dirty': True},
...  'id': 1,
...  'job': job, # full job info here
...  'meta_host': None,
...  'priority': 0,
...  'status': 'Queued',
...  'deleted': 0,
...  'execution_subdir': ''})
True
>>> data[2] == (
... {'active': 0,
...  'complete': 0,
...  'host': None,
...  'id': 3,
...  'job': job,
...  'meta_host': 'my_label',
...  'priority': 0,
...  'status': 'Queued',
...  'deleted': 0,
...  'execution_subdir': ''})
True
>>> rpc_interface.get_num_host_queue_entries(job=1)
4
>>> rpc_interface.get_hqe_percentage_complete(job=1)
0.0

# get_jobs_summary adds status counts to the rest of the get_jobs info
>>> data = rpc_interface.get_jobs_summary()
>>> counts = data[0]['status_counts']
>>> counts
{'Queued': 4}

# abort the job
>>> rpc_interface.abort_host_queue_entries(job__id=1)
>>> data = rpc_interface.get_jobs_summary(id=1)
>>> data[0]['status_counts']
{'Abort': 4}

# Remove the two hosts in my_label
>>> rpc_interface.delete_host('my_label_host1')
>>> rpc_interface.delete_host('my_label_host2')


# extra querying parameters
# #########################

# get_* methods can take query_start and query_limit arguments to implement
# paging and a sort_by argument to specify the sort column
>>> data = rpc_interface.get_hosts(query_limit=1)
>>> [host['hostname'] for host in data]
['host1']
>>> data = rpc_interface.get_hosts(query_start=1, query_limit=1)
>>> [host['hostname'] for host in data]
['host2']

# sort_by = ['-hostname'] indicates sorting in descending order by hostname
>>> data = rpc_interface.get_hosts(sort_by=['-hostname'])
>>> [host['hostname'] for host in data]
['host2', 'host1']


# advanced usage
# ##############

# synch_count
>>> rpc_interface.create_job(name='my_job',
...                          priority='Low',
...                          control_file=cf_info['control_file'],
...                          control_type='Server',
...                          synch_count=2,
...                          hosts=['host1', 'host2'])
2

>>> data = rpc_interface.get_jobs(id=2)
>>> data[0]['synch_count']
2

# get hosts ACL'd to a user
>>> hosts = rpc_interface.get_hosts(acl_group__users__login='debug_user')
>>> sorted([host['hostname'] for host in hosts])
['host1', 'host2']

>>> rpc_interface.add_acl_group(name='mygroup')
3
>>> rpc_interface.acl_group_add_users('mygroup', ['debug_user'])
>>> rpc_interface.acl_group_add_hosts('mygroup', ['host1'])
>>> data = rpc_interface.get_acl_groups(name='Everyone')[0]
>>> data['users'], data['hosts']
(['debug_user', 'showard'], ['host2'])
>>> data = rpc_interface.get_acl_groups(name='mygroup')[0]
>>> data['users'], data['hosts']
(['debug_user'], ['host1'])

>>> hosts = rpc_interface.get_hosts(acl_group__users__login='debug_user')
>>> sorted([host['hostname'] for host in hosts])
['host1', 'host2']
>>> hosts = rpc_interface.get_hosts(acl_group__users__login='showard')
>>> [host['hostname'] for host in hosts]
['host2']

>>> rpc_interface.delete_acl_group('mygroup')
>>> data = rpc_interface.get_acl_groups(name='Everyone')[0]
>>> sorted(data['hosts'])
['host1', 'host2']