cli/skylab_migration2.py - platform/external/autotest - Gitiles

 #!/usr/bin/env python2


 # user facing commands:
 #
 #  -- write_statjson_hostnames
 #  -- do_quick_add


 # pretty_dump_board:

 # Given the name of a board, dump it to a directory with
 # the following structure:

 # data.dir
 #   - hostname1
 #   - hostname2
 #   - hostname3

 # NEXT, report all the ways in which hostnames of that board are not valid
 # jsonified protobufs for adding a new DUT.
 #
 # this involves shelling out to skylab validate-new-dut-json


 # adding a dut that's already present
 # $ skylab quick-add-dut /tmp/json
 # Deployment ID:         d604b46b-87c7-4a4d-8b8a-ed97565c4797
 # Status:                DUT_DEPLOYMENT_STATUS_FAILED
 # Inventory change URL:
 # Deploy task URL:
 # Message:               failed to add DUT(s) to fleet: add dut to fleet: inventory store commit: nothing to commit


 # adding a dut successfully
 # $ skylab quick-add-dut /tmp/json
 # Deployment ID:         8e27b3ff-39ad-4b92-9bea-3f804a9d7bf9
 # Status:                DUT_DEPLOYMENT_STATUS_FAILED
 # Inventory change URL:  https://chrome-internal-review.googlesource.com/c/chromeos/infra_internal/skylab_inventory/+/1940714
 # Deploy task URL:
 # Message:               missing deploy task ID in deploy request entry


 # $ atest host rename --for-migration --non-interactive chromeos2-row1-rack7-host1
 # Successfully renamed:
 # chromeos2-row1-rack7-host1 to chromeos2-row1-rack7-host1-migrated-do-not-use


 # unsuccessful lock

 # $ atest host mod --lock -r 'migration to skylab' chromeos2-row1-rack7-host1-migrated-do-not-use
 # Operation modify_host failed:
 #     ValidationError: {'locked': u'Host chromeos2-row1-rack7-host1-migrated-do-not-use already locked by pprabhu on 2019-08-29 16:48:51.'}
 #    1


 # successful lock


 from __future__ import print_function
 from __future__ import unicode_literals
 import os
 import sys
 import subprocess
 import pipes
 import os.path
 import warnings
 import json
 import tempfile
 import shutil

 TEXT = (type(b""), type(u""))
 NONETYPE = type(None)


 def flush_sync(fh):
     fh.flush()
     os.fsync(fh)
     return


 # accepts: shell command, rest of args
 # returns: exit_status, stdout, stderr
 def shell_capture_all(cmd, *rest):
     shellcmd = ("bash", "-c", cmd, "bash",) + rest
     pr = subprocess.Popen(
         shellcmd,
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
     )
     stdout, stderr = pr.communicate()
     return pr.returncode, stdout, stderr


 # accepts: name of board
 # returns: list of hostnames, error message (None if no error)
 def get_all_hosts_for_board(board):
     # extract just the hostnames for all the hosts in the board
     cmd = "set -o pipefail; ( atest host list --label=%%%LABEL%%% | awk '{print $1}' )"
     cmd = cmd.replace("%%%LABEL%%%", pipes.quote("board:%s" % board))
     returncode, out, err = shell_capture_all(cmd)
     if returncode != 0:
         return None, "get_all_hosts_for_board: " + err
     hostnames = [x.strip() for x in out.split()]
     if hostnames and hostnames[0] == "Host":
         hostnames.pop(0)
     return hostnames, None


 # make a directory and test if we can write to it as defensively as possible
 # free vars: %%%DEST%%%
 MAKE_DIR_CMD = r"""
 mkdir -p %%%DEST%%%
 cd %%%DEST%%% && touch ./writeable && rm ./writeable
 """


 # free vars: %%%HOSTNAME%%% %%%DEST%%%
 STATJSON_CMD = r"""
 atest host statjson %%%HOSTNAME%%% 1>%%%DEST%%%/%%%HOSTNAME%%%
 """

 # accepts: path to directory
 # returns: error message (None if successfully made dir)
 def mkdirp(dirpath):
     cmd = MAKE_DIR_CMD.replace('%%%DEST%%%', pipes.quote(dirpath))
     returncode, out, err = shell_capture_all(cmd)
     if returncode == 0:
         return None
     else:
         return err


 # accepts: list of hostnames, output directory (path)
 # returns: successful hosts, failed hosts, error message (None if no error)
 def write_statjson_hostnames(hostnames, outdir):
     successful_hosts = []
     failed_hosts = []
     err = mkdirp(outdir)
     if err is not None:
         return None, None, err
     for hostname in hostnames:
         cmd = STATJSON_CMD
         cmd = cmd.replace('%%%HOSTNAME%%%', pipes.quote(hostname))
         cmd = cmd.replace('%%%DEST%%%', pipes.quote(outdir))
         returncode, out, err = shell_capture_all(cmd)
         if returncode == 0:
             successful_hosts.append(hostname)
         else:
             failed_hosts.append(hostname)
     return successful_hosts, failed_hosts, None


 # accepts: board name, output directory
 # returns: successful hosts, failed hosts, error message
 def write_statjson_board(board, outdir):
     hostnames, err = get_all_hosts_for_board(board)
     if err is not None:
         return None, None, err
     successful_hosts, failed_hosts, err = write_statjson_hostnames(hostnames, outdir)
     return successful_hosts, failed_hosts, err


 # free vars: %%%PATH%%%
 VALIDATE_CMD = r"""
 skylab validate-new-dut-json %%%PATH%%%
 """


 # accepts: directory with hostname files
 # returns: dictionary of the form below, error message (or None if no error)
 #
 # {
 #    hostname: error message (or None if no error)
 # }
 def validate_output(hostname_dir):
     try:
         paths = os.listdir(hostname_dir)
     except OSError:
         return None, ("bad directory: %s" % hostname_dir)

     # defensively populate result dictionary with errors
     # so that we don't erroneously conclude that an unvisited
     # hostname was validated
     result = {}
     for path in paths:
         result[path] = "DID NOT PROCESS"

     for path in paths:
         cmd = VALIDATE_CMD
         cmd = cmd.replace('%%%PATH%%%', pipes.quote(os.path.join(hostname_dir, path)))
         returncode, out, err = shell_capture_all(cmd)
         if returncode != 0:
             result[path] = "failed to validate (errcode %s): %s" % (returncode, err)
         elif os.path.exists(os.path.join(hostname_dir, path)):
             result[path] = None
         else:
             warnings.warn("nonexistent path %s" % path)
             result[path] = "file does not exist"

     return result, None


 # accepts: name of board, output directory
 # returns: dictionary of form below, error message (or None if no error)
 #
 # {
 #    hostname: error-message or NOFILE if no file or None if no error
 # }
 def process_board(board, output_dir):
     queried_hosts, unqueried_hosts, err = write_statjson_board(board, output_dir)
     if err is not None:
         return None, err
     validate_result, err = validate_output(output_dir)
     if err is not None:
         return None, err
     result = {}

     for hostname in queried_hosts:
         try:
             result[hostname] = validate_result[hostname]
         except KeyError:
             warnings.warn("hostname (%s) not present in validate_result")

     for hostname in unqueried_hosts:
         result["hostname"] = "NOFILE"

     return result, None


 # accepts: name of board, output directory
 # returns: number of bad hosts
 # emits:   prints error message for every bad host
 def pretty_process_board(board, output_dir):
     result, err = process_board(board, output_dir)
     if err is not None:
         print(err)
         return 1
     bad_results = {}
     for k in result:
         if result[k] is not None:
             bad_results[k] = result[k]
     # no bad results --> nothing printed
     for k in bad_result:
         print(k, result[k])
     return len(bad_results)


 # accepts: path to output directory
 # returns: combined json object, error message (None if no error)
 # NOTE: the directory not existing is a fatal error
 #       processing a file that is invalid json AFTER the output directory
 #       has been validated produces a warning. the invalid json situation
 #       should be impossible, but also isn't enough to prevent assemble_output_dir
 #       from doing something reasonable.
 def assemble_output_dir(output_dir):
     _, err = validate_output(output_dir)
     if err is not None:
         return None, err
     out = []
     items = None
     try:
         items = os.listdir(output_dir)
     except OSError:
         return None, ("directory %s does not exist or is not readable" % output_dir)
     for item in items:
         obj = None
         try:
             with open(os.path.join(output_dir, item), "r") as fh:
                 try:
                     obj = json.load(fh)
                 except ValueError:
                     warnings.warn("file %s does not contain valid JSON" % item)
                     continue
         except IOError:
             warnings.warn("file %s somehow doesn't exist" % item)
             continue
         out.append(obj)
     return out, None


 # accepts: single json dictionary
 # returns: error message (None if valid)
 def validate_single_dut_json(obj):
     with tempfile.NamedTemporaryFile(delete=True) as fh:
         json.dump(obj, fh)
         flush_sync(fh)
         cmd = VALIDATE_CMD
         cmd = cmd.replace('%%%PATH%%%', pipes.quote(fh.name))
         returncode, out, err = shell_capture_all(cmd)
         if returncode == 0:
             return None
         else:
             return err


 # accepts: json obj
 # returns: hostname, error message (None if no error)
 def get_hostname_from_dut_json(obj):
     try:
         common = obj["common"]
     except KeyError:
         return None, "dut has no common element"
     if not isinstance(common, dict):
         return None, ("common block must be dict not %s" % type(common))
     try:
         return common["hostname"], None
     except KeyError:
         return None, "common block has no hostname element"


 # accepts: path to file
 # returns: hostname mapping, error message
 # hostname mapping has the following form
 #
 # {
 #    hostname -> new_dut_info_json
 # }
 #
 # malformed entries are not included in the map
 def load_hostname_map_file(filepath):
     obj = None
     try:
         with open(filepath, "r") as fh:
             try:
                 obj = json.load(fh)
             except ValueError:
                 return None, ("file does not contain JSON %s" % filepath)
     except IOError:
         return None, ("cannot load hostname map from nonexistent file %s" % filepath)

     out = {}

     # return a singleton map if the toplevel entry is a dictionary
     if isinstance(obj, dict):
         err = validate_single_dut_json(obj)
         if err is not None:
             return None, err
         hostname, err = get_hostname_from_dut_json(obj)
         if err is not None:
             return None, err
         return {hostname: obj}, None

     # validate all elements if the toplevel entry is a list
     elif isinstance(obj, list):
         for subobj in obj:
             err = validate_single_dut_json(subobj)
             if err is not None:
                 warnings.warn(err)
                 continue
             hostname, err = get_hostname_from_dut_json(subobj)
             if err is not None:
                 warnings.warn(err)
                 continue
             # warn if we get a duplicate, but don't halt execution
             if hostname in out:
                 warnings.warn(("duplicate hostname %s" % hostname))
             out[hostname] = subobj

     if len(out) == 0:
         return out, "out cannot be empty"

     return out, None


 # accepts: path to directory
 # returns: hostnaming mapping, error message
 # hostname mapping has the following form
 #
 # {
 #    hostname -> new_dut_info_json
 # }
 #
 # malformed entries are not included in the map
 def load_hostname_map(dirpath):
     items = None
     try:
         items = os.listdir(dirpath)
     except OSError:
         return None, ("cannot load from nonexistent directory %s" % dirpath)
     if len(items) == 0:
         return None, ("nothing in directory %s" % dirpath)
     out = {}
     for item in items:
         hostname_map, err = load_hostname_map_file(os.path.join(dirpath, item))
         if err is not None:
             warnings.warn(err)
             continue

         for hostname in hostname_map:
             if hostname in out:
                 warnings.warn("load_hostname_map: duplicate hostname %s" % hostname)
                 continue
             out[hostname] = hostname_map[hostname]

     return out, None


 # free vars: %%%DIR%%%
 QUICK_ADD_DUTS_CMD = r"""
 skylab quick-add-duts %%%DIR%%%/*
 """


 # accepts: list of hostnames, hostname_dirpath
 # returns: error message (None if successful)
 # NOTE: the quick-add-duts API is atomic
 # emits: missing hostnames when there are missing hostnames
 def do_quick_add_duts(hostnames, dirpath):
     # validation
     if isinstance(hostnames, TEXT):
         return "hostnames cannot be %s" % type(hostnames)
     # defensively prevent hostnames from being modified
     hostnames = tuple(hostnames)
     for hostname in hostnames:
         if hostname.startswith("."):
             return "hostname cannot start with '.' (%s)" % hostname
         if not hostname:
             return "hostname cannot be falsey (%s)" % hostname


     hostnames_map, err = load_hostname_map(dirpath)
     if err is not None:
         return err

     # check that every hostname is in the map before trying
     missing_hostnames = set([])
     for hostname in hostnames:
         if hostname not in hostnames_map:
             missing_hostnames.add(hostname)

     if missing_hostnames:
         for hostname in sorted(missing_hostnames):
             print(("MISSING %s" % hostname))
         return "%s missing hostnames" % len(missing_hostnames)

     try:
         # construct temporary directory of dut files.
         tdir = tempfile.mkdtemp()
         for hostname in hostnames:
             newpath = os.path.join(tdir, hostname)
             with open(newpath, "w") as fh:
                 json.dump(obj=hostnames_map[hostname], fp=fh)

         # paranoia, check number of files.
         num_files = len(os.listdir(tdir))
         if num_files != len(hostnames):
             return "internal error. hostnames: %s, files: %s, tdir: %s" % (len(hostnames), num_files, tdir)

         # validate directory contents before proceeding
         _, err = validate_output(tdir)
         if err is not None:
             return err

         # execute the command, transfer file contents to skylab
         # NOTE: this step is atomic
         # note... skylab quick-add-duts will claim there's no deploy task ID
         # when it is successful
         # we need to parse the output to tell what happened
         #
         # right now, I use the presence of the magic strings 'nothing to commit'
         # and 'missing deploy task ID in request entry' to diagnose what happened
         # but this is fragile.
         cmd = QUICK_ADD_DUTS_CMD
         cmd = cmd.replace('%%%DIR%%%', pipes.quote(tdir))
         returncode, out, err = shell_capture_all(cmd)
         if " nothing to commit" in out:
             warnings.warn("nothing to commit ... no change made to inventory")
             return None
         elif "missing deploy task ID in deploy request entry" in out:
             print("SUCCESS!")
             print(out)
             return None
         else:
             return ("%s\n%s" % (out, err))
     finally:
         print(("path to tempdir: %s" % tdir))


 # free vars: %%%MLIST%%%
 ATEST_LOCK_RENAME_CMD = r"""

 tdir="$(mktemp -d)"
 test -d "$tdir" || exit 10

 # produce the unmangled hostnames
 cat -- %%%MLIST%%% | sed -e 's/-migrated-do-not-use$//' > "$tdir"/unmangled
 cat -- "$tdir"/unmangled | sed -e 's/$/-migrated-do-not-use//' > "$tdir"/mangled

 # just attempt to lock, discard results
 # first process unmangled names
 atest host mod --lock -r 'migration to skylab' --mlist "$tdir"/unmangled 2>&1
 # then process mangled names
 atest host mod --lock -r 'migration to skylab' --mlist "$tdir"/mangled   2>&1

 # next attempt to rename only the unmangled names
 atest host rename --for-migration --non-interactive --mlist "$tdir"/unmangled 2>&1

 """


 # accepts: hostnames
 # returns: error message (None if no error)
 # emits:   output
 def atest_lock_rename(hostnames):
     if isinstance(hostnames, TEXT):
         return "hostnames cannot be text"

     with tempfile.NamedTemporaryFile(delete=True) as fh:
         for hostname in hostnames:
             fh.write(hostname.strip())
             fh.write("\n")
         flush_sync(fh)
         cmd = ATEST_LOCK_RENAME_CMD
         cmd = cmd.replace('%%%MLIST%%%', pipes.quote(fh.name))
         returncode, out, err = shell_capture_all(cmd)

         print(out)
         return None
	#!/usr/bin/env python2


	# user facing commands:
	#
	# -- write_statjson_hostnames
	# -- do_quick_add



	# pretty_dump_board:

	# Given the name of a board, dump it to a directory with
	# the following structure:

	# data.dir
	# - hostname1
	# - hostname2
	# - hostname3

	# NEXT, report all the ways in which hostnames of that board are not valid
	# jsonified protobufs for adding a new DUT.
	#
	# this involves shelling out to skylab validate-new-dut-json



	# adding a dut that's already present
	# $ skylab quick-add-dut /tmp/json
	# Deployment ID: d604b46b-87c7-4a4d-8b8a-ed97565c4797
	# Status: DUT_DEPLOYMENT_STATUS_FAILED
	# Inventory change URL:
	# Deploy task URL:
	# Message: failed to add DUT(s) to fleet: add dut to fleet: inventory store commit: nothing to commit


	# adding a dut successfully
	# $ skylab quick-add-dut /tmp/json
	# Deployment ID: 8e27b3ff-39ad-4b92-9bea-3f804a9d7bf9
	# Status: DUT_DEPLOYMENT_STATUS_FAILED
	# Inventory change URL: https://chrome-internal-review.googlesource.com/c/chromeos/infra_internal/skylab_inventory/+/1940714
	# Deploy task URL:
	# Message: missing deploy task ID in deploy request entry



	# $ atest host rename --for-migration --non-interactive chromeos2-row1-rack7-host1
	# Successfully renamed:
	# chromeos2-row1-rack7-host1 to chromeos2-row1-rack7-host1-migrated-do-not-use


	# unsuccessful lock

	# $ atest host mod --lock -r 'migration to skylab' chromeos2-row1-rack7-host1-migrated-do-not-use
	# Operation modify_host failed:
	# ValidationError: {'locked': u'Host chromeos2-row1-rack7-host1-migrated-do-not-use already locked by pprabhu on 2019-08-29 16:48:51.'}
	# 1


	# successful lock



	from __future__ import print_function
	from __future__ import unicode_literals
	import os
	import sys
	import subprocess
	import pipes
	import os.path
	import warnings
	import json
	import tempfile
	import shutil

	TEXT = (type(b""), type(u""))
	NONETYPE = type(None)


	def flush_sync(fh):
	fh.flush()
	os.fsync(fh)
	return


	# accepts: shell command, rest of args
	# returns: exit_status, stdout, stderr
	def shell_capture_all(cmd, *rest):
	shellcmd = ("bash", "-c", cmd, "bash",) + rest
	pr = subprocess.Popen(
	shellcmd,
	stdout=subprocess.PIPE,
	stderr=subprocess.PIPE,
	)
	stdout, stderr = pr.communicate()
	return pr.returncode, stdout, stderr


	# accepts: name of board
	# returns: list of hostnames, error message (None if no error)
	def get_all_hosts_for_board(board):
	# extract just the hostnames for all the hosts in the board
	cmd = "set -o pipefail; ( atest host list --label=%%%LABEL%%% \| awk '{print $1}' )"
	cmd = cmd.replace("%%%LABEL%%%", pipes.quote("board:%s" % board))
	returncode, out, err = shell_capture_all(cmd)
	if returncode != 0:
	return None, "get_all_hosts_for_board: " + err
	hostnames = [x.strip() for x in out.split()]
	if hostnames and hostnames[0] == "Host":
	hostnames.pop(0)
	return hostnames, None


	# make a directory and test if we can write to it as defensively as possible
	# free vars: %%%DEST%%%
	MAKE_DIR_CMD = r"""
	mkdir -p %%%DEST%%%
	cd %%%DEST%%% && touch ./writeable && rm ./writeable
	"""


	# free vars: %%%HOSTNAME%%% %%%DEST%%%
	STATJSON_CMD = r"""
	atest host statjson %%%HOSTNAME%%% 1>%%%DEST%%%/%%%HOSTNAME%%%
	"""

	# accepts: path to directory
	# returns: error message (None if successfully made dir)
	def mkdirp(dirpath):
	cmd = MAKE_DIR_CMD.replace('%%%DEST%%%', pipes.quote(dirpath))
	returncode, out, err = shell_capture_all(cmd)
	if returncode == 0:
	return None
	else:
	return err


	# accepts: list of hostnames, output directory (path)
	# returns: successful hosts, failed hosts, error message (None if no error)
	def write_statjson_hostnames(hostnames, outdir):
	successful_hosts = []
	failed_hosts = []
	err = mkdirp(outdir)
	if err is not None:
	return None, None, err
	for hostname in hostnames:
	cmd = STATJSON_CMD
	cmd = cmd.replace('%%%HOSTNAME%%%', pipes.quote(hostname))
	cmd = cmd.replace('%%%DEST%%%', pipes.quote(outdir))
	returncode, out, err = shell_capture_all(cmd)
	if returncode == 0:
	successful_hosts.append(hostname)
	else:
	failed_hosts.append(hostname)
	return successful_hosts, failed_hosts, None


	# accepts: board name, output directory
	# returns: successful hosts, failed hosts, error message
	def write_statjson_board(board, outdir):
	hostnames, err = get_all_hosts_for_board(board)
	if err is not None:
	return None, None, err
	successful_hosts, failed_hosts, err = write_statjson_hostnames(hostnames, outdir)
	return successful_hosts, failed_hosts, err


	# free vars: %%%PATH%%%
	VALIDATE_CMD = r"""
	skylab validate-new-dut-json %%%PATH%%%
	"""


	# accepts: directory with hostname files
	# returns: dictionary of the form below, error message (or None if no error)
	#
	# {
	# hostname: error message (or None if no error)
	# }
	def validate_output(hostname_dir):
	try:
	paths = os.listdir(hostname_dir)
	except OSError:
	return None, ("bad directory: %s" % hostname_dir)

	# defensively populate result dictionary with errors
	# so that we don't erroneously conclude that an unvisited
	# hostname was validated
	result = {}
	for path in paths:
	result[path] = "DID NOT PROCESS"

	for path in paths:
	cmd = VALIDATE_CMD
	cmd = cmd.replace('%%%PATH%%%', pipes.quote(os.path.join(hostname_dir, path)))
	returncode, out, err = shell_capture_all(cmd)
	if returncode != 0:
	result[path] = "failed to validate (errcode %s): %s" % (returncode, err)
	elif os.path.exists(os.path.join(hostname_dir, path)):
	result[path] = None
	else:
	warnings.warn("nonexistent path %s" % path)
	result[path] = "file does not exist"

	return result, None



	# accepts: name of board, output directory
	# returns: dictionary of form below, error message (or None if no error)
	#
	# {
	# hostname: error-message or NOFILE if no file or None if no error
	# }
	def process_board(board, output_dir):
	queried_hosts, unqueried_hosts, err = write_statjson_board(board, output_dir)
	if err is not None:
	return None, err
	validate_result, err = validate_output(output_dir)
	if err is not None:
	return None, err
	result = {}

	for hostname in queried_hosts:
	try:
	result[hostname] = validate_result[hostname]
	except KeyError:
	warnings.warn("hostname (%s) not present in validate_result")

	for hostname in unqueried_hosts:
	result["hostname"] = "NOFILE"

	return result, None


	# accepts: name of board, output directory
	# returns: number of bad hosts
	# emits: prints error message for every bad host
	def pretty_process_board(board, output_dir):
	result, err = process_board(board, output_dir)
	if err is not None:
	print(err)
	return 1
	bad_results = {}
	for k in result:
	if result[k] is not None:
	bad_results[k] = result[k]
	# no bad results --> nothing printed
	for k in bad_result:
	print(k, result[k])
	return len(bad_results)


	# accepts: path to output directory
	# returns: combined json object, error message (None if no error)
	# NOTE: the directory not existing is a fatal error
	# processing a file that is invalid json AFTER the output directory
	# has been validated produces a warning. the invalid json situation
	# should be impossible, but also isn't enough to prevent assemble_output_dir
	# from doing something reasonable.
	def assemble_output_dir(output_dir):
	_, err = validate_output(output_dir)
	if err is not None:
	return None, err
	out = []
	items = None
	try:
	items = os.listdir(output_dir)
	except OSError:
	return None, ("directory %s does not exist or is not readable" % output_dir)
	for item in items:
	obj = None
	try:
	with open(os.path.join(output_dir, item), "r") as fh:
	try:
	obj = json.load(fh)
	except ValueError:
	warnings.warn("file %s does not contain valid JSON" % item)
	continue
	except IOError:
	warnings.warn("file %s somehow doesn't exist" % item)
	continue
	out.append(obj)
	return out, None


	# accepts: single json dictionary
	# returns: error message (None if valid)
	def validate_single_dut_json(obj):
	with tempfile.NamedTemporaryFile(delete=True) as fh:
	json.dump(obj, fh)
	flush_sync(fh)
	cmd = VALIDATE_CMD
	cmd = cmd.replace('%%%PATH%%%', pipes.quote(fh.name))
	returncode, out, err = shell_capture_all(cmd)
	if returncode == 0:
	return None
	else:
	return err


	# accepts: json obj
	# returns: hostname, error message (None if no error)
	def get_hostname_from_dut_json(obj):
	try:
	common = obj["common"]
	except KeyError:
	return None, "dut has no common element"
	if not isinstance(common, dict):
	return None, ("common block must be dict not %s" % type(common))
	try:
	return common["hostname"], None
	except KeyError:
	return None, "common block has no hostname element"


	# accepts: path to file
	# returns: hostname mapping, error message
	# hostname mapping has the following form
	#
	# {
	# hostname -> new_dut_info_json
	# }
	#
	# malformed entries are not included in the map
	def load_hostname_map_file(filepath):
	obj = None
	try:
	with open(filepath, "r") as fh:
	try:
	obj = json.load(fh)
	except ValueError:
	return None, ("file does not contain JSON %s" % filepath)
	except IOError:
	return None, ("cannot load hostname map from nonexistent file %s" % filepath)

	out = {}

	# return a singleton map if the toplevel entry is a dictionary
	if isinstance(obj, dict):
	err = validate_single_dut_json(obj)
	if err is not None:
	return None, err
	hostname, err = get_hostname_from_dut_json(obj)
	if err is not None:
	return None, err
	return {hostname: obj}, None

	# validate all elements if the toplevel entry is a list
	elif isinstance(obj, list):
	for subobj in obj:
	err = validate_single_dut_json(subobj)
	if err is not None:
	warnings.warn(err)
	continue
	hostname, err = get_hostname_from_dut_json(subobj)
	if err is not None:
	warnings.warn(err)
	continue
	# warn if we get a duplicate, but don't halt execution
	if hostname in out:
	warnings.warn(("duplicate hostname %s" % hostname))
	out[hostname] = subobj

	if len(out) == 0:
	return out, "out cannot be empty"

	return out, None


	# accepts: path to directory
	# returns: hostnaming mapping, error message
	# hostname mapping has the following form
	#
	# {
	# hostname -> new_dut_info_json
	# }
	#
	# malformed entries are not included in the map
	def load_hostname_map(dirpath):
	items = None
	try:
	items = os.listdir(dirpath)
	except OSError:
	return None, ("cannot load from nonexistent directory %s" % dirpath)
	if len(items) == 0:
	return None, ("nothing in directory %s" % dirpath)
	out = {}
	for item in items:
	hostname_map, err = load_hostname_map_file(os.path.join(dirpath, item))
	if err is not None:
	warnings.warn(err)
	continue

	for hostname in hostname_map:
	if hostname in out:
	warnings.warn("load_hostname_map: duplicate hostname %s" % hostname)
	continue
	out[hostname] = hostname_map[hostname]

	return out, None


	# free vars: %%%DIR%%%
	QUICK_ADD_DUTS_CMD = r"""
	skylab quick-add-duts %%%DIR%%%/*
	"""


	# accepts: list of hostnames, hostname_dirpath
	# returns: error message (None if successful)
	# NOTE: the quick-add-duts API is atomic
	# emits: missing hostnames when there are missing hostnames
	def do_quick_add_duts(hostnames, dirpath):
	# validation
	if isinstance(hostnames, TEXT):
	return "hostnames cannot be %s" % type(hostnames)
	# defensively prevent hostnames from being modified
	hostnames = tuple(hostnames)
	for hostname in hostnames:
	if hostname.startswith("."):
	return "hostname cannot start with '.' (%s)" % hostname
	if not hostname:
	return "hostname cannot be falsey (%s)" % hostname


	hostnames_map, err = load_hostname_map(dirpath)
	if err is not None:
	return err

	# check that every hostname is in the map before trying
	missing_hostnames = set([])
	for hostname in hostnames:
	if hostname not in hostnames_map:
	missing_hostnames.add(hostname)

	if missing_hostnames:
	for hostname in sorted(missing_hostnames):
	print(("MISSING %s" % hostname))
	return "%s missing hostnames" % len(missing_hostnames)

	try:
	# construct temporary directory of dut files.
	tdir = tempfile.mkdtemp()
	for hostname in hostnames:
	newpath = os.path.join(tdir, hostname)
	with open(newpath, "w") as fh:
	json.dump(obj=hostnames_map[hostname], fp=fh)

	# paranoia, check number of files.
	num_files = len(os.listdir(tdir))
	if num_files != len(hostnames):
	return "internal error. hostnames: %s, files: %s, tdir: %s" % (len(hostnames), num_files, tdir)

	# validate directory contents before proceeding
	_, err = validate_output(tdir)
	if err is not None:
	return err

	# execute the command, transfer file contents to skylab
	# NOTE: this step is atomic
	# note... skylab quick-add-duts will claim there's no deploy task ID
	# when it is successful
	# we need to parse the output to tell what happened
	#
	# right now, I use the presence of the magic strings 'nothing to commit'
	# and 'missing deploy task ID in request entry' to diagnose what happened
	# but this is fragile.
	cmd = QUICK_ADD_DUTS_CMD
	cmd = cmd.replace('%%%DIR%%%', pipes.quote(tdir))
	returncode, out, err = shell_capture_all(cmd)
	if " nothing to commit" in out:
	warnings.warn("nothing to commit ... no change made to inventory")
	return None
	elif "missing deploy task ID in deploy request entry" in out:
	print("SUCCESS!")
	print(out)
	return None
	else:
	return ("%s\n%s" % (out, err))
	finally:
	print(("path to tempdir: %s" % tdir))


	# free vars: %%%MLIST%%%
	ATEST_LOCK_RENAME_CMD = r"""

	tdir="$(mktemp -d)"
	test -d "$tdir" \|\| exit 10

	# produce the unmangled hostnames
	cat -- %%%MLIST%%% \| sed -e 's/-migrated-do-not-use$//' > "$tdir"/unmangled
	cat -- "$tdir"/unmangled \| sed -e 's/$/-migrated-do-not-use//' > "$tdir"/mangled

	# just attempt to lock, discard results
	# first process unmangled names
	atest host mod --lock -r 'migration to skylab' --mlist "$tdir"/unmangled 2>&1
	# then process mangled names
	atest host mod --lock -r 'migration to skylab' --mlist "$tdir"/mangled 2>&1

	# next attempt to rename only the unmangled names
	atest host rename --for-migration --non-interactive --mlist "$tdir"/unmangled 2>&1

	"""


	# accepts: hostnames
	# returns: error message (None if no error)
	# emits: output
	def atest_lock_rename(hostnames):
	if isinstance(hostnames, TEXT):
	return "hostnames cannot be text"

	with tempfile.NamedTemporaryFile(delete=True) as fh:
	for hostname in hostnames:
	fh.write(hostname.strip())
	fh.write("\n")
	flush_sync(fh)
	cmd = ATEST_LOCK_RENAME_CMD
	cmd = cmd.replace('%%%MLIST%%%', pipes.quote(fh.name))
	returncode, out, err = shell_capture_all(cmd)

	print(out)
	return None