client/bin/fsinfo.py

"""This module gives the mkfs creation options for an existing filesystem.

tune2fs or xfs_growfs is called according to the filesystem. The results,
filesystem tunables, are parsed and mapped to corresponding mkfs options.
"""
import os, re, tempfile
import common
from autotest_lib.client.common_lib import error, utils


def opt_string2dict(opt_string):
    """Breaks the mkfs.ext* option string into dictionary."""
    # Example string: '-j -q -i 8192 -b 4096'. There may be extra whitespaces.
    opt_dict = {}

    for item in opt_string.split('-'):
        item = item.strip()
        if ' ' in item:
            (opt, value) = item.split(' ', 1)
            opt_dict['-%s' % opt] = value
        elif item != '':
            opt_dict['-%s' % item] = None
    # Convert all the digit strings to int.
    for key, value in opt_dict.iteritems():
        if value and value.isdigit():
            opt_dict[key] = int(value)

    return opt_dict


def parse_mke2fs_conf(fs_type, conf_file='/etc/mke2fs.conf'):
    """Parses mke2fs config file for default settings."""
    # Please see /ect/mke2fs.conf for an example.
    default_opt = {}
    fs_opt = {}
    current_fs_type = ''
    current_section = ''
    f = open(conf_file, 'r')
    for line in f:
        if '[defaults]' == line.strip():
            current_section = '[defaults]'
        elif '[fs_types]' == line.strip():
            current_section = '[fs_types]'
        elif current_section == '[defaults]':
            components = line.split('=', 1)
            if len(components) == 2:
                default_opt[components[0].strip()] = components[1].strip()
        elif current_section == '[fs_types]':
            m = re.search('(\w+) = {', line)
            if m:
                current_fs_type = m.group(1)
            else:
                components = line.split('=', 1)
                if len(components) == 2 and current_fs_type == fs_type:
                    default_opt[components[0].strip()] = components[1].strip()
    f.close()

    # fs_types options override the defaults options
    for key, value in fs_opt.iteritems():
        default_opt[key] = value

    # Convert all the digit strings to int.
    for key, value in default_opt.iteritems():
        if value and value.isdigit():
            default_opt[key] = int(value)

    return default_opt


def convert_conf_opt(default_opt):
    conf_opt_mapping = {'blocksize': '-b',
                        'inode_ratio': '-i',
                        'inode_size': '-I'}
    mkfs_opt = {}

    # Here we simply concatenate the feature string while we really need
    # to do the better and/or operations.
    if 'base_features' in default_opt:
        mkfs_opt['-O'] = default_opt['base_features']
    if 'default_features' in default_opt:
        mkfs_opt['-O'] += ',%s' % default_opt['default_features']
    if 'features' in default_opt:
        mkfs_opt['-O'] += ',%s' % default_opt['features']

    for key, value in conf_opt_mapping.iteritems():
        if key in default_opt:
            mkfs_opt[value] = default_opt[key]

    if '-O' in mkfs_opt:
        mkfs_opt['-O'] = mkfs_opt['-O'].split(',')

    return mkfs_opt


def merge_ext_features(conf_feature, user_feature):
    user_feature_list = user_feature.split(',')

    merged_feature = []
    # Removes duplicate entries in conf_list.
    for item in conf_feature:
        if item not in merged_feature:
            merged_feature.append(item)

    # User options override config options.
    for item in user_feature_list:
        if item[0] == '^':
            if item[1:] in merged_feature:
                merged_feature.remove(item[1:])
            else:
                merged_feature.append(item)
        elif item not in merged_feature:
            merged_feature.append(item)
    return merged_feature


def ext_tunables(dev):
    """Call tune2fs -l and parse the result."""
    cmd = 'tune2fs -l %s' % dev
    try:
        out = utils.system_output(cmd)
    except error.CmdError:
        tools_dir = os.path.join(os.environ['AUTODIR'], 'tools')
        cmd = '%s/tune2fs.ext4dev -l %s' % (tools_dir, dev)
        out = utils.system_output(cmd)
    # Load option mappings
    tune2fs_dict = {}
    for line in out.splitlines():
        components = line.split(':', 1)
        if len(components) == 2:
            value = components[1].strip()
            option = components[0]
            if value.isdigit():
                tune2fs_dict[option] = int(value)
            else:
                tune2fs_dict[option] = value

    return tune2fs_dict


def ext_mkfs_options(tune2fs_dict, mkfs_option):
    """Map the tune2fs options to mkfs options."""

    def __inode_count(tune_dict, k):
        return (tune_dict['Block count']/tune_dict[k] + 1) * (
            tune_dict['Block size'])

    def __block_count(tune_dict, k):
        return int(100*tune_dict[k]/tune_dict['Block count'] + 1)

    def __volume_name(tune_dict, k):
        if tune_dict[k] != '<none>':
            return tune_dict[k]
        else:
            return ''

    # mappings between fs features and mkfs options
    ext_mapping = {'Blocks per group': '-g',
                   'Block size': '-b',
                   'Filesystem features': '-O',
                   'Filesystem OS type': '-o',
                   'Filesystem revision #': '-r',
                   'Filesystem volume name': '-L',
                   'Flex block group size': '-G',
                   'Fragment size': '-f',
                   'Inode count': '-i',
                   'Inode size': '-I',
                   'Journal inode': '-j',
                   'Reserved block count': '-m'}

    conversions = {
        'Journal inode': lambda d, k: None,
        'Filesystem volume name': __volume_name,
        'Reserved block count': __block_count,
        'Inode count': __inode_count,
        'Filesystem features': lambda d, k: re.sub(' ', ',', d[k]),
        'Filesystem revision #': lambda d, k: d[k][0]}

    for key, value in ext_mapping.iteritems():
        if key not in tune2fs_dict:
            continue
        if key in conversions:
            mkfs_option[value] = conversions[key](tune2fs_dict, key)
        else:
            mkfs_option[value] = tune2fs_dict[key]


def xfs_tunables(dev):
    """Call xfs_grow -n to get filesystem tunables."""
    # Have to mount the filesystem to call xfs_grow.
    tmp_mount_dir = tempfile.mkdtemp()
    cmd = 'mount %s %s' % (dev, tmp_mount_dir)
    utils.system_output(cmd)
    xfs_growfs = os.path.join(os.environ['AUTODIR'], 'tools', 'xfs_growfs')
    cmd = '%s -n %s' % (xfs_growfs, dev)
    try:
        out = utils.system_output(cmd)
    finally:
        # Clean.
        cmd = 'umount %s' % dev
        utils.system_output(cmd, ignore_status=True)
        os.rmdir(tmp_mount_dir)

    ## The output format is given in report_info (xfs_growfs.c)
    ## "meta-data=%-22s isize=%-6u agcount=%u, agsize=%u blks\n"
    ## "                 =%-22s sectsz=%-5u attr=%u\n"
    ## "data         =%-22s bsize=%-6u blocks=%llu, imaxpct=%u\n"
    ## "                 =%-22s sunit=%-6u swidth=%u blks\n"
    ## "naming     =version %-14u bsize=%-6u\n"
    ## "log            =%-22s bsize=%-6u blocks=%u, version=%u\n"
    ## "                 =%-22s sectsz=%-5u sunit=%u blks, lazy-count=%u\n"
    ## "realtime =%-22s extsz=%-6u blocks=%llu, rtextents=%llu\n"

    tune2fs_dict = {}
    # Flag for extracting naming version number
    keep_version = False
    for line in out.splitlines():
        m = re.search('^([-\w]+)', line)
        if m:
            main_tag = m.group(1)
        pairs = line.split()
        for pair in pairs:
            # naming: version needs special treatment
            if pair == '=version':
                # 1 means the next pair is the version number we want
                keep_version = True
                continue
            if keep_version:
                tune2fs_dict['naming: version'] = pair
                # Resets the flag since we have logged the version
                keep_version = False
                continue
            # Ignores the strings without '=', such as 'blks'
            if '=' not in pair:
                continue
            key, value = pair.split('=')
            tagged_key = '%s: %s' % (main_tag, key)
            if re.match('[0-9]+', value):
                tune2fs_dict[tagged_key] = int(value.rstrip(','))
            else:
                tune2fs_dict[tagged_key] = value.rstrip(',')

    return tune2fs_dict


def xfs_mkfs_options(tune2fs_dict, mkfs_option):
    """Maps filesystem tunables to their corresponding mkfs options."""

    # Mappings
    xfs_mapping = {'meta-data: isize': '-i size',
                   'meta-data: agcount': '-d agcount',
                   'meta-data: sectsz': '-s size',
                   'meta-data: attr': '-i attr',
                   'data: bsize': '-b size',
                   'data: imaxpct': '-i maxpct',
                   'data: sunit': '-d sunit',
                   'data: swidth': '-d swidth',
                   'data: unwritten': '-d unwritten',
                   'naming: version': '-n version',
                   'naming: bsize': '-n size',
                   'log: version': '-l version',
                   'log: sectsz': '-l sectsize',
                   'log: sunit': '-l sunit',
                   'log: lazy-count': '-l lazy-count',
                   'realtime: extsz': '-r extsize',
                   'realtime: blocks': '-r size',
                   'realtime: rtextents': '-r rtdev'}

    mkfs_option['-l size'] = tune2fs_dict['log: bsize'] * (
        tune2fs_dict['log: blocks'])

    for key, value in xfs_mapping.iteritems():
        mkfs_option[value] = tune2fs_dict[key]


def compare_features(needed_feature, current_feature):
    """Compare two ext* feature lists."""
    if len(needed_feature) != len(current_feature):
        return False
    for feature in current_feature:
        if feature not in needed_feature:
            return False
    return True


def match_ext_options(fs_type, dev, needed_options):
    """Compare the current ext* filesystem tunables with needed ones."""
    # mkfs.ext* will load default options from /etc/mke2fs.conf
    conf_opt = parse_mke2fs_conf(fs_type)
    # We need to convert the conf options to mkfs options.
    conf_mkfs_opt = convert_conf_opt(conf_opt)
    # Breaks user mkfs option string to dictionary.
    needed_opt_dict = opt_string2dict(needed_options)
    # Removes ignored options.
    ignored_option = ['-c', '-q', '-E', '-F']
    for opt in ignored_option:
        if opt in needed_opt_dict:
            del needed_opt_dict[opt]

   # User options override config options.
    needed_opt = conf_mkfs_opt
    for key, value in needed_opt_dict.iteritems():
        if key == '-N' or key == '-T':
            raise Exception('-N/T is not allowed.')
        elif key == '-O':
            needed_opt[key] = merge_ext_features(needed_opt[key], value)
        else:
            needed_opt[key] = value

    # '-j' option will add 'has_journal' feature.
    if '-j' in needed_opt and 'has_journal' not in needed_opt['-O']:
        needed_opt['-O'].append('has_journal')
    # 'extents' will be shown as 'extent' in the outcome of tune2fs
    if 'extents' in needed_opt['-O']:
        needed_opt['-O'].append('extent')
        needed_opt['-O'].remove('extents')
    # large_file is a byproduct of resize_inode.
    if 'large_file' not in needed_opt['-O'] and (
        'resize_inode' in needed_opt['-O']):
        needed_opt['-O'].append('large_file')

    current_opt = {}
    tune2fs_dict = ext_tunables(dev)
    ext_mkfs_options(tune2fs_dict, current_opt)

    # Does the match
    for key, value in needed_opt.iteritems():
        if key == '-O':
            if not compare_features(value, current_opt[key].split(',')):
                return False
        elif key not in current_opt or value != current_opt[key]:
            return False
    return True


def match_xfs_options(dev, needed_options):
    """Compare the current ext* filesystem tunables with needed ones."""
    tmp_mount_dir = tempfile.mkdtemp()
    cmd = 'mount %s %s' % (dev, tmp_mount_dir)
    utils.system_output(cmd)
    xfs_growfs = os.path.join(os.environ['AUTODIR'], 'tools', 'xfs_growfs')
    cmd = '%s -n %s' % (xfs_growfs, dev)
    try:
        current_option = utils.system_output(cmd)
    finally:
        # Clean.
        cmd = 'umount %s' % dev
        utils.system_output(cmd, ignore_status=True)
        os.rmdir(tmp_mount_dir)

    # '-N' has the same effect as '-n' in mkfs.ext*. Man mkfs.xfs for details.
    cmd = 'mkfs.xfs %s -N -f %s' % (needed_options, dev)
    needed_out = utils.system_output(cmd)
    # 'mkfs.xfs -N' produces slightly different result than 'xfs_growfs -n'
    needed_out = re.sub('internal log', 'internal    ', needed_out)
    if current_option == needed_out:
        return True
    else:
        return False


def match_mkfs_option(fs_type, dev, needed_options):
    """Compare the current filesystem tunables with needed ones."""
    if fs_type.startswith('ext'):
        ret = match_ext_options(fs_type, dev, needed_options)
    elif fs_type == 'xfs':
        ret = match_xfs_options(dev, needed_options)
    else:
        ret = False

    return ret