runltp

#!/bin/sh
################################################################################
##                                                                            ##
## Copyright (c) International Business Machines  Corp., 2001                 ##
##                                                                            ##
## This program is free software;  you can redistribute it and#or modify      ##
## it under the terms of the GNU General Public License as published by       ##
## the Free Software Foundation; either version 2 of the License, or          ##
## (at your option) any later version.                                        ##
##                                                                            ##
## This program is distributed in the hope that it will be useful, but        ##
## WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY ##
## or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License   ##
## for more details.                                                          ##
##                                                                            ##
## You should have received a copy of the GNU General Public License          ##
## along with this program;  if not, write to the Free Software               ##
## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA    ##
##                                                                            ##
################################################################################
# File: runltp
#
# Description:  This script can be used to the tests in the LTP test suite
#
# Authors:      Manoj Iyer - manjo@mail.utexas.edu
#               Robbe Williamson - robbiew@us.ibm.com
#               
# History:      Oct 07 2003 - Modified - Manoj Iyer
#               - use functions
#               - clean up on script exit
#               - error checking etc.
#                
#               Oct 08 2003 - Modified - Manoj Iyer
#               - fixed bug in creating results directory
#               - all checks should be enlclosed in " " to avoid bash error
#               - exit with error if ltp-pan is not found in pan directory
#
#               Jul 22 2007 - Modified - Ricardo Salveti de Araujo
#               - added support to put more then one file at CMDLINE (-f)
#               - added a new option, that the user can pass the address of
#               the command file, and it'll use wget to get it (-w)
#               - now -s does the grep at the selected command files (default,
#               -f or -w)
#
#               Jul 23 2007 - Modified - Ricardo Salveti de Araujo
#               - added flag to get the command file that has all failed tests
#
#               Sep 11 2007 - Modified - Subrata Modak
#               - added option to create Failed File if it is not an absolute path
#               - added option to create Output File if it is not an absolute path
#               - added option to create Failed File compulsory, even if user has not mentioned it
#
#               Sep 14 2007 - Modified - Ricardo Salveti de Araujo
#               - cleaning and removing duplicated code
#
#               Oct 27 2007 - Modified - Ricardo Salveti de Araujo and Subrata Modak
#               - better ways to integrate "ltp/bin/genload/stress" with "ltp/runltp"
#               Nov 24 2007 - Modified - Subrata Modak
#               - Added a new option to generate output in HTML format also. Also retaining
#                 the original test format
#               Nov 28 2007 - Modified - Subrata Modak
#               - Added a new option to mail back LTP reports
#               May 19 2008 - Modified - Subrata Modak
#               - Added capability for default Log file generation
#                Aug 17 2009 - Modified - Subrata Modak
#                - Added Fault Injection generation Capability through -F Option
#
#################################################################################

die()
{
    error "$*"
    exit 1
}

error()
{
    echo "$RUNLTP: ERROR : $*" >&2
}

info()
{
    [ "x$QUIET_MODE" != x1 ] && echo "$RUNLTP: INFO : $*"
}

warning()
{
    echo "$RUNLTP: WARNING : $*" >&2
}

setup()
{
    OLDCWD=$(pwd)
    RUNLTP=${0##*/}
    export LTPROOT=$(dirname "$0")
    if ! cd "$LTPROOT"; then
        die "unable to change directory to $LTPROOT"
    fi
    export TMPBASE="/tmp"
    export PATH="$LTPROOT/testcases/bin:$PATH"

    if [ ! -d "$LTPROOT/testcases/bin" -o ! -x "$LTPROOT/bin/ltp-pan" ] ; then
        die "test suite not installed correctly; please read INSTALL"
    fi
}

version_of_ltp()
{
    cat "$LTPROOT/Version"
    return 0
}

usage() 
{
    cat <<EOF >&2

    usage: $RUNLTP [ -a EMAIL_TO ] [ -c NUM_PROCS ] [ -C FAILCMDFILE ] [ -d TMPDIR ]
    [ -D NUM_PROCS,NUM_FILES,NUM_BYTES,CLEAN_FLAG ] -e [ -f CMDFILES(,...) ] [ -g HTMLFILE]
    [ -i NUM_PROCS ] [ -l LOGFILE ] [ -m NUM_PROCS,CHUNKS,BYTES,HANGUP_FLAG ] 
    -N -n [ -o OUTPUTFILE ] -p -q [ -r LTPROOT ] [ -s PATTERN ] [ -t DURATION ]
    -v [ -w CMDFILEADDR ] [ -x INSTANCES ] [ -b DEVICE ] [-B DEVICE_FS_TYPE]
        [ -F LOOPS,PERCENTAGE ]
                
    -a EMAIL_TO     EMAIL all your Reports to this E-mail Address
    -c NUM_PROCS    Run LTP under additional background CPU load
                    [NUM_PROCS = no. of processes creating the CPU Load by spinning over sqrt()
                                 (Defaults to 1 when value)]
    -C FAILCMDFILE  Command file with all failed test cases.
    -d TMPDIR       Directory where temporary files will be created.
    -D NUM_PROCS,NUM_FILES,NUM_BYTES,CLEAN_FLAG
                    Run LTP under additional background Load on Secondary Storage (Seperate by comma)
                    [NUM_PROCS   = no. of processes creating Storage Load by spinning over write()]
                    [NUM_FILES   = Write() to these many files (Defaults to 1 when value 0 or undefined)]
                    [NUM_BYTES   = write these many bytes (defaults to 1GB, when value 0 or undefined)]
                    [CLEAN_FLAG  = unlink file to which random data written, when value 1]
    -e              Prints the date of the current LTP release
    -f CMDFILES     Execute user defined list of testcases (separate with ',')
        -F LOOPS,PERCENTAGE Induce PERCENTAGE Fault in the Kernel Subsystems, and, run each test for LOOPS loop
    -g HTMLFILE     Create an additional HTML output format
    -h              Help. Prints all available options.
    -i NUM_PROCS    Run LTP under additional background Load on IO Bus
                    [NUM_PROCS   = no. of processes creating IO Bus Load by spinning over sync()]
    -l LOGFILE      Log results of test in a logfile.
    -m NUM_PROCS,CHUNKS,BYTES,HANGUP_FLAG
                    Run LTP under additional background Load on Main memory (Seperate by comma)
                    [NUM_PROCS   = no. of processes creating main Memory Load by spinning over malloc()]
                    [CHUNKS      = malloc these many chunks (default is 1 when value 0 or undefined)]
                    [BYTES       = malloc CHUNKS of BYTES bytes (default is 256MB when value 0 or undefined) ]
                    [HANGUP_FLAG = hang in a sleep loop after memory allocated, when value 1]
        -M CHECK_TYPE
                [CHECK_TYPE=1 => Full Memory Leak Check tracing children as well]
                [CHECK_TYPE=2 => Thread Concurrency Check tracing children as well]
                [CHECK_TYPE=3 => Full Memory Leak & Thread Concurrency Check tracing children as well]
    -N              Run all the networking tests. 
    -n              Run LTP with network traffic in background.
    -o OUTPUTFILE   Redirect test output to a file.
    -p              Human readable format logfiles. 
    -q              Print less verbose output to screen.
    -r LTPROOT      Fully qualified path where testsuite is installed.
    -s PATTERN      Only run test cases which match PATTERN.
    -S SKIPFILE     Skip tests specified in SKIPFILE
    -t DURATION     Execute the testsuite for given duration. Examples:
                      -t 60s = 60 seconds
                      -t 45m = 45 minutes
                      -t 24h = 24 hours
                      -t 2d  = 2 days
    -T REPETITION   Execute the testsuite for REPETITION no. of times
    -w CMDFILEADDR  Uses wget to get the user's list of testcases.
    -x INSTANCES    Run multiple instances of this testsuite.
    -b DEVICE       Some tests require an unmounted block device
                    to run correctly.
    -B DEVICE_FS_TYPE The file system of test block devices.


    Example:

        $RUNLTP -c 2 -i 2 -m 2,4,10240,1 -D 2,10,10240,1 -p -q \
                -l /tmp/result-log.$$ -o /tmp/result-output.$$ \
                -C /tmp/result-failed.$$ -d $PWD


EOF
    exit 0
}

main()
{
    local CMDFILES=""
    local PRETTY_PRT=""
    local ALT_DIR_OUT=0
    local ALT_DIR_RES=0
    local ALT_HTML_OUT=0
    local ALT_EMAIL_OUT=0
    local RUN_NETEST=0
    local RUN_REPEATED=0
    local QUIET_MODE=0
    local PAN_QUIET_MODE=""
    local NETPIPE=0
    local GENLOAD=0
    local MEMSIZE=0
    local DURATION=""
    local CMDFILEADDR=""
    local FAILCMDFILE=""
    local INJECT_KERNEL_FAULT=""
    local INJECT_KERNEL_FAULT_PERCENTAGE=""
    local INJECT_FAULT_LOOPS_PER_TEST=""
    local VALGRIND_CHECK=""
    local VALGRIND_CHECK_TYPE=""
    local LOGFILE_NAME=""
    local LOGFILE=""
    local OUTPUTFILE_NAME=""
    local OUTPUTFILE=""
    local HTMLFILE_NAME=""
    local HTMLFILE=""
    local EMAIL_TO=""
    local SCENFILES=""
    local TAG_RESTRICT_STRING=""
    local PAN_COMMAND=""
    local DEFAULT_FILE_NAME_GENERATION_TIME=`date +"%Y_%b_%d-%Hh_%Mm_%Ss"`
    version_date=$(cat "$LTPROOT/Version")

    while getopts a:c:C:d:D:f:F:ehi:g:l:m:M:Nno:pqr:s:S:t:T:w:x:b:B: arg; do
        case "$arg" in
        a)  EMAIL_TO=$OPTARG
            ALT_EMAIL_OUT=1;;
        c)       
            NUM_PROCS=$(($OPTARG))
            if   [ "$NUM_PROCS" -eq 0 ]; then
                # User Did not Define the Value ,or, User Defined Zero,
                # hence, prevent from creating infinite processes
                NUM_PROCS=1
            fi
            "$LTPROOT/testcases/bin/genload" --cpu $NUM_PROCS >/dev/null 2>&1 &
            GENLOAD=1 ;;

        C)  
            case $OPTARG in
            /*)
                FAILCMDFILE="-C $OPTARG" ;;
            *)    
                FAILCMDFILE="-C $LTPROOT/output/$OPTARG"
                ALT_DIR_OUT=1 ;;
            esac ;;
                   
        d)  # convert the user path to absolute path.
            export TMPBASE=$(cd $(dirname "${OPTARG}"); pwd)/$(basename "${OPTARG}")
            ;;

        D)  NUM_PROCS=1; NUM_FILES=1; NUM_BYTES=$((1024 * 1024 * 1024)); CLEAN_FLAG=0
            ARGUMENT_LIST=$OPTARG
            TOTAL_ARGUMENTS=1                                             # Initial Assume
            for ARGUMENT in `echo "$ARGUMENT_LIST" | sed 's/,/\n/g'`      # Store all value in a Loop
            do
                case $TOTAL_ARGUMENTS in
                1) NUM_PROCS="$ARGUMENT" ;;
                2) NUM_FILES="$ARGUMENT" ;;
                3) NUM_BYTES="$ARGUMENT" ;;
                4) CLEAN_FLAG="$ARGUMENT" ;;
                esac
                : $((TOTAL_ARGUMENTS += 1))
            done
            # just to get the default values if the user passed 0
            if [ "$NUM_PROCS" -eq 0 ]; then
                NUM_PROCS=1
            fi
            if [ "$NUM_FILES" -eq 0 ]; then
                NUM_FILES=1
            fi
            if [ "$NUM_BYTES" -eq 0 ]; then
                NUM_BYTES=$((1024 * 1024 * 1024))
            fi
            if [ "$CLEAN_FLAG" -ne 1 ]; then
                CLEAN_FLAG=0
            fi
            if [ "$CLEAN_FLAG" -eq 1 ]; then
                # Do not unlink file in this case
                "$LTPROOT/testcases/bin/genload" --hdd $NUM_PROCS --hdd-files \
                    $NUM_FILES --hdd-bytes $NUM_BYTES >/dev/null 2>&1 &
            else
                # Cleanup otherwise
                "$LTPROOT/testcases/bin/genload" --hdd $NUM_PROCS --hdd-files \
                    $NUM_FILES --hdd-bytes $NUM_BYTES --hdd-noclean >/dev/null 2>&1 &
            fi
            GENLOAD=1;;

        e)  # Print out the version of LTP
            version_of_ltp
            ;;
        f)  # Execute user defined set of testcases.
            # Can be more then one file, just separate it with ',', like:
            # -f nfs,commands,/tmp/testfile
            CMDFILES=$OPTARG;;
        F)        INJECT_KERNEL_FAULT=1
                #Seperate out the NO_OF_LOOPS & FAULT_PERCENTAGE
                INJECT_FAULT_LOOPS_PER_TEST=`echo $OPTARG |cut -d',' -f1 | tr -d '\n' | tr -d ' '`
                INJECT_KERNEL_FAULT_PERCENTAGE=`echo $OPTARG |cut -d',' -f2 | tr -d '\n' | tr -d ' '`
                if [ ! $INJECT_FAULT_LOOPS_PER_TEST ]; then
                    warning "Loops not properly defined. Defaulting to 5..."
                    export INJECT_FAULT_LOOPS_PER_TEST=5
                fi
                if [ ! $INJECT_KERNEL_FAULT_PERCENTAGE ]; then
                    warning "Fault percentage not properly defined. Defaulting to 10..."
                    export INJECT_KERNEL_FAULT_PERCENTAGE=10
                fi;;
        g)  HTMLFILE_NAME="$OPTARG"
            case $OPTARG in
            /*)
                HTMLFILE="$OPTARG";;
            *)
                HTMLFILE="$LTPROOT/output/$OPTARG";;
            esac
            ALT_DIR_OUT=1 
            ALT_HTML_OUT=1;;
        h)  usage;;
        
        i)
            NUM_PROCS=$(($OPTARG))
            if   [ "$NUM_PROCS" -eq 0 ]; then
                # User Did not Define the Value ,or, User Defined Zero,
                # hence, prevent from creating infinite processes
                NUM_PROCS=1
            fi
            "$LTPROOT/testcases/bin/genload" --io $NUM_PROCS >/dev/null 2>&1 &
            GENLOAD=1 ;;
    
        l)      
            LOGFILE_NAME="$OPTARG"
            case $OPTARG in
            /*)
                LOGFILE="-l $OPTARG" ;;
            *)    
                LOGFILE="-l $LTPROOT/results/$OPTARG"
                ALT_DIR_RES=1 ;;
            esac ;;
    
        m)  NUM_PROCS=1; CHUNKS=1; BYTES=$((256 * 1024 * 1024)); HANGUP_FLAG=0
            ARGUMENT_LIST=$OPTARG
            TOTAL_ARGUMENTS=1                                             # Initial Assume
            for ARGUMENT in `echo "$ARGUMENT_LIST" | sed 's/,/\n/g'`      # Store all value in a Loop
            do
                case $TOTAL_ARGUMENTS in
                1) NUM_PROCS="$ARGUMENT" ;;
                2) CHUNKS="$ARGUMENT" ;;
                3) BYTES="$ARGUMENT" ;;
                4) HANGUP_FLAG="$ARGUMENT" ;;
                esac
                : $(( TOTAL_ARGUMENTS += 1 ))
            done
            # just to get the default values if the user passed 0
            [ "x$NUM_PROCS" = x0 ] && NUM_PROCS=1
            [ "x$CHUNKS" != x0 ] && CHUNKS=1
            [ "x$BYTES" = x0 ] && BYTES=$((256 * 1024 * 1024))
            [ "x$HANGUP_FLAG" != x1 ] && HANGUP_FLAG=0
            if [ "$HANGUP_FLAG" -eq 1 ]; then
                # Hang in a Sleep loop after memory allocated
                "$LTPROOT/testcases/bin/genload" --vm $NUM_PROCS --vm-chunks \
                    $CHUNKS --vm-bytes $BYTES --vm-hang >/dev/null 2>&1 &
            else
                # Otherwise Do not Hangup
                "$LTPROOT/testcases/bin/genload" --vm $NUM_PROCS --vm-chunks \
                    $CHUNKS --vm-bytes $BYTES >/dev/null 2>&1 &
            fi
            GENLOAD=1;;
        M)
            VALGRIND_CHECK_TYPE="$OPTARG";;

        N)  RUN_NETEST=1;;
    
        n)  
            "$LTPROOT/testcases/bin/netpipe.sh"
            NETPIPE=1;;
    
        o)  OUTPUTFILE_NAME="$OPTARG"
            case $OPTARG in
            /*)
                OUTPUTFILE="-o $OPTARG";;
            *)
                OUTPUTFILE="-o $LTPROOT/output/$OPTARG"
                ALT_DIR_OUT=1 ;;
            esac
            ;;
    
        p)  PRETTY_PRT=" -p ";;
   
        q)
            PAN_QUIET_MODE=" -q "
            QUIET_MODE=1
            ;;
    
        r)  LTPROOT=$OPTARG;;
    
        s)  TAG_RESTRICT_STRING=$OPTARG;;

        S)  case $OPTARG in
            /*)
                SKIPFILE=$OPTARG;;
            *)
                SKIPFILE="$LTPROOT/$OPTARG";;
            esac
            ;;
    
        t)  # In case you want to specify the time 
            # to run from the command line 
            # (2m = two minutes, 2h = two hours, etc)
            DURATION="-t $OPTARG" ;;

        T)  # In case you want the testcases to runsequentially RUN_REPEATED times
            RUN_REPEATED=$OPTARG;;

        w)  CMDFILEADDR=$OPTARG;;
   
        x)  # number of ltp's to run
            warning "Concurrent invocations of ltp-pan (-x option) can cause"
            warning "unpredictable failures, because some tests cannot be run"
            warning "in parallel."
            warning "Pausing for 10 seconds."
            sleep 10 
            INSTANCES="-x $OPTARG"
            ;;
        b) DEVICE=$OPTARG;;
        B) DEVICE_FS_TYPE=$OPTARG;;
        \?) usage;;
        esac
    done

    ## It would be nice to create a default log file even if the user has not mentioned
    if [ ! "$LOGFILE" ]; then                                ## User has not mentioned about Log File name
       LOGFILE_NAME=$DEFAULT_FILE_NAME_GENERATION_TIME
       LOGFILE="-l $LTPROOT/results/LTP_RUN_ON-$LOGFILE_NAME.log"
       ALT_DIR_RES=1
       PRETTY_PRT=" -p "
    fi
   
    ## It would be nice if a Failed File is compulsorily created (gives User better Idea of Tests that failed)  

    ## User didn't request for a failed file to be created.
    if [ -n "$FAILCMDFILE" ]; then                           
         ALT_DIR_OUT=1
         # User didn't request that an output log be created.
         if [ -n "$OUTPUTFILE" ]; then
            # User didn't request that a log file be created.
            if [ -n "$LOGFILE" ]; then
               FAILCMDFILE_TAG="$DEFAULT_FILE_NAME_GENERATION_TIME"
            # User explicitly requested a log filename.
            else
               FAILCMDFILE_TAG=${LOGFILE_NAME##*/}
            fi
         # User requested for a specific output log filename.
         else
             FAILCMDFILE_TAG="${OUTPUTFILE_NAME##*/}"
         fi
         FAILCMDFILE="-C '$LTPROOT/output/LTP_RUN_ON-${FAILCMDFILE_TAG}.failed'"
    fi

    if [ "$ALT_HTML_OUT" -eq 1 ] ; then                      ## User wants the HTML version of the output
                                                             ## which the HTML parser will require
       if [ "x$OUTPUTFILE" != x ]; then                      ## User has not mentioned about the Outputfile name, then we need to definitely generate one
          OUTPUTFILE_NAME=$DEFAULT_FILE_NAME_GENERATION_TIME
          OUTPUTFILE="-o $LTPROOT/output/LTP_RUN_ON-$OUTPUTFILE_NAME.output"
          ALT_DIR_OUT=1
          if [ "x$HTMLFILE" != x ] ; then                    ## User has not mentioned HTML File name, We need to create one
             HTMLFILE_NAME=${OUTPUTFILE_NAME##*/}
             HTMLFILE="$LTPROOT/output/$HTMLFILE_NAME.html"
          fi
       fi
    fi

    # If we need, create the output directory
    if [ "$ALT_DIR_OUT" -eq 1 ] ; then
        if ! (test -d "$LTPROOT/output" || mkdir -p "$LTPROOT/output") ; then
           die "failed to create $LTPROOT/output"
        fi
    fi
    # If we need, create the results directory
    if [ "$ALT_DIR_RES" -eq 1 ] ; then
        if ! (test -d "$LTPROOT/results" || mkdir -p "$LTPROOT/results"); then
           die "failed to create $LTPROOT/results"
        fi
    fi

    # Added -m 777 for tests that call tst_tmpdir() and try to 
    #  write to it as user nobody
    if ! mkdir -m 777 -p $TMPBASE ; then
        die "unable to make temporary directory $TMPBASE"
    fi
    # use mktemp to create "safe" temporary directories
    export TMPTEMPLATE="${TMPBASE}/ltp-XXXXXXXXXX"
    if ! export TMP=$(mktemp -d "$TMPTEMPLATE" 2>&1); then
        die "unable to make temp directory: $TMP"
    fi
    # To be invoked by tst_tmpdir()
    # write to it as user nobody
    export TMPDIR=$TMP

    if chmod 777 "$TMP" ; then
        die "unable to chmod 777 $TMP ... aborting"
    fi
    
    if ! cd $TMP ; then
        die "could not cd ${TMP} ... exiting"
    fi
    
    if [ -n $INSTANCES ] ; then
        INSTANCES="$INSTANCES -O ${TMP}"
    fi
    
    if [ "$RUN_NETEST" -eq 1 ] ; then

        if [ -z "$RHOST" -o -z "$PASSWD" ] ; then

            if [ -z "$RHOST" ] ; then
                echo \
                "INFO: Enter RHOST = 'name of the remote host machine'"
                echo -n "-> "
                read RHOST
            fi

            if [ -z "$PASSWD" ] ; then
                echo
                echo \
                "INFO: Enter PASSWD = 'root passwd of the remote host machine'"
                echo -n "-> "
                read PASSWD
            fi
            export RHOST=$RHOST
            export PASSWD=$PASSWD
        fi
    fi
    
    # If user does not provide a command file select a default set of testcases
    # to execute.
    if [ -z "$CMDFILES" -a -z "$CMDFILEADDR" ] ; then

        info "No command files were provided; executing the following default "
        info "test scenario's:"

        SCENFILES="syscalls fs fsx dio io mm ipc sched math nptl pty containers"
        SCENFILES="$SCENFILES fs_bind controllers filecaps cap_bounds"
        SCENFILES="$SCENFILES fcntl-locktests connectors admin_tools timers"
        SCENFILES="$SCENFILES power_management_tests numa hugetlb commands"

        case "$(uname -m)" in
        *86*)
            # No sense in even trying to run hyperthreading tests if it
            # isn't an Intel platform.
            SCENFILES="$SCENFILES hyperthreading"
            ;;
        esac

        info "No command files were provided; executing the following default "
        info "test scenario's:"
        if [ "x$QUIET_MODE" != x1 ] ; then
            for SCENARIO in $SCENFILES; do
                echo "	$SCENARIO"
            done
        fi

    elif [ "x$CMDFILEADDR" = x ] ; then
        SCENFILES=$(echo "$CMDFILES" | awk 'BEGIN { RS=","; } { print }')
    fi

    if [ "x$SCENFILES" != x ] ; then

        for SCENARIO in "$SCENFILES"; do
        
            # Most likely not an absolute path.
            if [ ! -f "$SCENARIO" ] ; then 
                # Relative path from $OLDCWD.
                if [ -f "$OLDCWD/$SCENFILES" ] ; then 
                    SCENFILE="$OLDCWD/$SCENARIO"
                # Let's assume it's just a runtest filename -- we'll figure
                # out whether or not this assumption is correct in a
                # second ;)...
                else
                    SCENFILE="$LTPROOT/runtest/$SCENARIO"
                fi
            fi
            # Print out a user readable message that makes sense, because the
            # `unable to create command file' one isn't helpful in this regard
            # if the file doesn't exist...
            if [ ! -f "$SCENFILE" ] ; then
                die "scenario file - $SCENFILE - doesn't exist."
            elif ! cat "$SCENFILE" >> "$TMP/alltests"; then
                die "unable add $SCENFILE to command file \`$TMP/alltests'"
            fi

        done

    elif [ "x$CMDFILEADDR" != x ] ; then
        if ! type wget >/dev/null; then
            die "You must have wget to get $CMDFILEADDR"
        elif wget -q "$CMDFILEADDR" -O "$TMP/wgetcmdfile" ; then
            die "the file transfer with wget failed (address $CMDFILEADDR)"
        elif ! cat "$TMP/wgetcmdfile" >> "$TMP/alltests" ; then
            die "unable to add scenario - $SCENFILE - to the command file"
        fi
    fi

    if [ "$RUN_NETEST" -eq 1 ] ; then

        for SCENARIO in tcp_cmds multicast rpc nfs ; do

            SCENFILE="$LTPROOT/runtest/$SCENARIO"
            if [ ! -e "$SCENFILE" ] ; then
                die "missing scenario file: $SCENFILE"
            fi
                         
            if ! cat "$SCENFILE" >> ${TMP}/alltests ; then
                die "unable to add scenario - $SCENFILE - to $TMP/alltests"
            fi

        done

    fi

    # The fsx-linux tests use the SCRATCHDEV environment variable as a location
    # that can be reformatted and run on.  Set SCRATCHDEV if you want to run 
    # these tests.  As a safeguard, this is disabled.
    unset SCRATCHDEV
    if [ "x$SCRATCHDEV" != x ] ; then
        if ! cat "$LTPROOT/runtest/fsx" >> "$TMP/alltests"; then
            die "unable to create fsx-linux tests command file: $TMP/alltests"
        fi
    fi

    # If enabled, execute only test cases that match the PATTERN
    if [ -n "$TAG_RESTRICT_STRING" ]; then
        mv -f "$TMP/alltests" "$TMP/alltests.orig"
        # Filter out all non-restricted tags.
        grep "$TAG_RESTRICT_STRING" "$TMP/alltests.orig" > "$TMP/alltests"
    fi

    # Blacklist or skip tests if a SKIPFILE was specified with -S
    if [ -n "$SKIPFILE" ]; then
        for file in $( cat $SKIPFILE ); do
            sed -i "/$file/d" "${TMP}/alltests"
        done
    fi
    
    # check for required users and groups
    if "$LTPROOT/IDcheck.sh" &>/dev/null ; then
        warning "missing some required users and/or groups; some testcases will fail"
    fi
    # display versions of installed software
    if [ -z "$QUIET_MODE" ] ; then
        if ! "$LTPROOT/ver_linux" ; then
            die "unable to display versions of software installed"
        fi
    fi

    if [ "x$DEVICE" != x ]; then
        if ! sed -i "s|DEVICE|$DEVICE|" "$TMP/alltests" ; then
            die "the sed operation on alltests failed (DEVICE)"
        elif ! mnt_pnt=$(mktemp -d --tmpdir=${TMP} mnt_pnt.XXXXXX); then
            die "Failed to create a temporary mountpoint"
        fi
        if [ -n "$DEVICE_FS_TYPE" ]; then
            mount -t $DEVICE_FS_TYPE $DEVICE $mnt_pnt
        else
            mount $DEVICE $mnt_pnt
        fi

        if [ $? -ne 0 ]; then
            die "can't mount block device $DEVICE."
        fi

        if [ -z "$DEVICE_FS_TYPE" ]; then
            DEVICE_FS_TYPE=$(awk "{if (\\\$1 == \"$DEVICE\") print \\\$3; }" /proc/mounts)
            info "determining the file system $DEVICE_FS_TYPE on block device $DEVICE"
        fi

        if ! umount $DEVICE; then
            die "can't umount $DEVICE"
        fi
    else
        info "removing testcases which require block devices."
        if ! sed -i "/DEVICE/d" "$TMP/alltests"; then
            die "the sed operation on alltests failed (DEVICE)"
        fi
    fi

    if [ "x$DEVICE_FS_TYPE" != x ]; then
        if ! sed -i "s|DEVICE_FS_TYPE|$DEVICE_FS_TYPE|" "$TMP/alltests" ; then
            die "the sed operation on alltests failed (DEVICE_FS_TYPE)"
        fi
    fi

    if [ $RUN_REPEATED -gt 1 ]; then # You need to specify at least more than 1 sequential run, else it runs default
        info "PAN will run these test cases $RUN_REPEATED times."
        info "Test Tags will be prefixed with iteration"
        inc=1
        ## Remove all whitespace.
        sed -e '/^$/ d' -e 's/^[ ,\t]*//' -e '/^#/ d' < "$TMP/alltests" > "$TMP/alltests.temp"
        ## .temp is the baseline file
        sed 's/^[0-9,a-z,A-Z]*/'"$inc"'_ITERATION_&/' < "$TMP/alltests.temp" > "$TMP/alltests"
        while [ $inc -lt $RUN_REPEATED ] ; do
            : $(( inc += 1 ))
            # Append the iteration number.
            sed 's/^[0-9,a-z,A-Z]*/'"$inc"'_ITERATION_&/' < "$TMP/alltests.temp" >> "$TMP/alltests"
        done
    fi

    info "Test start time: $(date)"
    PAN_COMMAND="\"${LTPROOT}/bin/ltp-pan\" $PAN_QUIET_MODE -e -S $INSTANCES"
    PAN_COMMAND="$PAN_COMMAND $DURATION -a $$ -n $$ $PRETTY_PRT"
    PAN_COMMAND="$PAN_COMMAND -f \"$TMP/alltests\" $LOGFILE $OUTPUTFILE "
    PAN_COMMAND="$PAN_COMMAND $FAILCMDFILE"
    info "COMMAND:    $PAN_COMMAND"
    if [ -n "$TAG_RESTRICT_STRING" ] ; then
        info "Restricting testcase tags to: $TAG_RESTRICT_STRING"
    fi

    ## Display the Output/Log/Failed/HTML file names here
    info "LOG File: $(cut -b4- < "$LOGFILE")"

    [ -n "$OUTPUTFILE" ] && info "OUTPUT File: $(cut -b4- < "$OUTPUTFILE")"
    info "FAILED COMMAND File: $(cut -b4- < "$FAILCMDFILE")"

    [ -n "$HTMLFILE" ] && info "HTML File: $HTMLFILE"

    # User wants testing with Kernel Fault Injection
    if [ "x$INJECT_KERNEL_FAULT" = x1 ] ; then
        #See if Debugfs is mounted, and
        #Fault Injection Framework available through Debugfs
        if [ -d "/sys/kernel/debug/fail_io_timeout" -o \
             -d "/sys/kernel/debug/fail_make_request" -o \
             -d "/sys/kernel/debug/fail_page_alloc" -o \
             -d "/sys/kernel/debug/failslab" ]; then
            # If atleast one of the Framework is available
            # Go ahead to Inject Fault & Create required
            # Command Files for LTP run
            info "Running tests with Fault Injection Enabled in the Kernel..."
            "${LTPROOT}/bin/create_kernel_faults_in_loops_and_probability.pl"\
                "${TMP}/alltests" $INJECT_FAULT_LOOPS_PER_TEST \
                $INJECT_KERNEL_FAULT_PERCENTAGE > "${TMP}/alltests.tmp"
            cp "${TMP}/alltests.tmp" "${TMP}/alltests"
            rm -f "${TMP}/alltests.tmp"
        else
            warning "Fault injection not enabled in the kernel; will run"
            warning "testcases normally."
        fi
    ## Valgrind Check will work only when Kernel Fault Injection is not expected,
    ## We do not want to test Faults when valgrind is running
    elif [ "x$VALGRIND_CHECK_TYPE" = x1 ]; then
        if ! type valgrind 2>/dev/null ; then
            error "valgrind isn't available"
        else
            case "$VALGRIND_CHECK_TYPE" in
            [1-3])
                "${LTPROOT}/bin/create_valgrind_check.pl" "$TMP/alltests" \
                    $VALGRIND_CHECK_TYPE > "$TMP/alltests.tmp"
                cp "$TMP/alltests.tmp" "$TMP/alltests"
                rm -rf "$TMP/alltests.tmp"
                ;;
            *)
            error "Invalid valgrind memory check type: $VALGRIND_CHECK_TYPE.";;
            esac
        fi
    fi

    info "Running tests......."
    test_start_time=$(date)

    # Some tests need to run inside the "bin" directory.
    cd "${LTPROOT}/testcases/bin"
    "${PAN_COMMAND}"
    #"${LTPROOT}/bin/ltp-pan" $PAN_QUIET_MODE -e -S $INSTANCES $DURATION -a $$ \
    #    -n $$ $PRETTY_PRT -f "$TMP/alltests" "$LOGFILE" "$OUTPUTFILE" \
    #    "$FAILCMDFILE"
     
    if [ $? -eq 0 ]; then
        info "ltp-pan reported all tests PASS'ed"
        LTP_EXIT_VALUE=0
    else
        info "ltp-pan reported some tests FAIL'ed"
        LTP_EXIT_VALUE=1
    fi
    cd ..
    info "LTP Version: $version_date"

    # Punt out 
    if [ "$ALT_HTML_OUT" -eq 1 ] ; then        #User wants the HTML output to be created, it then needs to be generated
       export LTP_VERSION=$version_date
       export TEST_START_TIME=$test_start_time
       export TEST_END_TIME=$(date)
       OUTPUT_DIRECTORY=$(cut -c4- < "$OUTPUTFILE")
       LOGS_DIRECTORY="$LTPROOT/results"
       export TEST_OUTPUT_DIRECTORY="$LTPROOT/output"
       export TEST_LOGS_DIRECTORY=$LOGS_DIRECTORY
       info "Generating HTML Output.....!!"
       ( perl $LTPROOT/bin/genhtml.pl $LTPROOT/bin/html_report_header.txt \
         test_start test_end test_output execution_status \
         "$OUTPUT_DIRECTORY" > "$HTMLFILE" )
       info "Generated HTML Output.....!!"
       info "Location: $HTMLFILE"
    fi
 
    # Send email report
    if [ "$ALT_EMAIL_OUT" -eq 1 ] ; then

       if [ ! -r "$HTMLFILE_NAME" -a ! -r "$OUTPUTFILE_NAME" -a ! -r "$LOGFILE_NAME" ] ; then
           ## User does not have output/logs/html-output, nothing to be
           ## mailed in this situation
           info "Nothing to be mailed"
       elif ! MUTT=$(which mutt 2>/dev/null) && ! MAIL=$(which mail 2>/dev/null) ; then
           error "********************************************"
           error "Cannot find a working copy of mail or mutt."
           error "Will not email report."
           error "********************************************"
       elif ! type gzip 2>/dev/null; then
           error "********************************************"
           error "Cannot find a working copy of gzip"
           error "Will not email report."
           error "********************************************"
       else

           TAR_FILE_NAME=LTP_RUN_$version_date$DEFAULT_FILE_NAME_GENERATION_TIME.tar

           # Got HTML file.
           for f in "$HTMLFILE_NAME" "$OUTPUTFILE_NAME" ; do

               if [ -r "$f" -a "$(readlink -f "$(dirname "$f")")" != "$(readlink -f "$LTPROOT/output")" ] ; then

                   test -d "$LTPROOT/output" || mkdir -p "$LTPROOT/output"
                   if [ $? -eq 0 ] ; then
                       cp $HTMLFILE_NAME "$LTPROOT/output/."
                   else
                       error "Failed to create \`$LTPROOT/output/'."
                       error "Skipping tar for all files in $LTPROOT/output/."
                   fi

               fi

           done

           if [ -r "$LOGFILE_NAME" -a "$(readlink -f "$(dirname "$LOGFILE_NAME")")" != "$(readlink -f "$LTPROOT/results")" ] ; then
               test -d mkdir -p "$LTPROOT/results"                ## We need to create this Directory
                   cp "$LOGFILE_NAME" "$LTPROOT/results/."
               fi
               test -d "$LTPROOT/output" || mkdir -p "$LTPROOT/output"
               if [ $? -eq 0 ] ; then
                   cp "$LOGFILE_NAME" "$LTPROOT/results/."
               else
                   error "Failed to create \`$LTPROOT/results/'."
                   error "Skipping tar for all files in $LTPROOT/results/."
               fi
           fi
           # Got output log(s)
           if [ -d "$LTPROOT/output" ] ; then
               if tar -cf  "./$TAR_FILE_NAME" "$LTPROOT/output"; then
                   info "Created TAR File: ./$TAR_FILE_NAME successfully, added $LTPROOT/output"
               else
                   error "Cannot Create TAR File: ./$TAR_FILE_NAME for adding $LTPROOT/output"
               fi
           fi # END: output log(s)
           # Got results file(s)
           if [ -d "$LTPROOT/results" ] ; then
               if tar -uf "./$TAR_FILE_NAME" "$LTPROOT/results"; then
                   info "Updated TAR File: ./$TAR_FILE_NAME successfully, added $LTPROOT/results"
               else
                   error "Cannot Update TAR File: ./$TAR_FILE_NAME for adding $LTPROOT/results"
               fi
           fi # END: results file(s)
           if [ -e "$LTPROOT/nohup.out" ] ; then
               if tar -uf "./$TAR_FILE_NAME" "$LTPROOT/nohup.out"; then
                   info "Updated TAR File: ./$TAR_FILE_NAME successfully, added $LTPROOT/nohup.out"
               else
                   error "Cannot Update TAR File: ./$TAR_FILE_NAME for adding $LTPROOT/nohup.out"
               fi
           fi
           if gzip "./$TAR_FILE_NAME"; then

               info "Gunzipped TAR File: ./$TAR_FILE_NAME"

               # Use mutt(1)
               if [ -n "$MUTT" ] ; then
                   info "Starting mailing reports to: $EMAIL_TO, file: ./$TAR_FILE_NAME.gz"
                   mutt -a ./$TAR_FILE_NAME.gz -s "LTP Reports on $test_start_time" $EMAIL_TO < /dev/null
                   if [ $? -eq 0 ]; then
                       info "Reports Successfully mailed to: $EMAIL_TO"
                   else
                       error "Email operation failed"
                   fi
               # Use mail(1)
               elif [ -n "$MAIL" ] ; then
                   echo "Starting mailing reports to: $EMAIL_TO, file: ./$TAR_FILE_NAME.gz"
                   uuencode ./$TAR_FILE_NAME.gz $TAR_FILE_NAME.gz | mail  $EMAIL_TO -s "LTP Reports on $test_start_time"
                   if [ $? -eq 0 ]; then
                       info "Reports Successfully mailed to: $EMAIL_TO"
                   else
                       error "Email operation failed"
                   fi

               else
                   die "Programmer error; please report bug (\$MAIL and \$MUTT were both zero-length strings)."
               fi # END: Send email via client mail or mutt.
           else
               error "Cannot Gunzip TAR File: ./$TAR_FILE_NAME"
           fi

       fi
       # END: Got something to email? // got a client to email with? //
       #      got gzip? // send report.

    fi # END: Send email report?
   
    info "Test end time: $(date)"
    
    [ "x$GENLOAD" = x1 ] && (killall -9 genload >/dev/null 2>&1)
    [ "x$NETPIPE" = x1 ] && (killall -9 NPtcp >/dev/null 2>&1)
    
    if [ "x$ALT_DIR_OUT" = x1 -o "x$ALT_DIR_RES" = x1 ] ; then
        cat <<-EOF
        
       ###############################################################"
        
            Done executing testcases."
            LTP Version:  $version_date
       ###############################################################"
       
        EOF
    fi
    exit $LTP_EXIT_VALUE

}

cleanup()
{
    rm -rf "${TMP}"
}

trap "cleanup" 0
setup
main "$@"