uname -n >> $MOUNT/recon    if [ ! -z "$CLIENTS" ]; then        $PDSH $CLIENTS "df $MOUNT; uname -n >> $MOUNT/recon" > /dev/null    fi    echo Connected clients:    cat $MOUNT/recon    ls -l $MOUNT/recon > /dev/null    rm $MOUNT/recon}facet_failover() {    facet=$1    echo "Failing $facet on node `facet_active_host $facet`"    shutdown_facet $facet    reboot_facet $facet    client_df &    DFPID=$!    echo "df pid is $DFPID"    change_active $facet    TO=`facet_active_host $facet`    echo "Failover $facet to $TO"    wait_for $facet    mount_facet $facet || error "Restart of $facet failed"}obd_name() {    local facet=$1}replay_barrier() {    local facet=$1    do_facet $facet sync    df $MOUNT    local svc=${facet}_svc    do_facet $facet $LCTL --device %${!svc} readonly    do_facet $facet $LCTL --device %${!svc} notransno    do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"    $LCTL mark "local REPLAY BARRIER on ${!svc}"}replay_barrier_nodf() {    local facet=$1    echo running=${running}    do_facet $facet sync    local svc=${facet}_svc    echo Replay barrier on ${!svc}    do_facet $facet $LCTL --device %${!svc} readonly    do_facet $facet $LCTL --device %${!svc} notransno    do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"    $LCTL mark "local REPLAY BARRIER on ${!svc}"}mds_evict_client() {    UUID=`cat /proc/fs/lustre/mdc/${mds_svc}-mdc-*/uuid`    do_facet mds "echo $UUID > /proc/fs/lustre/mds/${mds_svc}/evict_client"}ost_evict_client() {    UUID=`cat /proc/fs/lustre/osc/${ost1_svc}-osc-*/uuid`    do_facet ost1 "echo $UUID > /proc/fs/lustre/obdfilter/${ost1_svc}/evict_client"}fail() {    facet_failover $* || error "failover: $?"    df $MOUNT || error "post-failover df: $?"}fail_nodf() {    local facet=$1    facet_failover $facet}fail_abort() {    local facet=$1    stop $facet    change_active $facet    mount_facet $facet -o abort_recovery    df $MOUNT || echo "first df failed: $?"    sleep 1    df $MOUNT || error "post-failover df: $?"}do_lmc() {    echo There is no lmc.  This is mountconf, baby.    exit 1}h2gm () {    if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else        ID=`$PDSH $1 $GMNALNID -l | cut -d\  -f2`        echo $ID"@gm"    fi}h2ptl() {   if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else       ID=`xtprocadmin -n $1 2>/dev/null | egrep -v 'NID' | awk '{print $1}'`       if [ -z "$ID" ]; then           echo "Could not get a ptl id for $1..."           exit 1       fi       echo $ID"@ptl"   fi}declare -fx h2ptlh2tcp() {    if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else        echo $1"@tcp"     fi}declare -fx h2tcph2elan() {    if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else        if type __h2elan >/dev/null 2>&1; then            ID=$(__h2elan $1)        else            ID=`echo $1 | sed 's/[^0-9]*//g'`        fi        echo $ID"@elan"    fi}declare -fx h2elanh2openib() {    if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else        ID=`echo $1 | sed 's/[^0-9]*//g'`        echo $ID"@openib"    fi}declare -fx h2openibfacet_host() {    local facet=$1    varname=${facet}_HOST    if [ -z "${!varname}" ]; then        if [ "${facet:0:3}" == "ost" ]; then            eval ${facet}_HOST=${ost_HOST}        fi    fi    echo -n ${!varname}}facet_active() {    local facet=$1    local activevar=${facet}active    if [ -f ./${facet}active ] ; then        source ./${facet}active    fi    active=${!activevar}    if [ -z "$active" ] ; then         echo -n ${facet}    else        echo -n ${active}    fi}facet_active_host() {    local facet=$1    local active=`facet_active $facet`    if [ "$facet" == client ]; then        echo $HOSTNAME    else        echo `facet_host $active`    fi}change_active() {    local facet=$1    failover=${facet}failover     host=`facet_host $failover`    [ -z "$host" ] && return    curactive=`facet_active $facet`    if [ -z "${curactive}" -o "$curactive" == "$failover" ] ; then        eval export ${facet}active=$facet    else        eval export ${facet}active=$failover    fi    # save the active host for this facet    activevar=${facet}active    echo "$activevar=${!activevar}" > ./$activevar}do_node() {    HOST=$1    shift    local myPDSH=$PDSH    if [ "$HOST" = "$HOSTNAME" ]; then        myPDSH="no_dsh"    elif [ -z "$myPDSH" -o "$myPDSH" = "no_dsh" ]; then        echo "cannot run remote command on $HOST with $myPDSH"        return 128    fi    if $VERBOSE; then        echo "CMD: $HOST $@" >&2        $myPDSH $HOST $LCTL mark "$@" > /dev/null 2>&1 || :    fi    if [ "$myPDSH" = "rsh" ]; then# we need this because rsh does not return exit code of an executed command	local command_status="$TMP/cs"	rsh $HOST ":> $command_status"	rsh $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin;		    cd $RPWD; sh -c \"$@\") || 		    echo command failed >$command_status"	[ -n "$($myPDSH $HOST cat $command_status)" ] && return 1 || true        return 0    fi    $myPDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; sh -c \"$@\")" | sed "s/^${HOST}: //"    return ${PIPESTATUS[0]}}do_facet() {    facet=$1    shift    HOST=`facet_active_host $facet`    [ -z $HOST ] && echo No host defined for facet ${facet} && exit 1    do_node $HOST "$@"}add() {    local facet=$1    shift    # make sure its not already running    stop ${facet} -f    rm -f ${facet}active    do_facet ${facet} $MKFS $*}ostdevname() {    num=$1    DEVNAME=OSTDEV$num    #if $OSTDEVn isn't defined, default is $OSTDEVBASE + num    eval DEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}}    echo -n $DEVPTR}########## MountConf setupstopall() {    # make sure we are using the primary server, so test-framework will    # be able to clean up properly.    activemds=`facet_active mds`    if [ $activemds != "mds" ]; then        fail mds    fi        # assume client mount is local     grep " $MOUNT " /proc/mounts && zconf_umount $HOSTNAME $MOUNT $*    grep " $MOUNT2 " /proc/mounts && zconf_umount $HOSTNAME $MOUNT2 $*    [ "$CLIENTONLY" ] && return    stop mds -f    for num in `seq $OSTCOUNT`; do        stop ost$num -f    done    return 0}cleanupall() {    stopall $*    unload_modules}formatall() {    [ "$FSTYPE" ] && FSTYPE_OPT="--backfstype $FSTYPE"    stopall    # We need ldiskfs here, may as well load them all    load_modules    [ "$CLIENTONLY" ] && return    echo Formatting mds, osts    if $VERBOSE; then        add mds $MDS_MKFS_OPTS $FSTYPE_OPT --reformat $MDSDEV || exit 10    else        add mds $MDS_MKFS_OPTS $FSTYPE_OPT --reformat $MDSDEV > /dev/null || exit 10    fi    for num in `seq $OSTCOUNT`; do        if $VERBOSE; then            add ost$num $OST_MKFS_OPTS $FSTYPE_OPT --reformat `ostdevname $num` || exit 10        else            add ost$num $OST_MKFS_OPTS $FSTYPE_OPT --reformat `ostdevname $num` > /dev/null || exit 10        fi    done}mount_client() {    grep " $1 " /proc/mounts || zconf_mount $HOSTNAME $*}remount_client(){	zconf_umount `hostname` $1 || error "umount failed"	zconf_mount `hostname` $1 || error "mount failed"}set_obd_timeout() {    local facet=$1    local timeout=$2    do_facet $facet lsmod | grep -q obdclass || \        do_facet $facet "modprobe obdclass"    do_facet $facet "sysctl -w lustre.timeout=$timeout"}setupall() {    load_modules    if [ -z "$CLIENTONLY" ]; then        echo Setup mdt, osts        echo $REFORMAT | grep -q "reformat" \	    || do_facet mds "$TUNEFS --writeconf $MDSDEV"        set_obd_timeout mds $TIMEOUT        start mds $MDSDEV $MDS_MOUNT_OPTS        for num in `seq $OSTCOUNT`; do            DEVNAME=`ostdevname $num`            set_obd_timeout ost$num $TIMEOUT            start ost$num $DEVNAME $OST_MOUNT_OPTS        done    fi    [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE    mount_client $MOUNT    if [ "$MOUNT_2" ]; then        mount_client $MOUNT2    fi    sleep 5}mounted_lustre_filesystems() {	awk '($3 ~ "lustre" && $1 ~ ":") { print $2 }' /proc/mounts}check_and_setup_lustre() {    MOUNTED="`mounted_lustre_filesystems`"    if [ -z "$MOUNTED" ]; then        [ "$REFORMAT" ] && formatall        setupall        MOUNTED="`mounted_lustre_filesystems`"        [ -z "$MOUNTED" ] && error "NAME=$NAME not mounted"        export I_MOUNTED=yes    fi    if [ "$ONLY" == "setup" ]; then        exit 0    fi}cleanup_and_setup_lustre() {    if [ "$ONLY" == "cleanup" -o "`mount | grep $MOUNT`" ]; then        sysctl -w lnet.debug=0 || true        cleanupall        if [ "$ONLY" == "cleanup" ]; then     	    exit 0        fi    fi    check_and_setup_lustre}check_and_cleanup_lustre() {    if [ "`mount | grep $MOUNT`" ]; then        [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]*    fi    if [ "$I_MOUNTED" = "yes" ]; then        cleanupall -f || error "cleanup failed"    fi    unset I_MOUNTED}####### # General functionscheck_network() {    local NETWORK=0    local WAIT=0    local MAX=$2    while [ $NETWORK -eq 0 ]; do        ping -c 1 -w 3 $1 > /dev/null        if [ $? -eq 0 ]; then            NETWORK=1        else            WAIT=$((WAIT + 5))            echo "waiting for $1, $((MAX - WAIT)) secs left"            sleep 5        fi        if [ $WAIT -gt $MAX ]; then            echo "Network not available"            exit 1        fi    done}check_port() {    while( !($DSH2 $1 "netstat -tna | grep -q $2") ) ; do        sleep 9    done}no_dsh() {    shift    eval $@}comma_list() {    # the sed converts spaces to commas, but leaves the last space    # alone, so the line doesn't end with a comma.    echo "$*" | tr -s " " "\n" | sort -b -u | tr "\n" " " | sed 's/ \([^$]\)/,\1/g'}absolute_path() {    (cd `dirname $1`; echo $PWD/`basename $1`)}################################### Adaptive Timeouts funcsat_is_valid() {    if [ -z "$AT_MAX_PATH" ]; then        AT_MAX_PATH=$(do_facet mds "find /sys/ -name at_max")        [ -z "$AT_MAX_PATH" ] && echo "missing /sys/.../at_max " && return 1    fi    return 0}at_is_enabled() {    at_is_valid || error "invalid call"    # only check mds, we assume at_max is the same on all nodes    local at_max=$(do_facet mds "cat $AT_MAX_PATH")    if [ $at_max -eq 0 ]; then        return 1    else        return 0    fi}at_max_get() {    at_is_valid || error "invalid call"    do_facet $1 "cat $AT_MAX_PATH"}at_max_set() {    local at_max=$1    shift    at_is_valid || error "invalid call"    for facet in $@; do        if [ $facet == "ost" ]; then            for i in `seq $OSTCOUNT`; do                do_facet ost$i "echo $at_max > $AT_MAX_PATH"            done        else            do_facet $facet "echo $at_max > $AT_MAX_PATH"        fi    done}################################### OBD_FAIL funcsdrop_request() {# OBD_FAIL_MDS_ALL_REQUEST_NET    RC=0    do_facet mds sysctl -w lustre.fail_loc=0x123    do_facet client "$1" || RC=$?    do_facet mds sysctl -w lustre.fail_loc=0    return $RC}drop_reply() {# OBD_FAIL_MDS_ALL_REPLY_NET    RC=0    do_facet mds sysctl -w lustre.fail_loc=0x122    do_facet client "$@" || RC=$?    do_facet mds sysctl -w lustre.fail_loc=0    return $RC}drop_reint_reply() {# OBD_FAIL_MDS_REINT_NET_REP