#!/bin/bash
# vim:ts=4:sts=4:sw=0:et:
#
# Copyright (c) 2006-2007 Cisco Systems. All rights reserved.
# Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved.
#
# This Software is licensed under one of the following licenses:
#
# 1) under the terms of the "Common Public License 1.0" a copy of which is
#    available from the Open Source Initiative, see
#    http://www.opensource.org/licenses/cpl.php.
#
# 2) under the terms of the "The BSD License" a copy of which is
#    available from the Open Source Initiative, see
#    http://www.opensource.org/licenses/bsd-license.php.
#
# 3) under the terms of the "GNU General Public License (GPL) Version 2" a
#    copy of which is available from the Open Source Initiative, see
#    http://www.opensource.org/licenses/gpl-license.php.
#
# Licensee has the right to choose one of the above licenses.
#
# Redistributions of source code must retain the above copyright
# notice and one of the license notices.
#
# Redistributions in binary form must reproduce both the above copyright
# notice, one of the license notices in the documentation
# and/or other materials provided with the distribution.
#
#
# Description: Test health of HCA

# For colored text
green='\E[32m'
red='\E[31m'
cyan='\E[36m'

bold_start='\033[1m'
bold_stop='\033[0m'

INSTALL_PREFIX=/usr

MTHCA_DRIVER_NEEDED=mthca
HERMON_DRIVER_NEEDED=mlx5_
CONNECTIB_DRIVER_NEEDED=mlx5_
DRIVER_NEEDED=$MTHCA_DRIVER_NEEDED
INFINI_CLASS_PATH=/sys/class/infiniband
G_LSPCI_OUTPUT_FILE=/tmp/hca_self_test_lspci.output


# Color echo
cecho () {
    message=${1}     # argument 1 - message
    color=${2}       # argument 2 - color
    echo -e "$color"
    echo "$message"
    tput sgr0        # Reset to normal
    return
}

# Exit code
EXIT_CODE=0

# Check whether the script is being run as root and exit if otherwise
if [ `id -g` -ne 0 ]; then
    echo "Error: hca_self_test must be run as root."
    EXIT_CODE=1
    exit $EXIT_CODE
fi

echo
g_pkg_cmd="rpm"
IS_SUSE=false

if [ -f /etc/debian_version ]; then
    INSTALL_PREFIX=/opt/topspin
    g_pkg_cmd="dpkg"
fi

if [ -f /etc/SuSE-release ]; then
    IS_SUSE=true
fi

for cmd in  lspci cat id $g_pkg_cmd uname grep ls awk egrep modprobe; do
    cmd_exist=`which $cmd 2> /dev/null`
    if [ "$cmd_exist" == "" ]; then
            echo "Error: $cmd tool was not found in the PATH"
            exit 1
    fi
done


HCA_LSPCI_NAME="InfiniBand"
HCA_LSPCI_NAME_NEW="Infiniband controller"
VPI_LSPCI_NAME="Network controller"
NIC_LSPCI_NAME="Ethernet controller"
MEM_CON_LSPCI_NAME="Memory controller"

MEL_LSPCI_NAME_EXP="${HCA_LSPCI_NAME}|${HCA_LSPCI_NAME_NEW}|${VPI_LSPCI_NAME}|${NIC_LSPCI_NAME}"

echo "---- Performing Adapter Device Self Test ----"

# HCA/PCI check
NUM_IB_DEV=`lspci 2> /dev/null | grep -E "(${MEL_LSPCI_NAME_EXP})(\s\[[0-9]+\])?: Mellanox Technolog" | wc -l`
NUM_MEM_CON=`lspci 2> /dev/null | grep "${MEM_CON_LSPCI_NAME}(\s\[[0-9]+\])?: Mellanox Technolog" | wc -l`

let "NUM_HCAS=$NUM_IB_DEV + $NUM_MEM_CON"
echo "Number of CAs Detected ................. $NUM_HCAS";

if [ $NUM_HCAS -ne 0 ]; then
    if [ $NUM_MEM_CON -ne 0 ]; then
        echo -e "PCI Device Check ....................... ${red}FAIL"
        tput sgr0
        echo "    REASON: jumper set on CA or CA hardware failure"
        EXIT_CODE=1
    else
        echo -e "PCI Device Check ....................... ${green}PASS"
        tput sgr0
    fi
else
    echo -e "PCI Device Check ....................... ${red}FAIL"
    tput sgr0
    echo "    REASON: no CAs in the system"
    EXIT_CODE=1
    exit $EXIT_CODE
fi


mlx5_core_ko=`modinfo mlx5_core 2> /dev/null | grep filename | awk '{print $NF}'`
mlx5_core_ko=`readlink -f $mlx5_core_ko 2> /dev/null`

# check for inbox driver
is_inbox=0
if [[ $mlx5_core_ko =~ "/kernel/drivers" ]]; then
    is_inbox=1
fi

if [ $g_pkg_cmd = "dpkg" ]; then
    RPM_CHECK_FAIL=0
    RPM_CUR_BOOTED_KER=1

    RPM_USR_VER=`dpkg -s libibverbs1 2> /dev/null | wc -l`
    RPM_KER_VER=`dpkg -l 2> /dev/null | grep -E "ofed-kernel|mlnx[-_]en" | wc -l`
    RPM_KER_NAME=`modinfo -F vermagic $mlx5_core_ko | awk '{print$1}'`
    if [ $RPM_KER_VER -eq 0 ] && [ "$RPM_KER_NAME" != "" ]; then
        RPM_KER_VER=1
    fi
    BOOTED_KER=`uname -r`
    if [ "$BOOTED_KER" != "$RPM_KER_NAME" ]; then
        RPM_CUR_BOOTED_KER=0
    fi
else
    # RPM check
    RPM_CHECK_FAIL=0
    RPM_USR_VER=`rpm -q libibverbs 2> /dev/null | wc -l`
    RPM_KER_VER=`rpm -qa 2> /dev/null | grep -E "kernel-ib|ofa_kernel|mlnx[-_]en|bluefield" | wc -l`
    KER_RPM=`rpm -qf $mlx5_core_ko 2> /dev/null | grep -E "kernel-ib|ofa_kernel|bluefield"`

    if [ ! -z "$KER_RPM" ]; then
        RPM_KER_NAME=`modinfo -F vermagic $mlx5_core_ko | awk '{print$1}'`
        RPM_CUR_BOOTED_KER=1
    else
        if (rpm -q ofa_kernel_dkms > /dev/null 2>&1); then
            IS_DKMS=1
            RPM_KER_NAME=`rpm -q --queryformat "[%{NAME}-%{VERSION}]\n" ofa_kernel_dkms`
            RPM_CUR_BOOTED_KER=1
        else
            RPM_CUR_BOOTED_KER=0
        fi
    fi
fi

OFED_VERSION=$(ofed_info 2> /dev/null | head -1)
RPM_KER_ARCH=`uname -m`
BOOTED_KER=`uname -r`

if [ $IS_SUSE = true ]; then
    # Have to munge uname output a bit.
    BOOTED_KER=`uname -r | sed s@-@_@g`
fi

if [ $RPM_KER_VER -eq 0 ] && [ $is_inbox -eq 1 ]; then
    RPM_KER_VER=1
fi

if [ $RPM_USR_VER -eq 0 ] && [ $RPM_KER_VER -eq 0 ]; then
    echo -e "Host Driver RPM Check .................. ${red}FAIL"
    tput sgr0
    echo "    REASON: no RPMs found"
    RPM_CHECK_FAIL=1
    EXIT_CODE=1
elif [ $RPM_USR_VER -eq 0 ]; then
    echo -e "Host Driver RPM Check .................. ${red}FAIL"
    tput sgr0
    echo "    REASON: no user level RPMs found"
    RPM_CHECK_FAIL=1
    EXIT_CODE=1
elif [ $RPM_KER_VER -eq 0 ]; then
    echo -e "Host Driver RPM Check .................. ${red}FAIL"
    tput sgr0
    echo "    REASON: no kernel level RPMs found"
    RPM_CHECK_FAIL=1
    EXIT_CODE=1
fi

# if not inbox check if driver kernel RPM matches the booted kernel

if [ $is_inbox -eq 0 ] && [ $RPM_KER_VER -ne 0 ]; then
    if [ $RPM_CUR_BOOTED_KER -eq 0 ]; then
        echo -e "Host Driver RPM Check .................. ${red}FAIL"
        tput sgr0
        echo "    REASON: no RPMs found for currently booted kernel $BOOTED_KER"
        RPM_CHECK_FAIL=1
        EXIT_CODE=1
    fi
fi

echo "Kernel Arch ............................ $RPM_KER_ARCH"
if [ "$RPM_KER_NAME" != "" ]; then
    echo "Host Driver Version .................... $OFED_VERSION $RPM_KER_NAME"
else
    echo "Host Driver Version .................... NA"
fi

if [ $RPM_CHECK_FAIL -eq 0 ]; then
    echo -e "Host Driver RPM Check .................. ${green}PASS"
    tput sgr0
fi


function get_curr_ca_pci_info () {
    loop_cnt=$1

    if [ "${g_lspci_was_ran}" == "" ]; then
        lspci 2> /dev/null | grep -E "(${MEL_LSPCI_NAME_EXP}|${MEM_CON_LSPCI_NAME})(\s\[[0-9]+\])?: Mellanox Technolog" > ${G_LSPCI_OUTPUT_FILE}
        g_lspci_was_ran=1
    fi
    current_pci_info=$(cat ${G_LSPCI_OUTPUT_FILE} | head -$(expr $loop_cnt + 1) | tail -1)
    echo ${current_pci_info}
}

function get_ca_type() {
    loop_cnt=$1

    current_pci_info=`get_curr_ca_pci_info $loop_cnt`

    if [[ ${current_pci_info} =~ ${HCA_LSPCI_NAME} ]] || [[ ${current_pci_info} =~ ${HCA_LSPCI_NAME_NEW} ]] || [[ ${current_pci_info} =~ ${MEM_CON_LSPCI_NAME} ]]; then
        ca_type=HCA
    elif [[ ${current_pci_info}  =~ ${VPI_LSPCI_NAME} ]]; then
        ca_type=VPI
    elif [[ ${current_pci_info}  =~ ${NIC_LSPCI_NAME} ]]; then
        ca_type=NIC
    else
        ca_type="N/A"
    fi
    echo "${ca_type}"
}
#get the device ID
function get_device_id {
    loop_cnt=$1

    current_pci_info=`get_curr_ca_pci_info $loop_cnt`
    PCI_DEV=$(echo $current_pci_info | awk '{print $1}')

    HexDevice_ID=$(lspci -D -n -d "15b3:"  2> /dev/null | grep $PCI_DEV |  tail -1 | cut -d ":" -f5 | cut -d " " -f1)
    if [ "$HexDevice_ID" != "" ]; then
        HexDevice_ID=0x$HexDevice_ID
        let "tmp=$HexDevice_ID"
        Device_ID=$(echo $tmp)
    else
        Device_ID=$(mstflint -d $PCI_DEV q 2> /dev/null | grep "Device ID" | awk '{print $3}')
    fi
    echo $Device_ID
}
#get the HCA NAME

g_connectx="ConnectX"
g_connectx3="ConnectX-3"
g_connectx3_pro="ConnectX-3_Pro"
g_connectib="Connect-IB"
g_connectx4="ConnectX-4"
g_connectx4lx="ConnectX-4_Lx"
g_connectx5="ConnectX-5"
g_connectx5bf="ConnectX-5_BF"
g_InfiniHost_III_Ex="InfiniHost_III_Ex"
g_InfiniHost_III_Ex_memfree="InfiniHost_III_Ex_m"
g_InfiniHost_III_Lx="InfiniHost_III_Lx"
g_InfiniHost="InfiniHost"


function  get_hca_name {
    dev_id=$1
    if [ $dev_id -eq 25208 ]; then
        ret_val=$g_InfiniHost_III_Ex
    elif [ $dev_id -eq 25218  ]; then
        ret_val=$g_InfiniHost_III_Ex_memfree
    elif [ $dev_id -eq 24204 -o $dev_id -eq 25204 ]; then
        ret_val=$g_InfiniHost_III_Lx
    elif [ $dev_id -eq 23108 ]; then
        ret_val=$g_InfiniHost
    elif [ $dev_id -eq 4099 ]; then
        ret_val=$g_connectx3
    elif [ $dev_id -eq 4103 ]; then
        ret_val=$g_connectx3_pro
    elif [ $dev_id -eq 4113 ]; then
        ret_val=$g_connectib
    elif [ $dev_id -eq 4115 ]; then
        ret_val=$g_connectx4
    elif [ $dev_id -eq 4117 ]; then
        ret_val=$g_connectx4lx
    elif [ $dev_id -eq 4119 ]; then
        ret_val=$g_connectx5
    elif [ $dev_id -eq 41682 ]; then
        ret_val=$g_connectx5bf
    else
        ret_val=$g_connectx4
    fi
    echo $ret_val

}

#get the Driver Name
function get_driver {
    loop_cnt=$1
    driver_need=""
    Device_ID=$(get_device_id $LOOP_COUNT)
    if [ "$Device_ID" != "" ]; then
        hca_name=$(get_hca_name $Device_ID)
		case "$hca_name" in
			$g_connectx|$g_connectx3|$g_connectx3_pro)
                driver_need=$HERMON_DRIVER_NEEDED
				;;
			$g_connectib|$g_connectx4|$g_connectx4lx|$g_connectx5|$g_connectx5bf)
				driver_need=$CONNECTIB_DRIVER_NEEDED
				;;
			*)
				driver_need=$MTHCA_DRIVER_NEEDED
				;;
		esac
    fi
    echo $driver_need
}


function compare_fw  {
    found=$1
    needed=$2

    n_1=$(echo $needed | cut -f1 -d"." | cut -b 2-)
    n_2=$(echo $needed | cut -f2 -d".")
    n_3=$(echo $needed | cut -f3 -d".")
    f_1=$(echo $found  | cut -f1 -d"." | cut -b 2-)
    f_2=$(echo $found  | cut -f2 -d".")
    f_3=$(echo $found  | cut -f3 -d".")

    if [ $n_1 -gt $f_1 ]; then
        echo "required"
    elif [ $f_1 -gt $n_1 ]; then
        echo "found"
    elif [ $n_2 -gt $f_2 ]; then
        echo "required"
    elif [ $f_2 -gt $n_2 ]; then
        echo "found"
   elif [ $n_3 -gt $f_3 ]; then
        echo "required"
    elif [ $f_3 -gt $n_3 ]; then
        echo "found"
    fi

}



# HCA firmware check
echo_fw_check () {
    LOOP_COUNT=$1
    ca_type=$2
    result=$3
    echo -e "Firmware Check on CA #$LOOP_COUNT (${ca_type}) .......... ${result}"
    tput sgr0
}
if [ $NUM_HCAS -ne 0 ]; then
    mlx_dev_num=0
    mlx5_dev_num=0
    mthca_dev_num=0
    LOOP_COUNT=0

    # To take care of more than one HCA
    while [ $LOOP_COUNT -lt $NUM_HCAS ]
    do
        ca_type=$(get_ca_type $LOOP_COUNT)
        if [ $RPM_USR_VER -ne 0 ] && [ $RPM_CUR_BOOTED_KER -ne 0 ]; then

            #default mthca0
            device_num=$mthca_dev_num

            ## get the Device Id
            PCI_DEVICE=$(lspci 2> /dev/null | grep Mellanox | head -$(expr $LOOP_COUNT + 1) | tail -1 | awk '{print $1}')
            Device_ID=$(get_device_id $LOOP_COUNT)
            if [ "$Device_ID" != "" ]; then
                hca_name=$(get_hca_name $Device_ID)
                if [ "$hca_name" != "" ]; then
                    # increase the mlx and mthca counter
                    case "$hca_name" in
                    "$g_connectx"|"$g_connectx3"|"$g_connectx3_pro")
                        DRIVER_NEEDED=$HERMON_DRIVER_NEEDED
                        device_num=$mlx_dev_num
                        let "mlx_dev_num=$mlx_dev_num + 1"
                    ;;
                    "$g_connectib"|"$g_connectx4"|"$g_connectx4lx"|"$g_connectx5"|"$g_connectx5bf")
                        DRIVER_NEEDED=$CONNECTIB_DRIVER_NEEDED
                        device_num=$mlx5_dev_num
                        let "mlx5_dev_num=$mlx5_dev_num + 1"
                    ;;
                    *)
                        let "mthca_dev_num=$mthca_dev_num + 1"
                    ;;
                    esac

                    FW_FOUND=v$(mstflint -d $PCI_DEVICE q 2> /dev/null | grep "FW Version" | awk '{print $3}')
                    if [ "$FW_FOUND" = "v" ]; then
                        if [ -f "$INFINI_CLASS_PATH/$DRIVER_NEEDED$device_num/fw_ver" ]; then
                            FW_FOUND=v`cat $INFINI_CLASS_PATH/$DRIVER_NEEDED$device_num/fw_ver 2> /dev/null`
                        else
                            echo_fw_check ${LOOP_COUNT} ${ca_type} "${red}FAIL"
                            echo "    REASON: CA #$LOOP_COUNT: failed to get firmware version"
                            EXIT_CODE=1
                            no_firmware=1
                        fi
                    fi

                    if [ "$no_firmware" != "1" ]; then
                        echo -e "Firmware on CA #$LOOP_COUNT ${ca_type} .................. $FW_FOUND"
                    fi
                else
                    echo_fw_check ${LOOP_COUNT} ${ca_type} "NA"
                fi
            else
                echo_fw_check ${LOOP_COUNT} ${ca_type} "NA"
            fi
        else
            echo_fw_check ${LOOP_COUNT} ${ca_type} "NA"
        fi
        let "LOOP_COUNT=$LOOP_COUNT + 1"
    done
else
    echo_fw_check ${LOOP_COUNT} ${ca_type} "${red}FAIL"
    echo "    REASON: no CAs in the system"
    EXIT_CODE=1
fi

# Check host driver initialization
HOST_DRIVER_INIT=0
if [ $NUM_HCAS -ne 0 ] && [ $RPM_CHECK_FAIL -eq 0 ]; then
    MODPROBE_OUT_FILE="/tmp/hca_self_test_modprobe.output"
    # Save the output of modprobe ib_ipoib in a tmp file
    modprobe ib_ipoib &> $MODPROBE_OUT_FILE
    let RET_CODE=$?
    if [ $RET_CODE -eq 0 ]; then
        echo -e "Host Driver Initialization ............. ${green}PASS"
        tput sgr0
        HOST_DRIVER_INIT=1
        # After successful initialization wait for IB SM sweep
        sleep 5
    else
        echo -e "Host Driver Initialization ............. ${red}FAIL"
        tput sgr0
        EXIT_CODE=1
        # "No such device"
        if [ `grep "No such device" $MODPROBE_OUT_FILE 2> /dev/null | wc -l` -ne 0 ]; then
            echo "    REASON: host driver initialization reported: No such device"
        fi
        # "No such file or directory"
        if [ `grep "No such file or directory" $MODPROBE_OUT_FILE 2> /dev/null | wc -l` -ne 0 ]; then
            echo "    REASON: host driver initialization reported: No such file or directory"
            echo "            It is possible that driver rpm might be missing file(s)"
        fi
        # "kernel-module version mismatch"
        if [ `grep "kernel-module version mismatch" $MODPROBE_OUT_FILE 2> /dev/null | wc -l` -ne 0 ]; then
            echo "    REASON: host driver initialization reported: kernel-module version mismatch"
        fi
        # "unresolved symbol"
        # Note: Could not test "unresolved symbol" error
        if [ `grep "unresolved symbol" $MODPROBE_OUT_FILE 2> /dev/null | wc -l` -ne 0 ]; then
            echo "    REASON: host driver initialization reported: unresolved symbol"
        fi
    fi
else
    echo "Host Driver Initialization ............. NA"
    EXIT_CODE=1
fi

function get_link_layer() {
    device_name=$1
    port_num=$2
    link_layer_file=$INFINI_CLASS_PATH/${device_name}/ports/${port_num}/link_layer
    # Default value
    link_layer=IB
    if [ -f ${link_layer_file} ]; then
        link_layer=`cat $link_layer_file 2> /dev/null`
    fi
    echo ${link_layer}
}

function report_port_state() {
    port_num=$1;
    LOOP_COUNT=$2
    device_name=$3
    ca_type=$4

    port_dir=$INFINI_CLASS_PATH/${device_name}/ports/${port_num}
    port_state_file=${port_dir}/state
    port_rate_file=${port_dir}/rate

if [ -f ${port_state_file} ]; then
    PORT_STATE=`awk -F": " '{print $2}' ${port_state_file} 2> /dev/null`
    link_layer=`get_link_layer ${device_name} ${port_num}`

    if [ "x$PORT_STATE" == "xACTIVE" ]; then
        PORT_SPEED=`awk -F\( '{print $2}' ${port_rate_file} 2> /dev/null | sed 's/)//'`
        state="${green}UP $PORT_SPEED"
    else
        if [ "x$PORT_STATE" == "xINIT" ]; then
            state="${cyan}INIT"
        else
            state="${red}DOWN"
        fi
    fi
    echo -e "Port State of Port #${port_num} on CA #$LOOP_COUNT (${ca_type})..... ${state} (${link_layer})"
    tput sgr0
fi

}

# Port info
if [ $HOST_DRIVER_INIT -eq 1 ]; then
    NUM_HCAS_PROC=`ls $INFINI_CLASS_PATH 2> /dev/null | wc -l`
    LOOP_COUNT=0
    NUM_PORT_ACTIVE=0

    mlx_dev_num=0
    mlx5_dev_num=0
    mthca_dev_num=0
    LOOP_COUNT=0

    # To take care of multiple HCAs
    while [ $LOOP_COUNT -lt $NUM_HCAS_PROC ]
    do
        driver_need=$(get_driver $LOOP_COUNT)
         if [ "$driver_need" != "" ]; then
             if [ "$driver_need" == "$HERMON_DRIVER_NEEDED" ]; then
                  device_num=$mlx_dev_num
                  let "mlx_dev_num=$mlx_dev_num + 1"
             elif [ "$driver_need" == "$CONNECTIB_DRIVER_NEEDED" ]; then
                  device_num=$mlx5_dev_num
                  let "mlx5_dev_num=$mlx5_dev_num + 1"
              else
                  device_num=$mthca_dev_num
                  let "mthca_dev_num=$mthca_dev_num + 1"
             fi

             if [ -f $INFINI_CLASS_PATH/$driver_need$device_num/ports/1/state ]; then
                let "NUM_PORT_ACTIVE+=`grep ACTIVE $INFINI_CLASS_PATH/$driver_need$device_num/ports/1/state 2> /dev/null | wc -l`"
             fi
             if [ -f $INFINI_CLASS_PATH/$driver_need$device_num/ports/2/state ]; then
                let "NUM_PORT_ACTIVE+=`grep ACTIVE $INFINI_CLASS_PATH/$driver_need$device_num/ports/2/state 2> /dev/null | wc -l`"
             fi
        fi

        let "LOOP_COUNT=$LOOP_COUNT + 1"
    done
    echo "Number of CA Ports Active .............. $NUM_PORT_ACTIVE"


    # Get each port state

    mlx_dev_num=0
    mthca_dev_num=0
    LOOP_COUNT=0
    mlx5_dev_num=0
    while [ $LOOP_COUNT -lt $NUM_HCAS_PROC ]
    do

        driver_need=$(get_driver $LOOP_COUNT)
        ca_type=$(get_ca_type $LOOP_COUNT)
        if [ "$driver_need" != "" ]; then
            if [ "$driver_need" == "$HERMON_DRIVER_NEEDED" ]; then
                device_num=$mlx_dev_num
                let "mlx_dev_num=$mlx_dev_num + 1"
            elif [ "$driver_need" == "$CONNECTIB_DRIVER_NEEDED" ]; then
                  device_num=$mlx5_dev_num
                  let "mlx5_dev_num=$mlx5_dev_num + 1"
            else
                device_num=$mthca_dev_num
                let "mthca_dev_num=$mthca_dev_num + 1"
            fi
            device_name=${driver_need}${device_num}
            report_port_state 1 ${LOOP_COUNT} ${device_name} ${ca_type}
            report_port_state 2 ${LOOP_COUNT} ${device_name} ${ca_type}
        fi
        let "LOOP_COUNT=$LOOP_COUNT + 1"
    done
else
    echo "Number of CA Ports Active .............. NA"
fi

# -D-

# Error counters check

echo_error_cnt () {
    LOOP_COUNT=$1
    ca_type=$2
    result=$3
    echo -e "Error Counter Check on CA #$LOOP_COUNT (${ca_type})...... ${result}"
    tput sgr0
}
if [ $HOST_DRIVER_INIT -eq 1 ]; then

    mlx_dev_num=0
    mthca_dev_num=0
    LOOP_COUNT=0
    mlx5_dev_num=0
    while [ $LOOP_COUNT -lt $NUM_HCAS_PROC ]; do

        driver_need=$(get_driver $LOOP_COUNT)
        ca_type=$(get_ca_type $LOOP_COUNT)
        if [ "$driver_need" != "" ]; then
            check_port1=1
            check_port2=1

            if [ "$driver_need" == "$HERMON_DRIVER_NEEDED" ]; then
                 device_num=$mlx_dev_num
                 let "mlx_dev_num=$mlx_dev_num + 1"
            elif [ "$driver_need" == "$CONNECTIB_DRIVER_NEEDED" ]; then
                  device_num=$mlx5_dev_num
                  let "mlx5_dev_num=$mlx5_dev_num + 1"
            else
                 device_num=$mthca_dev_num
                 let "mthca_dev_num=$mthca_dev_num + 1"
            fi

            for port in `seq 1 2`; do
                link_layer_file="$INFINI_CLASS_PATH/$driver_need$device_num/ports/$port/link_layer"
                if [ -f $link_layer_file ]; then
                    proto=`cat $link_layer_file 2> /dev/null`
                    if [ "$proto" == "Ethernet" ]; then
                        let "check_port$port=0"
                    fi
                fi
            done


            # Error counters check
            ERROR_COUNTER_PRINT=0
            if [ -f $INFINI_CLASS_PATH/$driver_need$device_num/ports/1/state ] && [ $check_port1 -eq 1 ]; then
                ERROR_COUNTER_PORT_1=0

                for i in $INFINI_CLASS_PATH/$driver_need$device_num/ports/1/counters/*error*; do
                    err_cnt=`cat $i 2> /dev/null`
                    RET_CODE=$?
                    if [ $RET_CODE -eq 0 ]; then
                        if [ $err_cnt -gt 20 ]; then
                            let "ERROR_COUNTER_PORT_1=$ERROR_COUNTER_PORT_1 + 1"
                        fi;
                    else
                        echo "-W- Failed to read $i file"
                    fi
                done
            fi
            if [ -f $INFINI_CLASS_PATH/$driver_need$device_num/ports/2/state ] && [ $check_port2 -eq 1 ]; then

                ERROR_COUNTER_PORT_2=0
                for i in $INFINI_CLASS_PATH/$driver_need$device_num/ports/2/counters/*error*; do
                    err_cnt=`cat $i 2> /dev/null`
                    RET_CODE=$?
                    if [ $RET_CODE -eq 0 ]; then
                        if [ $err_cnt -gt 20 ]; then

                            let "ERROR_COUNTER_PORT_2=$ERROR_COUNTER_PORT_2 + 1"
                        fi;
                    else
                        echo "-W- Failed to read $i file"
                    fi
                done

            else
                let ERROR_COUNTER_PORT_2=0
            fi

            let "ERROR_COUNTER=$ERROR_COUNTER_PORT_1 + $ERROR_COUNTER_PORT_2"
            # Print FAIL only once
            if [ $ERROR_COUNTER -ne 0 ] && [ $ERROR_COUNTER_PRINT -ne 1 ]; then
                echo_error_cnt ${LOOP_COUNT} ${ca_type} "${red}FAIL"
                echo "    REASON: found errors in the following counters"
                ERROR_COUNTER_PRINT=1
                EXIT_CODE=1
            fi

            # List the counters which are non-zero
            if [ $ERROR_COUNTER -ne 0 ]; then
                # Print only if error counters are non-zero of a specific IB port
                if [ $ERROR_COUNTER_PORT_1 -ne 0 ]; then
                    echo "      Errors in $INFINI_CLASS_PATH/$driver_need$device_num/ports/1/counters"

                    for i in $INFINI_CLASS_PATH/$driver_need$device_num/ports/1/counters/*error*; do
                        err_cnt=`cat $i 2> /dev/null`
                        RET_CODE=$?
                        if [ $RET_CODE -eq 0 ]; then
                            if [ $err_cnt -gt 20 ]; then
                                echo "         $(basename $i): $err_cnt";
                            fi;
                        else
                            echo "-W- Failed to read $i file"
                        fi
                    done

                fi

                if [ $ERROR_COUNTER_PORT_2 -ne 0 ]; then
                    echo "      Errors in $INFINI_CLASS_PATH/$driver_need$device_num/ports/2/counters"

                    for i in $INFINI_CLASS_PATH/$driver_need$device_num/ports/2/counters/*error*; do
                        err_cnt=`cat $i 2> /dev/null`
                        RET_CODE=$?
                        if [ $RET_CODE -eq 0 ]; then
                            if [ $err_cnt -gt 20 ]; then
                                echo "         $(basename $i): $err_cnt";
                            fi;
                        else
                            echo "-W- Failed to read $i file"
                        fi
                    done
                fi
            fi

            if [ $ERROR_COUNTER -eq 0 ]; then
                result="${green}PASS"
                if [ $check_port1 -ne 1 ] && [ $check_port2 -ne 1 ]; then
                    result="NA (Eth ports)"
                fi
                echo_error_cnt "${LOOP_COUNT}" "${ca_type}" "${result}"
            fi

            # Reset these variables for other HCAs
            let ERROR_COUNTER_PORT_1=0
            let ERROR_COUNTER_PORT_2=0


        else
            echo_error_cnt ${LOOP_COUNT} ${ca_type} "NA"
        fi
        let "LOOP_COUNT=$LOOP_COUNT + 1"
    done

else
    echo "Error Counter Check .................... NA"
fi


# Kernel syslog check
# Save the output of dmesg in a tmp file
if [ $HOST_DRIVER_INIT -eq 1 ]; then
    dmesg > /tmp/hca_self_test_dmesg.output
    VAPI_ERROR_COUNT=`egrep oom-\|"Out of Memory"\|tsIb\|VAPI\|THH_\|THHUL\|KERNEL_IB\|IB_NET\|MOD_LNX_SDP /tmp/hca_self_test_dmesg.output 2> /dev/null | grep -v 'SOCK: GETSOCKOPT unimplemented option <2>' | wc -l`
    OOPS_COUNT=`grep Oops /tmp/hca_self_test_dmesg.output 2> /dev/null | wc -l`
    KERNEL_PANIC_COUNT=`grep "Kernel panic" /tmp/hca_self_test_dmesg.output 2> /dev/null | wc -l`

    if [ $VAPI_ERROR_COUNT -eq 0 ] && [ $OOPS_COUNT -eq 0 ] && [ $KERNEL_PANIC_COUNT -eq 0 ]; then
        echo -e "Kernel Syslog Check .................... ${green}PASS"
        tput sgr0
    else
        echo -e "Kernel Syslog Check .................... ${red}FAIL"
        tput sgr0
        EXIT_CODE=1
        if [ $OOPS_COUNT -ne 0 ]; then
            echo "    REASON: Kernel syslog reported: Oops "
            grep Oops /tmp/hca_self_test_dmesg.output | uniq | awk -F'\n' '{print "      " $1 }'
        fi
        if [ $KERNEL_PANIC_COUNT -ne 0 ]; then
            echo "    REASON: Kernel syslog reported: Kernel panic "
            grep "Kernel panic" /tmp/hca_self_test_dmesg.output | uniq | awk -F'\n' '{print "      " $1 }'
        fi
        if [ $VAPI_ERROR_COUNT -ne 0 ]; then
            echo "    REASON: Kernel syslog reported: Driver messages "
            egrep oom-\|"Out of Memory"\|tsIb\|VAPI\|THH_\|THHUL\|KERNEL_IB\|IB_NET\|MOD_LNX_SDP /tmp/hca_self_test_dmesg.output | grep -v 'SOCK: GETSOCKOPT unimplemented option <2>' | uniq | awk -F'\n' '{print "      " $1 }'
        fi
    fi
else
    echo "Kernel Syslog Check .................... NA"
fi


#get the NODE Guide

if [ $NUM_HCAS -ne 0 ]; then
    mlx_dev_num=0
    mlx5_dev_num=0
    mthca_dev_num=0
    LOOP_COUNT=0

    # To take care of more than one HCA
    while [ $LOOP_COUNT -lt $NUM_HCAS ]
    do
        driver_need=$(get_driver $LOOP_COUNT)
        ca_type=$(get_ca_type $LOOP_COUNT)

        NODE_GUID="NA"
        if [ "$driver_need" != "" ]; then
            if [ "$driver_need" == "$HERMON_DRIVER_NEEDED" ]; then
                device_num=${mlx_dev_num}
                let "mlx_dev_num=$mlx_dev_num + 1"
            elif [ "$driver_need" == "$CONNECTIB_DRIVER_NEEDED" ]; then
                  device_num=$mlx5_dev_num
                  let "mlx5_dev_num=$mlx5_dev_num + 1"
            else
                device_num=${mthca_dev_num}
                let "mthca_dev_num=$mthca_dev_num + 1"
            fi

            if [ -f "$INFINI_CLASS_PATH/$driver_need$device_num/node_guid" ]; then
                NODE_GUID=$(sed 's/\([0-9a-f]\)\([0-9a-f]\)\([0-9a-f]\)\([0-9a-f]\)/\1\2:\3\4/g' < $INFINI_CLASS_PATH/$driver_need$device_num/node_guid)
            else
                PCI_DEVICE=$(lspci 2> /dev/null | grep Mellanox | head -$(expr $LOOP_COUNT + 1) | tail -1 | awk '{print $1}')
                NODE_GUID1=$(mstflint -d $PCI_DEVICE q 2> /dev/null | grep "GUIDs:" | awk '{print $2}' | sed 's/\([0-9a-f]\)\([0-9a-f]\)\([0-9a-f]\)\([0-9a-f]\)/\1\2:\3\4:/g' | cut -b -23)

                if [ "$NODE_GUID1" != "" ]; then
                    NODE_GUID=${NODE_GUID}
                fi
            fi
        fi
        echo "Node GUID on CA #$LOOP_COUNT (${ca_type}) ............... ${NODE_GUID}"
        let "LOOP_COUNT=$LOOP_COUNT + 1"
     done

fi
echo "------------------ DONE ---------------------"
echo
#rm -f /tmp/hca_self_test_modprobe.output
rm -f /tmp/hca_self_test_dmesg.output
rm -f ${G_LSPCI_OUTPUT_FILE}
exit $EXIT_CODE
