Initial commit

customcd/files/root/rescuebru/faultmonitor.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+# faultmonitor.sh
+# Rescue CD Backup & Restore Utility fault monitoring script
+# version 7.2.5
+# Rod Wright 1/19/2024
+# Refer to CHANGELOG file for details.
+# -----------------------------------------------------------------------------
+
+CAPACITY_WARN=90
+STATUS_FILE="/root/faultstatus.txt"
+
+function checkraid() {
+    # check for RAID failure
+    # takes one argument, a full raid device name
+    local device=$1
+    if [ "$device" != "none" ]; then
+        if mdadm -D $device | grep -q degraded ; then
+            return 1
+        else
+            return 0
+        fi
+    else
+        return 0
+    fi
+}
+
+function checksmart() {
+    # check for RAID member SMART failure
+    # takes a list of RAID partition full device names as positional parameters
+    for drive in $* ; do
+        if echo $drive|grep -q nvme ; then
+            nvmesmartstat=`nvme smart-log $drive|grep critical_warning|cut -d ":" -f2|sed 's/^ *//g'`
+            if [ $nvmesmartstat -ne 0 ]; then return 1 ;fi
+        else
+            smartctl -q silent $drive
+            local smartstat=$(($? & 8))
+            if [ $smartstat -ne 0 ]; then return 1 ;fi
+        fi
+    done
+    return 0
+}
+
+function checkcapacity() {
+    # check for RAID usage above a threshold
+    # takes two arguments, a RAID device and a percentage
+    local device=$1
+    local threshold=$2
+    if [ "$device" != "none" ]; then
+        local pctstrg=`df $device |grep $device|awk '{print $5}'`
+        local pct=${pctstrg//%}
+        if [ $pct -ge $threshold ] ; then
+            return 1
+        else
+            return 0
+        fi
+    else
+        return 0
+    fi
+}
+
+function flashuid() {
+    # turn flashing UID light on or off
+    # takes one argument, the word "on" or "off"
+    # determine if ipmi hardware is supported and return failure if not
+    if ! ipmitool chassis status >/dev/null 2>&1; then return 1 ; fi
+    if [ "$1" = "on" ]; then
+        local ipmiarg="force"
+    elif [ "$1" = "off" ]; then
+        local ipmiarg="0"
+    else
+        # invalid argument make no change and return failure
+        return 1
+    fi
+    ipmitool chassis identify $ipmiarg
+}
+
+function update_faultstatusfile() {
+    # maintain the fault status file
+    if [ "$raiddevname" != "none" ]; then
+        if [ $raidfail ]; then
+            local raidline="RAID Status :   \033[1;31mFAIL\033[0m"
+        else
+            local raidline="RAID Status :     \033[1;32mOK\033[0m"
+        fi
+    else
+        local raidline="RAID Status :   \033[1;33mWAIT\033[0m"
+    fi
+    if [ $smartfail ]; then
+        local smartline="Disk Status :   \033[1;31mFAIL\033[0m"
+    else
+        local smartline="Disk Status :     \033[1;32mOK\033[0m"
+    fi
+    if [ "$raiddevname" != "none" ]; then
+        if [ $lowspace ]; then
+            local capline="Free Space  :    \033[1;31mLOW\033[0m"
+        else
+            local capline="Free Space  :     \033[1;32mOK\033[0m"
+        fi
+    else
+        local capline="Free Space  :   \033[1;33mWAIT\033[0m"
+    fi
+    printf "$raidline \n$smartline \n$capline \n">$STATUS_FILE
+}
+
+
+# generate a list of RAID component drives
+for raiddrv in `cat /proc/mdstat|grep -o -e "sd[a-z]|nvme[0-9]n[0-9]"`; do
+    raidmembers=$raidmembers" /dev/$raiddrv"
+done
+unset raidfail
+unset smartfail
+unset lowspace
+
+# main loop
+while true ; do
+    # determine the RAID device name and see if it's mounted
+    if mount | grep -q "/dev/md" ; then
+        raiddevname=`mount |grep "/dev/md"|awk '{print $1}'`
+    else
+        raiddevname="none"
+    fi
+    
+    # run the tests
+    if ! checkraid $raiddevname ; then raidfail=1 ; else unset raidfail ; fi
+    if ! checksmart $raidmembers ; then smartfail=1 ; else unset smartfail ; fi
+    if ! checkcapacity $raiddevname $CAPACITY_WARN ; then lowspace=1 ; else unset lowspace ; fi
+    if [ $raidfail -o $smartfail -o $lowspace ]; then flashuid on ; else flashuid off ; fi
+    update_faultstatusfile
+    sleep 60
+done