#!/bin/bash
### PROTONYX: LOAD MASTERCONF ###
. /etc/protonyx.conf
################################
. /etc/kerbynet.conf
if [ "$DEBUG" = 0 ]; then
  ERROUT=/dev/null
else
  ERROUT="$DBGFILE"
fi
function pingDOWN {
#echo xxx1 >> "$ERROUT"
  GW=$1
  echo PROBEDOWN
  TC=`cat $GW/TC 2>>"$ERROUT"`
  [ -z "$TC" ] && TC=1
  TO=$((Timeout*TC))
  echo Timeout $TO
#echo xxx2 >> "$ERROUT"
##echo pd=$ProbesDOWN
  for R in `seq 1 $ProbesDOWN` ; do
#echo xxx3 >> "$ERROUT"
#echo gw=$GW
GWNUM=`echo $GW |sed 's/^0*//'`
#echo newgw=$GWNUM
#echo success=${SUCCESS[$GWNUM]}
#echo ips=$IPS
    for I in ${SUCCESS[$GWNUM]} $IPS ; do
#echo xxx4 >> "$ERROUT"
      IPP=${IP[$I]}
      ip ru add tos 0x04 to ${IP[$I]} ta 1$GW
      if ping -Q 0x04 -c 1 -w $TO ${IP[$I]} ; then
        ip ru del tos 0x04 2>>"$ERROUT"
        SUCCESS[$GWNUM]=$I
        return 0
      fi
      ip ru del tos 0x04 2>>"$ERROUT"
    done
#echo xxx5 >> "$ERROUT"
  done
#echo xxx6 >> "$ERROUT"
  return 1
}
function pingUP {
  GW=$1
  GWNUM=`echo $GW |sed 's/^0*//'`
  echo PROBEUP
  for I in ${SUCCESS[$GWNUM]} $IPS ; do
    IPP=${IP[$I]}
    ip ru add tos 0x04 to ${IP[$I]} ta 1$GW
    R=0
    ERROR=""
    while [ $R -lt $ProbesUP -a -z "$ERROR" ] ; do
      if ! ping -Q 0x04 -c 1 -w $Timeout ${IP[$I]} ; then
        ERROR=yes
      else
        R=$((R+1))
      fi
      
    done
    ip ru del tos 0x04 2>>"$ERROUT"
    if [ -z "$ERROR" ] ; then
      SUCCESS[$GWNUM]=$I
      return 0
    fi
  done
  return 1
}
function KillPPPD {
  xPPP="$1"
  xI=0
  PID=`ps -ef |grep -w pppd |grep $xPPP| awk '{print $2}'`
  while [ -n "$PID" -a $xI -lt 15 ] ; do
    kill $PID
    sleep 1
    xI=$((xI+1))
    PID=`ps -ef |grep -w pppd |grep $xPPP| awk '{print $2}'`
  done
  if [ -n "$PID" ] ; then
     kill -9 $PID
     sleep 1
     rm -f /var/run/$xPPP.pid
  fi
}
function SetTime {
  NOW=`date +%s`
  TIME=`cat $STATUSDIR/$G/Time 2>>"$ERROUT"`
  echo $NOW > $STATUSDIR/$G/Time
  if [ -n "$TIME" ] ; then
     TIME=$((NOW-TIME))
  else
     TIME=0
  fi
  [ -r $STATUSDIR/Outage ] || echo 0 > $STATUSDIR/Outage
  [ -r $STATUSDIR/UpTime ] || echo 0 > $STATUSDIR/UpTime
}
function SetFailure {
  SetTime
  UPTIME=`cat $STATUSDIR/$G/UpTime 2>>"$ERROUT"`
  [ -z "$UPTIME" ] && UPTIME=0
  UPTIME=$((UPTIME+TIME))
  echo $UPTIME > $STATUSDIR/$G/UpTime
  FAILURE=`cat $STATUSDIR/$G/Failure 2>>"$ERROUT"`
  [ -z "$FAILURE" ] && FAILURE=0
  FAILURE=$((FAILURE+1))
  echo $FAILURE > $STATUSDIR/$G/Failure
  W=`cat $G/IP 2>>"$ERROUT"`
  if [ -z "$W" ] ; then
    W="Interface: `cat $G/Interface 2>>"$ERROUT"`"
  else
    W="Gateway: $W"
  fi
  DESCRIPTION="`cat $G/Description 2>>"$ERROUT"` ($W) [Last Uptime: `$SCRIPTS/tick2time $TIME`]"
  echo "DOWN: $DESCRIPTION"
  logger -t NetBalancer "FAULT: $DESCRIPTION"
  ### PROTONYX POINT-START ###
  GWNUM=`echo $GW |sed 's/^0*//'`
  if [ $GWNUM -ge 6 ] ; then
     GWNAME=$((GWNUM-1))
  else
    if [ $GWNUM -eq 5 ] ; then
       GWNAME="VLL"
    else
       GWNAME="$GWNUM"
    fi
  fi
  DATEEPOCH=`date +%s`
  echo "$DATEEPOCH" line$GWNAME down >> "$QUALITYPATH"/"$RRDEVENTFILE" 2>&1
  if [ -d /opt/alerts ]; then
    # proceed as long as the zabbix user exists
    id -u zabbix > /dev/null 2>&1
    if [ $? -eq 0 ]; then
      if [ $GWNUM -ne 5 ]; then
        #touch /opt/alerts/legs/line$GWNAME/disconnects/$DATEEPOCH
        if [ ! -d /opt/alerts/legs/$GW/disconnects ]; then
	  if [ ! -d /opt/alerts/legs/$GW ]; then
	    if [ ! -d /opt/alerts/legs ]; then
	      mkdir /opt/alerts/legs
	      chown zabbix:zabbix /opt/alerts/legs
	    fi
	    mkdir /opt/alerts/legs/$GW
	    chown zabbix:zabbix /opt/alerts/legs/$GW
	  fi
          mkdir -p /opt/alerts/legs/$GW/disconnects
	  chown zabbix:zabbix /opt/alerts/legs/$GW/disconnects
        fi
        touch /opt/alerts/legs/$GW/disconnects/$DATEEPOCH
        #echo $DATEEPOCH > /opt/alerts/legs/line$GWNAME/down
        echo $DATEEPOCH > /opt/alerts/legs/$GW/down
      fi
    fi
  fi
  echo `date '+%Y-%m-%d %H:%M:%S'` line$GWNAME down >> /opt/quality/legStatus.log
  NOWDATE=`date`
  $SCRIPTS/nb_setnexthop
  /opt/zs-scripts/mailtest.sh "Line-$GWNAME has gone down on $NOWDATE" "$SITENAME"
  #/opt/zs-scripts/notify_zabbix.sh $GWNAME DOWN
  ### PROTONYX POINT-END  ###
  #$SCRIPTS/nb_setnexthop
  if ! ip ro |grep -q default ; then
    logger -t NetBalancer "WARNING: connectivity unavailable"
  fi
}
PIDS=`ps -e |grep -w failoverd|awk '{print $1}'`
#echo kolos1 >>"$ERROUT"
for P in $PIDS ; do
  if [ "$P" != "$$" ] ; then
    kill  $P 2>>"$ERROUT" 
  fi
done
while ps -e | grep -w -v $$ | grep -q -w failoverd ; do
  sleep 0.5
done
#echo kolos2 >>"$ERROUT"
# [ "$NBSTARTUP" = yes ] && sleep 30

#echo kolos2,1 >>"$ERROUT"
CONFIG=$REGISTER/system/net/nb
STATUSDIR="$TMPSTATUS"/Gateways
ip ru del tos 0x04 2>>"$ERROUT"
ip ru del tos 0x04 2>>"$ERROUT"
ip ru del tos 0x04 2>>"$ERROUT"
ip ru del tos 0x04 2>>"$ERROUT"
#echo kolos2,8 >>"$ERROUT"
if [ "`cat $CONFIG/Enabled 2>>"$ERROUT"`" != yes ] ; then
  exit
fi
#echo kolos3 >>"$ERROUT"
if [ "`cat $CONFIG/FO/ICMPCheck 2>>"$ERROUT"`" != yes ] ; then
  logger -t NetBalancer "Failover monitor has been DISABLED"
  exit
else
  logger -t NetBalancer "Failover monitor has started"
fi
#echo kolos4 >>"$ERROUT"
ProbesDOWN="`cat $CONFIG/FO/ProbesDOWN 2>>"$ERROUT"`"
ProbesUP="`cat $CONFIG/FO/ProbesUP 2>>"$ERROUT"`"
Timeout="`cat $CONFIG/FO/Timeout 2>>"$ERROUT"`"
Pause="`cat $CONFIG/FO/Pause 2>>"$ERROUT"`"
PPPDRestart="`cat $CONFIG/FO/PPPDRestart 2>>"$ERROUT"`"
IP[1]="`cat $CONFIG/FO/IP1 2>>"$ERROUT"`"
IP[2]="`cat $CONFIG/FO/IP2 2>>"$ERROUT"`"
IP[3]="`cat $CONFIG/FO/IP3 2>>"$ERROUT"`"
EnabledIP[1]="`cat $CONFIG/FO/EnabledIP1 2>>"$ERROUT"`"
EnabledIP[2]="`cat $CONFIG/FO/EnabledIP2 2>>"$ERROUT"`"
EnabledIP[3]="`cat $CONFIG/FO/EnabledIP3 2>>"$ERROUT"`"
cd $CONFIG/Gateways
GWS=""
for G in `ls -d ??` ; do
  if [ "`cat $G/Enabled 2>>"$ERROUT"`" = yes ] ; then
    GWS="$GWS $G"
  else
    rm -f $STATUSDIR/$G/{Fault,Time,Outage,UpTime}
  fi
  GWNUM=`echo $G |sed 's/^0*//'`
  SUCCESS[$GWNUM]=""
done
IPS=""
for I in 1 2 3 ; do
  if [ "${EnabledIP[$I]}" = yes ] ; then
    IPS="$IPS $I"
  fi
done
if [ -z "$IPS" ] ; then
  logger -t NetBalancer "Failover monitor has not started because no Failover IP are enabled"
  exit
fi
#echo kolos5 >>"$ERROUT"
while true ; do
  #echo kolos6 >>"$ERROUT"
  for G in $GWS ; do
    #echo "++++++++ TESTING `cat $G/Description 2>>"$ERROUT"`"
    #echo kolos7 >>"$ERROUT"
    if ip ro sh ta 1$G | grep -q ^default  ; then
      #echo kolos7.1 >>"$ERROUT"
      if [ -r $STATUSDIR/$G/Fault ] ; then
         #echo kolos7.11 >>"$ERROUT"
         if pingUP $G ; then
           rm -f $STATUSDIR/$G/Fault
           W=`cat $G/IP 2>>"$ERROUT"`
           if [ -z "$W" ] ; then
             W="Interface: `cat $G/Interface 2>>"$ERROUT"`"
           else
             W="Gateway: $W"
           fi
           SetTime
           OUTAGE=`cat $STATUSDIR/$G/Outage 2>>"$ERROUT"`
           [ -z "$OUTAGE" ] && OUTAGE=0
           OUTAGE=$((OUTAGE+TIME))
           echo $OUTAGE > $STATUSDIR/$G/Outage
           DESCRIPTION="`cat $G/Description 2>>"$ERROUT"` ($W) [Last Downtime: `$SCRIPTS/tick2time $TIME`]"
           echo "UP: $DESCRIPTION"
           logger -t NetBalancer "RECOVERED: $DESCRIPTION"
           ### PROTONYX POINT-START ###
	   GWNUM=`echo $GW |sed 's/^0*//'`
           if [ $GWNUM -ge 6 ] ; then
             GWNAME=$((GWNUM-1))
           else
             if [ $GWNUM -eq 5 ] ; then
               GWNAME="VLL"
             else
               GWNAME="$GWNUM"
             fi
           fi
	   echo `date +%s` line$GWNAME up >> "$QUALITYPATH"/"$RRDEVENTFILE" 2>&1
	   #rm -rf /opt/alerts/legs/line$GWNAME/down
	   if [ -d /opt/alerts ]; then
	     if [ $GWNUM -ne 5 ]; then
	       rm -rf /opt/alerts/legs/$GW/down
	     fi
	   fi
	   echo `date '+%Y-%m-%d %H:%M:%S'` line$GWNAME up >> /opt/quality/legStatus.log
           NOWDATE=`date`
           $SCRIPTS/nb_setnexthop
           /opt/zs-scripts/mailtest.sh "Line-$GWNAME has come back up on $NOWDATE" "$SITENAME"
	   #/opt/zs-scripts/notify_zabbix.sh $GWNAME UP 
           ### PROTONYX POINT-END  ###
           #$SCRIPTS/nb_setnexthop
         fi
      else
         #echo kolos7.12 >>"$ERROUT"
         if ! pingDOWN $G ; then
#echo kolos7.121 >>"$ERROUT"
           touch $STATUSDIR/$G/Fault
           SetFailure 
#echo kolos7.121.1 >>"$ERROUT"
           if [ "$PPPDRestart" = yes ] ; then
              INTERFACE=`cat $G/Interface 2>>"$ERROUT"`
              if [ "${INTERFACE:0:3}" = ppp ] ; then 
                KillPPPD $INTERFACE
                logger -t NetBalancer "Restarting $INTERFACE (`cat $REGISTER/system/net/interfaces/$INTERFACE/Description 2>>"$ERROUT"`) PPPoE/3G connection"
              fi
           fi
         else
#echo kolos7.122 >>"$ERROUT"
           [ -r "$STATUSDIR/$G/Time" ] || SetTime 
#echo kolos7.123 >>"$ERROUT"
         fi
      fi
    else
      #echo kolos7.2 >>"$ERROUT"
      if ! [ -r $STATUSDIR/$G/Fault ] ; then
        #echo kolos7.21 >>"$ERROUT"
        touch $STATUSDIR/$G/Fault
        logger -t NetBalancer "No link detected with the gateway (`cat $G/Description 2>>"$ERROUT"`)"
        SetFailure
      fi
    fi
  done
  sleep $Pause
done
