openvidu-deployment: Media Node health watchdog: do not count LiveKit failures while no master answers

The failure clock kept running during a master outage, so when a master answered
again the failure already looked older than the limit and the node was replaced at
once, before LiveKit could reconnect to the restored master. Seen live on AWS
elastic (ov-cloud-tester master-guard): after a 20 min master outage both Media
Nodes were replaced within 17 s of the master coming back. Now no failure is
counted while no master answers a Redis PING, so LiveKit gets the whole window
once a master is back.

openvidu-deployment/pro/elastic/aws/cf-openvidu-elastic.yaml

@@ -2173,8 +2173,8 @@ Resources:
                 #     the WARMUP_SEC after openvidu.service (re)started.
                 # Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage
                 # fails the health check of every Media Node at once, and a new node could not bootstrap
-                # either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until
+                # either; LiveKit failures are not counted until a master answers again. Once requested,
-                # the node goes away.
+                # the replacement is requested again every REPLACE_RETRY_SEC until the node goes away.
                 #
                 # Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...")
                 # Pause:    touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)
@@ -2313,6 +2313,14 @@ Resources:
                         esac
                         if openvidu_starting; then
                             FAIL_SINCE=""
+                        elif ! master_serving; then
+                            # A master outage fails the health check of every Media Node at once: not counted,
+                            # so LiveKit gets the whole window to reconnect once a master answers again
+                            FAIL_SINCE=""
+                            if [ -z "$HOLD_LOGGED_AT" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then
+                                log "LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted"
+                                HOLD_LOGGED_AT=$NOW
+                            fi
                         elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then
                             [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted"
                             SATURATED=1

openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.bicep

@@ -1068,8 +1068,8 @@ var openviduMediaHealthScript = '''
 #     the WARMUP_SEC after openvidu.service (re)started.
 # Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage
 # fails the health check of every Media Node at once, and a new node could not bootstrap
-# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until
+# either; LiveKit failures are not counted until a master answers again. Once requested,
-# the node goes away.
+# the replacement is requested again every REPLACE_RETRY_SEC until the node goes away.
 #
 # Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...")
 # Pause:    touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)
@@ -1208,6 +1208,14 @@ while true; do
         esac
         if openvidu_starting; then
             FAIL_SINCE=""
+        elif ! master_serving; then
+            # A master outage fails the health check of every Media Node at once: not counted,
+            # so LiveKit gets the whole window to reconnect once a master answers again
+            FAIL_SINCE=""
+            if [ -z "$HOLD_LOGGED_AT" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then
+                log "LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted"
+                HOLD_LOGGED_AT=$NOW
+            fi
         elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then
             [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted"
             SATURATED=1

openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf

@@ -1429,8 +1429,8 @@ EOF
 #     the WARMUP_SEC after openvidu.service (re)started.
 # Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage
 # fails the health check of every Media Node at once, and a new node could not bootstrap
-# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until
+# either; LiveKit failures are not counted until a master answers again. Once requested,
-# the node goes away.
+# the replacement is requested again every REPLACE_RETRY_SEC until the node goes away.
 #
 # Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...")
 # Pause:    touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)
@@ -1569,6 +1569,14 @@ while true; do
         esac
         if openvidu_starting; then
             FAIL_SINCE=""
+        elif ! master_serving; then
+            # A master outage fails the health check of every Media Node at once: not counted,
+            # so LiveKit gets the whole window to reconnect once a master answers again
+            FAIL_SINCE=""
+            if [ -z "$HOLD_LOGGED_AT" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then
+                log "LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted"
+                HOLD_LOGGED_AT=$NOW
+            fi
         elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then
             [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted"
             SATURATED=1

openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml

@@ -2816,8 +2816,8 @@ Resources:
                 #     the WARMUP_SEC after openvidu.service (re)started.
                 # Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage
                 # fails the health check of every Media Node at once, and a new node could not bootstrap
-                # either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until
+                # either; LiveKit failures are not counted until a master answers again. Once requested,
-                # the node goes away.
+                # the replacement is requested again every REPLACE_RETRY_SEC until the node goes away.
                 #
                 # Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...")
                 # Pause:    touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)
@@ -2956,6 +2956,14 @@ Resources:
                         esac
                         if openvidu_starting; then
                             FAIL_SINCE=""
+                        elif ! master_serving; then
+                            # A master outage fails the health check of every Media Node at once: not counted,
+                            # so LiveKit gets the whole window to reconnect once a master answers again
+                            FAIL_SINCE=""
+                            if [ -z "$HOLD_LOGGED_AT" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then
+                                log "LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted"
+                                HOLD_LOGGED_AT=$NOW
+                            fi
                         elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then
                             [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted"
                             SATURATED=1

openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep

@@ -1461,8 +1461,8 @@ var openviduMediaHealthScript = '''
 #     the WARMUP_SEC after openvidu.service (re)started.
 # Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage
 # fails the health check of every Media Node at once, and a new node could not bootstrap
-# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until
+# either; LiveKit failures are not counted until a master answers again. Once requested,
-# the node goes away.
+# the replacement is requested again every REPLACE_RETRY_SEC until the node goes away.
 #
 # Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...")
 # Pause:    touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)
@@ -1601,6 +1601,14 @@ while true; do
         esac
         if openvidu_starting; then
             FAIL_SINCE=""
+        elif ! master_serving; then
+            # A master outage fails the health check of every Media Node at once: not counted,
+            # so LiveKit gets the whole window to reconnect once a master answers again
+            FAIL_SINCE=""
+            if [ -z "$HOLD_LOGGED_AT" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then
+                log "LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted"
+                HOLD_LOGGED_AT=$NOW
+            fi
         elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then
             [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted"
             SATURATED=1

openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf

@@ -1813,8 +1813,8 @@ EOF
 #     the WARMUP_SEC after openvidu.service (re)started.
 # Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage
 # fails the health check of every Media Node at once, and a new node could not bootstrap
-# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until
+# either; LiveKit failures are not counted until a master answers again. Once requested,
-# the node goes away.
+# the replacement is requested again every REPLACE_RETRY_SEC until the node goes away.
 #
 # Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...")
 # Pause:    touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)
@@ -1953,6 +1953,14 @@ while true; do
         esac
         if openvidu_starting; then
             FAIL_SINCE=""
+        elif ! master_serving; then
+            # A master outage fails the health check of every Media Node at once: not counted,
+            # so LiveKit gets the whole window to reconnect once a master answers again
+            FAIL_SINCE=""
+            if [ -z "$HOLD_LOGGED_AT" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then
+                log "LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted"
+                HOLD_LOGGED_AT=$NOW
+            fi
         elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then
             [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted"
             SATURATED=1