openvidu-deployment: Media Node health watchdog: withdraw the unhealthy mark when the replacement request fails

The replacement script marks the node unhealthy (/run/openvidu-unhealthy) before it asks
the cloud for a new node, so that the node then leaves without waiting for its sessions.
When the request failed (a cloud API error, a missing permission) the mark stayed: the
node remained a member and, if LiveKit recovered, a later normal scale-in would drop its
sessions instead of draining them. The mark is now withdrawn on any failed exit of the
request.
pull/917/head
Piwccle 2026-10-16 13:02:11 +02:00
parent ba72e6b969
commit 5d0a4ac300
8 changed files with 22 additions and 4 deletions

View File

@ -2353,6 +2353,9 @@ Resources:
echo "$(date -u +%FT%TZ) $1" >> /var/lib/openvidu-media-health/replace-requests.log
# Let stop_media_node.sh know this node is unhealthy, so it does not wait for sessions
touch /run/openvidu-unhealthy
# Withdrawn if the request fails: the node then stays, and if it recovers
# a later scale-in must still wait for its sessions
trap '[ $? -eq 0 ] || rm -f /run/openvidu-unhealthy' EXIT
# Token for IMDSv2
TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")

View File

@ -1263,6 +1263,9 @@ var media_replaceScriptMediaTemplate = '''
mkdir -p /var/lib/openvidu-media-health
echo "$(date -u +%FT%TZ) $1" >> /var/lib/openvidu-media-health/replace-requests.log
touch /run/openvidu-unhealthy
# Withdrawn if the request fails: the node then stays, and if it recovers
# a later scale-in must still wait for its sessions
trap '[ $? -eq 0 ] || rm -f /run/openvidu-unhealthy' EXIT
RESOURCE_GROUP_NAME=${resourceGroupName}
VM_SCALE_SET_NAME=${vmScaleSetName}

File diff suppressed because one or more lines are too long

View File

@ -1610,6 +1610,9 @@ mkdir -p /var/lib/openvidu-media-health
echo "$(date -u +%FT%TZ) $REASON" >> /var/lib/openvidu-media-health/replace-requests.log
# Tell the drain script this node is being replaced so it skips waiting for sessions.
touch /run/openvidu-unhealthy
# Withdrawn if the request fails: the node then stays, and if it recovers
# a later scale-in must still wait for its sessions
trap '[ $? -eq 0 ] || rm -f /run/openvidu-unhealthy' EXIT
META="http://metadata.google.internal/computeMetadata/v1"
meta() { curl -s -H "Metadata-Flavor: Google" "$META/$1"; }

View File

@ -2996,6 +2996,9 @@ Resources:
echo "$(date -u +%FT%TZ) $1" >> /var/lib/openvidu-media-health/replace-requests.log
# Let stop_media_node.sh know this node is unhealthy, so it does not wait for sessions
touch /run/openvidu-unhealthy
# Withdrawn if the request fails: the node then stays, and if it recovers
# a later scale-in must still wait for its sessions
trap '[ $? -eq 0 ] || rm -f /run/openvidu-unhealthy' EXIT
# Token for IMDSv2
TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")

View File

@ -1656,6 +1656,9 @@ var media_replaceScriptMediaTemplate = '''
mkdir -p /var/lib/openvidu-media-health
echo "$(date -u +%FT%TZ) $1" >> /var/lib/openvidu-media-health/replace-requests.log
touch /run/openvidu-unhealthy
# Withdrawn if the request fails: the node then stays, and if it recovers
# a later scale-in must still wait for its sessions
trap '[ $? -eq 0 ] || rm -f /run/openvidu-unhealthy' EXIT
RESOURCE_GROUP_NAME=${resourceGroupName}
VM_SCALE_SET_NAME=${vmScaleSetName}

File diff suppressed because one or more lines are too long

View File

@ -1994,6 +1994,9 @@ mkdir -p /var/lib/openvidu-media-health
echo "$(date -u +%FT%TZ) $REASON" >> /var/lib/openvidu-media-health/replace-requests.log
# Tell the drain script this node is being replaced so it skips waiting for sessions.
touch /run/openvidu-unhealthy
# Withdrawn if the request fails: the node then stays, and if it recovers
# a later scale-in must still wait for its sessions
trap '[ $? -eq 0 ] || rm -f /run/openvidu-unhealthy' EXIT
META="http://metadata.google.internal/computeMetadata/v1"
meta() { curl -s -H "Metadata-Flavor: Google" "$META/$1"; }