mirror of https://github.com/OpenVidu/openvidu.git
openvidu-deployment: Media Node health watchdog: withdraw the unhealthy mark when the replacement request fails
The replacement script marks the node unhealthy (/run/openvidu-unhealthy) before it asks the cloud for a new node, so that the node then leaves without waiting for its sessions. When the request failed (a cloud API error, a missing permission) the mark stayed: the node remained a member and, if LiveKit recovered, a later normal scale-in would drop its sessions instead of draining them. The mark is now withdrawn on any failed exit of the request.pull/917/head
parent
ba72e6b969
commit
5d0a4ac300
|
|
@ -2353,6 +2353,9 @@ Resources:
|
|||
echo "$(date -u +%FT%TZ) $1" >> /var/lib/openvidu-media-health/replace-requests.log
|
||||
# Let stop_media_node.sh know this node is unhealthy, so it does not wait for sessions
|
||||
touch /run/openvidu-unhealthy
|
||||
# Withdrawn if the request fails: the node then stays, and if it recovers
|
||||
# a later scale-in must still wait for its sessions
|
||||
trap '[ $? -eq 0 ] || rm -f /run/openvidu-unhealthy' EXIT
|
||||
|
||||
# Token for IMDSv2
|
||||
TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
|
||||
|
|
|
|||
|
|
@ -1263,6 +1263,9 @@ var media_replaceScriptMediaTemplate = '''
|
|||
mkdir -p /var/lib/openvidu-media-health
|
||||
echo "$(date -u +%FT%TZ) $1" >> /var/lib/openvidu-media-health/replace-requests.log
|
||||
touch /run/openvidu-unhealthy
|
||||
# Withdrawn if the request fails: the node then stays, and if it recovers
|
||||
# a later scale-in must still wait for its sessions
|
||||
trap '[ $? -eq 0 ] || rm -f /run/openvidu-unhealthy' EXIT
|
||||
|
||||
RESOURCE_GROUP_NAME=${resourceGroupName}
|
||||
VM_SCALE_SET_NAME=${vmScaleSetName}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -1610,6 +1610,9 @@ mkdir -p /var/lib/openvidu-media-health
|
|||
echo "$(date -u +%FT%TZ) $REASON" >> /var/lib/openvidu-media-health/replace-requests.log
|
||||
# Tell the drain script this node is being replaced so it skips waiting for sessions.
|
||||
touch /run/openvidu-unhealthy
|
||||
# Withdrawn if the request fails: the node then stays, and if it recovers
|
||||
# a later scale-in must still wait for its sessions
|
||||
trap '[ $? -eq 0 ] || rm -f /run/openvidu-unhealthy' EXIT
|
||||
|
||||
META="http://metadata.google.internal/computeMetadata/v1"
|
||||
meta() { curl -s -H "Metadata-Flavor: Google" "$META/$1"; }
|
||||
|
|
|
|||
|
|
@ -2996,6 +2996,9 @@ Resources:
|
|||
echo "$(date -u +%FT%TZ) $1" >> /var/lib/openvidu-media-health/replace-requests.log
|
||||
# Let stop_media_node.sh know this node is unhealthy, so it does not wait for sessions
|
||||
touch /run/openvidu-unhealthy
|
||||
# Withdrawn if the request fails: the node then stays, and if it recovers
|
||||
# a later scale-in must still wait for its sessions
|
||||
trap '[ $? -eq 0 ] || rm -f /run/openvidu-unhealthy' EXIT
|
||||
|
||||
# Token for IMDSv2
|
||||
TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
|
||||
|
|
|
|||
|
|
@ -1656,6 +1656,9 @@ var media_replaceScriptMediaTemplate = '''
|
|||
mkdir -p /var/lib/openvidu-media-health
|
||||
echo "$(date -u +%FT%TZ) $1" >> /var/lib/openvidu-media-health/replace-requests.log
|
||||
touch /run/openvidu-unhealthy
|
||||
# Withdrawn if the request fails: the node then stays, and if it recovers
|
||||
# a later scale-in must still wait for its sessions
|
||||
trap '[ $? -eq 0 ] || rm -f /run/openvidu-unhealthy' EXIT
|
||||
|
||||
RESOURCE_GROUP_NAME=${resourceGroupName}
|
||||
VM_SCALE_SET_NAME=${vmScaleSetName}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -1994,6 +1994,9 @@ mkdir -p /var/lib/openvidu-media-health
|
|||
echo "$(date -u +%FT%TZ) $REASON" >> /var/lib/openvidu-media-health/replace-requests.log
|
||||
# Tell the drain script this node is being replaced so it skips waiting for sessions.
|
||||
touch /run/openvidu-unhealthy
|
||||
# Withdrawn if the request fails: the node then stays, and if it recovers
|
||||
# a later scale-in must still wait for its sessions
|
||||
trap '[ $? -eq 0 ] || rm -f /run/openvidu-unhealthy' EXIT
|
||||
|
||||
META="http://metadata.google.internal/computeMetadata/v1"
|
||||
meta() { curl -s -H "Metadata-Flavor: Google" "$META/$1"; }
|
||||
|
|
|
|||
Loading…
Reference in New Issue