mirror of https://github.com/OpenVidu/openvidu.git
openvidu-deployment: replace unhealthy Media Nodes (AWS, GCP, Azure elastic and HA)
Every Media Node runs openvidu-media-health, a watchdog that asks the cloud to replace it with a new node (keeping the Media Node count) when its bootstrap failed or never finished, or when LiveKit fails its health check on :7880 for 5 minutes (down) or 10 minutes (errors or timeouts; not counted while the CPU is saturated). Nothing is replaced while no master answers a Redis PING, so a master outage cannot replace the whole media fleet. Drain scripts pause the watchdog, and skip waiting for sessions on a node being replaced. - AWS: the node marks itself Unhealthy in its Auto Scaling group (the lifecycle hook drains it and a new instance is launched). aws-cli is installed before the watchdog starts; set_as_unhealthy.sh is gone. - GCP: the node asks its managed instance group to recreate it. Media autohealing (TCP 7880) now waits 3600 s before judging a new node and needs 5 minutes of failures; HA gets the same autohealing plus the firewall rule for the health check ranges. - Azure: the node reimages itself (new OS disk, bootstrap runs again, same instance, scale-in protection kept) instead of deleting itself, which shrank the scale set. The .json twins are rebuilt from the .bicep.media-node-health
Piwccle
parent
b372409ccf
commit
bf54f336e0
|
|
@ -2105,7 +2105,6 @@ Resources:
|
|||
SECRETS_WAIT_ATTEMPTS=$((SECRETS_WAIT_ATTEMPTS + 1))
|
||||
if [[ $SECRETS_WAIT_ATTEMPTS -ge $SECRETS_WAIT_MAX ]]; then
|
||||
echo "Error: timed out after 30 minutes waiting for the master node to generate the shared secrets"
|
||||
/usr/local/bin/set_as_unhealthy.sh
|
||||
exit 1
|
||||
fi
|
||||
sleep 5
|
||||
|
|
@ -2157,28 +2156,238 @@ Resources:
|
|||
mode: "000755"
|
||||
owner: "root"
|
||||
group: "root"
|
||||
'/usr/local/bin/set_as_unhealthy.sh':
|
||||
'/usr/local/bin/openvidu-media-health.sh':
|
||||
content: |
|
||||
#!/bin/bash
|
||||
# OpenVidu Media Node health watchdog.
|
||||
#
|
||||
# Asks the cloud to replace this Media Node with a new one, keeping the number of
|
||||
# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when:
|
||||
# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a
|
||||
# cool-down, so a persistent failure does not churn nodes and can be inspected,
|
||||
# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot,
|
||||
# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC
|
||||
# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC
|
||||
# while it answers an error or times out. Soft failures are not counted while the CPU
|
||||
# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in
|
||||
# the WARMUP_SEC after openvidu.service (re)started.
|
||||
# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage
|
||||
# fails the health check of every Media Node at once, and a new node could not bootstrap
|
||||
# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until
|
||||
# the node goes away.
|
||||
#
|
||||
# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...")
|
||||
# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)
|
||||
# Logs: journalctl -u openvidu-media-health
|
||||
#
|
||||
# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and
|
||||
# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed
|
||||
# by an opening brace, nor two opening braces in a row.
|
||||
|
||||
STATE_DIR=/var/lib/openvidu-media-health
|
||||
INSTALLED_MARKER="$STATE_DIR/installed"
|
||||
BOOTSTRAP_FAILED_MARKER="$STATE_DIR/bootstrap-failed"
|
||||
REPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh
|
||||
ENV_FILE=/etc/openvidu/media-health.env
|
||||
PAUSE_FILE=/etc/openvidu/media-health.disabled
|
||||
DRAIN_PAUSE_FILE=/run/openvidu-media-health.paused
|
||||
|
||||
log() { echo "[media-health] $*"; }
|
||||
|
||||
# Seconds since boot: monotonic, unaffected by clock steps
|
||||
uptime_sec() { cut -d. -f1 /proc/uptime; }
|
||||
|
||||
# Busy CPU percentage since the previous call, in BUSY (0 on the first call)
|
||||
cpu_busy() {
|
||||
read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat
|
||||
total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal))
|
||||
idle=$((c_idle + c_iowait))
|
||||
BUSY=0
|
||||
if [ -n "$PREV_TOTAL" ] && [ "$total" -gt "$PREV_TOTAL" ]; then
|
||||
BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL)))
|
||||
fi
|
||||
PREV_TOTAL=$total
|
||||
PREV_IDLE=$idle
|
||||
}
|
||||
|
||||
# openvidu.service entered the active state less than WARMUP_SEC ago
|
||||
openvidu_starting() {
|
||||
since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null)
|
||||
case "$since" in ''|0|*[!0-9]*) return 1 ;; esac
|
||||
[ $(($(uptime_sec) - since / 1000000)) -lt "$WARMUP_SEC" ]
|
||||
}
|
||||
|
||||
# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare
|
||||
# TCP connect would also succeed against a wedged Redis.
|
||||
master_serving() {
|
||||
[ -n "$MASTER_ADDRS" ] || return 1
|
||||
for addr in $MASTER_ADDRS; do
|
||||
host=$(echo "$addr" | cut -d: -f1)
|
||||
port=$(echo "$addr" | cut -d: -f2)
|
||||
reply=$(timeout 5 bash -c 'exec 3<>"/dev/tcp/$0/$1" && printf "PING\r\n" >&3 && head -c 7 <&3' "$host" "$port" 2>/dev/null)
|
||||
case "$reply" in
|
||||
+PONG*|-NOAUTH*) return 0 ;;
|
||||
esac
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Request the replacement of this node ($1 = reason). Gated on a master answering, except
|
||||
# in the bootstrap phase before the master addresses are known.
|
||||
request_replacement() {
|
||||
now=$(uptime_sec)
|
||||
if [ -n "$REQUESTED_AT" ] && [ $((now - REQUESTED_AT)) -lt "$REPLACE_RETRY_SEC" ]; then
|
||||
return
|
||||
fi
|
||||
if [ -n "$MASTER_ADDRS" ] || [ -f "$INSTALLED_MARKER" ]; then
|
||||
if ! master_serving; then
|
||||
if [ -z "$HOLD_LOGGED_AT" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then
|
||||
log "$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node"
|
||||
HOLD_LOGGED_AT=$now
|
||||
fi
|
||||
return
|
||||
fi
|
||||
fi
|
||||
log "$1: requesting the replacement of this node"
|
||||
if "$REPLACE_SCRIPT" "$1"; then
|
||||
REQUESTED_AT=$now
|
||||
log "replacement requested"
|
||||
else
|
||||
log "the replacement request failed, retrying in $INTERVAL_SEC s"
|
||||
fi
|
||||
}
|
||||
|
||||
mkdir -p "$STATE_DIR"
|
||||
REQUESTED_AT=""
|
||||
HOLD_LOGGED_AT=""
|
||||
FAILED_SEEN_AT=""
|
||||
FAIL_SINCE=""
|
||||
SATURATED=""
|
||||
PREV_TOTAL=""
|
||||
PREV_IDLE=""
|
||||
log "started"
|
||||
|
||||
while true; do
|
||||
PROBE_URL=http://127.0.0.1:7880/
|
||||
INTERVAL_SEC=30
|
||||
HARD_UNHEALTHY_AFTER_SEC=300
|
||||
SOFT_UNHEALTHY_AFTER_SEC=600
|
||||
CPU_SATURATED_PCT=90
|
||||
WARMUP_SEC=900
|
||||
BOOTSTRAP_DEADLINE_SEC=5400
|
||||
BOOTSTRAP_FAILURE_COOLDOWN_SEC=600
|
||||
REPLACE_RETRY_SEC=600
|
||||
MASTER_ADDRS=""
|
||||
[ -f "$ENV_FILE" ] && . "$ENV_FILE"
|
||||
NOW=$(uptime_sec)
|
||||
cpu_busy
|
||||
|
||||
if [ -f "$PAUSE_FILE" ] || [ -f "$DRAIN_PAUSE_FILE" ]; then
|
||||
FAIL_SINCE=""
|
||||
elif [ -f "$BOOTSTRAP_FAILED_MARKER" ]; then
|
||||
if [ -z "$FAILED_SEEN_AT" ]; then
|
||||
FAILED_SEEN_AT=$NOW
|
||||
log "bootstrap failed ($(head -c 300 "$BOOTSTRAP_FAILED_MARKER")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s"
|
||||
fi
|
||||
if [ $((NOW - FAILED_SEEN_AT)) -ge "$BOOTSTRAP_FAILURE_COOLDOWN_SEC" ]; then
|
||||
request_replacement "bootstrap failed: $(head -c 300 "$BOOTSTRAP_FAILED_MARKER")"
|
||||
fi
|
||||
elif [ ! -f "$INSTALLED_MARKER" ]; then
|
||||
if [ "$NOW" -ge "$BOOTSTRAP_DEADLINE_SEC" ]; then
|
||||
request_replacement "bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot"
|
||||
fi
|
||||
elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 "$PROBE_URL"; then
|
||||
[ -n "$FAIL_SINCE" ] && log "LiveKit healthy again after $((NOW - FAIL_SINCE)) s"
|
||||
FAIL_SINCE=""
|
||||
SATURATED=""
|
||||
else
|
||||
RC=$?
|
||||
case "$RC" in
|
||||
7|52|56) KIND=hard ;;
|
||||
*) KIND=soft ;;
|
||||
esac
|
||||
if openvidu_starting; then
|
||||
FAIL_SINCE=""
|
||||
elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then
|
||||
[ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted"
|
||||
SATURATED=1
|
||||
FAIL_SINCE=""
|
||||
else
|
||||
SATURATED=""
|
||||
if [ -z "$FAIL_SINCE" ]; then
|
||||
FAIL_SINCE=$NOW
|
||||
log "LiveKit health check failed (curl exit $RC)"
|
||||
fi
|
||||
LIMIT=$SOFT_UNHEALTHY_AFTER_SEC
|
||||
[ "$KIND" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC
|
||||
if [ $((NOW - FAIL_SINCE)) -ge "$LIMIT" ]; then
|
||||
request_replacement "LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
sleep "$INTERVAL_SEC"
|
||||
done
|
||||
mode: "000755"
|
||||
owner: "root"
|
||||
group: "root"
|
||||
'/usr/local/bin/openvidu-media-replace.sh':
|
||||
content: !Sub |
|
||||
#!/bin/bash
|
||||
set -e
|
||||
# Ask the Auto Scaling group to replace this Media Node with a new one, keeping the
|
||||
# count. Called by openvidu-media-health.sh (same cgroup): it never stops or signals
|
||||
# the watchdog, which re-requests until this node goes away.
|
||||
mkdir -p /var/lib/openvidu-media-health
|
||||
echo "$(date -u +%FT%TZ) $1" >> /var/lib/openvidu-media-health/replace-requests.log
|
||||
# Let stop_media_node.sh know this node is unhealthy, so it does not wait for sessions
|
||||
touch /run/openvidu-unhealthy
|
||||
|
||||
# Token for IMDSv2
|
||||
TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
|
||||
|
||||
# Get own instance ID
|
||||
INSTANCE_ID=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-id)
|
||||
|
||||
# Set instance as unhealthy
|
||||
aws autoscaling set-instance-health \
|
||||
--region ${AWS::Region} \
|
||||
--instance-id "$INSTANCE_ID" \
|
||||
--health-status Unhealthy
|
||||
# set-instance-health Unhealthy makes the ASG terminate this node through the existing
|
||||
# lifecycle hook (drain) and launch a replacement. Idempotent; retry transient API errors.
|
||||
for attempt in $(seq 1 5); do
|
||||
if aws autoscaling set-instance-health \
|
||||
--region ${AWS::Region} \
|
||||
--instance-id "$INSTANCE_ID" \
|
||||
--health-status Unhealthy; then
|
||||
exit 0
|
||||
fi
|
||||
echo "[openvidu-media-replace] set-instance-health failed (attempt $attempt/5), retrying in 12s" >&2
|
||||
sleep 12
|
||||
done
|
||||
exit 1
|
||||
mode: "000755"
|
||||
owner: "root"
|
||||
group: "root"
|
||||
'/etc/systemd/system/openvidu-media-health.service':
|
||||
content: |
|
||||
[Unit]
|
||||
Description=OpenVidu Media Node health watchdog
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/local/bin/openvidu-media-health.sh
|
||||
Restart=always
|
||||
RestartSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
mode: "000644"
|
||||
owner: "root"
|
||||
group: "root"
|
||||
'/usr/local/bin/stop_media_node.sh':
|
||||
content: !Sub |
|
||||
#!/bin/bash
|
||||
set -e
|
||||
# Drain: pause and stop the health watchdog so it does not replace this node
|
||||
touch /run/openvidu-media-health.paused
|
||||
systemctl stop openvidu-media-health 2>/dev/null || true
|
||||
# Token for IMDSv2
|
||||
TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
|
||||
|
||||
|
|
@ -2197,6 +2406,10 @@ Resources:
|
|||
done
|
||||
TIME_PASSED=0
|
||||
HEARTBEAT_MAX=1800
|
||||
# An unhealthy node being replaced should not wait for sessions: kill now
|
||||
if [ -f /run/openvidu-unhealthy ]; then
|
||||
docker ps -q | xargs -r docker kill >/dev/null 2>&1 || true
|
||||
fi
|
||||
# Wait for running containers to not be openvidu, ingress, egress or an openvidu agent
|
||||
while [ "$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)" -gt 0 ] || \
|
||||
[ "$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)" = "true" ] || \
|
||||
|
|
@ -2257,11 +2470,22 @@ Resources:
|
|||
#!/bin/bash
|
||||
set -eu -o pipefail
|
||||
|
||||
# Record a media-node bootstrap failure so the health watchdog (started below)
|
||||
# replaces this node after a cool-down. Route user-data failure exits through this.
|
||||
ov_media_bootstrap_failed() {
|
||||
echo "[OpenVidu] media node bootstrap failed: $1"
|
||||
mkdir -p /var/lib/openvidu-media-health
|
||||
echo "$1" > /var/lib/openvidu-media-health/bootstrap-failed
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout
|
||||
|
||||
apt-get update && apt-get install -y \
|
||||
python3-pip \
|
||||
ec2-instance-connect
|
||||
ec2-instance-connect \
|
||||
curl \
|
||||
unzip
|
||||
|
||||
CFN_BOOTSTRAP_VERSION=2.0-39
|
||||
# Detect Ubuntu version and install cfn-bootstrap accordingly
|
||||
|
|
@ -2276,8 +2500,26 @@ Resources:
|
|||
|
||||
export HOME="/root"
|
||||
|
||||
# aws-cli v2, hoisted from install.sh so the watchdog's replace script can call
|
||||
# the ASG API even if bootstrap hangs before install.sh runs
|
||||
AWS_CLI_VERSION=2.36.40
|
||||
if ! command -v aws &> /dev/null; then
|
||||
curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-${!AWS_CLI_VERSION}.zip" -o "awscliv2.zip"
|
||||
unzip -qq awscliv2.zip
|
||||
./aws/install
|
||||
rm -rf awscliv2.zip aws
|
||||
fi
|
||||
|
||||
# Health watchdog config: master address for the anti-storm Redis PING guard
|
||||
mkdir -p /etc/openvidu
|
||||
echo 'MASTER_ADDRS="${OpenViduMasterNode.PrivateIp}:7000"' > /etc/openvidu/media-health.env
|
||||
|
||||
# Start the health watchdog before install.sh so a hung bootstrap is covered too
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now openvidu-media-health
|
||||
|
||||
# Install OpenVidu
|
||||
/usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu"; /usr/local/bin/set_as_unhealthy.sh; exit 1; }
|
||||
/usr/local/bin/install.sh || ov_media_bootstrap_failed "install.sh failed"
|
||||
|
||||
# Wait for the master node to be healthy before starting (bounded: up to 1800s at 5s)
|
||||
MASTER_NODE_IP=${OpenViduMasterNode.PrivateIp}
|
||||
|
|
@ -2290,13 +2532,15 @@ Resources:
|
|||
sleep 5
|
||||
done
|
||||
if [ "$MASTER_HEALTHY" != "true" ]; then
|
||||
echo "[OpenVidu] master node did not become healthy after 30 minutes"
|
||||
/usr/local/bin/set_as_unhealthy.sh
|
||||
exit 1
|
||||
ov_media_bootstrap_failed "master node did not become healthy after 30 minutes"
|
||||
fi
|
||||
|
||||
# Start OpenVidu
|
||||
systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; /usr/local/bin/set_as_unhealthy.sh; exit 1; }
|
||||
systemctl start openvidu || ov_media_bootstrap_failed "error starting OpenVidu"
|
||||
|
||||
# Bootstrap finished: switch the watchdog from the bootstrap deadline to runtime checks
|
||||
mkdir -p /var/lib/openvidu-media-health
|
||||
touch /var/lib/openvidu-media-health/installed
|
||||
|
||||
BlockDeviceMappings:
|
||||
- DeviceName: /dev/sda1
|
||||
|
|
|
|||
|
|
@ -1002,6 +1002,11 @@ var stop_media_nodesScriptMediaTemplate = '''
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Pause the health watchdog before draining so it never reimages a node that is being
|
||||
# drained for scale-in (synchronous; the drain runs outside the watchdog's cgroup).
|
||||
touch /run/openvidu-media-health.paused
|
||||
systemctl stop openvidu-media-health 2>/dev/null || true
|
||||
|
||||
if ! (set -o noclobber ; echo > /tmp/global.lock) ; then
|
||||
exit 1 # the global.lock already exists
|
||||
fi
|
||||
|
|
@ -1017,6 +1022,11 @@ if [ -x "$(command -v docker)" ]; then
|
|||
docker container kill --signal=SIGQUIT "$agent_container" 2>/dev/null || true
|
||||
done
|
||||
|
||||
# If this node is being replaced because it is unhealthy, do not wait for sessions to end.
|
||||
if [ -f /run/openvidu-unhealthy ]; then
|
||||
docker ps -q | xargs -r docker kill >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
# Wait for running containers to not be openvidu, ingress, egress or an openvidu agent
|
||||
while [ "$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)" -gt 0 ] || \
|
||||
[ "$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)" = "true" ] || \
|
||||
|
|
@ -1042,25 +1052,244 @@ az tag update --resource-id $RESOURCE_ID --operation replace --tags "STATUS"="HE
|
|||
az vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID
|
||||
'''
|
||||
|
||||
var delete_mediaNode_ScriptMediaTemplate = '''
|
||||
var openviduMediaHealthScript = '''
|
||||
#!/bin/bash
|
||||
set -e
|
||||
# OpenVidu Media Node health watchdog.
|
||||
#
|
||||
# Asks the cloud to replace this Media Node with a new one, keeping the number of
|
||||
# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when:
|
||||
# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a
|
||||
# cool-down, so a persistent failure does not churn nodes and can be inspected,
|
||||
# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot,
|
||||
# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC
|
||||
# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC
|
||||
# while it answers an error or times out. Soft failures are not counted while the CPU
|
||||
# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in
|
||||
# the WARMUP_SEC after openvidu.service (re)started.
|
||||
# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage
|
||||
# fails the health check of every Media Node at once, and a new node could not bootstrap
|
||||
# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until
|
||||
# the node goes away.
|
||||
#
|
||||
# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...")
|
||||
# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)
|
||||
# Logs: journalctl -u openvidu-media-health
|
||||
#
|
||||
# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and
|
||||
# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed
|
||||
# by an opening brace, nor two opening braces in a row.
|
||||
|
||||
az login --identity
|
||||
STATE_DIR=/var/lib/openvidu-media-health
|
||||
INSTALLED_MARKER="$STATE_DIR/installed"
|
||||
BOOTSTRAP_FAILED_MARKER="$STATE_DIR/bootstrap-failed"
|
||||
REPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh
|
||||
ENV_FILE=/etc/openvidu/media-health.env
|
||||
PAUSE_FILE=/etc/openvidu/media-health.disabled
|
||||
DRAIN_PAUSE_FILE=/run/openvidu-media-health.paused
|
||||
|
||||
log() { echo "[media-health] $*"; }
|
||||
|
||||
# Seconds since boot: monotonic, unaffected by clock steps
|
||||
uptime_sec() { cut -d. -f1 /proc/uptime; }
|
||||
|
||||
# Busy CPU percentage since the previous call, in BUSY (0 on the first call)
|
||||
cpu_busy() {
|
||||
read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat
|
||||
total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal))
|
||||
idle=$((c_idle + c_iowait))
|
||||
BUSY=0
|
||||
if [ -n "$PREV_TOTAL" ] && [ "$total" -gt "$PREV_TOTAL" ]; then
|
||||
BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL)))
|
||||
fi
|
||||
PREV_TOTAL=$total
|
||||
PREV_IDLE=$idle
|
||||
}
|
||||
|
||||
# openvidu.service entered the active state less than WARMUP_SEC ago
|
||||
openvidu_starting() {
|
||||
since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null)
|
||||
case "$since" in ''|0|*[!0-9]*) return 1 ;; esac
|
||||
[ $(($(uptime_sec) - since / 1000000)) -lt "$WARMUP_SEC" ]
|
||||
}
|
||||
|
||||
# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare
|
||||
# TCP connect would also succeed against a wedged Redis.
|
||||
master_serving() {
|
||||
[ -n "$MASTER_ADDRS" ] || return 1
|
||||
for addr in $MASTER_ADDRS; do
|
||||
host=$(echo "$addr" | cut -d: -f1)
|
||||
port=$(echo "$addr" | cut -d: -f2)
|
||||
reply=$(timeout 5 bash -c 'exec 3<>"/dev/tcp/$0/$1" && printf "PING\r\n" >&3 && head -c 7 <&3' "$host" "$port" 2>/dev/null)
|
||||
case "$reply" in
|
||||
+PONG*|-NOAUTH*) return 0 ;;
|
||||
esac
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Request the replacement of this node ($1 = reason). Gated on a master answering, except
|
||||
# in the bootstrap phase before the master addresses are known.
|
||||
request_replacement() {
|
||||
now=$(uptime_sec)
|
||||
if [ -n "$REQUESTED_AT" ] && [ $((now - REQUESTED_AT)) -lt "$REPLACE_RETRY_SEC" ]; then
|
||||
return
|
||||
fi
|
||||
if [ -n "$MASTER_ADDRS" ] || [ -f "$INSTALLED_MARKER" ]; then
|
||||
if ! master_serving; then
|
||||
if [ -z "$HOLD_LOGGED_AT" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then
|
||||
log "$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node"
|
||||
HOLD_LOGGED_AT=$now
|
||||
fi
|
||||
return
|
||||
fi
|
||||
fi
|
||||
log "$1: requesting the replacement of this node"
|
||||
if "$REPLACE_SCRIPT" "$1"; then
|
||||
REQUESTED_AT=$now
|
||||
log "replacement requested"
|
||||
else
|
||||
log "the replacement request failed, retrying in $INTERVAL_SEC s"
|
||||
fi
|
||||
}
|
||||
|
||||
mkdir -p "$STATE_DIR"
|
||||
REQUESTED_AT=""
|
||||
HOLD_LOGGED_AT=""
|
||||
FAILED_SEEN_AT=""
|
||||
FAIL_SINCE=""
|
||||
SATURATED=""
|
||||
PREV_TOTAL=""
|
||||
PREV_IDLE=""
|
||||
log "started"
|
||||
|
||||
while true; do
|
||||
PROBE_URL=http://127.0.0.1:7880/
|
||||
INTERVAL_SEC=30
|
||||
HARD_UNHEALTHY_AFTER_SEC=300
|
||||
SOFT_UNHEALTHY_AFTER_SEC=600
|
||||
CPU_SATURATED_PCT=90
|
||||
WARMUP_SEC=900
|
||||
BOOTSTRAP_DEADLINE_SEC=5400
|
||||
BOOTSTRAP_FAILURE_COOLDOWN_SEC=600
|
||||
REPLACE_RETRY_SEC=600
|
||||
MASTER_ADDRS=""
|
||||
[ -f "$ENV_FILE" ] && . "$ENV_FILE"
|
||||
NOW=$(uptime_sec)
|
||||
cpu_busy
|
||||
|
||||
if [ -f "$PAUSE_FILE" ] || [ -f "$DRAIN_PAUSE_FILE" ]; then
|
||||
FAIL_SINCE=""
|
||||
elif [ -f "$BOOTSTRAP_FAILED_MARKER" ]; then
|
||||
if [ -z "$FAILED_SEEN_AT" ]; then
|
||||
FAILED_SEEN_AT=$NOW
|
||||
log "bootstrap failed ($(head -c 300 "$BOOTSTRAP_FAILED_MARKER")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s"
|
||||
fi
|
||||
if [ $((NOW - FAILED_SEEN_AT)) -ge "$BOOTSTRAP_FAILURE_COOLDOWN_SEC" ]; then
|
||||
request_replacement "bootstrap failed: $(head -c 300 "$BOOTSTRAP_FAILED_MARKER")"
|
||||
fi
|
||||
elif [ ! -f "$INSTALLED_MARKER" ]; then
|
||||
if [ "$NOW" -ge "$BOOTSTRAP_DEADLINE_SEC" ]; then
|
||||
request_replacement "bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot"
|
||||
fi
|
||||
elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 "$PROBE_URL"; then
|
||||
[ -n "$FAIL_SINCE" ] && log "LiveKit healthy again after $((NOW - FAIL_SINCE)) s"
|
||||
FAIL_SINCE=""
|
||||
SATURATED=""
|
||||
else
|
||||
RC=$?
|
||||
case "$RC" in
|
||||
7|52|56) KIND=hard ;;
|
||||
*) KIND=soft ;;
|
||||
esac
|
||||
if openvidu_starting; then
|
||||
FAIL_SINCE=""
|
||||
elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then
|
||||
[ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted"
|
||||
SATURATED=1
|
||||
FAIL_SINCE=""
|
||||
else
|
||||
SATURATED=""
|
||||
if [ -z "$FAIL_SINCE" ]; then
|
||||
FAIL_SINCE=$NOW
|
||||
log "LiveKit health check failed (curl exit $RC)"
|
||||
fi
|
||||
LIMIT=$SOFT_UNHEALTHY_AFTER_SEC
|
||||
[ "$KIND" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC
|
||||
if [ $((NOW - FAIL_SINCE)) -ge "$LIMIT" ]; then
|
||||
request_replacement "LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
sleep "$INTERVAL_SEC"
|
||||
done
|
||||
'''
|
||||
|
||||
var openviduMediaHealthUnit = '''
|
||||
[Unit]
|
||||
Description=OpenVidu Media Node health watchdog
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/local/bin/openvidu-media-health.sh
|
||||
Restart=always
|
||||
RestartSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
'''
|
||||
|
||||
var media_replaceScriptMediaTemplate = '''
|
||||
#!/bin/bash
|
||||
# openvidu-media-replace.sh "<reason>"
|
||||
# Requests the reimage of THIS Media Node, keeping the scale set capacity and the
|
||||
# scale-in protection: reimage provisions a new OS disk and the userData bootstrap runs
|
||||
# again on the same instance id. Called by the health watchdog; it never signals it.
|
||||
|
||||
mkdir -p /var/lib/openvidu-media-health
|
||||
echo "$(date -u +%FT%TZ) $1" >> /var/lib/openvidu-media-health/replace-requests.log
|
||||
touch /run/openvidu-unhealthy
|
||||
|
||||
RESOURCE_GROUP_NAME=${resourceGroupName}
|
||||
VM_SCALE_SET_NAME=${vmScaleSetName}
|
||||
BEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2021-02-01" | jq -r '.compute.resourceId')
|
||||
INSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')
|
||||
INSTANCE_RESOURCE_ID=$(curl -s -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2021-02-01" | jq -r '.compute.resourceId')
|
||||
INSTANCE_ID=$(echo "$INSTANCE_RESOURCE_ID" | awk -F'/' '{print $NF}')
|
||||
if [ -z "$INSTANCE_ID" ]; then
|
||||
echo "[media-replace] could not read this instance id from IMDS"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
az vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID
|
||||
n=0
|
||||
while true; do
|
||||
n=$((n + 1))
|
||||
if az login --identity >/dev/null 2>&1 && \
|
||||
az vmss reimage --resource-group "$RESOURCE_GROUP_NAME" --name "$VM_SCALE_SET_NAME" --instance-ids "$INSTANCE_ID" --no-wait; then
|
||||
echo "[media-replace] reimage requested for instance $INSTANCE_ID ($1)"
|
||||
exit 0
|
||||
fi
|
||||
if [ "$n" -ge 5 ]; then
|
||||
echo "[media-replace] reimage could not be requested after $n attempts"
|
||||
exit 1
|
||||
fi
|
||||
echo "[media-replace] reimage request failed (attempt $n), retrying in 12s"
|
||||
sleep 12
|
||||
done
|
||||
'''
|
||||
|
||||
|
||||
var userDataMediaNodeTemplate = '''
|
||||
#!/bin/bash
|
||||
set -eu -o pipefail
|
||||
|
||||
# Mark this media node's bootstrap as failed so the health watchdog replaces it
|
||||
ov_media_bootstrap_failed() {
|
||||
echo "[OpenVidu] media node bootstrap failed: $1"
|
||||
mkdir -p /var/lib/openvidu-media-health
|
||||
echo "$1" > /var/lib/openvidu-media-health/bootstrap-failed
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Introduce the scripts in the instance
|
||||
# install.sh
|
||||
echo ${base64install} | base64 -d > /usr/local/bin/install.sh
|
||||
|
|
@ -1070,9 +1299,12 @@ chmod +x /usr/local/bin/install.sh
|
|||
echo ${base64stop} | base64 -d > /usr/local/bin/stop_media_node.sh
|
||||
chmod +x /usr/local/bin/stop_media_node.sh
|
||||
|
||||
# delete_media_node.sh
|
||||
echo ${base64delete} | base64 -d > /usr/local/bin/delete_media_node.sh
|
||||
chmod +x /usr/local/bin/delete_media_node.sh
|
||||
# openvidu-media-health.sh (health watchdog), openvidu-media-replace.sh (reimage this node) and its unit
|
||||
echo ${base64mediaHealth} | base64 -d > /usr/local/bin/openvidu-media-health.sh
|
||||
chmod +x /usr/local/bin/openvidu-media-health.sh
|
||||
echo ${base64mediaReplace} | base64 -d > /usr/local/bin/openvidu-media-replace.sh
|
||||
chmod +x /usr/local/bin/openvidu-media-replace.sh
|
||||
echo ${base64mediaHealthUnit} | base64 -d > /etc/systemd/system/openvidu-media-health.service
|
||||
|
||||
echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout
|
||||
|
||||
|
|
@ -1097,8 +1329,16 @@ az vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME -
|
|||
|
||||
export HOME="/root"
|
||||
|
||||
# Start the Media Node health watchdog before install.sh (az CLI is installed and the node
|
||||
# is scale-in protected): it backstops a hung or failed bootstrap and, once running, reimages
|
||||
# the node if LiveKit stays unhealthy. Write MASTER_ADDRS first so its master guard is armed.
|
||||
mkdir -p /etc/openvidu
|
||||
echo 'MASTER_ADDRS="${privateIPMasterNode}:7000"' > /etc/openvidu/media-health.env
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now openvidu-media-health
|
||||
|
||||
# Install OpenVidu
|
||||
/usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu"; /usr/local/bin/delete_media_node.sh; exit 1; }
|
||||
/usr/local/bin/install.sh || ov_media_bootstrap_failed "install.sh failed"
|
||||
|
||||
# Gate 2: wait for master readiness before starting
|
||||
WAIT_INTERVAL=5
|
||||
|
|
@ -1112,16 +1352,18 @@ while true; do
|
|||
fi
|
||||
RETRIES=$((RETRIES + 1))
|
||||
if [ $RETRIES -ge $MAX_RETRIES ]; then
|
||||
echo "[OpenVidu] timed out after 30 min waiting for FINISH-MASTER-NODE"
|
||||
/usr/local/bin/delete_media_node.sh
|
||||
exit 1
|
||||
ov_media_bootstrap_failed "timed out after 30 min waiting for FINISH-MASTER-NODE"
|
||||
fi
|
||||
sleep $WAIT_INTERVAL
|
||||
done
|
||||
set -e
|
||||
|
||||
# Start OpenVidu
|
||||
systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; /usr/local/bin/delete_media_node.sh; }
|
||||
systemctl start openvidu || ov_media_bootstrap_failed "systemctl start openvidu failed"
|
||||
|
||||
# Bootstrap finished: the watchdog switches from bootstrap-deadline mode to runtime health checks
|
||||
mkdir -p /var/lib/openvidu-media-health
|
||||
touch /var/lib/openvidu-media-health/installed
|
||||
'''
|
||||
|
||||
var installScriptMedia = reduce(
|
||||
|
|
@ -1140,18 +1382,23 @@ var stop_media_nodesScriptMedia = reduce(
|
|||
|
||||
var base64stopMediaNode = base64(stop_media_nodesScriptMedia)
|
||||
|
||||
var delete_mediaNode_ScriptMedia = reduce(
|
||||
var media_replaceScriptMedia = reduce(
|
||||
items(stopMediaNodeParams),
|
||||
{ value: delete_mediaNode_ScriptMediaTemplate },
|
||||
{ value: media_replaceScriptMediaTemplate },
|
||||
(curr, next) => { value: replace(curr.value, '\${${next.key}}', next.value) }
|
||||
).value
|
||||
|
||||
var base64delete_mediaNode_ScriptMedia = base64(delete_mediaNode_ScriptMedia)
|
||||
var base64media_replaceScriptMedia = base64(media_replaceScriptMedia)
|
||||
var base64openviduMediaHealth = base64(openviduMediaHealthScript)
|
||||
var base64openviduMediaHealthUnit = base64(openviduMediaHealthUnit)
|
||||
|
||||
var userDataParamsMedia = {
|
||||
base64install: base64installMedia
|
||||
base64stop: base64stopMediaNode
|
||||
base64delete: base64delete_mediaNode_ScriptMedia
|
||||
base64mediaHealth: base64openviduMediaHealth
|
||||
base64mediaHealthUnit: base64openviduMediaHealthUnit
|
||||
base64mediaReplace: base64media_replaceScriptMedia
|
||||
privateIPMasterNode: netInterfaceMasterNode.properties.ipConfigurations[0].properties.privateIPAddress
|
||||
resourceGroupName: resourceGroup().name
|
||||
vmScaleSetName: '${stackName}-mediaNodeScaleSet'
|
||||
keyVaultName: keyVaultName
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -260,7 +260,9 @@ resource "google_compute_region_health_check" "media_node_health_check" {
|
|||
name = lower("${var.stackName}-media-node-health-check")
|
||||
region = var.region
|
||||
|
||||
# TCP-only, generous thresholds: auto-heal recreation kills live WebRTC sessions
|
||||
# TCP (not HTTP /): during a master/Redis outage LiveKit answers 406 on every media node at
|
||||
# once, so an HTTP check would recreate the whole fleet. Generous thresholds because auto-heal
|
||||
# recreation kills live WebRTC sessions; the on-node watchdog owns application-level health.
|
||||
tcp_health_check {
|
||||
port = 7880
|
||||
}
|
||||
|
|
@ -268,7 +270,7 @@ resource "google_compute_region_health_check" "media_node_health_check" {
|
|||
check_interval_sec = 30
|
||||
timeout_sec = 10
|
||||
healthy_threshold = 2
|
||||
unhealthy_threshold = 5
|
||||
unhealthy_threshold = 10
|
||||
|
||||
depends_on = [google_project_service.compute_api]
|
||||
}
|
||||
|
|
@ -289,10 +291,11 @@ resource "google_compute_region_instance_group_manager" "media_node_group" {
|
|||
port = 7880
|
||||
}
|
||||
|
||||
# initial_delay_sec generous so media nodes finish installing before health checks can recreate them
|
||||
# VM/process backstop only. initial_delay_sec at the 3600 s max so autohealing never recreates a
|
||||
# still-bootstrapping node; the marker-aware on-node watchdog owns bootstrap-failure detection.
|
||||
auto_healing_policies {
|
||||
health_check = google_compute_region_health_check.media_node_health_check.id
|
||||
initial_delay_sec = 600
|
||||
initial_delay_sec = 3600
|
||||
}
|
||||
|
||||
depends_on = [google_compute_instance.openvidu_master_node]
|
||||
|
|
@ -1334,6 +1337,12 @@ if ! flock -n 9; then
|
|||
exit 0
|
||||
fi
|
||||
|
||||
# Stop the health watchdog so it does not request a replacement while this node drains. The
|
||||
# pause file is honored on the watchdog's next loop regardless of when the unit stops; it is
|
||||
# cleared on reboot.
|
||||
touch /run/openvidu-media-health.paused
|
||||
systemctl stop openvidu-media-health 2>/dev/null || true
|
||||
|
||||
echo "Starting graceful shutdown of OpenVidu Media Node..."
|
||||
|
||||
INSTANCE_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google")
|
||||
|
|
@ -1351,6 +1360,11 @@ if [ -x "$(command -v docker)" ]; then
|
|||
docker container kill --signal=SIGQUIT "$agent_container" 2>/dev/null || true
|
||||
done
|
||||
|
||||
# A node being replaced because it is unhealthy must not wait for sessions to drain.
|
||||
if [ -f /run/openvidu-unhealthy ]; then
|
||||
docker ps -q | xargs -r docker kill >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
# Wait for running containers to not be openvidu, ingress, egress or an openvidu agent
|
||||
while [ "$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)" -gt 0 ] || \
|
||||
[ "$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)" = "true" ] || \
|
||||
|
|
@ -1397,12 +1411,256 @@ else
|
|||
echo "Instance $INSTANCE_NAME has been abandoned from MIG. Starting graceful shutdown..."
|
||||
/usr/local/bin/graceful_shutdown.sh
|
||||
fi
|
||||
EOF
|
||||
|
||||
media_health_watchdog_script = <<-EOF
|
||||
#!/bin/bash
|
||||
# OpenVidu Media Node health watchdog.
|
||||
#
|
||||
# Asks the cloud to replace this Media Node with a new one, keeping the number of
|
||||
# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when:
|
||||
# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a
|
||||
# cool-down, so a persistent failure does not churn nodes and can be inspected,
|
||||
# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot,
|
||||
# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC
|
||||
# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC
|
||||
# while it answers an error or times out. Soft failures are not counted while the CPU
|
||||
# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in
|
||||
# the WARMUP_SEC after openvidu.service (re)started.
|
||||
# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage
|
||||
# fails the health check of every Media Node at once, and a new node could not bootstrap
|
||||
# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until
|
||||
# the node goes away.
|
||||
#
|
||||
# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...")
|
||||
# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)
|
||||
# Logs: journalctl -u openvidu-media-health
|
||||
#
|
||||
# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and
|
||||
# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed
|
||||
# by an opening brace, nor two opening braces in a row.
|
||||
|
||||
STATE_DIR=/var/lib/openvidu-media-health
|
||||
INSTALLED_MARKER="$STATE_DIR/installed"
|
||||
BOOTSTRAP_FAILED_MARKER="$STATE_DIR/bootstrap-failed"
|
||||
REPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh
|
||||
ENV_FILE=/etc/openvidu/media-health.env
|
||||
PAUSE_FILE=/etc/openvidu/media-health.disabled
|
||||
DRAIN_PAUSE_FILE=/run/openvidu-media-health.paused
|
||||
|
||||
log() { echo "[media-health] $*"; }
|
||||
|
||||
# Seconds since boot: monotonic, unaffected by clock steps
|
||||
uptime_sec() { cut -d. -f1 /proc/uptime; }
|
||||
|
||||
# Busy CPU percentage since the previous call, in BUSY (0 on the first call)
|
||||
cpu_busy() {
|
||||
read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat
|
||||
total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal))
|
||||
idle=$((c_idle + c_iowait))
|
||||
BUSY=0
|
||||
if [ -n "$PREV_TOTAL" ] && [ "$total" -gt "$PREV_TOTAL" ]; then
|
||||
BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL)))
|
||||
fi
|
||||
PREV_TOTAL=$total
|
||||
PREV_IDLE=$idle
|
||||
}
|
||||
|
||||
# openvidu.service entered the active state less than WARMUP_SEC ago
|
||||
openvidu_starting() {
|
||||
since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null)
|
||||
case "$since" in ''|0|*[!0-9]*) return 1 ;; esac
|
||||
[ $(($(uptime_sec) - since / 1000000)) -lt "$WARMUP_SEC" ]
|
||||
}
|
||||
|
||||
# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare
|
||||
# TCP connect would also succeed against a wedged Redis.
|
||||
master_serving() {
|
||||
[ -n "$MASTER_ADDRS" ] || return 1
|
||||
for addr in $MASTER_ADDRS; do
|
||||
host=$(echo "$addr" | cut -d: -f1)
|
||||
port=$(echo "$addr" | cut -d: -f2)
|
||||
reply=$(timeout 5 bash -c 'exec 3<>"/dev/tcp/$0/$1" && printf "PING\r\n" >&3 && head -c 7 <&3' "$host" "$port" 2>/dev/null)
|
||||
case "$reply" in
|
||||
+PONG*|-NOAUTH*) return 0 ;;
|
||||
esac
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Request the replacement of this node ($1 = reason). Gated on a master answering, except
|
||||
# in the bootstrap phase before the master addresses are known.
|
||||
request_replacement() {
|
||||
now=$(uptime_sec)
|
||||
if [ -n "$REQUESTED_AT" ] && [ $((now - REQUESTED_AT)) -lt "$REPLACE_RETRY_SEC" ]; then
|
||||
return
|
||||
fi
|
||||
if [ -n "$MASTER_ADDRS" ] || [ -f "$INSTALLED_MARKER" ]; then
|
||||
if ! master_serving; then
|
||||
if [ -z "$HOLD_LOGGED_AT" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then
|
||||
log "$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node"
|
||||
HOLD_LOGGED_AT=$now
|
||||
fi
|
||||
return
|
||||
fi
|
||||
fi
|
||||
log "$1: requesting the replacement of this node"
|
||||
if "$REPLACE_SCRIPT" "$1"; then
|
||||
REQUESTED_AT=$now
|
||||
log "replacement requested"
|
||||
else
|
||||
log "the replacement request failed, retrying in $INTERVAL_SEC s"
|
||||
fi
|
||||
}
|
||||
|
||||
mkdir -p "$STATE_DIR"
|
||||
REQUESTED_AT=""
|
||||
HOLD_LOGGED_AT=""
|
||||
FAILED_SEEN_AT=""
|
||||
FAIL_SINCE=""
|
||||
SATURATED=""
|
||||
PREV_TOTAL=""
|
||||
PREV_IDLE=""
|
||||
log "started"
|
||||
|
||||
while true; do
|
||||
PROBE_URL=http://127.0.0.1:7880/
|
||||
INTERVAL_SEC=30
|
||||
HARD_UNHEALTHY_AFTER_SEC=300
|
||||
SOFT_UNHEALTHY_AFTER_SEC=600
|
||||
CPU_SATURATED_PCT=90
|
||||
WARMUP_SEC=900
|
||||
BOOTSTRAP_DEADLINE_SEC=5400
|
||||
BOOTSTRAP_FAILURE_COOLDOWN_SEC=600
|
||||
REPLACE_RETRY_SEC=600
|
||||
MASTER_ADDRS=""
|
||||
[ -f "$ENV_FILE" ] && . "$ENV_FILE"
|
||||
NOW=$(uptime_sec)
|
||||
cpu_busy
|
||||
|
||||
if [ -f "$PAUSE_FILE" ] || [ -f "$DRAIN_PAUSE_FILE" ]; then
|
||||
FAIL_SINCE=""
|
||||
elif [ -f "$BOOTSTRAP_FAILED_MARKER" ]; then
|
||||
if [ -z "$FAILED_SEEN_AT" ]; then
|
||||
FAILED_SEEN_AT=$NOW
|
||||
log "bootstrap failed ($(head -c 300 "$BOOTSTRAP_FAILED_MARKER")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s"
|
||||
fi
|
||||
if [ $((NOW - FAILED_SEEN_AT)) -ge "$BOOTSTRAP_FAILURE_COOLDOWN_SEC" ]; then
|
||||
request_replacement "bootstrap failed: $(head -c 300 "$BOOTSTRAP_FAILED_MARKER")"
|
||||
fi
|
||||
elif [ ! -f "$INSTALLED_MARKER" ]; then
|
||||
if [ "$NOW" -ge "$BOOTSTRAP_DEADLINE_SEC" ]; then
|
||||
request_replacement "bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot"
|
||||
fi
|
||||
elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 "$PROBE_URL"; then
|
||||
[ -n "$FAIL_SINCE" ] && log "LiveKit healthy again after $((NOW - FAIL_SINCE)) s"
|
||||
FAIL_SINCE=""
|
||||
SATURATED=""
|
||||
else
|
||||
RC=$?
|
||||
case "$RC" in
|
||||
7|52|56) KIND=hard ;;
|
||||
*) KIND=soft ;;
|
||||
esac
|
||||
if openvidu_starting; then
|
||||
FAIL_SINCE=""
|
||||
elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then
|
||||
[ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted"
|
||||
SATURATED=1
|
||||
FAIL_SINCE=""
|
||||
else
|
||||
SATURATED=""
|
||||
if [ -z "$FAIL_SINCE" ]; then
|
||||
FAIL_SINCE=$NOW
|
||||
log "LiveKit health check failed (curl exit $RC)"
|
||||
fi
|
||||
LIMIT=$SOFT_UNHEALTHY_AFTER_SEC
|
||||
[ "$KIND" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC
|
||||
if [ $((NOW - FAIL_SINCE)) -ge "$LIMIT" ]; then
|
||||
request_replacement "LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
sleep "$INTERVAL_SEC"
|
||||
done
|
||||
EOF
|
||||
|
||||
media_replace_script = <<-EOF
|
||||
#!/bin/bash
|
||||
# Requests the replacement of THIS media node while keeping the media node count: it asks the
|
||||
# regional MIG to recreate the instance (delete + create in place, same name, new instance id,
|
||||
# fresh disk). Called by the health watchdog and by the bootstrap-failure hook; it never stops
|
||||
# or signals the watchdog (its caller, in the same cgroup).
|
||||
|
||||
REASON="$1"
|
||||
mkdir -p /var/lib/openvidu-media-health
|
||||
echo "$(date -u +%FT%TZ) $REASON" >> /var/lib/openvidu-media-health/replace-requests.log
|
||||
# Tell the drain script this node is being replaced so it skips waiting for sessions.
|
||||
touch /run/openvidu-unhealthy
|
||||
|
||||
META="http://metadata.google.internal/computeMetadata/v1"
|
||||
meta() { curl -s -H "Metadata-Flavor: Google" "$META/$1"; }
|
||||
|
||||
SELF_NAME=$(meta "instance/name")
|
||||
PROJECT_ID=$(meta "project/project-id")
|
||||
ZONE=$(meta "instance/zone" | cut -d/ -f4)
|
||||
REGION=$(echo "$ZONE" | sed 's/-[a-z]$//')
|
||||
STACK_NAME=$(meta "instance/attributes/stackName")
|
||||
MIG_NAME=$(echo "$STACK_NAME" | tr '[:upper:]' '[:lower:]')-media-node-group
|
||||
|
||||
# Recreate keeps the MIG size. Retry transient API errors. A "not a member / already being
|
||||
# deleted, abandoned or recreated" error means the node is already leaving the group (e.g.
|
||||
# scaled in): nothing to do.
|
||||
i=0
|
||||
while [ "$i" -lt 5 ]; do
|
||||
i=$((i + 1))
|
||||
if OUT=$(gcloud compute instance-groups managed recreate-instances "$MIG_NAME" \
|
||||
--instances="$SELF_NAME" --region="$REGION" --project="$PROJECT_ID" --quiet 2>&1); then
|
||||
echo "[media-replace] recreate requested for $SELF_NAME"
|
||||
exit 0
|
||||
fi
|
||||
case "$OUT" in
|
||||
*"not a member"*|*"being deleted"*|*"being recreated"*|*abandon*|*ABANDON*)
|
||||
echo "[media-replace] $SELF_NAME already leaving the group, nothing to do: $OUT"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
echo "[media-replace] recreate attempt $i for $SELF_NAME failed: $OUT"
|
||||
sleep 12
|
||||
done
|
||||
echo "[media-replace] gave up requesting recreate of $SELF_NAME after $i attempts"
|
||||
exit 1
|
||||
EOF
|
||||
|
||||
media_health_unit = <<-EOF
|
||||
[Unit]
|
||||
Description=OpenVidu Media Node health watchdog
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/local/bin/openvidu-media-health.sh
|
||||
Restart=always
|
||||
RestartSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
user_data_media = <<-EOF
|
||||
#!/bin/bash -x
|
||||
set -eu -o pipefail
|
||||
|
||||
# Records a media-node bootstrap failure so the health watchdog replaces this node after a
|
||||
# cool-down, then aborts the bootstrap.
|
||||
ov_media_bootstrap_failed() {
|
||||
echo "[OpenVidu] media node bootstrap failed: $1"
|
||||
mkdir -p /var/lib/openvidu-media-health
|
||||
echo "$1" > /var/lib/openvidu-media-health/bootstrap-failed
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Check if installation already completed
|
||||
if [ -f /usr/local/bin/openvidu_install_counter.txt ]; then
|
||||
# Launch on reboot
|
||||
|
|
@ -1420,6 +1678,21 @@ ${local.graceful_shutdown_script}
|
|||
GRACEFUL_SHUTDOWN_EOF
|
||||
chmod +x /usr/local/bin/graceful_shutdown.sh
|
||||
|
||||
# Media node health watchdog + its replacement script and unit
|
||||
cat > /usr/local/bin/openvidu-media-health.sh << 'MEDIA_HEALTH_EOF'
|
||||
${local.media_health_watchdog_script}
|
||||
MEDIA_HEALTH_EOF
|
||||
chmod +x /usr/local/bin/openvidu-media-health.sh
|
||||
|
||||
cat > /usr/local/bin/openvidu-media-replace.sh << 'MEDIA_REPLACE_EOF'
|
||||
${local.media_replace_script}
|
||||
MEDIA_REPLACE_EOF
|
||||
chmod +x /usr/local/bin/openvidu-media-replace.sh
|
||||
|
||||
cat > /etc/systemd/system/openvidu-media-health.service << 'MEDIA_HEALTH_UNIT_EOF'
|
||||
${local.media_health_unit}
|
||||
MEDIA_HEALTH_UNIT_EOF
|
||||
|
||||
echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout
|
||||
|
||||
GCLOUD_VERSION=573.0.0
|
||||
|
|
@ -1437,14 +1710,28 @@ GRACEFUL_SHUTDOWN_EOF
|
|||
|
||||
export HOME="/root"
|
||||
|
||||
# Start the media node health watchdog now: gcloud (which its replacement needs) is installed,
|
||||
# and starting before install.sh lets the watchdog cover a hung bootstrap too. Write the master
|
||||
# address(es) first (before the installed marker) so the anti-storm guard is always populated.
|
||||
mkdir -p /etc/openvidu
|
||||
MASTER_NODE_PRIVATE_IP=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/masterNodePrivateIP" -H "Metadata-Flavor: Google")
|
||||
[ -n "$MASTER_NODE_PRIVATE_IP" ] || ov_media_bootstrap_failed "could not determine master node private IP"
|
||||
echo "MASTER_ADDRS=\"$MASTER_NODE_PRIVATE_IP:7000\"" > /etc/openvidu/media-health.env
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now openvidu-media-health
|
||||
|
||||
# Install OpenVidu Media Node
|
||||
/usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu Media Node"; exit 1; }
|
||||
/usr/local/bin/install.sh || ov_media_bootstrap_failed "install.sh failed"
|
||||
|
||||
# Mark installation as complete
|
||||
echo "installation_complete" > /usr/local/bin/openvidu_install_counter.txt
|
||||
|
||||
# Start OpenVidu
|
||||
systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; }
|
||||
systemctl start openvidu || ov_media_bootstrap_failed "systemctl start openvidu failed"
|
||||
|
||||
# Bootstrap finished: hand runtime replacement to the watchdog (never on a failure path).
|
||||
mkdir -p /var/lib/openvidu-media-health
|
||||
touch /var/lib/openvidu-media-health/installed
|
||||
|
||||
# Add cron job to check if instance is abandoned every minute
|
||||
cat > /usr/local/bin/check_abandoned.sh << 'CHECK_ABANDONED_EOF'
|
||||
|
|
|
|||
|
|
@ -2799,28 +2799,238 @@ Resources:
|
|||
mode: "000755"
|
||||
owner: "root"
|
||||
group: "root"
|
||||
'/usr/local/bin/set_as_unhealthy.sh':
|
||||
'/usr/local/bin/openvidu-media-health.sh':
|
||||
content: |
|
||||
#!/bin/bash
|
||||
# OpenVidu Media Node health watchdog.
|
||||
#
|
||||
# Asks the cloud to replace this Media Node with a new one, keeping the number of
|
||||
# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when:
|
||||
# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a
|
||||
# cool-down, so a persistent failure does not churn nodes and can be inspected,
|
||||
# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot,
|
||||
# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC
|
||||
# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC
|
||||
# while it answers an error or times out. Soft failures are not counted while the CPU
|
||||
# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in
|
||||
# the WARMUP_SEC after openvidu.service (re)started.
|
||||
# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage
|
||||
# fails the health check of every Media Node at once, and a new node could not bootstrap
|
||||
# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until
|
||||
# the node goes away.
|
||||
#
|
||||
# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...")
|
||||
# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)
|
||||
# Logs: journalctl -u openvidu-media-health
|
||||
#
|
||||
# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and
|
||||
# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed
|
||||
# by an opening brace, nor two opening braces in a row.
|
||||
|
||||
STATE_DIR=/var/lib/openvidu-media-health
|
||||
INSTALLED_MARKER="$STATE_DIR/installed"
|
||||
BOOTSTRAP_FAILED_MARKER="$STATE_DIR/bootstrap-failed"
|
||||
REPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh
|
||||
ENV_FILE=/etc/openvidu/media-health.env
|
||||
PAUSE_FILE=/etc/openvidu/media-health.disabled
|
||||
DRAIN_PAUSE_FILE=/run/openvidu-media-health.paused
|
||||
|
||||
log() { echo "[media-health] $*"; }
|
||||
|
||||
# Seconds since boot: monotonic, unaffected by clock steps
|
||||
uptime_sec() { cut -d. -f1 /proc/uptime; }
|
||||
|
||||
# Busy CPU percentage since the previous call, in BUSY (0 on the first call)
|
||||
cpu_busy() {
|
||||
read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat
|
||||
total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal))
|
||||
idle=$((c_idle + c_iowait))
|
||||
BUSY=0
|
||||
if [ -n "$PREV_TOTAL" ] && [ "$total" -gt "$PREV_TOTAL" ]; then
|
||||
BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL)))
|
||||
fi
|
||||
PREV_TOTAL=$total
|
||||
PREV_IDLE=$idle
|
||||
}
|
||||
|
||||
# openvidu.service entered the active state less than WARMUP_SEC ago
|
||||
openvidu_starting() {
|
||||
since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null)
|
||||
case "$since" in ''|0|*[!0-9]*) return 1 ;; esac
|
||||
[ $(($(uptime_sec) - since / 1000000)) -lt "$WARMUP_SEC" ]
|
||||
}
|
||||
|
||||
# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare
|
||||
# TCP connect would also succeed against a wedged Redis.
|
||||
master_serving() {
|
||||
[ -n "$MASTER_ADDRS" ] || return 1
|
||||
for addr in $MASTER_ADDRS; do
|
||||
host=$(echo "$addr" | cut -d: -f1)
|
||||
port=$(echo "$addr" | cut -d: -f2)
|
||||
reply=$(timeout 5 bash -c 'exec 3<>"/dev/tcp/$0/$1" && printf "PING\r\n" >&3 && head -c 7 <&3' "$host" "$port" 2>/dev/null)
|
||||
case "$reply" in
|
||||
+PONG*|-NOAUTH*) return 0 ;;
|
||||
esac
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Request the replacement of this node ($1 = reason). Gated on a master answering, except
|
||||
# in the bootstrap phase before the master addresses are known.
|
||||
request_replacement() {
|
||||
now=$(uptime_sec)
|
||||
if [ -n "$REQUESTED_AT" ] && [ $((now - REQUESTED_AT)) -lt "$REPLACE_RETRY_SEC" ]; then
|
||||
return
|
||||
fi
|
||||
if [ -n "$MASTER_ADDRS" ] || [ -f "$INSTALLED_MARKER" ]; then
|
||||
if ! master_serving; then
|
||||
if [ -z "$HOLD_LOGGED_AT" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then
|
||||
log "$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node"
|
||||
HOLD_LOGGED_AT=$now
|
||||
fi
|
||||
return
|
||||
fi
|
||||
fi
|
||||
log "$1: requesting the replacement of this node"
|
||||
if "$REPLACE_SCRIPT" "$1"; then
|
||||
REQUESTED_AT=$now
|
||||
log "replacement requested"
|
||||
else
|
||||
log "the replacement request failed, retrying in $INTERVAL_SEC s"
|
||||
fi
|
||||
}
|
||||
|
||||
mkdir -p "$STATE_DIR"
|
||||
REQUESTED_AT=""
|
||||
HOLD_LOGGED_AT=""
|
||||
FAILED_SEEN_AT=""
|
||||
FAIL_SINCE=""
|
||||
SATURATED=""
|
||||
PREV_TOTAL=""
|
||||
PREV_IDLE=""
|
||||
log "started"
|
||||
|
||||
while true; do
|
||||
PROBE_URL=http://127.0.0.1:7880/
|
||||
INTERVAL_SEC=30
|
||||
HARD_UNHEALTHY_AFTER_SEC=300
|
||||
SOFT_UNHEALTHY_AFTER_SEC=600
|
||||
CPU_SATURATED_PCT=90
|
||||
WARMUP_SEC=900
|
||||
BOOTSTRAP_DEADLINE_SEC=5400
|
||||
BOOTSTRAP_FAILURE_COOLDOWN_SEC=600
|
||||
REPLACE_RETRY_SEC=600
|
||||
MASTER_ADDRS=""
|
||||
[ -f "$ENV_FILE" ] && . "$ENV_FILE"
|
||||
NOW=$(uptime_sec)
|
||||
cpu_busy
|
||||
|
||||
if [ -f "$PAUSE_FILE" ] || [ -f "$DRAIN_PAUSE_FILE" ]; then
|
||||
FAIL_SINCE=""
|
||||
elif [ -f "$BOOTSTRAP_FAILED_MARKER" ]; then
|
||||
if [ -z "$FAILED_SEEN_AT" ]; then
|
||||
FAILED_SEEN_AT=$NOW
|
||||
log "bootstrap failed ($(head -c 300 "$BOOTSTRAP_FAILED_MARKER")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s"
|
||||
fi
|
||||
if [ $((NOW - FAILED_SEEN_AT)) -ge "$BOOTSTRAP_FAILURE_COOLDOWN_SEC" ]; then
|
||||
request_replacement "bootstrap failed: $(head -c 300 "$BOOTSTRAP_FAILED_MARKER")"
|
||||
fi
|
||||
elif [ ! -f "$INSTALLED_MARKER" ]; then
|
||||
if [ "$NOW" -ge "$BOOTSTRAP_DEADLINE_SEC" ]; then
|
||||
request_replacement "bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot"
|
||||
fi
|
||||
elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 "$PROBE_URL"; then
|
||||
[ -n "$FAIL_SINCE" ] && log "LiveKit healthy again after $((NOW - FAIL_SINCE)) s"
|
||||
FAIL_SINCE=""
|
||||
SATURATED=""
|
||||
else
|
||||
RC=$?
|
||||
case "$RC" in
|
||||
7|52|56) KIND=hard ;;
|
||||
*) KIND=soft ;;
|
||||
esac
|
||||
if openvidu_starting; then
|
||||
FAIL_SINCE=""
|
||||
elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then
|
||||
[ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted"
|
||||
SATURATED=1
|
||||
FAIL_SINCE=""
|
||||
else
|
||||
SATURATED=""
|
||||
if [ -z "$FAIL_SINCE" ]; then
|
||||
FAIL_SINCE=$NOW
|
||||
log "LiveKit health check failed (curl exit $RC)"
|
||||
fi
|
||||
LIMIT=$SOFT_UNHEALTHY_AFTER_SEC
|
||||
[ "$KIND" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC
|
||||
if [ $((NOW - FAIL_SINCE)) -ge "$LIMIT" ]; then
|
||||
request_replacement "LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
sleep "$INTERVAL_SEC"
|
||||
done
|
||||
mode: "000755"
|
||||
owner: "root"
|
||||
group: "root"
|
||||
'/usr/local/bin/openvidu-media-replace.sh':
|
||||
content: !Sub |
|
||||
#!/bin/bash
|
||||
set -e
|
||||
# Ask the Auto Scaling group to replace this Media Node with a new one, keeping the
|
||||
# count. Called by openvidu-media-health.sh (same cgroup): it never stops or signals
|
||||
# the watchdog, which re-requests until this node goes away.
|
||||
mkdir -p /var/lib/openvidu-media-health
|
||||
echo "$(date -u +%FT%TZ) $1" >> /var/lib/openvidu-media-health/replace-requests.log
|
||||
# Let stop_media_node.sh know this node is unhealthy, so it does not wait for sessions
|
||||
touch /run/openvidu-unhealthy
|
||||
|
||||
# Token for IMDSv2
|
||||
TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
|
||||
|
||||
# Get own instance ID
|
||||
INSTANCE_ID=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-id)
|
||||
|
||||
# Set instance as unhealthy
|
||||
aws autoscaling set-instance-health \
|
||||
--region ${AWS::Region} \
|
||||
--instance-id "$INSTANCE_ID" \
|
||||
--health-status Unhealthy
|
||||
# set-instance-health Unhealthy makes the ASG terminate this node through the existing
|
||||
# lifecycle hook (drain) and launch a replacement. Idempotent; retry transient API errors.
|
||||
for attempt in $(seq 1 5); do
|
||||
if aws autoscaling set-instance-health \
|
||||
--region ${AWS::Region} \
|
||||
--instance-id "$INSTANCE_ID" \
|
||||
--health-status Unhealthy; then
|
||||
exit 0
|
||||
fi
|
||||
echo "[openvidu-media-replace] set-instance-health failed (attempt $attempt/5), retrying in 12s" >&2
|
||||
sleep 12
|
||||
done
|
||||
exit 1
|
||||
mode: "000755"
|
||||
owner: "root"
|
||||
group: "root"
|
||||
'/etc/systemd/system/openvidu-media-health.service':
|
||||
content: |
|
||||
[Unit]
|
||||
Description=OpenVidu Media Node health watchdog
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/local/bin/openvidu-media-health.sh
|
||||
Restart=always
|
||||
RestartSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
mode: "000644"
|
||||
owner: "root"
|
||||
group: "root"
|
||||
'/usr/local/bin/stop_media_node.sh':
|
||||
content: !Sub |
|
||||
#!/bin/bash
|
||||
set -e
|
||||
# Drain: pause and stop the health watchdog so it does not replace this node
|
||||
touch /run/openvidu-media-health.paused
|
||||
systemctl stop openvidu-media-health 2>/dev/null || true
|
||||
# Token for IMDSv2
|
||||
TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
|
||||
|
||||
|
|
@ -2839,6 +3049,10 @@ Resources:
|
|||
done
|
||||
TIME_PASSED=0
|
||||
HEARTBEAT_MAX=1800
|
||||
# An unhealthy node being replaced should not wait for sessions: kill now
|
||||
if [ -f /run/openvidu-unhealthy ]; then
|
||||
docker ps -q | xargs -r docker kill >/dev/null 2>&1 || true
|
||||
fi
|
||||
# Wait for running containers to not be openvidu, ingress, egress or an openvidu agent
|
||||
while [ "$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)" -gt 0 ] || \
|
||||
[ "$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)" = "true" ] || \
|
||||
|
|
@ -2898,11 +3112,22 @@ Resources:
|
|||
#!/bin/bash
|
||||
set -eu -o pipefail
|
||||
|
||||
# Record a media-node bootstrap failure so the health watchdog (started below)
|
||||
# replaces this node after a cool-down. Route user-data failure exits through this.
|
||||
ov_media_bootstrap_failed() {
|
||||
echo "[OpenVidu] media node bootstrap failed: $1"
|
||||
mkdir -p /var/lib/openvidu-media-health
|
||||
echo "$1" > /var/lib/openvidu-media-health/bootstrap-failed
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout
|
||||
|
||||
apt-get update && apt-get install -y \
|
||||
python3-pip \
|
||||
ec2-instance-connect
|
||||
ec2-instance-connect \
|
||||
curl \
|
||||
unzip
|
||||
|
||||
CFN_BOOTSTRAP_VERSION=2.0-39
|
||||
# Detect Ubuntu version and install cfn-bootstrap accordingly
|
||||
|
|
@ -2917,13 +3142,38 @@ Resources:
|
|||
|
||||
export HOME="/root"
|
||||
|
||||
# aws-cli v2, hoisted from install.sh so the watchdog's replace script can call
|
||||
# the ASG API even if bootstrap hangs before install.sh runs
|
||||
AWS_CLI_VERSION=2.36.40
|
||||
if ! command -v aws &> /dev/null; then
|
||||
curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-${!AWS_CLI_VERSION}.zip" -o "awscliv2.zip"
|
||||
unzip -qq awscliv2.zip
|
||||
./aws/install
|
||||
rm -rf awscliv2.zip aws
|
||||
fi
|
||||
|
||||
# Start the health watchdog before install.sh so a hung bootstrap is covered too
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now openvidu-media-health
|
||||
|
||||
# Install OpenVidu
|
||||
/usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu"; /usr/local/bin/set_as_unhealthy.sh; exit 1; }
|
||||
/usr/local/bin/install.sh || ov_media_bootstrap_failed "install.sh failed"
|
||||
|
||||
MASTER_IP_1=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-1-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none')
|
||||
MASTER_IP_2=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-2-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none')
|
||||
MASTER_IP_3=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-3-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none')
|
||||
MASTER_IP_4=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-4-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none')
|
||||
|
||||
# Health watchdog config: master addresses for the anti-storm Redis PING guard
|
||||
mkdir -p /etc/openvidu
|
||||
MASTER_ADDRS=""
|
||||
for MASTER_IP in "$MASTER_IP_1" "$MASTER_IP_2" "$MASTER_IP_3" "$MASTER_IP_4"; do
|
||||
if [ "$MASTER_IP" != "none" ] && [ -n "$MASTER_IP" ]; then
|
||||
MASTER_ADDRS="$MASTER_ADDRS $MASTER_IP:7001"
|
||||
fi
|
||||
done
|
||||
echo "MASTER_ADDRS=\"$(echo $MASTER_ADDRS | xargs)\"" > /etc/openvidu/media-health.env
|
||||
|
||||
MASTER_HEALTHY=false
|
||||
for i in $(seq 1 360); do
|
||||
for MASTER_IP in "$MASTER_IP_1" "$MASTER_IP_2" "$MASTER_IP_3" "$MASTER_IP_4"; do
|
||||
|
|
@ -2938,13 +3188,15 @@ Resources:
|
|||
sleep 5
|
||||
done
|
||||
if [ "$MASTER_HEALTHY" != "true" ]; then
|
||||
echo "[OpenVidu] no master node became healthy after 30 minutes"
|
||||
/usr/local/bin/set_as_unhealthy.sh
|
||||
exit 1
|
||||
ov_media_bootstrap_failed "no master node became healthy after 30 minutes"
|
||||
fi
|
||||
|
||||
# Start OpenVidu
|
||||
systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; /usr/local/bin/set_as_unhealthy.sh; exit 1; }
|
||||
systemctl start openvidu || ov_media_bootstrap_failed "error starting OpenVidu"
|
||||
|
||||
# Bootstrap finished: switch the watchdog from the bootstrap deadline to runtime checks
|
||||
mkdir -p /var/lib/openvidu-media-health
|
||||
touch /var/lib/openvidu-media-health/installed
|
||||
|
||||
# Wait for the app
|
||||
# /usr/local/bin/check_app_ready.sh
|
||||
|
|
|
|||
|
|
@ -1395,6 +1395,11 @@ var stop_media_nodesScriptMediaTemplate = '''
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Pause the health watchdog before draining so it never reimages a node that is being
|
||||
# drained for scale-in (synchronous; the drain runs outside the watchdog's cgroup).
|
||||
touch /run/openvidu-media-health.paused
|
||||
systemctl stop openvidu-media-health 2>/dev/null || true
|
||||
|
||||
if ! (set -o noclobber ; echo > /tmp/global.lock) ; then
|
||||
exit 1 # the global.lock already exists
|
||||
fi
|
||||
|
|
@ -1410,6 +1415,11 @@ if [ -x "$(command -v docker)" ]; then
|
|||
docker container kill --signal=SIGQUIT "$agent_container" 2>/dev/null || true
|
||||
done
|
||||
|
||||
# If this node is being replaced because it is unhealthy, do not wait for sessions to end.
|
||||
if [ -f /run/openvidu-unhealthy ]; then
|
||||
docker ps -q | xargs -r docker kill >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
# Wait for running containers to not be openvidu, ingress or egress
|
||||
while [ "$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)" -gt 0 ] || \
|
||||
[ "$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)" = "true" ] || \
|
||||
|
|
@ -1435,25 +1445,244 @@ az tag update --resource-id $RESOURCE_ID --operation replace --tags "STATUS"="HE
|
|||
az vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID
|
||||
'''
|
||||
|
||||
var delete_mediaNode_ScriptMediaTemplate = '''
|
||||
var openviduMediaHealthScript = '''
|
||||
#!/bin/bash
|
||||
set -e
|
||||
# OpenVidu Media Node health watchdog.
|
||||
#
|
||||
# Asks the cloud to replace this Media Node with a new one, keeping the number of
|
||||
# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when:
|
||||
# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a
|
||||
# cool-down, so a persistent failure does not churn nodes and can be inspected,
|
||||
# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot,
|
||||
# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC
|
||||
# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC
|
||||
# while it answers an error or times out. Soft failures are not counted while the CPU
|
||||
# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in
|
||||
# the WARMUP_SEC after openvidu.service (re)started.
|
||||
# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage
|
||||
# fails the health check of every Media Node at once, and a new node could not bootstrap
|
||||
# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until
|
||||
# the node goes away.
|
||||
#
|
||||
# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...")
|
||||
# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)
|
||||
# Logs: journalctl -u openvidu-media-health
|
||||
#
|
||||
# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and
|
||||
# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed
|
||||
# by an opening brace, nor two opening braces in a row.
|
||||
|
||||
az login --identity --client-id ${mediaIdentityClientId}
|
||||
STATE_DIR=/var/lib/openvidu-media-health
|
||||
INSTALLED_MARKER="$STATE_DIR/installed"
|
||||
BOOTSTRAP_FAILED_MARKER="$STATE_DIR/bootstrap-failed"
|
||||
REPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh
|
||||
ENV_FILE=/etc/openvidu/media-health.env
|
||||
PAUSE_FILE=/etc/openvidu/media-health.disabled
|
||||
DRAIN_PAUSE_FILE=/run/openvidu-media-health.paused
|
||||
|
||||
log() { echo "[media-health] $*"; }
|
||||
|
||||
# Seconds since boot: monotonic, unaffected by clock steps
|
||||
uptime_sec() { cut -d. -f1 /proc/uptime; }
|
||||
|
||||
# Busy CPU percentage since the previous call, in BUSY (0 on the first call)
|
||||
cpu_busy() {
|
||||
read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat
|
||||
total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal))
|
||||
idle=$((c_idle + c_iowait))
|
||||
BUSY=0
|
||||
if [ -n "$PREV_TOTAL" ] && [ "$total" -gt "$PREV_TOTAL" ]; then
|
||||
BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL)))
|
||||
fi
|
||||
PREV_TOTAL=$total
|
||||
PREV_IDLE=$idle
|
||||
}
|
||||
|
||||
# openvidu.service entered the active state less than WARMUP_SEC ago
|
||||
openvidu_starting() {
|
||||
since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null)
|
||||
case "$since" in ''|0|*[!0-9]*) return 1 ;; esac
|
||||
[ $(($(uptime_sec) - since / 1000000)) -lt "$WARMUP_SEC" ]
|
||||
}
|
||||
|
||||
# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare
|
||||
# TCP connect would also succeed against a wedged Redis.
|
||||
master_serving() {
|
||||
[ -n "$MASTER_ADDRS" ] || return 1
|
||||
for addr in $MASTER_ADDRS; do
|
||||
host=$(echo "$addr" | cut -d: -f1)
|
||||
port=$(echo "$addr" | cut -d: -f2)
|
||||
reply=$(timeout 5 bash -c 'exec 3<>"/dev/tcp/$0/$1" && printf "PING\r\n" >&3 && head -c 7 <&3' "$host" "$port" 2>/dev/null)
|
||||
case "$reply" in
|
||||
+PONG*|-NOAUTH*) return 0 ;;
|
||||
esac
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Request the replacement of this node ($1 = reason). Gated on a master answering, except
|
||||
# in the bootstrap phase before the master addresses are known.
|
||||
request_replacement() {
|
||||
now=$(uptime_sec)
|
||||
if [ -n "$REQUESTED_AT" ] && [ $((now - REQUESTED_AT)) -lt "$REPLACE_RETRY_SEC" ]; then
|
||||
return
|
||||
fi
|
||||
if [ -n "$MASTER_ADDRS" ] || [ -f "$INSTALLED_MARKER" ]; then
|
||||
if ! master_serving; then
|
||||
if [ -z "$HOLD_LOGGED_AT" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then
|
||||
log "$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node"
|
||||
HOLD_LOGGED_AT=$now
|
||||
fi
|
||||
return
|
||||
fi
|
||||
fi
|
||||
log "$1: requesting the replacement of this node"
|
||||
if "$REPLACE_SCRIPT" "$1"; then
|
||||
REQUESTED_AT=$now
|
||||
log "replacement requested"
|
||||
else
|
||||
log "the replacement request failed, retrying in $INTERVAL_SEC s"
|
||||
fi
|
||||
}
|
||||
|
||||
mkdir -p "$STATE_DIR"
|
||||
REQUESTED_AT=""
|
||||
HOLD_LOGGED_AT=""
|
||||
FAILED_SEEN_AT=""
|
||||
FAIL_SINCE=""
|
||||
SATURATED=""
|
||||
PREV_TOTAL=""
|
||||
PREV_IDLE=""
|
||||
log "started"
|
||||
|
||||
while true; do
|
||||
PROBE_URL=http://127.0.0.1:7880/
|
||||
INTERVAL_SEC=30
|
||||
HARD_UNHEALTHY_AFTER_SEC=300
|
||||
SOFT_UNHEALTHY_AFTER_SEC=600
|
||||
CPU_SATURATED_PCT=90
|
||||
WARMUP_SEC=900
|
||||
BOOTSTRAP_DEADLINE_SEC=5400
|
||||
BOOTSTRAP_FAILURE_COOLDOWN_SEC=600
|
||||
REPLACE_RETRY_SEC=600
|
||||
MASTER_ADDRS=""
|
||||
[ -f "$ENV_FILE" ] && . "$ENV_FILE"
|
||||
NOW=$(uptime_sec)
|
||||
cpu_busy
|
||||
|
||||
if [ -f "$PAUSE_FILE" ] || [ -f "$DRAIN_PAUSE_FILE" ]; then
|
||||
FAIL_SINCE=""
|
||||
elif [ -f "$BOOTSTRAP_FAILED_MARKER" ]; then
|
||||
if [ -z "$FAILED_SEEN_AT" ]; then
|
||||
FAILED_SEEN_AT=$NOW
|
||||
log "bootstrap failed ($(head -c 300 "$BOOTSTRAP_FAILED_MARKER")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s"
|
||||
fi
|
||||
if [ $((NOW - FAILED_SEEN_AT)) -ge "$BOOTSTRAP_FAILURE_COOLDOWN_SEC" ]; then
|
||||
request_replacement "bootstrap failed: $(head -c 300 "$BOOTSTRAP_FAILED_MARKER")"
|
||||
fi
|
||||
elif [ ! -f "$INSTALLED_MARKER" ]; then
|
||||
if [ "$NOW" -ge "$BOOTSTRAP_DEADLINE_SEC" ]; then
|
||||
request_replacement "bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot"
|
||||
fi
|
||||
elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 "$PROBE_URL"; then
|
||||
[ -n "$FAIL_SINCE" ] && log "LiveKit healthy again after $((NOW - FAIL_SINCE)) s"
|
||||
FAIL_SINCE=""
|
||||
SATURATED=""
|
||||
else
|
||||
RC=$?
|
||||
case "$RC" in
|
||||
7|52|56) KIND=hard ;;
|
||||
*) KIND=soft ;;
|
||||
esac
|
||||
if openvidu_starting; then
|
||||
FAIL_SINCE=""
|
||||
elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then
|
||||
[ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted"
|
||||
SATURATED=1
|
||||
FAIL_SINCE=""
|
||||
else
|
||||
SATURATED=""
|
||||
if [ -z "$FAIL_SINCE" ]; then
|
||||
FAIL_SINCE=$NOW
|
||||
log "LiveKit health check failed (curl exit $RC)"
|
||||
fi
|
||||
LIMIT=$SOFT_UNHEALTHY_AFTER_SEC
|
||||
[ "$KIND" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC
|
||||
if [ $((NOW - FAIL_SINCE)) -ge "$LIMIT" ]; then
|
||||
request_replacement "LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
sleep "$INTERVAL_SEC"
|
||||
done
|
||||
'''
|
||||
|
||||
var openviduMediaHealthUnit = '''
|
||||
[Unit]
|
||||
Description=OpenVidu Media Node health watchdog
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/local/bin/openvidu-media-health.sh
|
||||
Restart=always
|
||||
RestartSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
'''
|
||||
|
||||
var media_replaceScriptMediaTemplate = '''
|
||||
#!/bin/bash
|
||||
# openvidu-media-replace.sh "<reason>"
|
||||
# Requests the reimage of THIS Media Node, keeping the scale set capacity and the
|
||||
# scale-in protection: reimage provisions a new OS disk and the userData bootstrap runs
|
||||
# again on the same instance id. Called by the health watchdog; it never signals it.
|
||||
|
||||
mkdir -p /var/lib/openvidu-media-health
|
||||
echo "$(date -u +%FT%TZ) $1" >> /var/lib/openvidu-media-health/replace-requests.log
|
||||
touch /run/openvidu-unhealthy
|
||||
|
||||
RESOURCE_GROUP_NAME=${resourceGroupName}
|
||||
VM_SCALE_SET_NAME=${vmScaleSetName}
|
||||
BEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2021-02-01" | jq -r '.compute.resourceId')
|
||||
INSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')
|
||||
INSTANCE_RESOURCE_ID=$(curl -s -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2021-02-01" | jq -r '.compute.resourceId')
|
||||
INSTANCE_ID=$(echo "$INSTANCE_RESOURCE_ID" | awk -F'/' '{print $NF}')
|
||||
if [ -z "$INSTANCE_ID" ]; then
|
||||
echo "[media-replace] could not read this instance id from IMDS"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
az vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID
|
||||
n=0
|
||||
while true; do
|
||||
n=$((n + 1))
|
||||
if az login --identity --client-id ${mediaIdentityClientId} >/dev/null 2>&1 && \
|
||||
az vmss reimage --resource-group "$RESOURCE_GROUP_NAME" --name "$VM_SCALE_SET_NAME" --instance-ids "$INSTANCE_ID" --no-wait; then
|
||||
echo "[media-replace] reimage requested for instance $INSTANCE_ID ($1)"
|
||||
exit 0
|
||||
fi
|
||||
if [ "$n" -ge 5 ]; then
|
||||
echo "[media-replace] reimage could not be requested after $n attempts"
|
||||
exit 1
|
||||
fi
|
||||
echo "[media-replace] reimage request failed (attempt $n), retrying in 12s"
|
||||
sleep 12
|
||||
done
|
||||
'''
|
||||
|
||||
|
||||
var userDataMediaNodeTemplate = '''
|
||||
#!/bin/bash -x
|
||||
set -eu -o pipefail
|
||||
|
||||
# Mark this media node's bootstrap as failed so the health watchdog replaces it
|
||||
ov_media_bootstrap_failed() {
|
||||
echo "[OpenVidu] media node bootstrap failed: $1"
|
||||
mkdir -p /var/lib/openvidu-media-health
|
||||
echo "$1" > /var/lib/openvidu-media-health/bootstrap-failed
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Introduce the scripts in the instance
|
||||
# install.sh
|
||||
echo ${base64install} | base64 -d > /usr/local/bin/install.sh
|
||||
|
|
@ -1463,9 +1692,12 @@ chmod +x /usr/local/bin/install.sh
|
|||
echo ${base64stop} | base64 -d > /usr/local/bin/stop_media_node.sh
|
||||
chmod +x /usr/local/bin/stop_media_node.sh
|
||||
|
||||
# delete_media_node.sh
|
||||
echo ${base64delete_mediaNode} | base64 -d > /usr/local/bin/delete_media_node.sh
|
||||
chmod +x /usr/local/bin/delete_media_node.sh
|
||||
# openvidu-media-health.sh (health watchdog), openvidu-media-replace.sh (reimage this node) and its unit
|
||||
echo ${base64mediaHealth} | base64 -d > /usr/local/bin/openvidu-media-health.sh
|
||||
chmod +x /usr/local/bin/openvidu-media-health.sh
|
||||
echo ${base64mediaReplace} | base64 -d > /usr/local/bin/openvidu-media-replace.sh
|
||||
chmod +x /usr/local/bin/openvidu-media-replace.sh
|
||||
echo ${base64mediaHealthUnit} | base64 -d > /etc/systemd/system/openvidu-media-health.service
|
||||
|
||||
echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout
|
||||
|
||||
|
|
@ -1490,8 +1722,16 @@ az vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME -
|
|||
|
||||
export HOME="/root"
|
||||
|
||||
# Start the Media Node health watchdog before install.sh (az CLI is installed and the node
|
||||
# is scale-in protected): it backstops a hung or failed bootstrap and, once running, reimages
|
||||
# the node if LiveKit stays unhealthy. Write MASTER_ADDRS first so its master guard is armed.
|
||||
mkdir -p /etc/openvidu
|
||||
echo 'MASTER_ADDRS="${privateIPMasterNode1}:7001 ${privateIPMasterNode2}:7001 ${privateIPMasterNode3}:7001 ${privateIPMasterNode4}:7001"' > /etc/openvidu/media-health.env
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now openvidu-media-health
|
||||
|
||||
# Install OpenVidu
|
||||
/usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu"; /usr/local/bin/delete_media_node.sh; }
|
||||
/usr/local/bin/install.sh || ov_media_bootstrap_failed "install.sh failed"
|
||||
|
||||
# Gate 2: wait for master readiness before starting
|
||||
WAIT_INTERVAL=5
|
||||
|
|
@ -1505,16 +1745,18 @@ while true; do
|
|||
fi
|
||||
RETRIES=$((RETRIES + 1))
|
||||
if [ $RETRIES -ge $MAX_RETRIES ]; then
|
||||
echo "[OpenVidu] timed out after 30 min waiting for FINISH-MASTER-NODE"
|
||||
/usr/local/bin/delete_media_node.sh
|
||||
exit 1
|
||||
ov_media_bootstrap_failed "timed out after 30 min waiting for FINISH-MASTER-NODE"
|
||||
fi
|
||||
sleep $WAIT_INTERVAL
|
||||
done
|
||||
set -e
|
||||
|
||||
# Start OpenVidu
|
||||
systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; /usr/local/bin/delete_media_node.sh; }
|
||||
systemctl start openvidu || ov_media_bootstrap_failed "systemctl start openvidu failed"
|
||||
|
||||
# Bootstrap finished: the watchdog switches from bootstrap-deadline mode to runtime health checks
|
||||
mkdir -p /var/lib/openvidu-media-health
|
||||
touch /var/lib/openvidu-media-health/installed
|
||||
'''
|
||||
|
||||
var installScriptMedia = reduce(
|
||||
|
|
@ -1529,20 +1771,28 @@ var stop_media_nodesScriptMedia = reduce(
|
|||
(curr, next) => { value: replace(curr.value, '\${${next.key}}', next.value) }
|
||||
).value
|
||||
|
||||
var delete_mediaNode_ScriptMedia = reduce(
|
||||
var media_replaceScriptMedia = reduce(
|
||||
items(stopMediaNodeParams),
|
||||
{ value: delete_mediaNode_ScriptMediaTemplate },
|
||||
{ value: media_replaceScriptMediaTemplate },
|
||||
(curr, next) => { value: replace(curr.value, '\${${next.key}}', next.value) }
|
||||
).value
|
||||
|
||||
var base64installMedia = base64(installScriptMedia)
|
||||
var base64stopMediaNode = base64(stop_media_nodesScriptMedia)
|
||||
var base64delete_mediaNode_ScriptMedia = base64(delete_mediaNode_ScriptMedia)
|
||||
var base64media_replaceScriptMedia = base64(media_replaceScriptMedia)
|
||||
var base64openviduMediaHealth = base64(openviduMediaHealthScript)
|
||||
var base64openviduMediaHealthUnit = base64(openviduMediaHealthUnit)
|
||||
|
||||
var userDataParamsMedia = {
|
||||
base64install: base64installMedia
|
||||
base64stop: base64stopMediaNode
|
||||
base64delete_mediaNode: base64delete_mediaNode_ScriptMedia
|
||||
base64mediaHealth: base64openviduMediaHealth
|
||||
base64mediaHealthUnit: base64openviduMediaHealthUnit
|
||||
base64mediaReplace: base64media_replaceScriptMedia
|
||||
privateIPMasterNode1: privateIPMasterNode1
|
||||
privateIPMasterNode2: privateIPMasterNode2
|
||||
privateIPMasterNode3: privateIPMasterNode3
|
||||
privateIPMasterNode4: privateIPMasterNode4
|
||||
mediaIdentityClientId: mediaIdentity.properties.clientId
|
||||
resourceGroupName: resourceGroup().name
|
||||
vmScaleSetName: '${stackName}-mediaNodeScaleSet'
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -194,6 +194,22 @@ resource "google_compute_firewall" "master_to_media_services" {
|
|||
depends_on = [google_project_service.compute_api]
|
||||
}
|
||||
|
||||
# Allow Google health-check probes (MIG media autohealing) to reach media nodes on 7880
|
||||
resource "google_compute_firewall" "health_check_to_media" {
|
||||
name = lower("${var.stackName}-health-check-to-media")
|
||||
network = "default"
|
||||
|
||||
allow {
|
||||
protocol = "tcp"
|
||||
ports = ["7880"]
|
||||
}
|
||||
|
||||
source_ranges = ["35.191.0.0/16", "130.211.0.0/22"]
|
||||
target_tags = [lower("${var.stackName}-media-node")]
|
||||
|
||||
depends_on = [google_project_service.compute_api]
|
||||
}
|
||||
|
||||
|
||||
# Regional static IP for the Network Load Balancer (conditional creation)
|
||||
resource "google_compute_address" "nlb_ip" {
|
||||
|
|
@ -940,6 +956,26 @@ resource "google_compute_instance_template" "media_node_template" {
|
|||
}
|
||||
}
|
||||
|
||||
# Health check for media node auto-healing
|
||||
resource "google_compute_region_health_check" "media_node_health_check" {
|
||||
name = lower("${var.stackName}-media-node-health-check")
|
||||
region = var.region
|
||||
|
||||
# TCP (not HTTP /): during a master/Redis outage LiveKit answers 406 on every media node at
|
||||
# once, so an HTTP check would recreate the whole fleet. Generous thresholds because auto-heal
|
||||
# recreation kills live WebRTC sessions; the on-node watchdog owns application-level health.
|
||||
tcp_health_check {
|
||||
port = 7880
|
||||
}
|
||||
|
||||
check_interval_sec = 30
|
||||
timeout_sec = 10
|
||||
healthy_threshold = 2
|
||||
unhealthy_threshold = 10
|
||||
|
||||
depends_on = [google_project_service.compute_api]
|
||||
}
|
||||
|
||||
# Managed Instance Group for Media Nodes
|
||||
resource "google_compute_region_instance_group_manager" "media_node_group" {
|
||||
name = lower("${var.stackName}-media-node-group")
|
||||
|
|
@ -955,6 +991,13 @@ resource "google_compute_region_instance_group_manager" "media_node_group" {
|
|||
name = "http"
|
||||
port = 7880
|
||||
}
|
||||
|
||||
# VM/process backstop only. initial_delay_sec at the 3600 s max so autohealing never recreates a
|
||||
# still-bootstrapping node; the marker-aware on-node watchdog owns bootstrap-failure detection.
|
||||
auto_healing_policies {
|
||||
health_check = google_compute_region_health_check.media_node_health_check.id
|
||||
initial_delay_sec = 3600
|
||||
}
|
||||
}
|
||||
|
||||
# Autoscaler for Media Nodes
|
||||
|
|
@ -1681,6 +1724,12 @@ if ! flock -n 9; then
|
|||
exit 0
|
||||
fi
|
||||
|
||||
# Stop the health watchdog so it does not request a replacement while this node drains. The
|
||||
# pause file is honored on the watchdog's next loop regardless of when the unit stops; it is
|
||||
# cleared on reboot.
|
||||
touch /run/openvidu-media-health.paused
|
||||
systemctl stop openvidu-media-health 2>/dev/null || true
|
||||
|
||||
echo "Starting graceful shutdown of OpenVidu Media Node..."
|
||||
|
||||
INSTANCE_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google")
|
||||
|
|
@ -1697,6 +1746,11 @@ if [ -x "$(command -v docker)" ]; then
|
|||
docker container kill --signal=SIGQUIT "$agent_container" 2>/dev/null || true
|
||||
done
|
||||
|
||||
# A node being replaced because it is unhealthy must not wait for sessions to drain.
|
||||
if [ -f /run/openvidu-unhealthy ]; then
|
||||
docker ps -q | xargs -r docker kill >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
# Wait for running containers to stop
|
||||
while [ "$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)" -gt 0 ] || \
|
||||
[ "$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)" = "true" ] || \
|
||||
|
|
@ -1740,12 +1794,256 @@ else
|
|||
echo "Instance $INSTANCE_NAME has been abandoned from MIG. Starting graceful shutdown..."
|
||||
/usr/local/bin/graceful_shutdown.sh
|
||||
fi
|
||||
EOF
|
||||
|
||||
media_health_watchdog_script = <<-EOF
|
||||
#!/bin/bash
|
||||
# OpenVidu Media Node health watchdog.
|
||||
#
|
||||
# Asks the cloud to replace this Media Node with a new one, keeping the number of
|
||||
# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when:
|
||||
# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a
|
||||
# cool-down, so a persistent failure does not churn nodes and can be inspected,
|
||||
# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot,
|
||||
# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC
|
||||
# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC
|
||||
# while it answers an error or times out. Soft failures are not counted while the CPU
|
||||
# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in
|
||||
# the WARMUP_SEC after openvidu.service (re)started.
|
||||
# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage
|
||||
# fails the health check of every Media Node at once, and a new node could not bootstrap
|
||||
# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until
|
||||
# the node goes away.
|
||||
#
|
||||
# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...")
|
||||
# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)
|
||||
# Logs: journalctl -u openvidu-media-health
|
||||
#
|
||||
# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and
|
||||
# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed
|
||||
# by an opening brace, nor two opening braces in a row.
|
||||
|
||||
STATE_DIR=/var/lib/openvidu-media-health
|
||||
INSTALLED_MARKER="$STATE_DIR/installed"
|
||||
BOOTSTRAP_FAILED_MARKER="$STATE_DIR/bootstrap-failed"
|
||||
REPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh
|
||||
ENV_FILE=/etc/openvidu/media-health.env
|
||||
PAUSE_FILE=/etc/openvidu/media-health.disabled
|
||||
DRAIN_PAUSE_FILE=/run/openvidu-media-health.paused
|
||||
|
||||
log() { echo "[media-health] $*"; }
|
||||
|
||||
# Seconds since boot: monotonic, unaffected by clock steps
|
||||
uptime_sec() { cut -d. -f1 /proc/uptime; }
|
||||
|
||||
# Busy CPU percentage since the previous call, in BUSY (0 on the first call)
|
||||
cpu_busy() {
|
||||
read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat
|
||||
total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal))
|
||||
idle=$((c_idle + c_iowait))
|
||||
BUSY=0
|
||||
if [ -n "$PREV_TOTAL" ] && [ "$total" -gt "$PREV_TOTAL" ]; then
|
||||
BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL)))
|
||||
fi
|
||||
PREV_TOTAL=$total
|
||||
PREV_IDLE=$idle
|
||||
}
|
||||
|
||||
# openvidu.service entered the active state less than WARMUP_SEC ago
|
||||
openvidu_starting() {
|
||||
since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null)
|
||||
case "$since" in ''|0|*[!0-9]*) return 1 ;; esac
|
||||
[ $(($(uptime_sec) - since / 1000000)) -lt "$WARMUP_SEC" ]
|
||||
}
|
||||
|
||||
# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare
|
||||
# TCP connect would also succeed against a wedged Redis.
|
||||
master_serving() {
|
||||
[ -n "$MASTER_ADDRS" ] || return 1
|
||||
for addr in $MASTER_ADDRS; do
|
||||
host=$(echo "$addr" | cut -d: -f1)
|
||||
port=$(echo "$addr" | cut -d: -f2)
|
||||
reply=$(timeout 5 bash -c 'exec 3<>"/dev/tcp/$0/$1" && printf "PING\r\n" >&3 && head -c 7 <&3' "$host" "$port" 2>/dev/null)
|
||||
case "$reply" in
|
||||
+PONG*|-NOAUTH*) return 0 ;;
|
||||
esac
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Request the replacement of this node ($1 = reason). Gated on a master answering, except
|
||||
# in the bootstrap phase before the master addresses are known.
|
||||
request_replacement() {
|
||||
now=$(uptime_sec)
|
||||
if [ -n "$REQUESTED_AT" ] && [ $((now - REQUESTED_AT)) -lt "$REPLACE_RETRY_SEC" ]; then
|
||||
return
|
||||
fi
|
||||
if [ -n "$MASTER_ADDRS" ] || [ -f "$INSTALLED_MARKER" ]; then
|
||||
if ! master_serving; then
|
||||
if [ -z "$HOLD_LOGGED_AT" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then
|
||||
log "$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node"
|
||||
HOLD_LOGGED_AT=$now
|
||||
fi
|
||||
return
|
||||
fi
|
||||
fi
|
||||
log "$1: requesting the replacement of this node"
|
||||
if "$REPLACE_SCRIPT" "$1"; then
|
||||
REQUESTED_AT=$now
|
||||
log "replacement requested"
|
||||
else
|
||||
log "the replacement request failed, retrying in $INTERVAL_SEC s"
|
||||
fi
|
||||
}
|
||||
|
||||
mkdir -p "$STATE_DIR"
|
||||
REQUESTED_AT=""
|
||||
HOLD_LOGGED_AT=""
|
||||
FAILED_SEEN_AT=""
|
||||
FAIL_SINCE=""
|
||||
SATURATED=""
|
||||
PREV_TOTAL=""
|
||||
PREV_IDLE=""
|
||||
log "started"
|
||||
|
||||
while true; do
|
||||
PROBE_URL=http://127.0.0.1:7880/
|
||||
INTERVAL_SEC=30
|
||||
HARD_UNHEALTHY_AFTER_SEC=300
|
||||
SOFT_UNHEALTHY_AFTER_SEC=600
|
||||
CPU_SATURATED_PCT=90
|
||||
WARMUP_SEC=900
|
||||
BOOTSTRAP_DEADLINE_SEC=5400
|
||||
BOOTSTRAP_FAILURE_COOLDOWN_SEC=600
|
||||
REPLACE_RETRY_SEC=600
|
||||
MASTER_ADDRS=""
|
||||
[ -f "$ENV_FILE" ] && . "$ENV_FILE"
|
||||
NOW=$(uptime_sec)
|
||||
cpu_busy
|
||||
|
||||
if [ -f "$PAUSE_FILE" ] || [ -f "$DRAIN_PAUSE_FILE" ]; then
|
||||
FAIL_SINCE=""
|
||||
elif [ -f "$BOOTSTRAP_FAILED_MARKER" ]; then
|
||||
if [ -z "$FAILED_SEEN_AT" ]; then
|
||||
FAILED_SEEN_AT=$NOW
|
||||
log "bootstrap failed ($(head -c 300 "$BOOTSTRAP_FAILED_MARKER")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s"
|
||||
fi
|
||||
if [ $((NOW - FAILED_SEEN_AT)) -ge "$BOOTSTRAP_FAILURE_COOLDOWN_SEC" ]; then
|
||||
request_replacement "bootstrap failed: $(head -c 300 "$BOOTSTRAP_FAILED_MARKER")"
|
||||
fi
|
||||
elif [ ! -f "$INSTALLED_MARKER" ]; then
|
||||
if [ "$NOW" -ge "$BOOTSTRAP_DEADLINE_SEC" ]; then
|
||||
request_replacement "bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot"
|
||||
fi
|
||||
elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 "$PROBE_URL"; then
|
||||
[ -n "$FAIL_SINCE" ] && log "LiveKit healthy again after $((NOW - FAIL_SINCE)) s"
|
||||
FAIL_SINCE=""
|
||||
SATURATED=""
|
||||
else
|
||||
RC=$?
|
||||
case "$RC" in
|
||||
7|52|56) KIND=hard ;;
|
||||
*) KIND=soft ;;
|
||||
esac
|
||||
if openvidu_starting; then
|
||||
FAIL_SINCE=""
|
||||
elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then
|
||||
[ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted"
|
||||
SATURATED=1
|
||||
FAIL_SINCE=""
|
||||
else
|
||||
SATURATED=""
|
||||
if [ -z "$FAIL_SINCE" ]; then
|
||||
FAIL_SINCE=$NOW
|
||||
log "LiveKit health check failed (curl exit $RC)"
|
||||
fi
|
||||
LIMIT=$SOFT_UNHEALTHY_AFTER_SEC
|
||||
[ "$KIND" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC
|
||||
if [ $((NOW - FAIL_SINCE)) -ge "$LIMIT" ]; then
|
||||
request_replacement "LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
sleep "$INTERVAL_SEC"
|
||||
done
|
||||
EOF
|
||||
|
||||
media_replace_script = <<-EOF
|
||||
#!/bin/bash
|
||||
# Requests the replacement of THIS media node while keeping the media node count: it asks the
|
||||
# regional MIG to recreate the instance (delete + create in place, same name, new instance id,
|
||||
# fresh disk). Called by the health watchdog and by the bootstrap-failure hook; it never stops
|
||||
# or signals the watchdog (its caller, in the same cgroup).
|
||||
|
||||
REASON="$1"
|
||||
mkdir -p /var/lib/openvidu-media-health
|
||||
echo "$(date -u +%FT%TZ) $REASON" >> /var/lib/openvidu-media-health/replace-requests.log
|
||||
# Tell the drain script this node is being replaced so it skips waiting for sessions.
|
||||
touch /run/openvidu-unhealthy
|
||||
|
||||
META="http://metadata.google.internal/computeMetadata/v1"
|
||||
meta() { curl -s -H "Metadata-Flavor: Google" "$META/$1"; }
|
||||
|
||||
SELF_NAME=$(meta "instance/name")
|
||||
PROJECT_ID=$(meta "project/project-id")
|
||||
ZONE=$(meta "instance/zone" | cut -d/ -f4)
|
||||
REGION=$(echo "$ZONE" | sed 's/-[a-z]$//')
|
||||
STACK_NAME=$(meta "instance/attributes/stackName")
|
||||
MIG_NAME=$(echo "$STACK_NAME" | tr '[:upper:]' '[:lower:]')-media-node-group
|
||||
|
||||
# Recreate keeps the MIG size. Retry transient API errors. A "not a member / already being
|
||||
# deleted, abandoned or recreated" error means the node is already leaving the group (e.g.
|
||||
# scaled in): nothing to do.
|
||||
i=0
|
||||
while [ "$i" -lt 5 ]; do
|
||||
i=$((i + 1))
|
||||
if OUT=$(gcloud compute instance-groups managed recreate-instances "$MIG_NAME" \
|
||||
--instances="$SELF_NAME" --region="$REGION" --project="$PROJECT_ID" --quiet 2>&1); then
|
||||
echo "[media-replace] recreate requested for $SELF_NAME"
|
||||
exit 0
|
||||
fi
|
||||
case "$OUT" in
|
||||
*"not a member"*|*"being deleted"*|*"being recreated"*|*abandon*|*ABANDON*)
|
||||
echo "[media-replace] $SELF_NAME already leaving the group, nothing to do: $OUT"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
echo "[media-replace] recreate attempt $i for $SELF_NAME failed: $OUT"
|
||||
sleep 12
|
||||
done
|
||||
echo "[media-replace] gave up requesting recreate of $SELF_NAME after $i attempts"
|
||||
exit 1
|
||||
EOF
|
||||
|
||||
media_health_unit = <<-EOF
|
||||
[Unit]
|
||||
Description=OpenVidu Media Node health watchdog
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/local/bin/openvidu-media-health.sh
|
||||
Restart=always
|
||||
RestartSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
user_data_media = <<-EOF
|
||||
#!/bin/bash -x
|
||||
set -eu -o pipefail
|
||||
|
||||
# Records a media-node bootstrap failure so the health watchdog replaces this node after a
|
||||
# cool-down, then aborts the bootstrap.
|
||||
ov_media_bootstrap_failed() {
|
||||
echo "[OpenVidu] media node bootstrap failed: $1"
|
||||
mkdir -p /var/lib/openvidu-media-health
|
||||
echo "$1" > /var/lib/openvidu-media-health/bootstrap-failed
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Check if installation already completed
|
||||
if [ -f /usr/local/bin/openvidu_install_counter.txt ]; then
|
||||
# Launch on reboot
|
||||
|
|
@ -1762,6 +2060,21 @@ ${local.graceful_shutdown_script}
|
|||
GRACEFUL_SHUTDOWN_EOF
|
||||
chmod +x /usr/local/bin/graceful_shutdown.sh
|
||||
|
||||
# Media node health watchdog + its replacement script and unit
|
||||
cat > /usr/local/bin/openvidu-media-health.sh << 'MEDIA_HEALTH_EOF'
|
||||
${local.media_health_watchdog_script}
|
||||
MEDIA_HEALTH_EOF
|
||||
chmod +x /usr/local/bin/openvidu-media-health.sh
|
||||
|
||||
cat > /usr/local/bin/openvidu-media-replace.sh << 'MEDIA_REPLACE_EOF'
|
||||
${local.media_replace_script}
|
||||
MEDIA_REPLACE_EOF
|
||||
chmod +x /usr/local/bin/openvidu-media-replace.sh
|
||||
|
||||
cat > /etc/systemd/system/openvidu-media-health.service << 'MEDIA_HEALTH_UNIT_EOF'
|
||||
${local.media_health_unit}
|
||||
MEDIA_HEALTH_UNIT_EOF
|
||||
|
||||
echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout
|
||||
|
||||
GCLOUD_VERSION=573.0.0
|
||||
|
|
@ -1779,14 +2092,29 @@ GRACEFUL_SHUTDOWN_EOF
|
|||
|
||||
export HOME="/root"
|
||||
|
||||
# Start the media node health watchdog now: gcloud (which its replacement needs) is installed,
|
||||
# and starting before install.sh lets the watchdog cover a hung bootstrap too. Write the master
|
||||
# address(es) first (before the installed marker) so the anti-storm guard is always populated.
|
||||
mkdir -p /etc/openvidu
|
||||
MASTER_NODE_PRIVATE_IP_LIST=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/masterNodePrivateIPList" -H "Metadata-Flavor: Google")
|
||||
[ -n "$MASTER_NODE_PRIVATE_IP_LIST" ] || ov_media_bootstrap_failed "could not determine master node private IP list"
|
||||
MASTER_ADDRS=$(echo "$MASTER_NODE_PRIVATE_IP_LIST" | tr ',' ' ' | xargs -n1 | sed 's/$/:7001/' | xargs)
|
||||
echo "MASTER_ADDRS=\"$MASTER_ADDRS\"" > /etc/openvidu/media-health.env
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now openvidu-media-health
|
||||
|
||||
# Install OpenVidu Media Node
|
||||
/usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu Media Node"; exit 1; }
|
||||
/usr/local/bin/install.sh || ov_media_bootstrap_failed "install.sh failed"
|
||||
|
||||
# Mark installation as complete
|
||||
echo "installation_complete" > /usr/local/bin/openvidu_install_counter.txt
|
||||
|
||||
# Start OpenVidu
|
||||
systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; }
|
||||
systemctl start openvidu || ov_media_bootstrap_failed "systemctl start openvidu failed"
|
||||
|
||||
# Bootstrap finished: hand runtime replacement to the watchdog (never on a failure path).
|
||||
mkdir -p /var/lib/openvidu-media-health
|
||||
touch /var/lib/openvidu-media-health/installed
|
||||
|
||||
# Add cron job to check if instance is abandoned every minute
|
||||
cat > /usr/local/bin/check_abandoned.sh << 'CHECK_ABANDONED_EOF'
|
||||
|
|
|
|||
Loading…
Reference in New Issue