From bf54f336e0fe078c20280c535b1d509ec871f36b Mon Sep 17 00:00:00 2001 From: Piwccle Date: Wed, 30 Sep 2026 17:30:23 +0200 Subject: [PATCH] openvidu-deployment: replace unhealthy Media Nodes (AWS, GCP, Azure elastic and HA) Every Media Node runs openvidu-media-health, a watchdog that asks the cloud to replace it with a new node (keeping the Media Node count) when its bootstrap failed or never finished, or when LiveKit fails its health check on :7880 for 5 minutes (down) or 10 minutes (errors or timeouts; not counted while the CPU is saturated). Nothing is replaced while no master answers a Redis PING, so a master outage cannot replace the whole media fleet. Drain scripts pause the watchdog, and skip waiting for sessions on a node being replaced. - AWS: the node marks itself Unhealthy in its Auto Scaling group (the lifecycle hook drains it and a new instance is launched). aws-cli is installed before the watchdog starts; set_as_unhealthy.sh is gone. - GCP: the node asks its managed instance group to recreate it. Media autohealing (TCP 7880) now waits 3600 s before judging a new node and needs 5 minutes of failures; HA gets the same autohealing plus the firewall rule for the health check ranges. - Azure: the node reimages itself (new OS disk, bootstrap runs again, same instance, scale-in protection kept) instead of deleting itself, which shrank the scale set. The .json twins are rebuilt from the .bicep. --- .../pro/elastic/aws/cf-openvidu-elastic.yaml | 272 +++++++++++++- .../elastic/azure/cf-openvidu-elastic.bicep | 285 ++++++++++++++- .../elastic/azure/cf-openvidu-elastic.json | 18 +- .../elastic/gcp/tf-gpc-openvidu-elastic.tf | 299 +++++++++++++++- .../pro/ha/aws/cf-openvidu-ha.yaml | 278 ++++++++++++++- .../pro/ha/azure/cf-openvidu-ha.bicep | 288 ++++++++++++++- .../pro/ha/azure/cf-openvidu-ha.json | 14 +- .../pro/ha/gcp/tf-gpc-openvidu-ha.tf | 332 +++++++++++++++++- 8 files changed, 1701 insertions(+), 85 deletions(-) diff --git a/openvidu-deployment/pro/elastic/aws/cf-openvidu-elastic.yaml b/openvidu-deployment/pro/elastic/aws/cf-openvidu-elastic.yaml index 7c27e14e2..771b00704 100644 --- a/openvidu-deployment/pro/elastic/aws/cf-openvidu-elastic.yaml +++ b/openvidu-deployment/pro/elastic/aws/cf-openvidu-elastic.yaml @@ -2105,7 +2105,6 @@ Resources: SECRETS_WAIT_ATTEMPTS=$((SECRETS_WAIT_ATTEMPTS + 1)) if [[ $SECRETS_WAIT_ATTEMPTS -ge $SECRETS_WAIT_MAX ]]; then echo "Error: timed out after 30 minutes waiting for the master node to generate the shared secrets" - /usr/local/bin/set_as_unhealthy.sh exit 1 fi sleep 5 @@ -2157,28 +2156,238 @@ Resources: mode: "000755" owner: "root" group: "root" - '/usr/local/bin/set_as_unhealthy.sh': + '/usr/local/bin/openvidu-media-health.sh': + content: | + #!/bin/bash + # OpenVidu Media Node health watchdog. + # + # Asks the cloud to replace this Media Node with a new one, keeping the number of + # Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when: + # - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a + # cool-down, so a persistent failure does not churn nodes and can be inspected, + # - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot, + # - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC + # while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC + # while it answers an error or times out. Soft failures are not counted while the CPU + # is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in + # the WARMUP_SEC after openvidu.service (re)started. + # Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage + # fails the health check of every Media Node at once, and a new node could not bootstrap + # either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until + # the node goes away. + # + # Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...") + # Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused) + # Logs: journalctl -u openvidu-media-health + # + # This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and + # Terraform heredocs, so it must never contain a dollar sign or a percent sign followed + # by an opening brace, nor two opening braces in a row. + + STATE_DIR=/var/lib/openvidu-media-health + INSTALLED_MARKER="$STATE_DIR/installed" + BOOTSTRAP_FAILED_MARKER="$STATE_DIR/bootstrap-failed" + REPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh + ENV_FILE=/etc/openvidu/media-health.env + PAUSE_FILE=/etc/openvidu/media-health.disabled + DRAIN_PAUSE_FILE=/run/openvidu-media-health.paused + + log() { echo "[media-health] $*"; } + + # Seconds since boot: monotonic, unaffected by clock steps + uptime_sec() { cut -d. -f1 /proc/uptime; } + + # Busy CPU percentage since the previous call, in BUSY (0 on the first call) + cpu_busy() { + read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat + total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal)) + idle=$((c_idle + c_iowait)) + BUSY=0 + if [ -n "$PREV_TOTAL" ] && [ "$total" -gt "$PREV_TOTAL" ]; then + BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL))) + fi + PREV_TOTAL=$total + PREV_IDLE=$idle + } + + # openvidu.service entered the active state less than WARMUP_SEC ago + openvidu_starting() { + since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null) + case "$since" in ''|0|*[!0-9]*) return 1 ;; esac + [ $(($(uptime_sec) - since / 1000000)) -lt "$WARMUP_SEC" ] + } + + # A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare + # TCP connect would also succeed against a wedged Redis. + master_serving() { + [ -n "$MASTER_ADDRS" ] || return 1 + for addr in $MASTER_ADDRS; do + host=$(echo "$addr" | cut -d: -f1) + port=$(echo "$addr" | cut -d: -f2) + reply=$(timeout 5 bash -c 'exec 3<>"/dev/tcp/$0/$1" && printf "PING\r\n" >&3 && head -c 7 <&3' "$host" "$port" 2>/dev/null) + case "$reply" in + +PONG*|-NOAUTH*) return 0 ;; + esac + done + return 1 + } + + # Request the replacement of this node ($1 = reason). Gated on a master answering, except + # in the bootstrap phase before the master addresses are known. + request_replacement() { + now=$(uptime_sec) + if [ -n "$REQUESTED_AT" ] && [ $((now - REQUESTED_AT)) -lt "$REPLACE_RETRY_SEC" ]; then + return + fi + if [ -n "$MASTER_ADDRS" ] || [ -f "$INSTALLED_MARKER" ]; then + if ! master_serving; then + if [ -z "$HOLD_LOGGED_AT" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then + log "$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node" + HOLD_LOGGED_AT=$now + fi + return + fi + fi + log "$1: requesting the replacement of this node" + if "$REPLACE_SCRIPT" "$1"; then + REQUESTED_AT=$now + log "replacement requested" + else + log "the replacement request failed, retrying in $INTERVAL_SEC s" + fi + } + + mkdir -p "$STATE_DIR" + REQUESTED_AT="" + HOLD_LOGGED_AT="" + FAILED_SEEN_AT="" + FAIL_SINCE="" + SATURATED="" + PREV_TOTAL="" + PREV_IDLE="" + log "started" + + while true; do + PROBE_URL=http://127.0.0.1:7880/ + INTERVAL_SEC=30 + HARD_UNHEALTHY_AFTER_SEC=300 + SOFT_UNHEALTHY_AFTER_SEC=600 + CPU_SATURATED_PCT=90 + WARMUP_SEC=900 + BOOTSTRAP_DEADLINE_SEC=5400 + BOOTSTRAP_FAILURE_COOLDOWN_SEC=600 + REPLACE_RETRY_SEC=600 + MASTER_ADDRS="" + [ -f "$ENV_FILE" ] && . "$ENV_FILE" + NOW=$(uptime_sec) + cpu_busy + + if [ -f "$PAUSE_FILE" ] || [ -f "$DRAIN_PAUSE_FILE" ]; then + FAIL_SINCE="" + elif [ -f "$BOOTSTRAP_FAILED_MARKER" ]; then + if [ -z "$FAILED_SEEN_AT" ]; then + FAILED_SEEN_AT=$NOW + log "bootstrap failed ($(head -c 300 "$BOOTSTRAP_FAILED_MARKER")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s" + fi + if [ $((NOW - FAILED_SEEN_AT)) -ge "$BOOTSTRAP_FAILURE_COOLDOWN_SEC" ]; then + request_replacement "bootstrap failed: $(head -c 300 "$BOOTSTRAP_FAILED_MARKER")" + fi + elif [ ! -f "$INSTALLED_MARKER" ]; then + if [ "$NOW" -ge "$BOOTSTRAP_DEADLINE_SEC" ]; then + request_replacement "bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot" + fi + elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 "$PROBE_URL"; then + [ -n "$FAIL_SINCE" ] && log "LiveKit healthy again after $((NOW - FAIL_SINCE)) s" + FAIL_SINCE="" + SATURATED="" + else + RC=$? + case "$RC" in + 7|52|56) KIND=hard ;; + *) KIND=soft ;; + esac + if openvidu_starting; then + FAIL_SINCE="" + elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then + [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted" + SATURATED=1 + FAIL_SINCE="" + else + SATURATED="" + if [ -z "$FAIL_SINCE" ]; then + FAIL_SINCE=$NOW + log "LiveKit health check failed (curl exit $RC)" + fi + LIMIT=$SOFT_UNHEALTHY_AFTER_SEC + [ "$KIND" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC + if [ $((NOW - FAIL_SINCE)) -ge "$LIMIT" ]; then + request_replacement "LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)" + fi + fi + fi + sleep "$INTERVAL_SEC" + done + mode: "000755" + owner: "root" + group: "root" + '/usr/local/bin/openvidu-media-replace.sh': content: !Sub | #!/bin/bash - set -e + # Ask the Auto Scaling group to replace this Media Node with a new one, keeping the + # count. Called by openvidu-media-health.sh (same cgroup): it never stops or signals + # the watchdog, which re-requests until this node goes away. + mkdir -p /var/lib/openvidu-media-health + echo "$(date -u +%FT%TZ) $1" >> /var/lib/openvidu-media-health/replace-requests.log + # Let stop_media_node.sh know this node is unhealthy, so it does not wait for sessions + touch /run/openvidu-unhealthy + # Token for IMDSv2 TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") # Get own instance ID INSTANCE_ID=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-id) - # Set instance as unhealthy - aws autoscaling set-instance-health \ - --region ${AWS::Region} \ - --instance-id "$INSTANCE_ID" \ - --health-status Unhealthy + # set-instance-health Unhealthy makes the ASG terminate this node through the existing + # lifecycle hook (drain) and launch a replacement. Idempotent; retry transient API errors. + for attempt in $(seq 1 5); do + if aws autoscaling set-instance-health \ + --region ${AWS::Region} \ + --instance-id "$INSTANCE_ID" \ + --health-status Unhealthy; then + exit 0 + fi + echo "[openvidu-media-replace] set-instance-health failed (attempt $attempt/5), retrying in 12s" >&2 + sleep 12 + done + exit 1 mode: "000755" owner: "root" group: "root" + '/etc/systemd/system/openvidu-media-health.service': + content: | + [Unit] + Description=OpenVidu Media Node health watchdog + After=network-online.target + Wants=network-online.target + + [Service] + Type=simple + ExecStart=/usr/local/bin/openvidu-media-health.sh + Restart=always + RestartSec=30 + + [Install] + WantedBy=multi-user.target + mode: "000644" + owner: "root" + group: "root" '/usr/local/bin/stop_media_node.sh': content: !Sub | #!/bin/bash set -e + # Drain: pause and stop the health watchdog so it does not replace this node + touch /run/openvidu-media-health.paused + systemctl stop openvidu-media-health 2>/dev/null || true # Token for IMDSv2 TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") @@ -2197,6 +2406,10 @@ Resources: done TIME_PASSED=0 HEARTBEAT_MAX=1800 + # An unhealthy node being replaced should not wait for sessions: kill now + if [ -f /run/openvidu-unhealthy ]; then + docker ps -q | xargs -r docker kill >/dev/null 2>&1 || true + fi # Wait for running containers to not be openvidu, ingress, egress or an openvidu agent while [ "$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)" -gt 0 ] || \ [ "$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)" = "true" ] || \ @@ -2257,11 +2470,22 @@ Resources: #!/bin/bash set -eu -o pipefail + # Record a media-node bootstrap failure so the health watchdog (started below) + # replaces this node after a cool-down. Route user-data failure exits through this. + ov_media_bootstrap_failed() { + echo "[OpenVidu] media node bootstrap failed: $1" + mkdir -p /var/lib/openvidu-media-health + echo "$1" > /var/lib/openvidu-media-health/bootstrap-failed + exit 1 + } + echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout apt-get update && apt-get install -y \ python3-pip \ - ec2-instance-connect + ec2-instance-connect \ + curl \ + unzip CFN_BOOTSTRAP_VERSION=2.0-39 # Detect Ubuntu version and install cfn-bootstrap accordingly @@ -2276,8 +2500,26 @@ Resources: export HOME="/root" + # aws-cli v2, hoisted from install.sh so the watchdog's replace script can call + # the ASG API even if bootstrap hangs before install.sh runs + AWS_CLI_VERSION=2.36.40 + if ! command -v aws &> /dev/null; then + curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-${!AWS_CLI_VERSION}.zip" -o "awscliv2.zip" + unzip -qq awscliv2.zip + ./aws/install + rm -rf awscliv2.zip aws + fi + + # Health watchdog config: master address for the anti-storm Redis PING guard + mkdir -p /etc/openvidu + echo 'MASTER_ADDRS="${OpenViduMasterNode.PrivateIp}:7000"' > /etc/openvidu/media-health.env + + # Start the health watchdog before install.sh so a hung bootstrap is covered too + systemctl daemon-reload + systemctl enable --now openvidu-media-health + # Install OpenVidu - /usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu"; /usr/local/bin/set_as_unhealthy.sh; exit 1; } + /usr/local/bin/install.sh || ov_media_bootstrap_failed "install.sh failed" # Wait for the master node to be healthy before starting (bounded: up to 1800s at 5s) MASTER_NODE_IP=${OpenViduMasterNode.PrivateIp} @@ -2290,13 +2532,15 @@ Resources: sleep 5 done if [ "$MASTER_HEALTHY" != "true" ]; then - echo "[OpenVidu] master node did not become healthy after 30 minutes" - /usr/local/bin/set_as_unhealthy.sh - exit 1 + ov_media_bootstrap_failed "master node did not become healthy after 30 minutes" fi # Start OpenVidu - systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; /usr/local/bin/set_as_unhealthy.sh; exit 1; } + systemctl start openvidu || ov_media_bootstrap_failed "error starting OpenVidu" + + # Bootstrap finished: switch the watchdog from the bootstrap deadline to runtime checks + mkdir -p /var/lib/openvidu-media-health + touch /var/lib/openvidu-media-health/installed BlockDeviceMappings: - DeviceName: /dev/sda1 diff --git a/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.bicep b/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.bicep index 38a79b97f..431f4e218 100644 --- a/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.bicep +++ b/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.bicep @@ -1002,6 +1002,11 @@ var stop_media_nodesScriptMediaTemplate = ''' #!/bin/bash set -e +# Pause the health watchdog before draining so it never reimages a node that is being +# drained for scale-in (synchronous; the drain runs outside the watchdog's cgroup). +touch /run/openvidu-media-health.paused +systemctl stop openvidu-media-health 2>/dev/null || true + if ! (set -o noclobber ; echo > /tmp/global.lock) ; then exit 1 # the global.lock already exists fi @@ -1017,6 +1022,11 @@ if [ -x "$(command -v docker)" ]; then docker container kill --signal=SIGQUIT "$agent_container" 2>/dev/null || true done + # If this node is being replaced because it is unhealthy, do not wait for sessions to end. + if [ -f /run/openvidu-unhealthy ]; then + docker ps -q | xargs -r docker kill >/dev/null 2>&1 || true + fi + # Wait for running containers to not be openvidu, ingress, egress or an openvidu agent while [ "$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)" -gt 0 ] || \ [ "$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)" = "true" ] || \ @@ -1042,25 +1052,244 @@ az tag update --resource-id $RESOURCE_ID --operation replace --tags "STATUS"="HE az vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID ''' -var delete_mediaNode_ScriptMediaTemplate = ''' +var openviduMediaHealthScript = ''' #!/bin/bash -set -e +# OpenVidu Media Node health watchdog. +# +# Asks the cloud to replace this Media Node with a new one, keeping the number of +# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when: +# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a +# cool-down, so a persistent failure does not churn nodes and can be inspected, +# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot, +# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC +# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC +# while it answers an error or times out. Soft failures are not counted while the CPU +# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in +# the WARMUP_SEC after openvidu.service (re)started. +# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage +# fails the health check of every Media Node at once, and a new node could not bootstrap +# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until +# the node goes away. +# +# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...") +# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused) +# Logs: journalctl -u openvidu-media-health +# +# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and +# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed +# by an opening brace, nor two opening braces in a row. -az login --identity +STATE_DIR=/var/lib/openvidu-media-health +INSTALLED_MARKER="$STATE_DIR/installed" +BOOTSTRAP_FAILED_MARKER="$STATE_DIR/bootstrap-failed" +REPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh +ENV_FILE=/etc/openvidu/media-health.env +PAUSE_FILE=/etc/openvidu/media-health.disabled +DRAIN_PAUSE_FILE=/run/openvidu-media-health.paused + +log() { echo "[media-health] $*"; } + +# Seconds since boot: monotonic, unaffected by clock steps +uptime_sec() { cut -d. -f1 /proc/uptime; } + +# Busy CPU percentage since the previous call, in BUSY (0 on the first call) +cpu_busy() { + read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat + total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal)) + idle=$((c_idle + c_iowait)) + BUSY=0 + if [ -n "$PREV_TOTAL" ] && [ "$total" -gt "$PREV_TOTAL" ]; then + BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL))) + fi + PREV_TOTAL=$total + PREV_IDLE=$idle +} + +# openvidu.service entered the active state less than WARMUP_SEC ago +openvidu_starting() { + since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null) + case "$since" in ''|0|*[!0-9]*) return 1 ;; esac + [ $(($(uptime_sec) - since / 1000000)) -lt "$WARMUP_SEC" ] +} + +# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare +# TCP connect would also succeed against a wedged Redis. +master_serving() { + [ -n "$MASTER_ADDRS" ] || return 1 + for addr in $MASTER_ADDRS; do + host=$(echo "$addr" | cut -d: -f1) + port=$(echo "$addr" | cut -d: -f2) + reply=$(timeout 5 bash -c 'exec 3<>"/dev/tcp/$0/$1" && printf "PING\r\n" >&3 && head -c 7 <&3' "$host" "$port" 2>/dev/null) + case "$reply" in + +PONG*|-NOAUTH*) return 0 ;; + esac + done + return 1 +} + +# Request the replacement of this node ($1 = reason). Gated on a master answering, except +# in the bootstrap phase before the master addresses are known. +request_replacement() { + now=$(uptime_sec) + if [ -n "$REQUESTED_AT" ] && [ $((now - REQUESTED_AT)) -lt "$REPLACE_RETRY_SEC" ]; then + return + fi + if [ -n "$MASTER_ADDRS" ] || [ -f "$INSTALLED_MARKER" ]; then + if ! master_serving; then + if [ -z "$HOLD_LOGGED_AT" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then + log "$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node" + HOLD_LOGGED_AT=$now + fi + return + fi + fi + log "$1: requesting the replacement of this node" + if "$REPLACE_SCRIPT" "$1"; then + REQUESTED_AT=$now + log "replacement requested" + else + log "the replacement request failed, retrying in $INTERVAL_SEC s" + fi +} + +mkdir -p "$STATE_DIR" +REQUESTED_AT="" +HOLD_LOGGED_AT="" +FAILED_SEEN_AT="" +FAIL_SINCE="" +SATURATED="" +PREV_TOTAL="" +PREV_IDLE="" +log "started" + +while true; do + PROBE_URL=http://127.0.0.1:7880/ + INTERVAL_SEC=30 + HARD_UNHEALTHY_AFTER_SEC=300 + SOFT_UNHEALTHY_AFTER_SEC=600 + CPU_SATURATED_PCT=90 + WARMUP_SEC=900 + BOOTSTRAP_DEADLINE_SEC=5400 + BOOTSTRAP_FAILURE_COOLDOWN_SEC=600 + REPLACE_RETRY_SEC=600 + MASTER_ADDRS="" + [ -f "$ENV_FILE" ] && . "$ENV_FILE" + NOW=$(uptime_sec) + cpu_busy + + if [ -f "$PAUSE_FILE" ] || [ -f "$DRAIN_PAUSE_FILE" ]; then + FAIL_SINCE="" + elif [ -f "$BOOTSTRAP_FAILED_MARKER" ]; then + if [ -z "$FAILED_SEEN_AT" ]; then + FAILED_SEEN_AT=$NOW + log "bootstrap failed ($(head -c 300 "$BOOTSTRAP_FAILED_MARKER")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s" + fi + if [ $((NOW - FAILED_SEEN_AT)) -ge "$BOOTSTRAP_FAILURE_COOLDOWN_SEC" ]; then + request_replacement "bootstrap failed: $(head -c 300 "$BOOTSTRAP_FAILED_MARKER")" + fi + elif [ ! -f "$INSTALLED_MARKER" ]; then + if [ "$NOW" -ge "$BOOTSTRAP_DEADLINE_SEC" ]; then + request_replacement "bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot" + fi + elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 "$PROBE_URL"; then + [ -n "$FAIL_SINCE" ] && log "LiveKit healthy again after $((NOW - FAIL_SINCE)) s" + FAIL_SINCE="" + SATURATED="" + else + RC=$? + case "$RC" in + 7|52|56) KIND=hard ;; + *) KIND=soft ;; + esac + if openvidu_starting; then + FAIL_SINCE="" + elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then + [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted" + SATURATED=1 + FAIL_SINCE="" + else + SATURATED="" + if [ -z "$FAIL_SINCE" ]; then + FAIL_SINCE=$NOW + log "LiveKit health check failed (curl exit $RC)" + fi + LIMIT=$SOFT_UNHEALTHY_AFTER_SEC + [ "$KIND" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC + if [ $((NOW - FAIL_SINCE)) -ge "$LIMIT" ]; then + request_replacement "LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)" + fi + fi + fi + sleep "$INTERVAL_SEC" +done +''' + +var openviduMediaHealthUnit = ''' +[Unit] +Description=OpenVidu Media Node health watchdog +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/openvidu-media-health.sh +Restart=always +RestartSec=30 + +[Install] +WantedBy=multi-user.target +''' + +var media_replaceScriptMediaTemplate = ''' +#!/bin/bash +# openvidu-media-replace.sh "" +# Requests the reimage of THIS Media Node, keeping the scale set capacity and the +# scale-in protection: reimage provisions a new OS disk and the userData bootstrap runs +# again on the same instance id. Called by the health watchdog; it never signals it. + +mkdir -p /var/lib/openvidu-media-health +echo "$(date -u +%FT%TZ) $1" >> /var/lib/openvidu-media-health/replace-requests.log +touch /run/openvidu-unhealthy RESOURCE_GROUP_NAME=${resourceGroupName} VM_SCALE_SET_NAME=${vmScaleSetName} -BEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2021-02-01" | jq -r '.compute.resourceId') -INSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}') +INSTANCE_RESOURCE_ID=$(curl -s -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2021-02-01" | jq -r '.compute.resourceId') +INSTANCE_ID=$(echo "$INSTANCE_RESOURCE_ID" | awk -F'/' '{print $NF}') +if [ -z "$INSTANCE_ID" ]; then + echo "[media-replace] could not read this instance id from IMDS" + exit 1 +fi - -az vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID +n=0 +while true; do + n=$((n + 1)) + if az login --identity >/dev/null 2>&1 && \ + az vmss reimage --resource-group "$RESOURCE_GROUP_NAME" --name "$VM_SCALE_SET_NAME" --instance-ids "$INSTANCE_ID" --no-wait; then + echo "[media-replace] reimage requested for instance $INSTANCE_ID ($1)" + exit 0 + fi + if [ "$n" -ge 5 ]; then + echo "[media-replace] reimage could not be requested after $n attempts" + exit 1 + fi + echo "[media-replace] reimage request failed (attempt $n), retrying in 12s" + sleep 12 +done ''' + var userDataMediaNodeTemplate = ''' #!/bin/bash set -eu -o pipefail +# Mark this media node's bootstrap as failed so the health watchdog replaces it +ov_media_bootstrap_failed() { + echo "[OpenVidu] media node bootstrap failed: $1" + mkdir -p /var/lib/openvidu-media-health + echo "$1" > /var/lib/openvidu-media-health/bootstrap-failed + exit 1 +} + # Introduce the scripts in the instance # install.sh echo ${base64install} | base64 -d > /usr/local/bin/install.sh @@ -1070,9 +1299,12 @@ chmod +x /usr/local/bin/install.sh echo ${base64stop} | base64 -d > /usr/local/bin/stop_media_node.sh chmod +x /usr/local/bin/stop_media_node.sh -# delete_media_node.sh -echo ${base64delete} | base64 -d > /usr/local/bin/delete_media_node.sh -chmod +x /usr/local/bin/delete_media_node.sh +# openvidu-media-health.sh (health watchdog), openvidu-media-replace.sh (reimage this node) and its unit +echo ${base64mediaHealth} | base64 -d > /usr/local/bin/openvidu-media-health.sh +chmod +x /usr/local/bin/openvidu-media-health.sh +echo ${base64mediaReplace} | base64 -d > /usr/local/bin/openvidu-media-replace.sh +chmod +x /usr/local/bin/openvidu-media-replace.sh +echo ${base64mediaHealthUnit} | base64 -d > /etc/systemd/system/openvidu-media-health.service echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout @@ -1097,8 +1329,16 @@ az vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME - export HOME="/root" +# Start the Media Node health watchdog before install.sh (az CLI is installed and the node +# is scale-in protected): it backstops a hung or failed bootstrap and, once running, reimages +# the node if LiveKit stays unhealthy. Write MASTER_ADDRS first so its master guard is armed. +mkdir -p /etc/openvidu +echo 'MASTER_ADDRS="${privateIPMasterNode}:7000"' > /etc/openvidu/media-health.env +systemctl daemon-reload +systemctl enable --now openvidu-media-health + # Install OpenVidu -/usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu"; /usr/local/bin/delete_media_node.sh; exit 1; } +/usr/local/bin/install.sh || ov_media_bootstrap_failed "install.sh failed" # Gate 2: wait for master readiness before starting WAIT_INTERVAL=5 @@ -1112,16 +1352,18 @@ while true; do fi RETRIES=$((RETRIES + 1)) if [ $RETRIES -ge $MAX_RETRIES ]; then - echo "[OpenVidu] timed out after 30 min waiting for FINISH-MASTER-NODE" - /usr/local/bin/delete_media_node.sh - exit 1 + ov_media_bootstrap_failed "timed out after 30 min waiting for FINISH-MASTER-NODE" fi sleep $WAIT_INTERVAL done set -e # Start OpenVidu -systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; /usr/local/bin/delete_media_node.sh; } +systemctl start openvidu || ov_media_bootstrap_failed "systemctl start openvidu failed" + +# Bootstrap finished: the watchdog switches from bootstrap-deadline mode to runtime health checks +mkdir -p /var/lib/openvidu-media-health +touch /var/lib/openvidu-media-health/installed ''' var installScriptMedia = reduce( @@ -1140,18 +1382,23 @@ var stop_media_nodesScriptMedia = reduce( var base64stopMediaNode = base64(stop_media_nodesScriptMedia) -var delete_mediaNode_ScriptMedia = reduce( +var media_replaceScriptMedia = reduce( items(stopMediaNodeParams), - { value: delete_mediaNode_ScriptMediaTemplate }, + { value: media_replaceScriptMediaTemplate }, (curr, next) => { value: replace(curr.value, '\${${next.key}}', next.value) } ).value -var base64delete_mediaNode_ScriptMedia = base64(delete_mediaNode_ScriptMedia) +var base64media_replaceScriptMedia = base64(media_replaceScriptMedia) +var base64openviduMediaHealth = base64(openviduMediaHealthScript) +var base64openviduMediaHealthUnit = base64(openviduMediaHealthUnit) var userDataParamsMedia = { base64install: base64installMedia base64stop: base64stopMediaNode - base64delete: base64delete_mediaNode_ScriptMedia + base64mediaHealth: base64openviduMediaHealth + base64mediaHealthUnit: base64openviduMediaHealthUnit + base64mediaReplace: base64media_replaceScriptMedia + privateIPMasterNode: netInterfaceMasterNode.properties.ipConfigurations[0].properties.privateIPAddress resourceGroupName: resourceGroup().name vmScaleSetName: '${stackName}-mediaNodeScaleSet' keyVaultName: keyVaultName diff --git a/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.json b/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.json index 63be97115..1d132eee3 100644 --- a/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.json +++ b/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.46.1.21595", - "templateHash": "250085011205479425" + "templateHash": "17689840301790847570" } }, "parameters": { @@ -278,13 +278,17 @@ "vmScaleSetName": "[format('{0}-mediaNodeScaleSet', parameters('stackName'))]", "storageAccountName": "[if(variables('isEmptyStorageAccountName'), uniqueString(resourceGroup().id), parameters('storageAccountName'))]" }, - "stop_media_nodesScriptMediaTemplate": "#!/bin/bash\nset -e\n\nif ! (set -o noclobber ; echo > /tmp/global.lock) ; then\n exit 1 # the global.lock already exists\nfi\n\n# Execute if docker is installed\nif [ -x \"$(command -v docker)\" ]; then\n\n echo \"Stopping media node services and waiting for termination...\"\n docker container kill --signal=SIGQUIT openvidu 2>/dev/null || true\n docker container kill --signal=SIGQUIT ingress 2>/dev/null || true\n docker container kill --signal=SIGQUIT egress 2>/dev/null || true\n for agent_container in $(docker ps --filter \"label=openvidu-agent=true\" --format '{{.Names}}' 2>/dev/null); do\n docker container kill --signal=SIGQUIT \"$agent_container\" 2>/dev/null || true\n done\n\n # Wait for running containers to not be openvidu, ingress, egress or an openvidu agent\n while [ \"$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)\" -gt 0 ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)\" = \"true\" ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' ingress 2>/dev/null)\" = \"true\" ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' egress 2>/dev/null)\" = \"true\" ]; do\n echo \"Waiting for containers to stop...\"\n sleep 5\n done\nfi\n\naz login --identity\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nSUBSCRIPTION_ID=${subscriptionId}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\nRESOURCE_ID=/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP_NAME/providers/Microsoft.Compute/virtualMachineScaleSets/$VM_SCALE_SET_NAME\nTIMESTAMP=$(date -u +\"%Y-%m-%dT%H:%M:%SZ\")\n\naz tag update --resource-id $RESOURCE_ID --operation replace --tags \"STATUS\"=\"HEALTHY\" \"InstanceDeleteTime\"=\"$TIMESTAMP\" \"storageAccount\"=\"${storageAccountName}\"\n\naz vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID\n", - "delete_mediaNode_ScriptMediaTemplate": "#!/bin/bash\nset -e\n\naz login --identity\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\n\n\naz vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID\n", - "userDataMediaNodeTemplate": "#!/bin/bash\nset -eu -o pipefail\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# stop_media_nodes.sh\necho ${base64stop} | base64 -d > /usr/local/bin/stop_media_node.sh\nchmod +x /usr/local/bin/stop_media_node.sh\n\n# delete_media_node.sh\necho ${base64delete} | base64 -d > /usr/local/bin/delete_media_node.sh\nchmod +x /usr/local/bin/delete_media_node.sh\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y jq\n\n# Install azure cli\nAZURE_CLI_VERSION=2.90.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity\n\n# Protect from scale in actions\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\naz vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-id $INSTANCE_ID --protect-from-scale-in true\n\nexport HOME=\"/root\"\n\n# Install OpenVidu\n/usr/local/bin/install.sh || { echo \"[OpenVidu] error installing OpenVidu\"; /usr/local/bin/delete_media_node.sh; exit 1; }\n\n# Gate 2: wait for master readiness before starting\nWAIT_INTERVAL=5\nMAX_RETRIES=360\nRETRIES=0\nset +e\nwhile true; do\n FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv 2>/dev/null)\n if [ \"$FINISH_MASTER_NODE\" == \"true\" ]; then\n break\n fi\n RETRIES=$((RETRIES + 1))\n if [ $RETRIES -ge $MAX_RETRIES ]; then\n echo \"[OpenVidu] timed out after 30 min waiting for FINISH-MASTER-NODE\"\n /usr/local/bin/delete_media_node.sh\n exit 1\n fi\n sleep $WAIT_INTERVAL\ndone\nset -e\n\n# Start OpenVidu\nsystemctl start openvidu || { echo \"[OpenVidu] error starting OpenVidu\"; /usr/local/bin/delete_media_node.sh; }\n", + "stop_media_nodesScriptMediaTemplate": "#!/bin/bash\nset -e\n\n# Pause the health watchdog before draining so it never reimages a node that is being\n# drained for scale-in (synchronous; the drain runs outside the watchdog's cgroup).\ntouch /run/openvidu-media-health.paused\nsystemctl stop openvidu-media-health 2>/dev/null || true\n\nif ! (set -o noclobber ; echo > /tmp/global.lock) ; then\n exit 1 # the global.lock already exists\nfi\n\n# Execute if docker is installed\nif [ -x \"$(command -v docker)\" ]; then\n\n echo \"Stopping media node services and waiting for termination...\"\n docker container kill --signal=SIGQUIT openvidu 2>/dev/null || true\n docker container kill --signal=SIGQUIT ingress 2>/dev/null || true\n docker container kill --signal=SIGQUIT egress 2>/dev/null || true\n for agent_container in $(docker ps --filter \"label=openvidu-agent=true\" --format '{{.Names}}' 2>/dev/null); do\n docker container kill --signal=SIGQUIT \"$agent_container\" 2>/dev/null || true\n done\n\n # If this node is being replaced because it is unhealthy, do not wait for sessions to end.\n if [ -f /run/openvidu-unhealthy ]; then\n docker ps -q | xargs -r docker kill >/dev/null 2>&1 || true\n fi\n\n # Wait for running containers to not be openvidu, ingress, egress or an openvidu agent\n while [ \"$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)\" -gt 0 ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)\" = \"true\" ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' ingress 2>/dev/null)\" = \"true\" ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' egress 2>/dev/null)\" = \"true\" ]; do\n echo \"Waiting for containers to stop...\"\n sleep 5\n done\nfi\n\naz login --identity\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nSUBSCRIPTION_ID=${subscriptionId}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\nRESOURCE_ID=/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP_NAME/providers/Microsoft.Compute/virtualMachineScaleSets/$VM_SCALE_SET_NAME\nTIMESTAMP=$(date -u +\"%Y-%m-%dT%H:%M:%SZ\")\n\naz tag update --resource-id $RESOURCE_ID --operation replace --tags \"STATUS\"=\"HEALTHY\" \"InstanceDeleteTime\"=\"$TIMESTAMP\" \"storageAccount\"=\"${storageAccountName}\"\n\naz vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID\n", + "openviduMediaHealthScript": "#!/bin/bash\n# OpenVidu Media Node health watchdog.\n#\n# Asks the cloud to replace this Media Node with a new one, keeping the number of\n# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when:\n# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a\n# cool-down, so a persistent failure does not churn nodes and can be inspected,\n# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot,\n# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC\n# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC\n# while it answers an error or times out. Soft failures are not counted while the CPU\n# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in\n# the WARMUP_SEC after openvidu.service (re)started.\n# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage\n# fails the health check of every Media Node at once, and a new node could not bootstrap\n# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until\n# the node goes away.\n#\n# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS=\"ip:port ...\")\n# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)\n# Logs: journalctl -u openvidu-media-health\n#\n# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and\n# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed\n# by an opening brace, nor two opening braces in a row.\n\nSTATE_DIR=/var/lib/openvidu-media-health\nINSTALLED_MARKER=\"$STATE_DIR/installed\"\nBOOTSTRAP_FAILED_MARKER=\"$STATE_DIR/bootstrap-failed\"\nREPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh\nENV_FILE=/etc/openvidu/media-health.env\nPAUSE_FILE=/etc/openvidu/media-health.disabled\nDRAIN_PAUSE_FILE=/run/openvidu-media-health.paused\n\nlog() { echo \"[media-health] $*\"; }\n\n# Seconds since boot: monotonic, unaffected by clock steps\nuptime_sec() { cut -d. -f1 /proc/uptime; }\n\n# Busy CPU percentage since the previous call, in BUSY (0 on the first call)\ncpu_busy() {\n read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat\n total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal))\n idle=$((c_idle + c_iowait))\n BUSY=0\n if [ -n \"$PREV_TOTAL\" ] && [ \"$total\" -gt \"$PREV_TOTAL\" ]; then\n BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL)))\n fi\n PREV_TOTAL=$total\n PREV_IDLE=$idle\n}\n\n# openvidu.service entered the active state less than WARMUP_SEC ago\nopenvidu_starting() {\n since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null)\n case \"$since\" in ''|0|*[!0-9]*) return 1 ;; esac\n [ $(($(uptime_sec) - since / 1000000)) -lt \"$WARMUP_SEC\" ]\n}\n\n# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare\n# TCP connect would also succeed against a wedged Redis.\nmaster_serving() {\n [ -n \"$MASTER_ADDRS\" ] || return 1\n for addr in $MASTER_ADDRS; do\n host=$(echo \"$addr\" | cut -d: -f1)\n port=$(echo \"$addr\" | cut -d: -f2)\n reply=$(timeout 5 bash -c 'exec 3<>\"/dev/tcp/$0/$1\" && printf \"PING\\r\\n\" >&3 && head -c 7 <&3' \"$host\" \"$port\" 2>/dev/null)\n case \"$reply\" in\n +PONG*|-NOAUTH*) return 0 ;;\n esac\n done\n return 1\n}\n\n# Request the replacement of this node ($1 = reason). Gated on a master answering, except\n# in the bootstrap phase before the master addresses are known.\nrequest_replacement() {\n now=$(uptime_sec)\n if [ -n \"$REQUESTED_AT\" ] && [ $((now - REQUESTED_AT)) -lt \"$REPLACE_RETRY_SEC\" ]; then\n return\n fi\n if [ -n \"$MASTER_ADDRS\" ] || [ -f \"$INSTALLED_MARKER\" ]; then\n if ! master_serving; then\n if [ -z \"$HOLD_LOGGED_AT\" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then\n log \"$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node\"\n HOLD_LOGGED_AT=$now\n fi\n return\n fi\n fi\n log \"$1: requesting the replacement of this node\"\n if \"$REPLACE_SCRIPT\" \"$1\"; then\n REQUESTED_AT=$now\n log \"replacement requested\"\n else\n log \"the replacement request failed, retrying in $INTERVAL_SEC s\"\n fi\n}\n\nmkdir -p \"$STATE_DIR\"\nREQUESTED_AT=\"\"\nHOLD_LOGGED_AT=\"\"\nFAILED_SEEN_AT=\"\"\nFAIL_SINCE=\"\"\nSATURATED=\"\"\nPREV_TOTAL=\"\"\nPREV_IDLE=\"\"\nlog \"started\"\n\nwhile true; do\n PROBE_URL=http://127.0.0.1:7880/\n INTERVAL_SEC=30\n HARD_UNHEALTHY_AFTER_SEC=300\n SOFT_UNHEALTHY_AFTER_SEC=600\n CPU_SATURATED_PCT=90\n WARMUP_SEC=900\n BOOTSTRAP_DEADLINE_SEC=5400\n BOOTSTRAP_FAILURE_COOLDOWN_SEC=600\n REPLACE_RETRY_SEC=600\n MASTER_ADDRS=\"\"\n [ -f \"$ENV_FILE\" ] && . \"$ENV_FILE\"\n NOW=$(uptime_sec)\n cpu_busy\n\n if [ -f \"$PAUSE_FILE\" ] || [ -f \"$DRAIN_PAUSE_FILE\" ]; then\n FAIL_SINCE=\"\"\n elif [ -f \"$BOOTSTRAP_FAILED_MARKER\" ]; then\n if [ -z \"$FAILED_SEEN_AT\" ]; then\n FAILED_SEEN_AT=$NOW\n log \"bootstrap failed ($(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s\"\n fi\n if [ $((NOW - FAILED_SEEN_AT)) -ge \"$BOOTSTRAP_FAILURE_COOLDOWN_SEC\" ]; then\n request_replacement \"bootstrap failed: $(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")\"\n fi\n elif [ ! -f \"$INSTALLED_MARKER\" ]; then\n if [ \"$NOW\" -ge \"$BOOTSTRAP_DEADLINE_SEC\" ]; then\n request_replacement \"bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot\"\n fi\n elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 \"$PROBE_URL\"; then\n [ -n \"$FAIL_SINCE\" ] && log \"LiveKit healthy again after $((NOW - FAIL_SINCE)) s\"\n FAIL_SINCE=\"\"\n SATURATED=\"\"\n else\n RC=$?\n case \"$RC\" in\n 7|52|56) KIND=hard ;;\n *) KIND=soft ;;\n esac\n if openvidu_starting; then\n FAIL_SINCE=\"\"\n elif [ \"$KIND\" = soft ] && [ \"$BUSY\" -ge \"$CPU_SATURATED_PCT\" ]; then\n [ -n \"$SATURATED\" ] || log \"LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted\"\n SATURATED=1\n FAIL_SINCE=\"\"\n else\n SATURATED=\"\"\n if [ -z \"$FAIL_SINCE\" ]; then\n FAIL_SINCE=$NOW\n log \"LiveKit health check failed (curl exit $RC)\"\n fi\n LIMIT=$SOFT_UNHEALTHY_AFTER_SEC\n [ \"$KIND\" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC\n if [ $((NOW - FAIL_SINCE)) -ge \"$LIMIT\" ]; then\n request_replacement \"LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)\"\n fi\n fi\n fi\n sleep \"$INTERVAL_SEC\"\ndone\n", + "openviduMediaHealthUnit": "[Unit]\nDescription=OpenVidu Media Node health watchdog\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/local/bin/openvidu-media-health.sh\nRestart=always\nRestartSec=30\n\n[Install]\nWantedBy=multi-user.target\n", + "media_replaceScriptMediaTemplate": "#!/bin/bash\n# openvidu-media-replace.sh \"\"\n# Requests the reimage of THIS Media Node, keeping the scale set capacity and the\n# scale-in protection: reimage provisions a new OS disk and the userData bootstrap runs\n# again on the same instance id. Called by the health watchdog; it never signals it.\n\nmkdir -p /var/lib/openvidu-media-health\necho \"$(date -u +%FT%TZ) $1\" >> /var/lib/openvidu-media-health/replace-requests.log\ntouch /run/openvidu-unhealthy\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nINSTANCE_RESOURCE_ID=$(curl -s -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo \"$INSTANCE_RESOURCE_ID\" | awk -F'/' '{print $NF}')\nif [ -z \"$INSTANCE_ID\" ]; then\n echo \"[media-replace] could not read this instance id from IMDS\"\n exit 1\nfi\n\nn=0\nwhile true; do\n n=$((n + 1))\n if az login --identity >/dev/null 2>&1 && \\\n az vmss reimage --resource-group \"$RESOURCE_GROUP_NAME\" --name \"$VM_SCALE_SET_NAME\" --instance-ids \"$INSTANCE_ID\" --no-wait; then\n echo \"[media-replace] reimage requested for instance $INSTANCE_ID ($1)\"\n exit 0\n fi\n if [ \"$n\" -ge 5 ]; then\n echo \"[media-replace] reimage could not be requested after $n attempts\"\n exit 1\n fi\n echo \"[media-replace] reimage request failed (attempt $n), retrying in 12s\"\n sleep 12\ndone\n", + "userDataMediaNodeTemplate": "#!/bin/bash\nset -eu -o pipefail\n\n# Mark this media node's bootstrap as failed so the health watchdog replaces it\nov_media_bootstrap_failed() {\n echo \"[OpenVidu] media node bootstrap failed: $1\"\n mkdir -p /var/lib/openvidu-media-health\n echo \"$1\" > /var/lib/openvidu-media-health/bootstrap-failed\n exit 1\n}\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# stop_media_nodes.sh\necho ${base64stop} | base64 -d > /usr/local/bin/stop_media_node.sh\nchmod +x /usr/local/bin/stop_media_node.sh\n\n# openvidu-media-health.sh (health watchdog), openvidu-media-replace.sh (reimage this node) and its unit\necho ${base64mediaHealth} | base64 -d > /usr/local/bin/openvidu-media-health.sh\nchmod +x /usr/local/bin/openvidu-media-health.sh\necho ${base64mediaReplace} | base64 -d > /usr/local/bin/openvidu-media-replace.sh\nchmod +x /usr/local/bin/openvidu-media-replace.sh\necho ${base64mediaHealthUnit} | base64 -d > /etc/systemd/system/openvidu-media-health.service\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y jq\n\n# Install azure cli\nAZURE_CLI_VERSION=2.90.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity\n\n# Protect from scale in actions\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\naz vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-id $INSTANCE_ID --protect-from-scale-in true\n\nexport HOME=\"/root\"\n\n# Start the Media Node health watchdog before install.sh (az CLI is installed and the node\n# is scale-in protected): it backstops a hung or failed bootstrap and, once running, reimages\n# the node if LiveKit stays unhealthy. Write MASTER_ADDRS first so its master guard is armed.\nmkdir -p /etc/openvidu\necho 'MASTER_ADDRS=\"${privateIPMasterNode}:7000\"' > /etc/openvidu/media-health.env\nsystemctl daemon-reload\nsystemctl enable --now openvidu-media-health\n\n# Install OpenVidu\n/usr/local/bin/install.sh || ov_media_bootstrap_failed \"install.sh failed\"\n\n# Gate 2: wait for master readiness before starting\nWAIT_INTERVAL=5\nMAX_RETRIES=360\nRETRIES=0\nset +e\nwhile true; do\n FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv 2>/dev/null)\n if [ \"$FINISH_MASTER_NODE\" == \"true\" ]; then\n break\n fi\n RETRIES=$((RETRIES + 1))\n if [ $RETRIES -ge $MAX_RETRIES ]; then\n ov_media_bootstrap_failed \"timed out after 30 min waiting for FINISH-MASTER-NODE\"\n fi\n sleep $WAIT_INTERVAL\ndone\nset -e\n\n# Start OpenVidu\nsystemctl start openvidu || ov_media_bootstrap_failed \"systemctl start openvidu failed\"\n\n# Bootstrap finished: the watchdog switches from bootstrap-deadline mode to runtime health checks\nmkdir -p /var/lib/openvidu-media-health\ntouch /var/lib/openvidu-media-health/installed\n", "stop_media_nodesScriptMedia": "[reduce(items(variables('stopMediaNodeParams')), createObject('value', variables('stop_media_nodesScriptMediaTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value]", "base64stopMediaNode": "[base64(variables('stop_media_nodesScriptMedia'))]", - "delete_mediaNode_ScriptMedia": "[reduce(items(variables('stopMediaNodeParams')), createObject('value', variables('delete_mediaNode_ScriptMediaTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value]", - "base64delete_mediaNode_ScriptMedia": "[base64(variables('delete_mediaNode_ScriptMedia'))]", + "media_replaceScriptMedia": "[reduce(items(variables('stopMediaNodeParams')), createObject('value', variables('media_replaceScriptMediaTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value]", + "base64media_replaceScriptMedia": "[base64(variables('media_replaceScriptMedia'))]", + "base64openviduMediaHealth": "[base64(variables('openviduMediaHealthScript'))]", + "base64openviduMediaHealthUnit": "[base64(variables('openviduMediaHealthUnit'))]", "isEmptyAutomationAccountName": "[equals(parameters('automationAccountName'), '')]", "ipExists": "[equals(parameters('publicIpAddressObject').newOrExistingOrNone, 'existing')]", "ipNew": "[equals(parameters('publicIpAddressObject').newOrExistingOrNone, 'new')]", @@ -471,7 +475,7 @@ } ] }, - "userData": "[base64(reduce(items(createObject('base64install', base64(reduce(items(createObject('privateIPMasterNode', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNoderNetInterface', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'keyVaultName', variables('keyVaultName'))), createObject('value', variables('installScriptTemplateMedia')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value), 'base64stop', variables('base64stopMediaNode'), 'base64delete', variables('base64delete_mediaNode_ScriptMedia'), 'resourceGroupName', resourceGroup().name, 'vmScaleSetName', format('{0}-mediaNodeScaleSet', parameters('stackName')), 'keyVaultName', variables('keyVaultName'))), createObject('value', variables('userDataMediaNodeTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value)]" + "userData": "[base64(reduce(items(createObject('base64install', base64(reduce(items(createObject('privateIPMasterNode', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNoderNetInterface', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'keyVaultName', variables('keyVaultName'))), createObject('value', variables('installScriptTemplateMedia')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value), 'base64stop', variables('base64stopMediaNode'), 'base64mediaHealth', variables('base64openviduMediaHealth'), 'base64mediaHealthUnit', variables('base64openviduMediaHealthUnit'), 'base64mediaReplace', variables('base64media_replaceScriptMedia'), 'privateIPMasterNode', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNoderNetInterface', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'resourceGroupName', resourceGroup().name, 'vmScaleSetName', format('{0}-mediaNodeScaleSet', parameters('stackName')), 'keyVaultName', variables('keyVaultName'))), createObject('value', variables('userDataMediaNodeTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value)]" } }, "dependsOn": [ diff --git a/openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf b/openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf index 54615afe4..7e39d3719 100644 --- a/openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf +++ b/openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf @@ -260,7 +260,9 @@ resource "google_compute_region_health_check" "media_node_health_check" { name = lower("${var.stackName}-media-node-health-check") region = var.region - # TCP-only, generous thresholds: auto-heal recreation kills live WebRTC sessions + # TCP (not HTTP /): during a master/Redis outage LiveKit answers 406 on every media node at + # once, so an HTTP check would recreate the whole fleet. Generous thresholds because auto-heal + # recreation kills live WebRTC sessions; the on-node watchdog owns application-level health. tcp_health_check { port = 7880 } @@ -268,7 +270,7 @@ resource "google_compute_region_health_check" "media_node_health_check" { check_interval_sec = 30 timeout_sec = 10 healthy_threshold = 2 - unhealthy_threshold = 5 + unhealthy_threshold = 10 depends_on = [google_project_service.compute_api] } @@ -289,10 +291,11 @@ resource "google_compute_region_instance_group_manager" "media_node_group" { port = 7880 } - # initial_delay_sec generous so media nodes finish installing before health checks can recreate them + # VM/process backstop only. initial_delay_sec at the 3600 s max so autohealing never recreates a + # still-bootstrapping node; the marker-aware on-node watchdog owns bootstrap-failure detection. auto_healing_policies { health_check = google_compute_region_health_check.media_node_health_check.id - initial_delay_sec = 600 + initial_delay_sec = 3600 } depends_on = [google_compute_instance.openvidu_master_node] @@ -1334,6 +1337,12 @@ if ! flock -n 9; then exit 0 fi +# Stop the health watchdog so it does not request a replacement while this node drains. The +# pause file is honored on the watchdog's next loop regardless of when the unit stops; it is +# cleared on reboot. +touch /run/openvidu-media-health.paused +systemctl stop openvidu-media-health 2>/dev/null || true + echo "Starting graceful shutdown of OpenVidu Media Node..." INSTANCE_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google") @@ -1351,6 +1360,11 @@ if [ -x "$(command -v docker)" ]; then docker container kill --signal=SIGQUIT "$agent_container" 2>/dev/null || true done + # A node being replaced because it is unhealthy must not wait for sessions to drain. + if [ -f /run/openvidu-unhealthy ]; then + docker ps -q | xargs -r docker kill >/dev/null 2>&1 || true + fi + # Wait for running containers to not be openvidu, ingress, egress or an openvidu agent while [ "$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)" -gt 0 ] || \ [ "$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)" = "true" ] || \ @@ -1397,12 +1411,256 @@ else echo "Instance $INSTANCE_NAME has been abandoned from MIG. Starting graceful shutdown..." /usr/local/bin/graceful_shutdown.sh fi +EOF + + media_health_watchdog_script = <<-EOF +#!/bin/bash +# OpenVidu Media Node health watchdog. +# +# Asks the cloud to replace this Media Node with a new one, keeping the number of +# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when: +# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a +# cool-down, so a persistent failure does not churn nodes and can be inspected, +# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot, +# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC +# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC +# while it answers an error or times out. Soft failures are not counted while the CPU +# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in +# the WARMUP_SEC after openvidu.service (re)started. +# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage +# fails the health check of every Media Node at once, and a new node could not bootstrap +# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until +# the node goes away. +# +# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...") +# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused) +# Logs: journalctl -u openvidu-media-health +# +# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and +# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed +# by an opening brace, nor two opening braces in a row. + +STATE_DIR=/var/lib/openvidu-media-health +INSTALLED_MARKER="$STATE_DIR/installed" +BOOTSTRAP_FAILED_MARKER="$STATE_DIR/bootstrap-failed" +REPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh +ENV_FILE=/etc/openvidu/media-health.env +PAUSE_FILE=/etc/openvidu/media-health.disabled +DRAIN_PAUSE_FILE=/run/openvidu-media-health.paused + +log() { echo "[media-health] $*"; } + +# Seconds since boot: monotonic, unaffected by clock steps +uptime_sec() { cut -d. -f1 /proc/uptime; } + +# Busy CPU percentage since the previous call, in BUSY (0 on the first call) +cpu_busy() { + read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat + total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal)) + idle=$((c_idle + c_iowait)) + BUSY=0 + if [ -n "$PREV_TOTAL" ] && [ "$total" -gt "$PREV_TOTAL" ]; then + BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL))) + fi + PREV_TOTAL=$total + PREV_IDLE=$idle +} + +# openvidu.service entered the active state less than WARMUP_SEC ago +openvidu_starting() { + since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null) + case "$since" in ''|0|*[!0-9]*) return 1 ;; esac + [ $(($(uptime_sec) - since / 1000000)) -lt "$WARMUP_SEC" ] +} + +# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare +# TCP connect would also succeed against a wedged Redis. +master_serving() { + [ -n "$MASTER_ADDRS" ] || return 1 + for addr in $MASTER_ADDRS; do + host=$(echo "$addr" | cut -d: -f1) + port=$(echo "$addr" | cut -d: -f2) + reply=$(timeout 5 bash -c 'exec 3<>"/dev/tcp/$0/$1" && printf "PING\r\n" >&3 && head -c 7 <&3' "$host" "$port" 2>/dev/null) + case "$reply" in + +PONG*|-NOAUTH*) return 0 ;; + esac + done + return 1 +} + +# Request the replacement of this node ($1 = reason). Gated on a master answering, except +# in the bootstrap phase before the master addresses are known. +request_replacement() { + now=$(uptime_sec) + if [ -n "$REQUESTED_AT" ] && [ $((now - REQUESTED_AT)) -lt "$REPLACE_RETRY_SEC" ]; then + return + fi + if [ -n "$MASTER_ADDRS" ] || [ -f "$INSTALLED_MARKER" ]; then + if ! master_serving; then + if [ -z "$HOLD_LOGGED_AT" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then + log "$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node" + HOLD_LOGGED_AT=$now + fi + return + fi + fi + log "$1: requesting the replacement of this node" + if "$REPLACE_SCRIPT" "$1"; then + REQUESTED_AT=$now + log "replacement requested" + else + log "the replacement request failed, retrying in $INTERVAL_SEC s" + fi +} + +mkdir -p "$STATE_DIR" +REQUESTED_AT="" +HOLD_LOGGED_AT="" +FAILED_SEEN_AT="" +FAIL_SINCE="" +SATURATED="" +PREV_TOTAL="" +PREV_IDLE="" +log "started" + +while true; do + PROBE_URL=http://127.0.0.1:7880/ + INTERVAL_SEC=30 + HARD_UNHEALTHY_AFTER_SEC=300 + SOFT_UNHEALTHY_AFTER_SEC=600 + CPU_SATURATED_PCT=90 + WARMUP_SEC=900 + BOOTSTRAP_DEADLINE_SEC=5400 + BOOTSTRAP_FAILURE_COOLDOWN_SEC=600 + REPLACE_RETRY_SEC=600 + MASTER_ADDRS="" + [ -f "$ENV_FILE" ] && . "$ENV_FILE" + NOW=$(uptime_sec) + cpu_busy + + if [ -f "$PAUSE_FILE" ] || [ -f "$DRAIN_PAUSE_FILE" ]; then + FAIL_SINCE="" + elif [ -f "$BOOTSTRAP_FAILED_MARKER" ]; then + if [ -z "$FAILED_SEEN_AT" ]; then + FAILED_SEEN_AT=$NOW + log "bootstrap failed ($(head -c 300 "$BOOTSTRAP_FAILED_MARKER")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s" + fi + if [ $((NOW - FAILED_SEEN_AT)) -ge "$BOOTSTRAP_FAILURE_COOLDOWN_SEC" ]; then + request_replacement "bootstrap failed: $(head -c 300 "$BOOTSTRAP_FAILED_MARKER")" + fi + elif [ ! -f "$INSTALLED_MARKER" ]; then + if [ "$NOW" -ge "$BOOTSTRAP_DEADLINE_SEC" ]; then + request_replacement "bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot" + fi + elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 "$PROBE_URL"; then + [ -n "$FAIL_SINCE" ] && log "LiveKit healthy again after $((NOW - FAIL_SINCE)) s" + FAIL_SINCE="" + SATURATED="" + else + RC=$? + case "$RC" in + 7|52|56) KIND=hard ;; + *) KIND=soft ;; + esac + if openvidu_starting; then + FAIL_SINCE="" + elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then + [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted" + SATURATED=1 + FAIL_SINCE="" + else + SATURATED="" + if [ -z "$FAIL_SINCE" ]; then + FAIL_SINCE=$NOW + log "LiveKit health check failed (curl exit $RC)" + fi + LIMIT=$SOFT_UNHEALTHY_AFTER_SEC + [ "$KIND" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC + if [ $((NOW - FAIL_SINCE)) -ge "$LIMIT" ]; then + request_replacement "LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)" + fi + fi + fi + sleep "$INTERVAL_SEC" +done +EOF + + media_replace_script = <<-EOF +#!/bin/bash +# Requests the replacement of THIS media node while keeping the media node count: it asks the +# regional MIG to recreate the instance (delete + create in place, same name, new instance id, +# fresh disk). Called by the health watchdog and by the bootstrap-failure hook; it never stops +# or signals the watchdog (its caller, in the same cgroup). + +REASON="$1" +mkdir -p /var/lib/openvidu-media-health +echo "$(date -u +%FT%TZ) $REASON" >> /var/lib/openvidu-media-health/replace-requests.log +# Tell the drain script this node is being replaced so it skips waiting for sessions. +touch /run/openvidu-unhealthy + +META="http://metadata.google.internal/computeMetadata/v1" +meta() { curl -s -H "Metadata-Flavor: Google" "$META/$1"; } + +SELF_NAME=$(meta "instance/name") +PROJECT_ID=$(meta "project/project-id") +ZONE=$(meta "instance/zone" | cut -d/ -f4) +REGION=$(echo "$ZONE" | sed 's/-[a-z]$//') +STACK_NAME=$(meta "instance/attributes/stackName") +MIG_NAME=$(echo "$STACK_NAME" | tr '[:upper:]' '[:lower:]')-media-node-group + +# Recreate keeps the MIG size. Retry transient API errors. A "not a member / already being +# deleted, abandoned or recreated" error means the node is already leaving the group (e.g. +# scaled in): nothing to do. +i=0 +while [ "$i" -lt 5 ]; do + i=$((i + 1)) + if OUT=$(gcloud compute instance-groups managed recreate-instances "$MIG_NAME" \ + --instances="$SELF_NAME" --region="$REGION" --project="$PROJECT_ID" --quiet 2>&1); then + echo "[media-replace] recreate requested for $SELF_NAME" + exit 0 + fi + case "$OUT" in + *"not a member"*|*"being deleted"*|*"being recreated"*|*abandon*|*ABANDON*) + echo "[media-replace] $SELF_NAME already leaving the group, nothing to do: $OUT" + exit 0 + ;; + esac + echo "[media-replace] recreate attempt $i for $SELF_NAME failed: $OUT" + sleep 12 +done +echo "[media-replace] gave up requesting recreate of $SELF_NAME after $i attempts" +exit 1 +EOF + + media_health_unit = <<-EOF +[Unit] +Description=OpenVidu Media Node health watchdog +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/openvidu-media-health.sh +Restart=always +RestartSec=30 + +[Install] +WantedBy=multi-user.target EOF user_data_media = <<-EOF #!/bin/bash -x set -eu -o pipefail +# Records a media-node bootstrap failure so the health watchdog replaces this node after a +# cool-down, then aborts the bootstrap. +ov_media_bootstrap_failed() { + echo "[OpenVidu] media node bootstrap failed: $1" + mkdir -p /var/lib/openvidu-media-health + echo "$1" > /var/lib/openvidu-media-health/bootstrap-failed + exit 1 +} + # Check if installation already completed if [ -f /usr/local/bin/openvidu_install_counter.txt ]; then # Launch on reboot @@ -1420,6 +1678,21 @@ ${local.graceful_shutdown_script} GRACEFUL_SHUTDOWN_EOF chmod +x /usr/local/bin/graceful_shutdown.sh + # Media node health watchdog + its replacement script and unit + cat > /usr/local/bin/openvidu-media-health.sh << 'MEDIA_HEALTH_EOF' +${local.media_health_watchdog_script} +MEDIA_HEALTH_EOF + chmod +x /usr/local/bin/openvidu-media-health.sh + + cat > /usr/local/bin/openvidu-media-replace.sh << 'MEDIA_REPLACE_EOF' +${local.media_replace_script} +MEDIA_REPLACE_EOF + chmod +x /usr/local/bin/openvidu-media-replace.sh + + cat > /etc/systemd/system/openvidu-media-health.service << 'MEDIA_HEALTH_UNIT_EOF' +${local.media_health_unit} +MEDIA_HEALTH_UNIT_EOF + echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout GCLOUD_VERSION=573.0.0 @@ -1437,14 +1710,28 @@ GRACEFUL_SHUTDOWN_EOF export HOME="/root" + # Start the media node health watchdog now: gcloud (which its replacement needs) is installed, + # and starting before install.sh lets the watchdog cover a hung bootstrap too. Write the master + # address(es) first (before the installed marker) so the anti-storm guard is always populated. + mkdir -p /etc/openvidu + MASTER_NODE_PRIVATE_IP=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/masterNodePrivateIP" -H "Metadata-Flavor: Google") + [ -n "$MASTER_NODE_PRIVATE_IP" ] || ov_media_bootstrap_failed "could not determine master node private IP" + echo "MASTER_ADDRS=\"$MASTER_NODE_PRIVATE_IP:7000\"" > /etc/openvidu/media-health.env + systemctl daemon-reload + systemctl enable --now openvidu-media-health + # Install OpenVidu Media Node - /usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu Media Node"; exit 1; } + /usr/local/bin/install.sh || ov_media_bootstrap_failed "install.sh failed" # Mark installation as complete echo "installation_complete" > /usr/local/bin/openvidu_install_counter.txt # Start OpenVidu - systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; } + systemctl start openvidu || ov_media_bootstrap_failed "systemctl start openvidu failed" + + # Bootstrap finished: hand runtime replacement to the watchdog (never on a failure path). + mkdir -p /var/lib/openvidu-media-health + touch /var/lib/openvidu-media-health/installed # Add cron job to check if instance is abandoned every minute cat > /usr/local/bin/check_abandoned.sh << 'CHECK_ABANDONED_EOF' diff --git a/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml b/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml index d6825a8d6..08fefc63d 100644 --- a/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml +++ b/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml @@ -2799,28 +2799,238 @@ Resources: mode: "000755" owner: "root" group: "root" - '/usr/local/bin/set_as_unhealthy.sh': + '/usr/local/bin/openvidu-media-health.sh': + content: | + #!/bin/bash + # OpenVidu Media Node health watchdog. + # + # Asks the cloud to replace this Media Node with a new one, keeping the number of + # Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when: + # - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a + # cool-down, so a persistent failure does not churn nodes and can be inspected, + # - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot, + # - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC + # while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC + # while it answers an error or times out. Soft failures are not counted while the CPU + # is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in + # the WARMUP_SEC after openvidu.service (re)started. + # Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage + # fails the health check of every Media Node at once, and a new node could not bootstrap + # either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until + # the node goes away. + # + # Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...") + # Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused) + # Logs: journalctl -u openvidu-media-health + # + # This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and + # Terraform heredocs, so it must never contain a dollar sign or a percent sign followed + # by an opening brace, nor two opening braces in a row. + + STATE_DIR=/var/lib/openvidu-media-health + INSTALLED_MARKER="$STATE_DIR/installed" + BOOTSTRAP_FAILED_MARKER="$STATE_DIR/bootstrap-failed" + REPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh + ENV_FILE=/etc/openvidu/media-health.env + PAUSE_FILE=/etc/openvidu/media-health.disabled + DRAIN_PAUSE_FILE=/run/openvidu-media-health.paused + + log() { echo "[media-health] $*"; } + + # Seconds since boot: monotonic, unaffected by clock steps + uptime_sec() { cut -d. -f1 /proc/uptime; } + + # Busy CPU percentage since the previous call, in BUSY (0 on the first call) + cpu_busy() { + read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat + total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal)) + idle=$((c_idle + c_iowait)) + BUSY=0 + if [ -n "$PREV_TOTAL" ] && [ "$total" -gt "$PREV_TOTAL" ]; then + BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL))) + fi + PREV_TOTAL=$total + PREV_IDLE=$idle + } + + # openvidu.service entered the active state less than WARMUP_SEC ago + openvidu_starting() { + since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null) + case "$since" in ''|0|*[!0-9]*) return 1 ;; esac + [ $(($(uptime_sec) - since / 1000000)) -lt "$WARMUP_SEC" ] + } + + # A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare + # TCP connect would also succeed against a wedged Redis. + master_serving() { + [ -n "$MASTER_ADDRS" ] || return 1 + for addr in $MASTER_ADDRS; do + host=$(echo "$addr" | cut -d: -f1) + port=$(echo "$addr" | cut -d: -f2) + reply=$(timeout 5 bash -c 'exec 3<>"/dev/tcp/$0/$1" && printf "PING\r\n" >&3 && head -c 7 <&3' "$host" "$port" 2>/dev/null) + case "$reply" in + +PONG*|-NOAUTH*) return 0 ;; + esac + done + return 1 + } + + # Request the replacement of this node ($1 = reason). Gated on a master answering, except + # in the bootstrap phase before the master addresses are known. + request_replacement() { + now=$(uptime_sec) + if [ -n "$REQUESTED_AT" ] && [ $((now - REQUESTED_AT)) -lt "$REPLACE_RETRY_SEC" ]; then + return + fi + if [ -n "$MASTER_ADDRS" ] || [ -f "$INSTALLED_MARKER" ]; then + if ! master_serving; then + if [ -z "$HOLD_LOGGED_AT" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then + log "$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node" + HOLD_LOGGED_AT=$now + fi + return + fi + fi + log "$1: requesting the replacement of this node" + if "$REPLACE_SCRIPT" "$1"; then + REQUESTED_AT=$now + log "replacement requested" + else + log "the replacement request failed, retrying in $INTERVAL_SEC s" + fi + } + + mkdir -p "$STATE_DIR" + REQUESTED_AT="" + HOLD_LOGGED_AT="" + FAILED_SEEN_AT="" + FAIL_SINCE="" + SATURATED="" + PREV_TOTAL="" + PREV_IDLE="" + log "started" + + while true; do + PROBE_URL=http://127.0.0.1:7880/ + INTERVAL_SEC=30 + HARD_UNHEALTHY_AFTER_SEC=300 + SOFT_UNHEALTHY_AFTER_SEC=600 + CPU_SATURATED_PCT=90 + WARMUP_SEC=900 + BOOTSTRAP_DEADLINE_SEC=5400 + BOOTSTRAP_FAILURE_COOLDOWN_SEC=600 + REPLACE_RETRY_SEC=600 + MASTER_ADDRS="" + [ -f "$ENV_FILE" ] && . "$ENV_FILE" + NOW=$(uptime_sec) + cpu_busy + + if [ -f "$PAUSE_FILE" ] || [ -f "$DRAIN_PAUSE_FILE" ]; then + FAIL_SINCE="" + elif [ -f "$BOOTSTRAP_FAILED_MARKER" ]; then + if [ -z "$FAILED_SEEN_AT" ]; then + FAILED_SEEN_AT=$NOW + log "bootstrap failed ($(head -c 300 "$BOOTSTRAP_FAILED_MARKER")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s" + fi + if [ $((NOW - FAILED_SEEN_AT)) -ge "$BOOTSTRAP_FAILURE_COOLDOWN_SEC" ]; then + request_replacement "bootstrap failed: $(head -c 300 "$BOOTSTRAP_FAILED_MARKER")" + fi + elif [ ! -f "$INSTALLED_MARKER" ]; then + if [ "$NOW" -ge "$BOOTSTRAP_DEADLINE_SEC" ]; then + request_replacement "bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot" + fi + elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 "$PROBE_URL"; then + [ -n "$FAIL_SINCE" ] && log "LiveKit healthy again after $((NOW - FAIL_SINCE)) s" + FAIL_SINCE="" + SATURATED="" + else + RC=$? + case "$RC" in + 7|52|56) KIND=hard ;; + *) KIND=soft ;; + esac + if openvidu_starting; then + FAIL_SINCE="" + elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then + [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted" + SATURATED=1 + FAIL_SINCE="" + else + SATURATED="" + if [ -z "$FAIL_SINCE" ]; then + FAIL_SINCE=$NOW + log "LiveKit health check failed (curl exit $RC)" + fi + LIMIT=$SOFT_UNHEALTHY_AFTER_SEC + [ "$KIND" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC + if [ $((NOW - FAIL_SINCE)) -ge "$LIMIT" ]; then + request_replacement "LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)" + fi + fi + fi + sleep "$INTERVAL_SEC" + done + mode: "000755" + owner: "root" + group: "root" + '/usr/local/bin/openvidu-media-replace.sh': content: !Sub | #!/bin/bash - set -e + # Ask the Auto Scaling group to replace this Media Node with a new one, keeping the + # count. Called by openvidu-media-health.sh (same cgroup): it never stops or signals + # the watchdog, which re-requests until this node goes away. + mkdir -p /var/lib/openvidu-media-health + echo "$(date -u +%FT%TZ) $1" >> /var/lib/openvidu-media-health/replace-requests.log + # Let stop_media_node.sh know this node is unhealthy, so it does not wait for sessions + touch /run/openvidu-unhealthy + # Token for IMDSv2 TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") # Get own instance ID INSTANCE_ID=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-id) - # Set instance as unhealthy - aws autoscaling set-instance-health \ - --region ${AWS::Region} \ - --instance-id "$INSTANCE_ID" \ - --health-status Unhealthy + # set-instance-health Unhealthy makes the ASG terminate this node through the existing + # lifecycle hook (drain) and launch a replacement. Idempotent; retry transient API errors. + for attempt in $(seq 1 5); do + if aws autoscaling set-instance-health \ + --region ${AWS::Region} \ + --instance-id "$INSTANCE_ID" \ + --health-status Unhealthy; then + exit 0 + fi + echo "[openvidu-media-replace] set-instance-health failed (attempt $attempt/5), retrying in 12s" >&2 + sleep 12 + done + exit 1 mode: "000755" owner: "root" group: "root" + '/etc/systemd/system/openvidu-media-health.service': + content: | + [Unit] + Description=OpenVidu Media Node health watchdog + After=network-online.target + Wants=network-online.target + + [Service] + Type=simple + ExecStart=/usr/local/bin/openvidu-media-health.sh + Restart=always + RestartSec=30 + + [Install] + WantedBy=multi-user.target + mode: "000644" + owner: "root" + group: "root" '/usr/local/bin/stop_media_node.sh': content: !Sub | #!/bin/bash set -e + # Drain: pause and stop the health watchdog so it does not replace this node + touch /run/openvidu-media-health.paused + systemctl stop openvidu-media-health 2>/dev/null || true # Token for IMDSv2 TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") @@ -2839,6 +3049,10 @@ Resources: done TIME_PASSED=0 HEARTBEAT_MAX=1800 + # An unhealthy node being replaced should not wait for sessions: kill now + if [ -f /run/openvidu-unhealthy ]; then + docker ps -q | xargs -r docker kill >/dev/null 2>&1 || true + fi # Wait for running containers to not be openvidu, ingress, egress or an openvidu agent while [ "$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)" -gt 0 ] || \ [ "$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)" = "true" ] || \ @@ -2898,11 +3112,22 @@ Resources: #!/bin/bash set -eu -o pipefail + # Record a media-node bootstrap failure so the health watchdog (started below) + # replaces this node after a cool-down. Route user-data failure exits through this. + ov_media_bootstrap_failed() { + echo "[OpenVidu] media node bootstrap failed: $1" + mkdir -p /var/lib/openvidu-media-health + echo "$1" > /var/lib/openvidu-media-health/bootstrap-failed + exit 1 + } + echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout apt-get update && apt-get install -y \ python3-pip \ - ec2-instance-connect + ec2-instance-connect \ + curl \ + unzip CFN_BOOTSTRAP_VERSION=2.0-39 # Detect Ubuntu version and install cfn-bootstrap accordingly @@ -2917,13 +3142,38 @@ Resources: export HOME="/root" + # aws-cli v2, hoisted from install.sh so the watchdog's replace script can call + # the ASG API even if bootstrap hangs before install.sh runs + AWS_CLI_VERSION=2.36.40 + if ! command -v aws &> /dev/null; then + curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-${!AWS_CLI_VERSION}.zip" -o "awscliv2.zip" + unzip -qq awscliv2.zip + ./aws/install + rm -rf awscliv2.zip aws + fi + + # Start the health watchdog before install.sh so a hung bootstrap is covered too + systemctl daemon-reload + systemctl enable --now openvidu-media-health + # Install OpenVidu - /usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu"; /usr/local/bin/set_as_unhealthy.sh; exit 1; } + /usr/local/bin/install.sh || ov_media_bootstrap_failed "install.sh failed" MASTER_IP_1=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-1-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') MASTER_IP_2=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-2-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') MASTER_IP_3=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-3-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') MASTER_IP_4=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-4-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + + # Health watchdog config: master addresses for the anti-storm Redis PING guard + mkdir -p /etc/openvidu + MASTER_ADDRS="" + for MASTER_IP in "$MASTER_IP_1" "$MASTER_IP_2" "$MASTER_IP_3" "$MASTER_IP_4"; do + if [ "$MASTER_IP" != "none" ] && [ -n "$MASTER_IP" ]; then + MASTER_ADDRS="$MASTER_ADDRS $MASTER_IP:7001" + fi + done + echo "MASTER_ADDRS=\"$(echo $MASTER_ADDRS | xargs)\"" > /etc/openvidu/media-health.env + MASTER_HEALTHY=false for i in $(seq 1 360); do for MASTER_IP in "$MASTER_IP_1" "$MASTER_IP_2" "$MASTER_IP_3" "$MASTER_IP_4"; do @@ -2938,13 +3188,15 @@ Resources: sleep 5 done if [ "$MASTER_HEALTHY" != "true" ]; then - echo "[OpenVidu] no master node became healthy after 30 minutes" - /usr/local/bin/set_as_unhealthy.sh - exit 1 + ov_media_bootstrap_failed "no master node became healthy after 30 minutes" fi # Start OpenVidu - systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; /usr/local/bin/set_as_unhealthy.sh; exit 1; } + systemctl start openvidu || ov_media_bootstrap_failed "error starting OpenVidu" + + # Bootstrap finished: switch the watchdog from the bootstrap deadline to runtime checks + mkdir -p /var/lib/openvidu-media-health + touch /var/lib/openvidu-media-health/installed # Wait for the app # /usr/local/bin/check_app_ready.sh diff --git a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep index fe2389826..242ff3a1a 100644 --- a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep +++ b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep @@ -1395,6 +1395,11 @@ var stop_media_nodesScriptMediaTemplate = ''' #!/bin/bash set -e +# Pause the health watchdog before draining so it never reimages a node that is being +# drained for scale-in (synchronous; the drain runs outside the watchdog's cgroup). +touch /run/openvidu-media-health.paused +systemctl stop openvidu-media-health 2>/dev/null || true + if ! (set -o noclobber ; echo > /tmp/global.lock) ; then exit 1 # the global.lock already exists fi @@ -1410,6 +1415,11 @@ if [ -x "$(command -v docker)" ]; then docker container kill --signal=SIGQUIT "$agent_container" 2>/dev/null || true done + # If this node is being replaced because it is unhealthy, do not wait for sessions to end. + if [ -f /run/openvidu-unhealthy ]; then + docker ps -q | xargs -r docker kill >/dev/null 2>&1 || true + fi + # Wait for running containers to not be openvidu, ingress or egress while [ "$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)" -gt 0 ] || \ [ "$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)" = "true" ] || \ @@ -1435,25 +1445,244 @@ az tag update --resource-id $RESOURCE_ID --operation replace --tags "STATUS"="HE az vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID ''' -var delete_mediaNode_ScriptMediaTemplate = ''' +var openviduMediaHealthScript = ''' #!/bin/bash -set -e +# OpenVidu Media Node health watchdog. +# +# Asks the cloud to replace this Media Node with a new one, keeping the number of +# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when: +# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a +# cool-down, so a persistent failure does not churn nodes and can be inspected, +# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot, +# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC +# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC +# while it answers an error or times out. Soft failures are not counted while the CPU +# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in +# the WARMUP_SEC after openvidu.service (re)started. +# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage +# fails the health check of every Media Node at once, and a new node could not bootstrap +# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until +# the node goes away. +# +# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...") +# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused) +# Logs: journalctl -u openvidu-media-health +# +# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and +# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed +# by an opening brace, nor two opening braces in a row. -az login --identity --client-id ${mediaIdentityClientId} +STATE_DIR=/var/lib/openvidu-media-health +INSTALLED_MARKER="$STATE_DIR/installed" +BOOTSTRAP_FAILED_MARKER="$STATE_DIR/bootstrap-failed" +REPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh +ENV_FILE=/etc/openvidu/media-health.env +PAUSE_FILE=/etc/openvidu/media-health.disabled +DRAIN_PAUSE_FILE=/run/openvidu-media-health.paused + +log() { echo "[media-health] $*"; } + +# Seconds since boot: monotonic, unaffected by clock steps +uptime_sec() { cut -d. -f1 /proc/uptime; } + +# Busy CPU percentage since the previous call, in BUSY (0 on the first call) +cpu_busy() { + read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat + total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal)) + idle=$((c_idle + c_iowait)) + BUSY=0 + if [ -n "$PREV_TOTAL" ] && [ "$total" -gt "$PREV_TOTAL" ]; then + BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL))) + fi + PREV_TOTAL=$total + PREV_IDLE=$idle +} + +# openvidu.service entered the active state less than WARMUP_SEC ago +openvidu_starting() { + since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null) + case "$since" in ''|0|*[!0-9]*) return 1 ;; esac + [ $(($(uptime_sec) - since / 1000000)) -lt "$WARMUP_SEC" ] +} + +# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare +# TCP connect would also succeed against a wedged Redis. +master_serving() { + [ -n "$MASTER_ADDRS" ] || return 1 + for addr in $MASTER_ADDRS; do + host=$(echo "$addr" | cut -d: -f1) + port=$(echo "$addr" | cut -d: -f2) + reply=$(timeout 5 bash -c 'exec 3<>"/dev/tcp/$0/$1" && printf "PING\r\n" >&3 && head -c 7 <&3' "$host" "$port" 2>/dev/null) + case "$reply" in + +PONG*|-NOAUTH*) return 0 ;; + esac + done + return 1 +} + +# Request the replacement of this node ($1 = reason). Gated on a master answering, except +# in the bootstrap phase before the master addresses are known. +request_replacement() { + now=$(uptime_sec) + if [ -n "$REQUESTED_AT" ] && [ $((now - REQUESTED_AT)) -lt "$REPLACE_RETRY_SEC" ]; then + return + fi + if [ -n "$MASTER_ADDRS" ] || [ -f "$INSTALLED_MARKER" ]; then + if ! master_serving; then + if [ -z "$HOLD_LOGGED_AT" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then + log "$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node" + HOLD_LOGGED_AT=$now + fi + return + fi + fi + log "$1: requesting the replacement of this node" + if "$REPLACE_SCRIPT" "$1"; then + REQUESTED_AT=$now + log "replacement requested" + else + log "the replacement request failed, retrying in $INTERVAL_SEC s" + fi +} + +mkdir -p "$STATE_DIR" +REQUESTED_AT="" +HOLD_LOGGED_AT="" +FAILED_SEEN_AT="" +FAIL_SINCE="" +SATURATED="" +PREV_TOTAL="" +PREV_IDLE="" +log "started" + +while true; do + PROBE_URL=http://127.0.0.1:7880/ + INTERVAL_SEC=30 + HARD_UNHEALTHY_AFTER_SEC=300 + SOFT_UNHEALTHY_AFTER_SEC=600 + CPU_SATURATED_PCT=90 + WARMUP_SEC=900 + BOOTSTRAP_DEADLINE_SEC=5400 + BOOTSTRAP_FAILURE_COOLDOWN_SEC=600 + REPLACE_RETRY_SEC=600 + MASTER_ADDRS="" + [ -f "$ENV_FILE" ] && . "$ENV_FILE" + NOW=$(uptime_sec) + cpu_busy + + if [ -f "$PAUSE_FILE" ] || [ -f "$DRAIN_PAUSE_FILE" ]; then + FAIL_SINCE="" + elif [ -f "$BOOTSTRAP_FAILED_MARKER" ]; then + if [ -z "$FAILED_SEEN_AT" ]; then + FAILED_SEEN_AT=$NOW + log "bootstrap failed ($(head -c 300 "$BOOTSTRAP_FAILED_MARKER")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s" + fi + if [ $((NOW - FAILED_SEEN_AT)) -ge "$BOOTSTRAP_FAILURE_COOLDOWN_SEC" ]; then + request_replacement "bootstrap failed: $(head -c 300 "$BOOTSTRAP_FAILED_MARKER")" + fi + elif [ ! -f "$INSTALLED_MARKER" ]; then + if [ "$NOW" -ge "$BOOTSTRAP_DEADLINE_SEC" ]; then + request_replacement "bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot" + fi + elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 "$PROBE_URL"; then + [ -n "$FAIL_SINCE" ] && log "LiveKit healthy again after $((NOW - FAIL_SINCE)) s" + FAIL_SINCE="" + SATURATED="" + else + RC=$? + case "$RC" in + 7|52|56) KIND=hard ;; + *) KIND=soft ;; + esac + if openvidu_starting; then + FAIL_SINCE="" + elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then + [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted" + SATURATED=1 + FAIL_SINCE="" + else + SATURATED="" + if [ -z "$FAIL_SINCE" ]; then + FAIL_SINCE=$NOW + log "LiveKit health check failed (curl exit $RC)" + fi + LIMIT=$SOFT_UNHEALTHY_AFTER_SEC + [ "$KIND" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC + if [ $((NOW - FAIL_SINCE)) -ge "$LIMIT" ]; then + request_replacement "LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)" + fi + fi + fi + sleep "$INTERVAL_SEC" +done +''' + +var openviduMediaHealthUnit = ''' +[Unit] +Description=OpenVidu Media Node health watchdog +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/openvidu-media-health.sh +Restart=always +RestartSec=30 + +[Install] +WantedBy=multi-user.target +''' + +var media_replaceScriptMediaTemplate = ''' +#!/bin/bash +# openvidu-media-replace.sh "" +# Requests the reimage of THIS Media Node, keeping the scale set capacity and the +# scale-in protection: reimage provisions a new OS disk and the userData bootstrap runs +# again on the same instance id. Called by the health watchdog; it never signals it. + +mkdir -p /var/lib/openvidu-media-health +echo "$(date -u +%FT%TZ) $1" >> /var/lib/openvidu-media-health/replace-requests.log +touch /run/openvidu-unhealthy RESOURCE_GROUP_NAME=${resourceGroupName} VM_SCALE_SET_NAME=${vmScaleSetName} -BEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2021-02-01" | jq -r '.compute.resourceId') -INSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}') +INSTANCE_RESOURCE_ID=$(curl -s -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2021-02-01" | jq -r '.compute.resourceId') +INSTANCE_ID=$(echo "$INSTANCE_RESOURCE_ID" | awk -F'/' '{print $NF}') +if [ -z "$INSTANCE_ID" ]; then + echo "[media-replace] could not read this instance id from IMDS" + exit 1 +fi - -az vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID +n=0 +while true; do + n=$((n + 1)) + if az login --identity --client-id ${mediaIdentityClientId} >/dev/null 2>&1 && \ + az vmss reimage --resource-group "$RESOURCE_GROUP_NAME" --name "$VM_SCALE_SET_NAME" --instance-ids "$INSTANCE_ID" --no-wait; then + echo "[media-replace] reimage requested for instance $INSTANCE_ID ($1)" + exit 0 + fi + if [ "$n" -ge 5 ]; then + echo "[media-replace] reimage could not be requested after $n attempts" + exit 1 + fi + echo "[media-replace] reimage request failed (attempt $n), retrying in 12s" + sleep 12 +done ''' + var userDataMediaNodeTemplate = ''' #!/bin/bash -x set -eu -o pipefail +# Mark this media node's bootstrap as failed so the health watchdog replaces it +ov_media_bootstrap_failed() { + echo "[OpenVidu] media node bootstrap failed: $1" + mkdir -p /var/lib/openvidu-media-health + echo "$1" > /var/lib/openvidu-media-health/bootstrap-failed + exit 1 +} + # Introduce the scripts in the instance # install.sh echo ${base64install} | base64 -d > /usr/local/bin/install.sh @@ -1463,9 +1692,12 @@ chmod +x /usr/local/bin/install.sh echo ${base64stop} | base64 -d > /usr/local/bin/stop_media_node.sh chmod +x /usr/local/bin/stop_media_node.sh -# delete_media_node.sh -echo ${base64delete_mediaNode} | base64 -d > /usr/local/bin/delete_media_node.sh -chmod +x /usr/local/bin/delete_media_node.sh +# openvidu-media-health.sh (health watchdog), openvidu-media-replace.sh (reimage this node) and its unit +echo ${base64mediaHealth} | base64 -d > /usr/local/bin/openvidu-media-health.sh +chmod +x /usr/local/bin/openvidu-media-health.sh +echo ${base64mediaReplace} | base64 -d > /usr/local/bin/openvidu-media-replace.sh +chmod +x /usr/local/bin/openvidu-media-replace.sh +echo ${base64mediaHealthUnit} | base64 -d > /etc/systemd/system/openvidu-media-health.service echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout @@ -1490,8 +1722,16 @@ az vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME - export HOME="/root" +# Start the Media Node health watchdog before install.sh (az CLI is installed and the node +# is scale-in protected): it backstops a hung or failed bootstrap and, once running, reimages +# the node if LiveKit stays unhealthy. Write MASTER_ADDRS first so its master guard is armed. +mkdir -p /etc/openvidu +echo 'MASTER_ADDRS="${privateIPMasterNode1}:7001 ${privateIPMasterNode2}:7001 ${privateIPMasterNode3}:7001 ${privateIPMasterNode4}:7001"' > /etc/openvidu/media-health.env +systemctl daemon-reload +systemctl enable --now openvidu-media-health + # Install OpenVidu -/usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu"; /usr/local/bin/delete_media_node.sh; } +/usr/local/bin/install.sh || ov_media_bootstrap_failed "install.sh failed" # Gate 2: wait for master readiness before starting WAIT_INTERVAL=5 @@ -1505,16 +1745,18 @@ while true; do fi RETRIES=$((RETRIES + 1)) if [ $RETRIES -ge $MAX_RETRIES ]; then - echo "[OpenVidu] timed out after 30 min waiting for FINISH-MASTER-NODE" - /usr/local/bin/delete_media_node.sh - exit 1 + ov_media_bootstrap_failed "timed out after 30 min waiting for FINISH-MASTER-NODE" fi sleep $WAIT_INTERVAL done set -e # Start OpenVidu -systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; /usr/local/bin/delete_media_node.sh; } +systemctl start openvidu || ov_media_bootstrap_failed "systemctl start openvidu failed" + +# Bootstrap finished: the watchdog switches from bootstrap-deadline mode to runtime health checks +mkdir -p /var/lib/openvidu-media-health +touch /var/lib/openvidu-media-health/installed ''' var installScriptMedia = reduce( @@ -1529,20 +1771,28 @@ var stop_media_nodesScriptMedia = reduce( (curr, next) => { value: replace(curr.value, '\${${next.key}}', next.value) } ).value -var delete_mediaNode_ScriptMedia = reduce( +var media_replaceScriptMedia = reduce( items(stopMediaNodeParams), - { value: delete_mediaNode_ScriptMediaTemplate }, + { value: media_replaceScriptMediaTemplate }, (curr, next) => { value: replace(curr.value, '\${${next.key}}', next.value) } ).value var base64installMedia = base64(installScriptMedia) var base64stopMediaNode = base64(stop_media_nodesScriptMedia) -var base64delete_mediaNode_ScriptMedia = base64(delete_mediaNode_ScriptMedia) +var base64media_replaceScriptMedia = base64(media_replaceScriptMedia) +var base64openviduMediaHealth = base64(openviduMediaHealthScript) +var base64openviduMediaHealthUnit = base64(openviduMediaHealthUnit) var userDataParamsMedia = { base64install: base64installMedia base64stop: base64stopMediaNode - base64delete_mediaNode: base64delete_mediaNode_ScriptMedia + base64mediaHealth: base64openviduMediaHealth + base64mediaHealthUnit: base64openviduMediaHealthUnit + base64mediaReplace: base64media_replaceScriptMedia + privateIPMasterNode1: privateIPMasterNode1 + privateIPMasterNode2: privateIPMasterNode2 + privateIPMasterNode3: privateIPMasterNode3 + privateIPMasterNode4: privateIPMasterNode4 mediaIdentityClientId: mediaIdentity.properties.clientId resourceGroupName: resourceGroup().name vmScaleSetName: '${stackName}-mediaNodeScaleSet' diff --git a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json index ab8b99784..3e5fd8c16 100644 --- a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json +++ b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.46.1.21595", - "templateHash": "12964488486587110828" + "templateHash": "18285008414247752522" } }, "parameters": { @@ -248,9 +248,13 @@ "base64restartMaster": "[base64(variables('restartScriptMaster'))]", "userDataTemplateMasterNode": "#!/bin/bash -x\nset -eu -o pipefail\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# after_install.sh\necho ${base64after_install} | base64 -d > /usr/local/bin/after_install.sh\nchmod +x /usr/local/bin/after_install.sh\n\n# get_public_ip.sh\necho ${base64get_public_ip} | base64 -d > /usr/local/bin/get_public_ip.sh\nchmod +x /usr/local/bin/get_public_ip.sh\n\n# update_config_from_secret.sh\necho ${base64update_config_from_secret} | base64 -d > /usr/local/bin/update_config_from_secret.sh\nchmod +x /usr/local/bin/update_config_from_secret.sh\n\n# update_secret_from_config.sh\necho ${base64update_secret_from_config} | base64 -d > /usr/local/bin/update_secret_from_config.sh\nchmod +x /usr/local/bin/update_secret_from_config.sh\n\n# get_value_from_config.sh\necho ${base64get_value_from_config} | base64 -d > /usr/local/bin/get_value_from_config.sh\nchmod +x /usr/local/bin/get_value_from_config.sh\n\n# store_secret.sh\necho ${base64store_secret} | base64 -d > /usr/local/bin/store_secret.sh\nchmod +x /usr/local/bin/store_secret.sh\n\n# check_app_ready.sh\necho ${base64check_app_ready} | base64 -d > /usr/local/bin/check_app_ready.sh\nchmod +x /usr/local/bin/check_app_ready.sh\n\n# restart.sh\necho ${base64restart} | base64 -d > /usr/local/bin/restart.sh\nchmod +x /usr/local/bin/restart.sh\n\necho ${base64config_blobStorage} | base64 -d > /usr/local/bin/config_blobStorage.sh\nchmod +x /usr/local/bin/config_blobStorage.sh\n\n# Install azure cli\nAZURE_CLI_VERSION=2.90.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity --client-id ${masterIdentityClientId} --allow-no-subscriptions\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\nexport HOME=\"/root\"\n\n# Install OpenVidu\n/usr/local/bin/install.sh || { echo \"[OpenVidu] error installing OpenVidu\"; exit 1; }\n\n#Config blob storage\n/usr/local/bin/config_blobStorage.sh || { echo \"[OpenVidu] error configuring Blob Storage\"; exit 1; }\n\n# Start OpenVidu\nsystemctl start openvidu || { echo \"[OpenVidu] error starting OpenVidu\"; exit 1; }\n\n# Update shared secret\n/usr/local/bin/after_install.sh || { echo \"[OpenVidu] error updating shared secret\"; exit 1; }\n\n# Launch on reboot\necho \"@reboot /usr/local/bin/restart.sh >> /var/log/openvidu-restart.log\" 2>&1 | crontab\n\n# check_app_ready.sh internally caps its wait at 1200s\n/usr/local/bin/check_app_ready.sh || { echo \"[OpenVidu] master node did not become healthy\"; exit 1; }\n\naz keyvault secret set --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --value \"true\"\n\nMASTER_NODE_NUM=${masterNodeNum}\nif [[ $MASTER_NODE_NUM -eq 4 ]]; then\n # Creating scale in lock\n set +e\n az storage blob upload --account-name ${storageAccountName} --container-name automation-locks --name lock.txt --file /dev/null --auth-mode key\n set -e\nfi\n", "installScriptTemplateMedia": "#!/bin/bash -x\nDOMAIN=\n\n# Install dependencies\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y \\\n curl \\\n unzip \\\n jq \\\n wget\n\n# Get own private IP\nPRIVATE_IP=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text\")\n\n# Gate 1: wait for master secrets and IPs before installing\nWAIT_INTERVAL=5\nMAX_RETRIES=360\nRETRIES=0\nset +e\nwhile true; do\n ALL_SECRETS_GENERATED=$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)\n MASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv 2>/dev/null)\n MASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv 2>/dev/null)\n MASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv 2>/dev/null)\n MASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv 2>/dev/null)\n\n if [ \"$ALL_SECRETS_GENERATED\" == \"true\" ] &&\n [ \"$MASTER_NODE_1_PRIVATE_IP\" != \"\" ] &&\n [ \"$MASTER_NODE_2_PRIVATE_IP\" != \"\" ] &&\n [ \"$MASTER_NODE_3_PRIVATE_IP\" != \"\" ] &&\n [ \"$MASTER_NODE_4_PRIVATE_IP\" != \"\" ]; then\n break\n fi\n\n RETRIES=$((RETRIES + 1))\n if [ $RETRIES -ge $MAX_RETRIES ]; then\n echo \"[OpenVidu] timed out after 30 min waiting for master nodes to generate secrets and publish their IPs\"\n exit 1\n fi\n\n sleep $WAIT_INTERVAL\ndone\nset -e\n\nMASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_PRIVATE_IP_LIST=\"$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP\"\nREDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name REDIS-PASSWORD --query value -o tsv)\nENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv)\nOPENVIDU_VERSION=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-VERSION --query value -o tsv)\n\n# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure\nINSTALLER_SCRIPT=\"/tmp/install_ov_media_node.sh\"\ncurl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o \"$INSTALLER_SCRIPT\" \"http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh\"\nif [ ! -s \"$INSTALLER_SCRIPT\" ]; then\n echo \"[OpenVidu] failed to download the media node installer script\"\n exit 1\nfi\n\n# Base command\nINSTALL_COMMAND=\"sh $INSTALLER_SCRIPT\"\n\n# Common arguments\nCOMMON_ARGS=(\n\"--no-tty\"\n\"--install\"\n\"--environment=azure\"\n\"--deployment-type='ha'\"\n\"--node-role='media-node'\"\n\"--master-node-private-ip-list=$MASTER_NODE_PRIVATE_IP_LIST\"\n\"--private-ip=$PRIVATE_IP\"\n\"--enabled-modules='$ENABLED_MODULES'\"\n\"--redis-password=$REDIS_PASSWORD\"\n)\n\n# Construct the final command with all arguments\nFINAL_COMMAND=\"$INSTALL_COMMAND $(printf \"%s \" \"${COMMON_ARGS[@]}\")\"\n\n# Install OpenVidu\nexec bash -c \"$FINAL_COMMAND\"\n", - "stop_media_nodesScriptMediaTemplate": "#!/bin/bash\nset -e\n\nif ! (set -o noclobber ; echo > /tmp/global.lock) ; then\n exit 1 # the global.lock already exists\nfi\n\n# Execute if docker is installed\nif [ -x \"$(command -v docker)\" ]; then\n\n echo \"Stopping media node services and waiting for termination...\"\n docker container kill --signal=SIGQUIT openvidu 2>/dev/null || true\n docker container kill --signal=SIGQUIT ingress 2>/dev/null || true\n docker container kill --signal=SIGQUIT egress 2>/dev/null || true\n for agent_container in $(docker ps --filter \"label=openvidu-agent=true\" --format '{{.Names}}' 2>/dev/null); do\n docker container kill --signal=SIGQUIT \"$agent_container\" 2>/dev/null || true\n done\n\n # Wait for running containers to not be openvidu, ingress or egress\n while [ \"$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)\" -gt 0 ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)\" = \"true\" ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' ingress 2>/dev/null)\" = \"true\" ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' egress 2>/dev/null)\" = \"true\" ]; do\n echo \"Waiting for containers to stop...\"\n sleep 5\n done\nfi\n\naz login --identity --client-id ${mediaIdentityClientId}\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nSUBSCRIPTION_ID=${subscriptionId}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\nRESOURCE_ID=/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP_NAME/providers/Microsoft.Compute/virtualMachineScaleSets/$VM_SCALE_SET_NAME\n\nTIMESTAMP=$(date -u +\"%Y-%m-%dT%H:%M:%SZ\")\naz tag update --resource-id $RESOURCE_ID --operation replace --tags \"STATUS\"=\"HEALTHY\" \"InstanceDeleteTime\"=\"$TIMESTAMP\" \"storageAccount\"=\"${storageAccountName}\"\n\naz vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID\n", - "delete_mediaNode_ScriptMediaTemplate": "#!/bin/bash\nset -e\n\naz login --identity --client-id ${mediaIdentityClientId}\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\n\n\naz vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID\n", - "userDataMediaNodeTemplate": "#!/bin/bash -x\nset -eu -o pipefail\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# stop_media_nodes.sh\necho ${base64stop} | base64 -d > /usr/local/bin/stop_media_node.sh\nchmod +x /usr/local/bin/stop_media_node.sh\n\n# delete_media_node.sh\necho ${base64delete_mediaNode} | base64 -d > /usr/local/bin/delete_media_node.sh\nchmod +x /usr/local/bin/delete_media_node.sh\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y jq\n\n# Install azure cli\nAZURE_CLI_VERSION=2.90.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity --client-id ${mediaIdentityClientId}\n\n# Protect from scale in actions\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\naz vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-id $INSTANCE_ID --protect-from-scale-in true\n\nexport HOME=\"/root\"\n\n# Install OpenVidu\n/usr/local/bin/install.sh || { echo \"[OpenVidu] error installing OpenVidu\"; /usr/local/bin/delete_media_node.sh; }\n\n# Gate 2: wait for master readiness before starting\nWAIT_INTERVAL=5\nMAX_RETRIES=360\nRETRIES=0\nset +e\nwhile true; do\n FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv 2>/dev/null)\n if [ \"$FINISH_MASTER_NODE\" == \"true\" ]; then\n break\n fi\n RETRIES=$((RETRIES + 1))\n if [ $RETRIES -ge $MAX_RETRIES ]; then\n echo \"[OpenVidu] timed out after 30 min waiting for FINISH-MASTER-NODE\"\n /usr/local/bin/delete_media_node.sh\n exit 1\n fi\n sleep $WAIT_INTERVAL\ndone\nset -e\n\n# Start OpenVidu\nsystemctl start openvidu || { echo \"[OpenVidu] error starting OpenVidu\"; /usr/local/bin/delete_media_node.sh; }\n", + "stop_media_nodesScriptMediaTemplate": "#!/bin/bash\nset -e\n\n# Pause the health watchdog before draining so it never reimages a node that is being\n# drained for scale-in (synchronous; the drain runs outside the watchdog's cgroup).\ntouch /run/openvidu-media-health.paused\nsystemctl stop openvidu-media-health 2>/dev/null || true\n\nif ! (set -o noclobber ; echo > /tmp/global.lock) ; then\n exit 1 # the global.lock already exists\nfi\n\n# Execute if docker is installed\nif [ -x \"$(command -v docker)\" ]; then\n\n echo \"Stopping media node services and waiting for termination...\"\n docker container kill --signal=SIGQUIT openvidu 2>/dev/null || true\n docker container kill --signal=SIGQUIT ingress 2>/dev/null || true\n docker container kill --signal=SIGQUIT egress 2>/dev/null || true\n for agent_container in $(docker ps --filter \"label=openvidu-agent=true\" --format '{{.Names}}' 2>/dev/null); do\n docker container kill --signal=SIGQUIT \"$agent_container\" 2>/dev/null || true\n done\n\n # If this node is being replaced because it is unhealthy, do not wait for sessions to end.\n if [ -f /run/openvidu-unhealthy ]; then\n docker ps -q | xargs -r docker kill >/dev/null 2>&1 || true\n fi\n\n # Wait for running containers to not be openvidu, ingress or egress\n while [ \"$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)\" -gt 0 ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)\" = \"true\" ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' ingress 2>/dev/null)\" = \"true\" ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' egress 2>/dev/null)\" = \"true\" ]; do\n echo \"Waiting for containers to stop...\"\n sleep 5\n done\nfi\n\naz login --identity --client-id ${mediaIdentityClientId}\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nSUBSCRIPTION_ID=${subscriptionId}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\nRESOURCE_ID=/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP_NAME/providers/Microsoft.Compute/virtualMachineScaleSets/$VM_SCALE_SET_NAME\n\nTIMESTAMP=$(date -u +\"%Y-%m-%dT%H:%M:%SZ\")\naz tag update --resource-id $RESOURCE_ID --operation replace --tags \"STATUS\"=\"HEALTHY\" \"InstanceDeleteTime\"=\"$TIMESTAMP\" \"storageAccount\"=\"${storageAccountName}\"\n\naz vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID\n", + "openviduMediaHealthScript": "#!/bin/bash\n# OpenVidu Media Node health watchdog.\n#\n# Asks the cloud to replace this Media Node with a new one, keeping the number of\n# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when:\n# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a\n# cool-down, so a persistent failure does not churn nodes and can be inspected,\n# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot,\n# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC\n# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC\n# while it answers an error or times out. Soft failures are not counted while the CPU\n# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in\n# the WARMUP_SEC after openvidu.service (re)started.\n# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage\n# fails the health check of every Media Node at once, and a new node could not bootstrap\n# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until\n# the node goes away.\n#\n# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS=\"ip:port ...\")\n# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)\n# Logs: journalctl -u openvidu-media-health\n#\n# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and\n# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed\n# by an opening brace, nor two opening braces in a row.\n\nSTATE_DIR=/var/lib/openvidu-media-health\nINSTALLED_MARKER=\"$STATE_DIR/installed\"\nBOOTSTRAP_FAILED_MARKER=\"$STATE_DIR/bootstrap-failed\"\nREPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh\nENV_FILE=/etc/openvidu/media-health.env\nPAUSE_FILE=/etc/openvidu/media-health.disabled\nDRAIN_PAUSE_FILE=/run/openvidu-media-health.paused\n\nlog() { echo \"[media-health] $*\"; }\n\n# Seconds since boot: monotonic, unaffected by clock steps\nuptime_sec() { cut -d. -f1 /proc/uptime; }\n\n# Busy CPU percentage since the previous call, in BUSY (0 on the first call)\ncpu_busy() {\n read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat\n total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal))\n idle=$((c_idle + c_iowait))\n BUSY=0\n if [ -n \"$PREV_TOTAL\" ] && [ \"$total\" -gt \"$PREV_TOTAL\" ]; then\n BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL)))\n fi\n PREV_TOTAL=$total\n PREV_IDLE=$idle\n}\n\n# openvidu.service entered the active state less than WARMUP_SEC ago\nopenvidu_starting() {\n since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null)\n case \"$since\" in ''|0|*[!0-9]*) return 1 ;; esac\n [ $(($(uptime_sec) - since / 1000000)) -lt \"$WARMUP_SEC\" ]\n}\n\n# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare\n# TCP connect would also succeed against a wedged Redis.\nmaster_serving() {\n [ -n \"$MASTER_ADDRS\" ] || return 1\n for addr in $MASTER_ADDRS; do\n host=$(echo \"$addr\" | cut -d: -f1)\n port=$(echo \"$addr\" | cut -d: -f2)\n reply=$(timeout 5 bash -c 'exec 3<>\"/dev/tcp/$0/$1\" && printf \"PING\\r\\n\" >&3 && head -c 7 <&3' \"$host\" \"$port\" 2>/dev/null)\n case \"$reply\" in\n +PONG*|-NOAUTH*) return 0 ;;\n esac\n done\n return 1\n}\n\n# Request the replacement of this node ($1 = reason). Gated on a master answering, except\n# in the bootstrap phase before the master addresses are known.\nrequest_replacement() {\n now=$(uptime_sec)\n if [ -n \"$REQUESTED_AT\" ] && [ $((now - REQUESTED_AT)) -lt \"$REPLACE_RETRY_SEC\" ]; then\n return\n fi\n if [ -n \"$MASTER_ADDRS\" ] || [ -f \"$INSTALLED_MARKER\" ]; then\n if ! master_serving; then\n if [ -z \"$HOLD_LOGGED_AT\" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then\n log \"$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node\"\n HOLD_LOGGED_AT=$now\n fi\n return\n fi\n fi\n log \"$1: requesting the replacement of this node\"\n if \"$REPLACE_SCRIPT\" \"$1\"; then\n REQUESTED_AT=$now\n log \"replacement requested\"\n else\n log \"the replacement request failed, retrying in $INTERVAL_SEC s\"\n fi\n}\n\nmkdir -p \"$STATE_DIR\"\nREQUESTED_AT=\"\"\nHOLD_LOGGED_AT=\"\"\nFAILED_SEEN_AT=\"\"\nFAIL_SINCE=\"\"\nSATURATED=\"\"\nPREV_TOTAL=\"\"\nPREV_IDLE=\"\"\nlog \"started\"\n\nwhile true; do\n PROBE_URL=http://127.0.0.1:7880/\n INTERVAL_SEC=30\n HARD_UNHEALTHY_AFTER_SEC=300\n SOFT_UNHEALTHY_AFTER_SEC=600\n CPU_SATURATED_PCT=90\n WARMUP_SEC=900\n BOOTSTRAP_DEADLINE_SEC=5400\n BOOTSTRAP_FAILURE_COOLDOWN_SEC=600\n REPLACE_RETRY_SEC=600\n MASTER_ADDRS=\"\"\n [ -f \"$ENV_FILE\" ] && . \"$ENV_FILE\"\n NOW=$(uptime_sec)\n cpu_busy\n\n if [ -f \"$PAUSE_FILE\" ] || [ -f \"$DRAIN_PAUSE_FILE\" ]; then\n FAIL_SINCE=\"\"\n elif [ -f \"$BOOTSTRAP_FAILED_MARKER\" ]; then\n if [ -z \"$FAILED_SEEN_AT\" ]; then\n FAILED_SEEN_AT=$NOW\n log \"bootstrap failed ($(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s\"\n fi\n if [ $((NOW - FAILED_SEEN_AT)) -ge \"$BOOTSTRAP_FAILURE_COOLDOWN_SEC\" ]; then\n request_replacement \"bootstrap failed: $(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")\"\n fi\n elif [ ! -f \"$INSTALLED_MARKER\" ]; then\n if [ \"$NOW\" -ge \"$BOOTSTRAP_DEADLINE_SEC\" ]; then\n request_replacement \"bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot\"\n fi\n elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 \"$PROBE_URL\"; then\n [ -n \"$FAIL_SINCE\" ] && log \"LiveKit healthy again after $((NOW - FAIL_SINCE)) s\"\n FAIL_SINCE=\"\"\n SATURATED=\"\"\n else\n RC=$?\n case \"$RC\" in\n 7|52|56) KIND=hard ;;\n *) KIND=soft ;;\n esac\n if openvidu_starting; then\n FAIL_SINCE=\"\"\n elif [ \"$KIND\" = soft ] && [ \"$BUSY\" -ge \"$CPU_SATURATED_PCT\" ]; then\n [ -n \"$SATURATED\" ] || log \"LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted\"\n SATURATED=1\n FAIL_SINCE=\"\"\n else\n SATURATED=\"\"\n if [ -z \"$FAIL_SINCE\" ]; then\n FAIL_SINCE=$NOW\n log \"LiveKit health check failed (curl exit $RC)\"\n fi\n LIMIT=$SOFT_UNHEALTHY_AFTER_SEC\n [ \"$KIND\" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC\n if [ $((NOW - FAIL_SINCE)) -ge \"$LIMIT\" ]; then\n request_replacement \"LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)\"\n fi\n fi\n fi\n sleep \"$INTERVAL_SEC\"\ndone\n", + "openviduMediaHealthUnit": "[Unit]\nDescription=OpenVidu Media Node health watchdog\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/local/bin/openvidu-media-health.sh\nRestart=always\nRestartSec=30\n\n[Install]\nWantedBy=multi-user.target\n", + "media_replaceScriptMediaTemplate": "#!/bin/bash\n# openvidu-media-replace.sh \"\"\n# Requests the reimage of THIS Media Node, keeping the scale set capacity and the\n# scale-in protection: reimage provisions a new OS disk and the userData bootstrap runs\n# again on the same instance id. Called by the health watchdog; it never signals it.\n\nmkdir -p /var/lib/openvidu-media-health\necho \"$(date -u +%FT%TZ) $1\" >> /var/lib/openvidu-media-health/replace-requests.log\ntouch /run/openvidu-unhealthy\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nINSTANCE_RESOURCE_ID=$(curl -s -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo \"$INSTANCE_RESOURCE_ID\" | awk -F'/' '{print $NF}')\nif [ -z \"$INSTANCE_ID\" ]; then\n echo \"[media-replace] could not read this instance id from IMDS\"\n exit 1\nfi\n\nn=0\nwhile true; do\n n=$((n + 1))\n if az login --identity --client-id ${mediaIdentityClientId} >/dev/null 2>&1 && \\\n az vmss reimage --resource-group \"$RESOURCE_GROUP_NAME\" --name \"$VM_SCALE_SET_NAME\" --instance-ids \"$INSTANCE_ID\" --no-wait; then\n echo \"[media-replace] reimage requested for instance $INSTANCE_ID ($1)\"\n exit 0\n fi\n if [ \"$n\" -ge 5 ]; then\n echo \"[media-replace] reimage could not be requested after $n attempts\"\n exit 1\n fi\n echo \"[media-replace] reimage request failed (attempt $n), retrying in 12s\"\n sleep 12\ndone\n", + "userDataMediaNodeTemplate": "#!/bin/bash -x\nset -eu -o pipefail\n\n# Mark this media node's bootstrap as failed so the health watchdog replaces it\nov_media_bootstrap_failed() {\n echo \"[OpenVidu] media node bootstrap failed: $1\"\n mkdir -p /var/lib/openvidu-media-health\n echo \"$1\" > /var/lib/openvidu-media-health/bootstrap-failed\n exit 1\n}\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# stop_media_nodes.sh\necho ${base64stop} | base64 -d > /usr/local/bin/stop_media_node.sh\nchmod +x /usr/local/bin/stop_media_node.sh\n\n# openvidu-media-health.sh (health watchdog), openvidu-media-replace.sh (reimage this node) and its unit\necho ${base64mediaHealth} | base64 -d > /usr/local/bin/openvidu-media-health.sh\nchmod +x /usr/local/bin/openvidu-media-health.sh\necho ${base64mediaReplace} | base64 -d > /usr/local/bin/openvidu-media-replace.sh\nchmod +x /usr/local/bin/openvidu-media-replace.sh\necho ${base64mediaHealthUnit} | base64 -d > /etc/systemd/system/openvidu-media-health.service\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y jq\n\n# Install azure cli\nAZURE_CLI_VERSION=2.90.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity --client-id ${mediaIdentityClientId}\n\n# Protect from scale in actions\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\naz vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-id $INSTANCE_ID --protect-from-scale-in true\n\nexport HOME=\"/root\"\n\n# Start the Media Node health watchdog before install.sh (az CLI is installed and the node\n# is scale-in protected): it backstops a hung or failed bootstrap and, once running, reimages\n# the node if LiveKit stays unhealthy. Write MASTER_ADDRS first so its master guard is armed.\nmkdir -p /etc/openvidu\necho 'MASTER_ADDRS=\"${privateIPMasterNode1}:7001 ${privateIPMasterNode2}:7001 ${privateIPMasterNode3}:7001 ${privateIPMasterNode4}:7001\"' > /etc/openvidu/media-health.env\nsystemctl daemon-reload\nsystemctl enable --now openvidu-media-health\n\n# Install OpenVidu\n/usr/local/bin/install.sh || ov_media_bootstrap_failed \"install.sh failed\"\n\n# Gate 2: wait for master readiness before starting\nWAIT_INTERVAL=5\nMAX_RETRIES=360\nRETRIES=0\nset +e\nwhile true; do\n FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv 2>/dev/null)\n if [ \"$FINISH_MASTER_NODE\" == \"true\" ]; then\n break\n fi\n RETRIES=$((RETRIES + 1))\n if [ $RETRIES -ge $MAX_RETRIES ]; then\n ov_media_bootstrap_failed \"timed out after 30 min waiting for FINISH-MASTER-NODE\"\n fi\n sleep $WAIT_INTERVAL\ndone\nset -e\n\n# Start OpenVidu\nsystemctl start openvidu || ov_media_bootstrap_failed \"systemctl start openvidu failed\"\n\n# Bootstrap finished: the watchdog switches from bootstrap-deadline mode to runtime health checks\nmkdir -p /var/lib/openvidu-media-health\ntouch /var/lib/openvidu-media-health/installed\n", + "base64openviduMediaHealth": "[base64(variables('openviduMediaHealthScript'))]", + "base64openviduMediaHealthUnit": "[base64(variables('openviduMediaHealthUnit'))]", "isEmptyAutomationAccountName": "[equals(parameters('automationAccountName'), '')]", "lbName": "[format('{0}-loadBalancer', parameters('stackName'))]", "lbFrontEndName": "LoadBalancerFrontEnd", @@ -604,7 +608,7 @@ } ] }, - "userData": "[base64(reduce(items(createObject('base64install', base64(reduce(items(createObject('privateIPMasterNode1', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface1', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'privateIPMasterNode2', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface2', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'privateIPMasterNode3', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface3', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'privateIPMasterNode4', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface4', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'keyVaultName', variables('keyVaultName'))), createObject('value', variables('installScriptTemplateMedia')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value), 'base64stop', base64(reduce(items(createObject('mediaIdentityClientId', reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}-media-identity', parameters('stackName'))), '2023-01-31').clientId, 'subscriptionId', subscription().subscriptionId, 'resourceGroupName', resourceGroup().name, 'vmScaleSetName', format('{0}-mediaNodeScaleSet', parameters('stackName')), 'storageAccountName', if(variables('isEmptyStorageAccountName'), uniqueString(resourceGroup().id), parameters('storageAccountName')))), createObject('value', variables('stop_media_nodesScriptMediaTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value), 'base64delete_mediaNode', base64(reduce(items(createObject('mediaIdentityClientId', reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}-media-identity', parameters('stackName'))), '2023-01-31').clientId, 'subscriptionId', subscription().subscriptionId, 'resourceGroupName', resourceGroup().name, 'vmScaleSetName', format('{0}-mediaNodeScaleSet', parameters('stackName')), 'storageAccountName', if(variables('isEmptyStorageAccountName'), uniqueString(resourceGroup().id), parameters('storageAccountName')))), createObject('value', variables('delete_mediaNode_ScriptMediaTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value), 'mediaIdentityClientId', reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}-media-identity', parameters('stackName'))), '2023-01-31').clientId, 'resourceGroupName', resourceGroup().name, 'vmScaleSetName', format('{0}-mediaNodeScaleSet', parameters('stackName')), 'keyVaultName', variables('keyVaultName'))), createObject('value', variables('userDataMediaNodeTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value)]" + "userData": "[base64(reduce(items(createObject('base64install', base64(reduce(items(createObject('privateIPMasterNode1', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface1', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'privateIPMasterNode2', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface2', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'privateIPMasterNode3', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface3', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'privateIPMasterNode4', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface4', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'keyVaultName', variables('keyVaultName'))), createObject('value', variables('installScriptTemplateMedia')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value), 'base64stop', base64(reduce(items(createObject('mediaIdentityClientId', reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}-media-identity', parameters('stackName'))), '2023-01-31').clientId, 'subscriptionId', subscription().subscriptionId, 'resourceGroupName', resourceGroup().name, 'vmScaleSetName', format('{0}-mediaNodeScaleSet', parameters('stackName')), 'storageAccountName', if(variables('isEmptyStorageAccountName'), uniqueString(resourceGroup().id), parameters('storageAccountName')))), createObject('value', variables('stop_media_nodesScriptMediaTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value), 'base64mediaHealth', variables('base64openviduMediaHealth'), 'base64mediaHealthUnit', variables('base64openviduMediaHealthUnit'), 'base64mediaReplace', base64(reduce(items(createObject('mediaIdentityClientId', reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}-media-identity', parameters('stackName'))), '2023-01-31').clientId, 'subscriptionId', subscription().subscriptionId, 'resourceGroupName', resourceGroup().name, 'vmScaleSetName', format('{0}-mediaNodeScaleSet', parameters('stackName')), 'storageAccountName', if(variables('isEmptyStorageAccountName'), uniqueString(resourceGroup().id), parameters('storageAccountName')))), createObject('value', variables('media_replaceScriptMediaTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value), 'privateIPMasterNode1', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface1', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'privateIPMasterNode2', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface2', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'privateIPMasterNode3', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface3', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'privateIPMasterNode4', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface4', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'mediaIdentityClientId', reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}-media-identity', parameters('stackName'))), '2023-01-31').clientId, 'resourceGroupName', resourceGroup().name, 'vmScaleSetName', format('{0}-mediaNodeScaleSet', parameters('stackName')), 'keyVaultName', variables('keyVaultName'))), createObject('value', variables('userDataMediaNodeTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value)]" } }, "dependsOn": [ diff --git a/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf b/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf index 32775678e..53d06efe4 100644 --- a/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf +++ b/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf @@ -194,6 +194,22 @@ resource "google_compute_firewall" "master_to_media_services" { depends_on = [google_project_service.compute_api] } +# Allow Google health-check probes (MIG media autohealing) to reach media nodes on 7880 +resource "google_compute_firewall" "health_check_to_media" { + name = lower("${var.stackName}-health-check-to-media") + network = "default" + + allow { + protocol = "tcp" + ports = ["7880"] + } + + source_ranges = ["35.191.0.0/16", "130.211.0.0/22"] + target_tags = [lower("${var.stackName}-media-node")] + + depends_on = [google_project_service.compute_api] +} + # Regional static IP for the Network Load Balancer (conditional creation) resource "google_compute_address" "nlb_ip" { @@ -940,6 +956,26 @@ resource "google_compute_instance_template" "media_node_template" { } } +# Health check for media node auto-healing +resource "google_compute_region_health_check" "media_node_health_check" { + name = lower("${var.stackName}-media-node-health-check") + region = var.region + + # TCP (not HTTP /): during a master/Redis outage LiveKit answers 406 on every media node at + # once, so an HTTP check would recreate the whole fleet. Generous thresholds because auto-heal + # recreation kills live WebRTC sessions; the on-node watchdog owns application-level health. + tcp_health_check { + port = 7880 + } + + check_interval_sec = 30 + timeout_sec = 10 + healthy_threshold = 2 + unhealthy_threshold = 10 + + depends_on = [google_project_service.compute_api] +} + # Managed Instance Group for Media Nodes resource "google_compute_region_instance_group_manager" "media_node_group" { name = lower("${var.stackName}-media-node-group") @@ -955,6 +991,13 @@ resource "google_compute_region_instance_group_manager" "media_node_group" { name = "http" port = 7880 } + + # VM/process backstop only. initial_delay_sec at the 3600 s max so autohealing never recreates a + # still-bootstrapping node; the marker-aware on-node watchdog owns bootstrap-failure detection. + auto_healing_policies { + health_check = google_compute_region_health_check.media_node_health_check.id + initial_delay_sec = 3600 + } } # Autoscaler for Media Nodes @@ -1681,6 +1724,12 @@ if ! flock -n 9; then exit 0 fi +# Stop the health watchdog so it does not request a replacement while this node drains. The +# pause file is honored on the watchdog's next loop regardless of when the unit stops; it is +# cleared on reboot. +touch /run/openvidu-media-health.paused +systemctl stop openvidu-media-health 2>/dev/null || true + echo "Starting graceful shutdown of OpenVidu Media Node..." INSTANCE_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google") @@ -1697,6 +1746,11 @@ if [ -x "$(command -v docker)" ]; then docker container kill --signal=SIGQUIT "$agent_container" 2>/dev/null || true done + # A node being replaced because it is unhealthy must not wait for sessions to drain. + if [ -f /run/openvidu-unhealthy ]; then + docker ps -q | xargs -r docker kill >/dev/null 2>&1 || true + fi + # Wait for running containers to stop while [ "$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)" -gt 0 ] || \ [ "$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)" = "true" ] || \ @@ -1740,12 +1794,256 @@ else echo "Instance $INSTANCE_NAME has been abandoned from MIG. Starting graceful shutdown..." /usr/local/bin/graceful_shutdown.sh fi +EOF + + media_health_watchdog_script = <<-EOF +#!/bin/bash +# OpenVidu Media Node health watchdog. +# +# Asks the cloud to replace this Media Node with a new one, keeping the number of +# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when: +# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a +# cool-down, so a persistent failure does not churn nodes and can be inspected, +# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot, +# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC +# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC +# while it answers an error or times out. Soft failures are not counted while the CPU +# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in +# the WARMUP_SEC after openvidu.service (re)started. +# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage +# fails the health check of every Media Node at once, and a new node could not bootstrap +# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until +# the node goes away. +# +# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...") +# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused) +# Logs: journalctl -u openvidu-media-health +# +# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and +# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed +# by an opening brace, nor two opening braces in a row. + +STATE_DIR=/var/lib/openvidu-media-health +INSTALLED_MARKER="$STATE_DIR/installed" +BOOTSTRAP_FAILED_MARKER="$STATE_DIR/bootstrap-failed" +REPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh +ENV_FILE=/etc/openvidu/media-health.env +PAUSE_FILE=/etc/openvidu/media-health.disabled +DRAIN_PAUSE_FILE=/run/openvidu-media-health.paused + +log() { echo "[media-health] $*"; } + +# Seconds since boot: monotonic, unaffected by clock steps +uptime_sec() { cut -d. -f1 /proc/uptime; } + +# Busy CPU percentage since the previous call, in BUSY (0 on the first call) +cpu_busy() { + read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat + total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal)) + idle=$((c_idle + c_iowait)) + BUSY=0 + if [ -n "$PREV_TOTAL" ] && [ "$total" -gt "$PREV_TOTAL" ]; then + BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL))) + fi + PREV_TOTAL=$total + PREV_IDLE=$idle +} + +# openvidu.service entered the active state less than WARMUP_SEC ago +openvidu_starting() { + since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null) + case "$since" in ''|0|*[!0-9]*) return 1 ;; esac + [ $(($(uptime_sec) - since / 1000000)) -lt "$WARMUP_SEC" ] +} + +# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare +# TCP connect would also succeed against a wedged Redis. +master_serving() { + [ -n "$MASTER_ADDRS" ] || return 1 + for addr in $MASTER_ADDRS; do + host=$(echo "$addr" | cut -d: -f1) + port=$(echo "$addr" | cut -d: -f2) + reply=$(timeout 5 bash -c 'exec 3<>"/dev/tcp/$0/$1" && printf "PING\r\n" >&3 && head -c 7 <&3' "$host" "$port" 2>/dev/null) + case "$reply" in + +PONG*|-NOAUTH*) return 0 ;; + esac + done + return 1 +} + +# Request the replacement of this node ($1 = reason). Gated on a master answering, except +# in the bootstrap phase before the master addresses are known. +request_replacement() { + now=$(uptime_sec) + if [ -n "$REQUESTED_AT" ] && [ $((now - REQUESTED_AT)) -lt "$REPLACE_RETRY_SEC" ]; then + return + fi + if [ -n "$MASTER_ADDRS" ] || [ -f "$INSTALLED_MARKER" ]; then + if ! master_serving; then + if [ -z "$HOLD_LOGGED_AT" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then + log "$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node" + HOLD_LOGGED_AT=$now + fi + return + fi + fi + log "$1: requesting the replacement of this node" + if "$REPLACE_SCRIPT" "$1"; then + REQUESTED_AT=$now + log "replacement requested" + else + log "the replacement request failed, retrying in $INTERVAL_SEC s" + fi +} + +mkdir -p "$STATE_DIR" +REQUESTED_AT="" +HOLD_LOGGED_AT="" +FAILED_SEEN_AT="" +FAIL_SINCE="" +SATURATED="" +PREV_TOTAL="" +PREV_IDLE="" +log "started" + +while true; do + PROBE_URL=http://127.0.0.1:7880/ + INTERVAL_SEC=30 + HARD_UNHEALTHY_AFTER_SEC=300 + SOFT_UNHEALTHY_AFTER_SEC=600 + CPU_SATURATED_PCT=90 + WARMUP_SEC=900 + BOOTSTRAP_DEADLINE_SEC=5400 + BOOTSTRAP_FAILURE_COOLDOWN_SEC=600 + REPLACE_RETRY_SEC=600 + MASTER_ADDRS="" + [ -f "$ENV_FILE" ] && . "$ENV_FILE" + NOW=$(uptime_sec) + cpu_busy + + if [ -f "$PAUSE_FILE" ] || [ -f "$DRAIN_PAUSE_FILE" ]; then + FAIL_SINCE="" + elif [ -f "$BOOTSTRAP_FAILED_MARKER" ]; then + if [ -z "$FAILED_SEEN_AT" ]; then + FAILED_SEEN_AT=$NOW + log "bootstrap failed ($(head -c 300 "$BOOTSTRAP_FAILED_MARKER")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s" + fi + if [ $((NOW - FAILED_SEEN_AT)) -ge "$BOOTSTRAP_FAILURE_COOLDOWN_SEC" ]; then + request_replacement "bootstrap failed: $(head -c 300 "$BOOTSTRAP_FAILED_MARKER")" + fi + elif [ ! -f "$INSTALLED_MARKER" ]; then + if [ "$NOW" -ge "$BOOTSTRAP_DEADLINE_SEC" ]; then + request_replacement "bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot" + fi + elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 "$PROBE_URL"; then + [ -n "$FAIL_SINCE" ] && log "LiveKit healthy again after $((NOW - FAIL_SINCE)) s" + FAIL_SINCE="" + SATURATED="" + else + RC=$? + case "$RC" in + 7|52|56) KIND=hard ;; + *) KIND=soft ;; + esac + if openvidu_starting; then + FAIL_SINCE="" + elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then + [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted" + SATURATED=1 + FAIL_SINCE="" + else + SATURATED="" + if [ -z "$FAIL_SINCE" ]; then + FAIL_SINCE=$NOW + log "LiveKit health check failed (curl exit $RC)" + fi + LIMIT=$SOFT_UNHEALTHY_AFTER_SEC + [ "$KIND" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC + if [ $((NOW - FAIL_SINCE)) -ge "$LIMIT" ]; then + request_replacement "LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)" + fi + fi + fi + sleep "$INTERVAL_SEC" +done +EOF + + media_replace_script = <<-EOF +#!/bin/bash +# Requests the replacement of THIS media node while keeping the media node count: it asks the +# regional MIG to recreate the instance (delete + create in place, same name, new instance id, +# fresh disk). Called by the health watchdog and by the bootstrap-failure hook; it never stops +# or signals the watchdog (its caller, in the same cgroup). + +REASON="$1" +mkdir -p /var/lib/openvidu-media-health +echo "$(date -u +%FT%TZ) $REASON" >> /var/lib/openvidu-media-health/replace-requests.log +# Tell the drain script this node is being replaced so it skips waiting for sessions. +touch /run/openvidu-unhealthy + +META="http://metadata.google.internal/computeMetadata/v1" +meta() { curl -s -H "Metadata-Flavor: Google" "$META/$1"; } + +SELF_NAME=$(meta "instance/name") +PROJECT_ID=$(meta "project/project-id") +ZONE=$(meta "instance/zone" | cut -d/ -f4) +REGION=$(echo "$ZONE" | sed 's/-[a-z]$//') +STACK_NAME=$(meta "instance/attributes/stackName") +MIG_NAME=$(echo "$STACK_NAME" | tr '[:upper:]' '[:lower:]')-media-node-group + +# Recreate keeps the MIG size. Retry transient API errors. A "not a member / already being +# deleted, abandoned or recreated" error means the node is already leaving the group (e.g. +# scaled in): nothing to do. +i=0 +while [ "$i" -lt 5 ]; do + i=$((i + 1)) + if OUT=$(gcloud compute instance-groups managed recreate-instances "$MIG_NAME" \ + --instances="$SELF_NAME" --region="$REGION" --project="$PROJECT_ID" --quiet 2>&1); then + echo "[media-replace] recreate requested for $SELF_NAME" + exit 0 + fi + case "$OUT" in + *"not a member"*|*"being deleted"*|*"being recreated"*|*abandon*|*ABANDON*) + echo "[media-replace] $SELF_NAME already leaving the group, nothing to do: $OUT" + exit 0 + ;; + esac + echo "[media-replace] recreate attempt $i for $SELF_NAME failed: $OUT" + sleep 12 +done +echo "[media-replace] gave up requesting recreate of $SELF_NAME after $i attempts" +exit 1 +EOF + + media_health_unit = <<-EOF +[Unit] +Description=OpenVidu Media Node health watchdog +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/openvidu-media-health.sh +Restart=always +RestartSec=30 + +[Install] +WantedBy=multi-user.target EOF user_data_media = <<-EOF #!/bin/bash -x set -eu -o pipefail +# Records a media-node bootstrap failure so the health watchdog replaces this node after a +# cool-down, then aborts the bootstrap. +ov_media_bootstrap_failed() { + echo "[OpenVidu] media node bootstrap failed: $1" + mkdir -p /var/lib/openvidu-media-health + echo "$1" > /var/lib/openvidu-media-health/bootstrap-failed + exit 1 +} + # Check if installation already completed if [ -f /usr/local/bin/openvidu_install_counter.txt ]; then # Launch on reboot @@ -1762,6 +2060,21 @@ ${local.graceful_shutdown_script} GRACEFUL_SHUTDOWN_EOF chmod +x /usr/local/bin/graceful_shutdown.sh + # Media node health watchdog + its replacement script and unit + cat > /usr/local/bin/openvidu-media-health.sh << 'MEDIA_HEALTH_EOF' +${local.media_health_watchdog_script} +MEDIA_HEALTH_EOF + chmod +x /usr/local/bin/openvidu-media-health.sh + + cat > /usr/local/bin/openvidu-media-replace.sh << 'MEDIA_REPLACE_EOF' +${local.media_replace_script} +MEDIA_REPLACE_EOF + chmod +x /usr/local/bin/openvidu-media-replace.sh + + cat > /etc/systemd/system/openvidu-media-health.service << 'MEDIA_HEALTH_UNIT_EOF' +${local.media_health_unit} +MEDIA_HEALTH_UNIT_EOF + echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout GCLOUD_VERSION=573.0.0 @@ -1779,14 +2092,29 @@ GRACEFUL_SHUTDOWN_EOF export HOME="/root" + # Start the media node health watchdog now: gcloud (which its replacement needs) is installed, + # and starting before install.sh lets the watchdog cover a hung bootstrap too. Write the master + # address(es) first (before the installed marker) so the anti-storm guard is always populated. + mkdir -p /etc/openvidu + MASTER_NODE_PRIVATE_IP_LIST=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/masterNodePrivateIPList" -H "Metadata-Flavor: Google") + [ -n "$MASTER_NODE_PRIVATE_IP_LIST" ] || ov_media_bootstrap_failed "could not determine master node private IP list" + MASTER_ADDRS=$(echo "$MASTER_NODE_PRIVATE_IP_LIST" | tr ',' ' ' | xargs -n1 | sed 's/$/:7001/' | xargs) + echo "MASTER_ADDRS=\"$MASTER_ADDRS\"" > /etc/openvidu/media-health.env + systemctl daemon-reload + systemctl enable --now openvidu-media-health + # Install OpenVidu Media Node - /usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu Media Node"; exit 1; } + /usr/local/bin/install.sh || ov_media_bootstrap_failed "install.sh failed" # Mark installation as complete echo "installation_complete" > /usr/local/bin/openvidu_install_counter.txt # Start OpenVidu - systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; } + systemctl start openvidu || ov_media_bootstrap_failed "systemctl start openvidu failed" + + # Bootstrap finished: hand runtime replacement to the watchdog (never on a failure path). + mkdir -p /var/lib/openvidu-media-health + touch /var/lib/openvidu-media-health/installed # Add cron job to check if instance is abandoned every minute cat > /usr/local/bin/check_abandoned.sh << 'CHECK_ABANDONED_EOF'