From 0b210fb24fd99f8598dafc085a0b6154bb81a7ac Mon Sep 17 00:00:00 2001 From: Piwccle Date: Tue, 6 Oct 2026 20:47:00 +0200 Subject: [PATCH] openvidu-deployment: Media Node health watchdog: do not count LiveKit failures while no master answers The failure clock kept running during a master outage, so when a master answered again the failure already looked older than the limit and the node was replaced at once, before LiveKit could reconnect to the restored master. Seen live on AWS elastic (ov-cloud-tester master-guard): after a 20 min master outage both Media Nodes were replaced within 17 s of the master coming back. Now no failure is counted while no master answers a Redis PING, so LiveKit gets the whole window once a master is back. --- .../pro/elastic/aws/cf-openvidu-elastic.yaml | 12 ++++++++++-- .../pro/elastic/azure/cf-openvidu-elastic.bicep | 12 ++++++++++-- .../pro/elastic/azure/cf-openvidu-elastic.json | 4 ++-- .../pro/elastic/gcp/tf-gpc-openvidu-elastic.tf | 12 ++++++++++-- openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml | 12 ++++++++++-- .../pro/ha/azure/cf-openvidu-ha.bicep | 12 ++++++++++-- openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json | 4 ++-- openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf | 12 ++++++++++-- 8 files changed, 64 insertions(+), 16 deletions(-) diff --git a/openvidu-deployment/pro/elastic/aws/cf-openvidu-elastic.yaml b/openvidu-deployment/pro/elastic/aws/cf-openvidu-elastic.yaml index ac48b124c..7cb959deb 100644 --- a/openvidu-deployment/pro/elastic/aws/cf-openvidu-elastic.yaml +++ b/openvidu-deployment/pro/elastic/aws/cf-openvidu-elastic.yaml @@ -2173,8 +2173,8 @@ Resources: # the WARMUP_SEC after openvidu.service (re)started. # Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage # fails the health check of every Media Node at once, and a new node could not bootstrap - # either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until - # the node goes away. + # either; LiveKit failures are not counted until a master answers again. Once requested, + # the replacement is requested again every REPLACE_RETRY_SEC until the node goes away. # # Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...") # Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused) @@ -2313,6 +2313,14 @@ Resources: esac if openvidu_starting; then FAIL_SINCE="" + elif ! master_serving; then + # A master outage fails the health check of every Media Node at once: not counted, + # so LiveKit gets the whole window to reconnect once a master answers again + FAIL_SINCE="" + if [ -z "$HOLD_LOGGED_AT" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then + log "LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted" + HOLD_LOGGED_AT=$NOW + fi elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted" SATURATED=1 diff --git a/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.bicep b/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.bicep index f1342edfc..51854cfb0 100644 --- a/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.bicep +++ b/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.bicep @@ -1068,8 +1068,8 @@ var openviduMediaHealthScript = ''' # the WARMUP_SEC after openvidu.service (re)started. # Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage # fails the health check of every Media Node at once, and a new node could not bootstrap -# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until -# the node goes away. +# either; LiveKit failures are not counted until a master answers again. Once requested, +# the replacement is requested again every REPLACE_RETRY_SEC until the node goes away. # # Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...") # Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused) @@ -1208,6 +1208,14 @@ while true; do esac if openvidu_starting; then FAIL_SINCE="" + elif ! master_serving; then + # A master outage fails the health check of every Media Node at once: not counted, + # so LiveKit gets the whole window to reconnect once a master answers again + FAIL_SINCE="" + if [ -z "$HOLD_LOGGED_AT" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then + log "LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted" + HOLD_LOGGED_AT=$NOW + fi elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted" SATURATED=1 diff --git a/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.json b/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.json index 76dbf7b74..a7a0e09c2 100644 --- a/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.json +++ b/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.46.1.21595", - "templateHash": "1384799452684335174" + "templateHash": "3671752928188054764" } }, "parameters": { @@ -279,7 +279,7 @@ "storageAccountName": "[if(variables('isEmptyStorageAccountName'), uniqueString(resourceGroup().id), parameters('storageAccountName'))]" }, "stop_media_nodesScriptMediaTemplate": "#!/bin/bash\nset -e\n\n# Pause the health watchdog before draining so it never reimages a node that is being\n# drained for scale-in (synchronous; the drain runs outside the watchdog's cgroup).\ntouch /run/openvidu-media-health.paused\nsystemctl stop openvidu-media-health 2>/dev/null || true\n\nif ! (set -o noclobber ; echo > /tmp/global.lock) ; then\n exit 1 # the global.lock already exists\nfi\n\n# Execute if docker is installed\nif [ -x \"$(command -v docker)\" ]; then\n\n echo \"Stopping media node services and waiting for termination...\"\n docker container kill --signal=SIGQUIT openvidu 2>/dev/null || true\n docker container kill --signal=SIGQUIT ingress 2>/dev/null || true\n docker container kill --signal=SIGQUIT egress 2>/dev/null || true\n for agent_container in $(docker ps --filter \"label=openvidu-agent=true\" --format '{{.Names}}' 2>/dev/null); do\n docker container kill --signal=SIGQUIT \"$agent_container\" 2>/dev/null || true\n done\n\n # If this node is being replaced because it is unhealthy, do not wait for sessions to end.\n if [ -f /run/openvidu-unhealthy ]; then\n docker ps -q | xargs -r docker kill >/dev/null 2>&1 || true\n fi\n\n # Wait for running containers to not be openvidu, ingress, egress or an openvidu agent\n while [ \"$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)\" -gt 0 ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)\" = \"true\" ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' ingress 2>/dev/null)\" = \"true\" ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' egress 2>/dev/null)\" = \"true\" ]; do\n echo \"Waiting for containers to stop...\"\n sleep 5\n done\nfi\n\naz login --identity\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nSUBSCRIPTION_ID=${subscriptionId}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\nRESOURCE_ID=/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP_NAME/providers/Microsoft.Compute/virtualMachineScaleSets/$VM_SCALE_SET_NAME\nTIMESTAMP=$(date -u +\"%Y-%m-%dT%H:%M:%SZ\")\n\naz tag update --resource-id $RESOURCE_ID --operation replace --tags \"STATUS\"=\"HEALTHY\" \"InstanceDeleteTime\"=\"$TIMESTAMP\" \"storageAccount\"=\"${storageAccountName}\"\n\naz vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID\n", - "openviduMediaHealthScript": "#!/bin/bash\n# OpenVidu Media Node health watchdog.\n#\n# Asks the cloud to replace this Media Node with a new one, keeping the number of\n# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when:\n# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a\n# cool-down, so a persistent failure does not churn nodes and can be inspected,\n# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot,\n# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC\n# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC\n# while it answers an error or times out. Soft failures are not counted while the CPU\n# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in\n# the WARMUP_SEC after openvidu.service (re)started.\n# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage\n# fails the health check of every Media Node at once, and a new node could not bootstrap\n# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until\n# the node goes away.\n#\n# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS=\"ip:port ...\")\n# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)\n# Logs: journalctl -u openvidu-media-health\n#\n# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and\n# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed\n# by an opening brace, nor two opening braces in a row.\n\nSTATE_DIR=/var/lib/openvidu-media-health\nINSTALLED_MARKER=\"$STATE_DIR/installed\"\nBOOTSTRAP_FAILED_MARKER=\"$STATE_DIR/bootstrap-failed\"\nREPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh\nENV_FILE=/etc/openvidu/media-health.env\nPAUSE_FILE=/etc/openvidu/media-health.disabled\nDRAIN_PAUSE_FILE=/run/openvidu-media-health.paused\n\nlog() { echo \"[media-health] $*\"; }\n\n# Seconds since boot: monotonic, unaffected by clock steps\nuptime_sec() { cut -d. -f1 /proc/uptime; }\n\n# Busy CPU percentage since the previous call, in BUSY (0 on the first call)\ncpu_busy() {\n read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat\n total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal))\n idle=$((c_idle + c_iowait))\n BUSY=0\n if [ -n \"$PREV_TOTAL\" ] && [ \"$total\" -gt \"$PREV_TOTAL\" ]; then\n BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL)))\n fi\n PREV_TOTAL=$total\n PREV_IDLE=$idle\n}\n\n# openvidu.service is running and entered the active state less than WARMUP_SEC ago\n# (a stopped or failed openvidu.service is not starting: no warm-up for it)\nopenvidu_starting() {\n case \"$(systemctl show -p ActiveState --value openvidu 2>/dev/null)\" in\n active|activating|reloading) ;;\n *) return 1 ;;\n esac\n since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null)\n case \"$since\" in ''|0|*[!0-9]*) return 1 ;; esac\n [ $(($(uptime_sec) - since / 1000000)) -lt \"$WARMUP_SEC\" ]\n}\n\n# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare\n# TCP connect would also succeed against a wedged Redis.\nmaster_serving() {\n [ -n \"$MASTER_ADDRS\" ] || return 1\n for addr in $MASTER_ADDRS; do\n host=$(echo \"$addr\" | cut -d: -f1)\n port=$(echo \"$addr\" | cut -d: -f2)\n reply=$(timeout 5 bash -c 'exec 3<>\"/dev/tcp/$0/$1\" && printf \"PING\\r\\n\" >&3 && head -c 7 <&3' \"$host\" \"$port\" 2>/dev/null)\n case \"$reply\" in\n +PONG*|-NOAUTH*) return 0 ;;\n esac\n done\n return 1\n}\n\n# Request the replacement of this node ($1 = reason). Gated on a master answering, except\n# in the bootstrap phase before the master addresses are known.\nrequest_replacement() {\n now=$(uptime_sec)\n if [ -n \"$REQUESTED_AT\" ] && [ $((now - REQUESTED_AT)) -lt \"$REPLACE_RETRY_SEC\" ]; then\n return\n fi\n if [ -n \"$MASTER_ADDRS\" ] || [ -f \"$INSTALLED_MARKER\" ]; then\n if ! master_serving; then\n if [ -z \"$HOLD_LOGGED_AT\" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then\n log \"$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node\"\n HOLD_LOGGED_AT=$now\n fi\n return\n fi\n fi\n log \"$1: requesting the replacement of this node\"\n if \"$REPLACE_SCRIPT\" \"$1\"; then\n REQUESTED_AT=$now\n log \"replacement requested\"\n else\n log \"the replacement request failed, retrying in $INTERVAL_SEC s\"\n fi\n}\n\nmkdir -p \"$STATE_DIR\"\nREQUESTED_AT=\"\"\nHOLD_LOGGED_AT=\"\"\nFAILED_SEEN_AT=\"\"\nFAIL_SINCE=\"\"\nSATURATED=\"\"\nPREV_TOTAL=\"\"\nPREV_IDLE=\"\"\nlog \"started\"\n\nwhile true; do\n PROBE_URL=http://127.0.0.1:7880/\n INTERVAL_SEC=30\n HARD_UNHEALTHY_AFTER_SEC=300\n SOFT_UNHEALTHY_AFTER_SEC=600\n CPU_SATURATED_PCT=90\n WARMUP_SEC=900\n BOOTSTRAP_DEADLINE_SEC=5400\n BOOTSTRAP_FAILURE_COOLDOWN_SEC=600\n REPLACE_RETRY_SEC=600\n MASTER_ADDRS=\"\"\n [ -f \"$ENV_FILE\" ] && . \"$ENV_FILE\"\n NOW=$(uptime_sec)\n cpu_busy\n\n if [ -f \"$PAUSE_FILE\" ] || [ -f \"$DRAIN_PAUSE_FILE\" ]; then\n FAIL_SINCE=\"\"\n elif [ -f \"$BOOTSTRAP_FAILED_MARKER\" ]; then\n if [ -z \"$FAILED_SEEN_AT\" ]; then\n FAILED_SEEN_AT=$NOW\n log \"bootstrap failed ($(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s\"\n fi\n if [ $((NOW - FAILED_SEEN_AT)) -ge \"$BOOTSTRAP_FAILURE_COOLDOWN_SEC\" ]; then\n request_replacement \"bootstrap failed: $(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")\"\n fi\n elif [ ! -f \"$INSTALLED_MARKER\" ]; then\n if [ \"$NOW\" -ge \"$BOOTSTRAP_DEADLINE_SEC\" ]; then\n request_replacement \"bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot\"\n fi\n elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 \"$PROBE_URL\"; then\n [ -n \"$FAIL_SINCE\" ] && log \"LiveKit healthy again after $((NOW - FAIL_SINCE)) s\"\n FAIL_SINCE=\"\"\n SATURATED=\"\"\n else\n RC=$?\n case \"$RC\" in\n 7|52|56) KIND=hard ;;\n *) KIND=soft ;;\n esac\n if openvidu_starting; then\n FAIL_SINCE=\"\"\n elif [ \"$KIND\" = soft ] && [ \"$BUSY\" -ge \"$CPU_SATURATED_PCT\" ]; then\n [ -n \"$SATURATED\" ] || log \"LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted\"\n SATURATED=1\n FAIL_SINCE=\"\"\n else\n SATURATED=\"\"\n if [ -z \"$FAIL_SINCE\" ]; then\n FAIL_SINCE=$NOW\n log \"LiveKit health check failed (curl exit $RC)\"\n fi\n LIMIT=$SOFT_UNHEALTHY_AFTER_SEC\n [ \"$KIND\" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC\n if [ $((NOW - FAIL_SINCE)) -ge \"$LIMIT\" ]; then\n request_replacement \"LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)\"\n fi\n fi\n fi\n sleep \"$INTERVAL_SEC\"\ndone\n", + "openviduMediaHealthScript": "#!/bin/bash\n# OpenVidu Media Node health watchdog.\n#\n# Asks the cloud to replace this Media Node with a new one, keeping the number of\n# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when:\n# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a\n# cool-down, so a persistent failure does not churn nodes and can be inspected,\n# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot,\n# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC\n# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC\n# while it answers an error or times out. Soft failures are not counted while the CPU\n# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in\n# the WARMUP_SEC after openvidu.service (re)started.\n# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage\n# fails the health check of every Media Node at once, and a new node could not bootstrap\n# either; LiveKit failures are not counted until a master answers again. Once requested,\n# the replacement is requested again every REPLACE_RETRY_SEC until the node goes away.\n#\n# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS=\"ip:port ...\")\n# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)\n# Logs: journalctl -u openvidu-media-health\n#\n# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and\n# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed\n# by an opening brace, nor two opening braces in a row.\n\nSTATE_DIR=/var/lib/openvidu-media-health\nINSTALLED_MARKER=\"$STATE_DIR/installed\"\nBOOTSTRAP_FAILED_MARKER=\"$STATE_DIR/bootstrap-failed\"\nREPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh\nENV_FILE=/etc/openvidu/media-health.env\nPAUSE_FILE=/etc/openvidu/media-health.disabled\nDRAIN_PAUSE_FILE=/run/openvidu-media-health.paused\n\nlog() { echo \"[media-health] $*\"; }\n\n# Seconds since boot: monotonic, unaffected by clock steps\nuptime_sec() { cut -d. -f1 /proc/uptime; }\n\n# Busy CPU percentage since the previous call, in BUSY (0 on the first call)\ncpu_busy() {\n read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat\n total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal))\n idle=$((c_idle + c_iowait))\n BUSY=0\n if [ -n \"$PREV_TOTAL\" ] && [ \"$total\" -gt \"$PREV_TOTAL\" ]; then\n BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL)))\n fi\n PREV_TOTAL=$total\n PREV_IDLE=$idle\n}\n\n# openvidu.service is running and entered the active state less than WARMUP_SEC ago\n# (a stopped or failed openvidu.service is not starting: no warm-up for it)\nopenvidu_starting() {\n case \"$(systemctl show -p ActiveState --value openvidu 2>/dev/null)\" in\n active|activating|reloading) ;;\n *) return 1 ;;\n esac\n since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null)\n case \"$since\" in ''|0|*[!0-9]*) return 1 ;; esac\n [ $(($(uptime_sec) - since / 1000000)) -lt \"$WARMUP_SEC\" ]\n}\n\n# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare\n# TCP connect would also succeed against a wedged Redis.\nmaster_serving() {\n [ -n \"$MASTER_ADDRS\" ] || return 1\n for addr in $MASTER_ADDRS; do\n host=$(echo \"$addr\" | cut -d: -f1)\n port=$(echo \"$addr\" | cut -d: -f2)\n reply=$(timeout 5 bash -c 'exec 3<>\"/dev/tcp/$0/$1\" && printf \"PING\\r\\n\" >&3 && head -c 7 <&3' \"$host\" \"$port\" 2>/dev/null)\n case \"$reply\" in\n +PONG*|-NOAUTH*) return 0 ;;\n esac\n done\n return 1\n}\n\n# Request the replacement of this node ($1 = reason). Gated on a master answering, except\n# in the bootstrap phase before the master addresses are known.\nrequest_replacement() {\n now=$(uptime_sec)\n if [ -n \"$REQUESTED_AT\" ] && [ $((now - REQUESTED_AT)) -lt \"$REPLACE_RETRY_SEC\" ]; then\n return\n fi\n if [ -n \"$MASTER_ADDRS\" ] || [ -f \"$INSTALLED_MARKER\" ]; then\n if ! master_serving; then\n if [ -z \"$HOLD_LOGGED_AT\" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then\n log \"$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node\"\n HOLD_LOGGED_AT=$now\n fi\n return\n fi\n fi\n log \"$1: requesting the replacement of this node\"\n if \"$REPLACE_SCRIPT\" \"$1\"; then\n REQUESTED_AT=$now\n log \"replacement requested\"\n else\n log \"the replacement request failed, retrying in $INTERVAL_SEC s\"\n fi\n}\n\nmkdir -p \"$STATE_DIR\"\nREQUESTED_AT=\"\"\nHOLD_LOGGED_AT=\"\"\nFAILED_SEEN_AT=\"\"\nFAIL_SINCE=\"\"\nSATURATED=\"\"\nPREV_TOTAL=\"\"\nPREV_IDLE=\"\"\nlog \"started\"\n\nwhile true; do\n PROBE_URL=http://127.0.0.1:7880/\n INTERVAL_SEC=30\n HARD_UNHEALTHY_AFTER_SEC=300\n SOFT_UNHEALTHY_AFTER_SEC=600\n CPU_SATURATED_PCT=90\n WARMUP_SEC=900\n BOOTSTRAP_DEADLINE_SEC=5400\n BOOTSTRAP_FAILURE_COOLDOWN_SEC=600\n REPLACE_RETRY_SEC=600\n MASTER_ADDRS=\"\"\n [ -f \"$ENV_FILE\" ] && . \"$ENV_FILE\"\n NOW=$(uptime_sec)\n cpu_busy\n\n if [ -f \"$PAUSE_FILE\" ] || [ -f \"$DRAIN_PAUSE_FILE\" ]; then\n FAIL_SINCE=\"\"\n elif [ -f \"$BOOTSTRAP_FAILED_MARKER\" ]; then\n if [ -z \"$FAILED_SEEN_AT\" ]; then\n FAILED_SEEN_AT=$NOW\n log \"bootstrap failed ($(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s\"\n fi\n if [ $((NOW - FAILED_SEEN_AT)) -ge \"$BOOTSTRAP_FAILURE_COOLDOWN_SEC\" ]; then\n request_replacement \"bootstrap failed: $(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")\"\n fi\n elif [ ! -f \"$INSTALLED_MARKER\" ]; then\n if [ \"$NOW\" -ge \"$BOOTSTRAP_DEADLINE_SEC\" ]; then\n request_replacement \"bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot\"\n fi\n elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 \"$PROBE_URL\"; then\n [ -n \"$FAIL_SINCE\" ] && log \"LiveKit healthy again after $((NOW - FAIL_SINCE)) s\"\n FAIL_SINCE=\"\"\n SATURATED=\"\"\n else\n RC=$?\n case \"$RC\" in\n 7|52|56) KIND=hard ;;\n *) KIND=soft ;;\n esac\n if openvidu_starting; then\n FAIL_SINCE=\"\"\n elif ! master_serving; then\n # A master outage fails the health check of every Media Node at once: not counted,\n # so LiveKit gets the whole window to reconnect once a master answers again\n FAIL_SINCE=\"\"\n if [ -z \"$HOLD_LOGGED_AT\" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then\n log \"LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted\"\n HOLD_LOGGED_AT=$NOW\n fi\n elif [ \"$KIND\" = soft ] && [ \"$BUSY\" -ge \"$CPU_SATURATED_PCT\" ]; then\n [ -n \"$SATURATED\" ] || log \"LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted\"\n SATURATED=1\n FAIL_SINCE=\"\"\n else\n SATURATED=\"\"\n if [ -z \"$FAIL_SINCE\" ]; then\n FAIL_SINCE=$NOW\n log \"LiveKit health check failed (curl exit $RC)\"\n fi\n LIMIT=$SOFT_UNHEALTHY_AFTER_SEC\n [ \"$KIND\" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC\n if [ $((NOW - FAIL_SINCE)) -ge \"$LIMIT\" ]; then\n request_replacement \"LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)\"\n fi\n fi\n fi\n sleep \"$INTERVAL_SEC\"\ndone\n", "openviduMediaHealthUnit": "[Unit]\nDescription=OpenVidu Media Node health watchdog\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/local/bin/openvidu-media-health.sh\nRestart=always\nRestartSec=30\n\n[Install]\nWantedBy=multi-user.target\n", "media_replaceScriptMediaTemplate": "#!/bin/bash\n# openvidu-media-replace.sh \"\"\n# Requests the reimage of THIS Media Node, keeping the scale set capacity and the\n# scale-in protection: reimage provisions a new OS disk and the userData bootstrap runs\n# again on the same instance id. Called by the health watchdog; it never signals it.\n\nmkdir -p /var/lib/openvidu-media-health\necho \"$(date -u +%FT%TZ) $1\" >> /var/lib/openvidu-media-health/replace-requests.log\ntouch /run/openvidu-unhealthy\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nINSTANCE_RESOURCE_ID=$(curl -s -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo \"$INSTANCE_RESOURCE_ID\" | awk -F'/' '{print $NF}')\nif [ -z \"$INSTANCE_ID\" ]; then\n echo \"[media-replace] could not read this instance id from IMDS\"\n exit 1\nfi\n\nn=0\nwhile true; do\n n=$((n + 1))\n if az login --identity >/dev/null 2>&1 && \\\n az vmss reimage --resource-group \"$RESOURCE_GROUP_NAME\" --name \"$VM_SCALE_SET_NAME\" --instance-ids \"$INSTANCE_ID\" --no-wait; then\n echo \"[media-replace] reimage requested for instance $INSTANCE_ID ($1)\"\n exit 0\n fi\n if [ \"$n\" -ge 5 ]; then\n echo \"[media-replace] reimage could not be requested after $n attempts\"\n exit 1\n fi\n echo \"[media-replace] reimage request failed (attempt $n), retrying in 12s\"\n sleep 12\ndone\n", "userDataMediaNodeTemplate": "#!/bin/bash\nset -eu -o pipefail\n\n# Mark this media node's bootstrap as failed so the health watchdog replaces it\nov_media_bootstrap_failed() {\n echo \"[OpenVidu] media node bootstrap failed: $1\"\n mkdir -p /var/lib/openvidu-media-health\n echo \"$1\" > /var/lib/openvidu-media-health/bootstrap-failed\n exit 1\n}\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# stop_media_nodes.sh\necho ${base64stop} | base64 -d > /usr/local/bin/stop_media_node.sh\nchmod +x /usr/local/bin/stop_media_node.sh\n\n# openvidu-media-health.sh (health watchdog), openvidu-media-replace.sh (reimage this node) and its unit\necho ${base64mediaHealth} | base64 -d > /usr/local/bin/openvidu-media-health.sh\nchmod +x /usr/local/bin/openvidu-media-health.sh\necho ${base64mediaReplace} | base64 -d > /usr/local/bin/openvidu-media-replace.sh\nchmod +x /usr/local/bin/openvidu-media-replace.sh\necho ${base64mediaHealthUnit} | base64 -d > /etc/systemd/system/openvidu-media-health.service\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y jq\n\n# Install azure cli\nAZURE_CLI_VERSION=2.90.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity\n\n# Protect from scale in actions\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\naz vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-id $INSTANCE_ID --protect-from-scale-in true\n\nexport HOME=\"/root\"\n\n# Start the Media Node health watchdog before install.sh (az CLI is installed and the node\n# is scale-in protected): it backstops a hung or failed bootstrap and, once running, reimages\n# the node if LiveKit stays unhealthy. Write MASTER_ADDRS first so its master guard is armed.\nmkdir -p /etc/openvidu\necho 'MASTER_ADDRS=\"${privateIPMasterNode}:7000\"' > /etc/openvidu/media-health.env\nsystemctl daemon-reload\nsystemctl enable --now openvidu-media-health\n\n# Install OpenVidu\n/usr/local/bin/install.sh || ov_media_bootstrap_failed \"install.sh failed\"\n\n# Gate 2: wait for master readiness before starting\nWAIT_INTERVAL=5\nMAX_RETRIES=360\nRETRIES=0\nset +e\nwhile true; do\n FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv 2>/dev/null)\n if [ \"$FINISH_MASTER_NODE\" == \"true\" ]; then\n break\n fi\n RETRIES=$((RETRIES + 1))\n if [ $RETRIES -ge $MAX_RETRIES ]; then\n ov_media_bootstrap_failed \"timed out after 30 min waiting for FINISH-MASTER-NODE\"\n fi\n sleep $WAIT_INTERVAL\ndone\nset -e\n\n# Start OpenVidu\nsystemctl start openvidu || ov_media_bootstrap_failed \"systemctl start openvidu failed\"\n\n# Bootstrap finished: the watchdog switches from bootstrap-deadline mode to runtime health checks\nmkdir -p /var/lib/openvidu-media-health\ntouch /var/lib/openvidu-media-health/installed\n", diff --git a/openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf b/openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf index 3b566402d..73c6523d2 100644 --- a/openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf +++ b/openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf @@ -1429,8 +1429,8 @@ EOF # the WARMUP_SEC after openvidu.service (re)started. # Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage # fails the health check of every Media Node at once, and a new node could not bootstrap -# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until -# the node goes away. +# either; LiveKit failures are not counted until a master answers again. Once requested, +# the replacement is requested again every REPLACE_RETRY_SEC until the node goes away. # # Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...") # Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused) @@ -1569,6 +1569,14 @@ while true; do esac if openvidu_starting; then FAIL_SINCE="" + elif ! master_serving; then + # A master outage fails the health check of every Media Node at once: not counted, + # so LiveKit gets the whole window to reconnect once a master answers again + FAIL_SINCE="" + if [ -z "$HOLD_LOGGED_AT" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then + log "LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted" + HOLD_LOGGED_AT=$NOW + fi elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted" SATURATED=1 diff --git a/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml b/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml index fdfd0a2ef..8c94f6b8b 100644 --- a/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml +++ b/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml @@ -2816,8 +2816,8 @@ Resources: # the WARMUP_SEC after openvidu.service (re)started. # Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage # fails the health check of every Media Node at once, and a new node could not bootstrap - # either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until - # the node goes away. + # either; LiveKit failures are not counted until a master answers again. Once requested, + # the replacement is requested again every REPLACE_RETRY_SEC until the node goes away. # # Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...") # Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused) @@ -2956,6 +2956,14 @@ Resources: esac if openvidu_starting; then FAIL_SINCE="" + elif ! master_serving; then + # A master outage fails the health check of every Media Node at once: not counted, + # so LiveKit gets the whole window to reconnect once a master answers again + FAIL_SINCE="" + if [ -z "$HOLD_LOGGED_AT" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then + log "LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted" + HOLD_LOGGED_AT=$NOW + fi elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted" SATURATED=1 diff --git a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep index ec3b25fe4..e061ad34e 100644 --- a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep +++ b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep @@ -1461,8 +1461,8 @@ var openviduMediaHealthScript = ''' # the WARMUP_SEC after openvidu.service (re)started. # Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage # fails the health check of every Media Node at once, and a new node could not bootstrap -# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until -# the node goes away. +# either; LiveKit failures are not counted until a master answers again. Once requested, +# the replacement is requested again every REPLACE_RETRY_SEC until the node goes away. # # Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...") # Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused) @@ -1601,6 +1601,14 @@ while true; do esac if openvidu_starting; then FAIL_SINCE="" + elif ! master_serving; then + # A master outage fails the health check of every Media Node at once: not counted, + # so LiveKit gets the whole window to reconnect once a master answers again + FAIL_SINCE="" + if [ -z "$HOLD_LOGGED_AT" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then + log "LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted" + HOLD_LOGGED_AT=$NOW + fi elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted" SATURATED=1 diff --git a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json index b1591abba..7bc32a6da 100644 --- a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json +++ b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.46.1.21595", - "templateHash": "9885917927692102096" + "templateHash": "11373208048589082219" } }, "parameters": { @@ -249,7 +249,7 @@ "userDataTemplateMasterNode": "#!/bin/bash -x\nset -eu -o pipefail\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# after_install.sh\necho ${base64after_install} | base64 -d > /usr/local/bin/after_install.sh\nchmod +x /usr/local/bin/after_install.sh\n\n# get_public_ip.sh\necho ${base64get_public_ip} | base64 -d > /usr/local/bin/get_public_ip.sh\nchmod +x /usr/local/bin/get_public_ip.sh\n\n# update_config_from_secret.sh\necho ${base64update_config_from_secret} | base64 -d > /usr/local/bin/update_config_from_secret.sh\nchmod +x /usr/local/bin/update_config_from_secret.sh\n\n# update_secret_from_config.sh\necho ${base64update_secret_from_config} | base64 -d > /usr/local/bin/update_secret_from_config.sh\nchmod +x /usr/local/bin/update_secret_from_config.sh\n\n# get_value_from_config.sh\necho ${base64get_value_from_config} | base64 -d > /usr/local/bin/get_value_from_config.sh\nchmod +x /usr/local/bin/get_value_from_config.sh\n\n# store_secret.sh\necho ${base64store_secret} | base64 -d > /usr/local/bin/store_secret.sh\nchmod +x /usr/local/bin/store_secret.sh\n\n# check_app_ready.sh\necho ${base64check_app_ready} | base64 -d > /usr/local/bin/check_app_ready.sh\nchmod +x /usr/local/bin/check_app_ready.sh\n\n# restart.sh\necho ${base64restart} | base64 -d > /usr/local/bin/restart.sh\nchmod +x /usr/local/bin/restart.sh\n\necho ${base64config_blobStorage} | base64 -d > /usr/local/bin/config_blobStorage.sh\nchmod +x /usr/local/bin/config_blobStorage.sh\n\n# Install azure cli\nAZURE_CLI_VERSION=2.90.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity --client-id ${masterIdentityClientId} --allow-no-subscriptions\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\nexport HOME=\"/root\"\n\n# Install OpenVidu\n/usr/local/bin/install.sh || { echo \"[OpenVidu] error installing OpenVidu\"; exit 1; }\n\n#Config blob storage\n/usr/local/bin/config_blobStorage.sh || { echo \"[OpenVidu] error configuring Blob Storage\"; exit 1; }\n\n# Start OpenVidu\nsystemctl start openvidu || { echo \"[OpenVidu] error starting OpenVidu\"; exit 1; }\n\n# Update shared secret\n/usr/local/bin/after_install.sh || { echo \"[OpenVidu] error updating shared secret\"; exit 1; }\n\n# Launch on reboot\necho \"@reboot /usr/local/bin/restart.sh >> /var/log/openvidu-restart.log\" 2>&1 | crontab\n\n# check_app_ready.sh internally caps its wait at 1200s\n/usr/local/bin/check_app_ready.sh || { echo \"[OpenVidu] master node did not become healthy\"; exit 1; }\n\naz keyvault secret set --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --value \"true\"\n\nMASTER_NODE_NUM=${masterNodeNum}\nif [[ $MASTER_NODE_NUM -eq 4 ]]; then\n # Creating scale in lock\n set +e\n az storage blob upload --account-name ${storageAccountName} --container-name automation-locks --name lock.txt --file /dev/null --auth-mode key\n set -e\nfi\n", "installScriptTemplateMedia": "#!/bin/bash -x\nDOMAIN=\n\n# Install dependencies\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y \\\n curl \\\n unzip \\\n jq \\\n wget\n\n# Get own private IP\nPRIVATE_IP=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text\")\n\n# Gate 1: wait for master secrets and IPs before installing\nWAIT_INTERVAL=5\nMAX_RETRIES=360\nRETRIES=0\nset +e\nwhile true; do\n ALL_SECRETS_GENERATED=$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)\n MASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv 2>/dev/null)\n MASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv 2>/dev/null)\n MASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv 2>/dev/null)\n MASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv 2>/dev/null)\n\n if [ \"$ALL_SECRETS_GENERATED\" == \"true\" ] &&\n [ \"$MASTER_NODE_1_PRIVATE_IP\" != \"\" ] &&\n [ \"$MASTER_NODE_2_PRIVATE_IP\" != \"\" ] &&\n [ \"$MASTER_NODE_3_PRIVATE_IP\" != \"\" ] &&\n [ \"$MASTER_NODE_4_PRIVATE_IP\" != \"\" ]; then\n break\n fi\n\n RETRIES=$((RETRIES + 1))\n if [ $RETRIES -ge $MAX_RETRIES ]; then\n echo \"[OpenVidu] timed out after 30 min waiting for master nodes to generate secrets and publish their IPs\"\n exit 1\n fi\n\n sleep $WAIT_INTERVAL\ndone\nset -e\n\nMASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_PRIVATE_IP_LIST=\"$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP\"\nREDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name REDIS-PASSWORD --query value -o tsv)\nENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv)\nOPENVIDU_VERSION=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-VERSION --query value -o tsv)\n\n# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure\nINSTALLER_SCRIPT=\"/tmp/install_ov_media_node.sh\"\ncurl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o \"$INSTALLER_SCRIPT\" \"http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh\"\nif [ ! -s \"$INSTALLER_SCRIPT\" ]; then\n echo \"[OpenVidu] failed to download the media node installer script\"\n exit 1\nfi\n\n# Base command\nINSTALL_COMMAND=\"sh $INSTALLER_SCRIPT\"\n\n# Common arguments\nCOMMON_ARGS=(\n\"--no-tty\"\n\"--install\"\n\"--environment=azure\"\n\"--deployment-type='ha'\"\n\"--node-role='media-node'\"\n\"--master-node-private-ip-list=$MASTER_NODE_PRIVATE_IP_LIST\"\n\"--private-ip=$PRIVATE_IP\"\n\"--enabled-modules='$ENABLED_MODULES'\"\n\"--redis-password=$REDIS_PASSWORD\"\n)\n\n# Construct the final command with all arguments\nFINAL_COMMAND=\"$INSTALL_COMMAND $(printf \"%s \" \"${COMMON_ARGS[@]}\")\"\n\n# Install OpenVidu\nexec bash -c \"$FINAL_COMMAND\"\n", "stop_media_nodesScriptMediaTemplate": "#!/bin/bash\nset -e\n\n# Pause the health watchdog before draining so it never reimages a node that is being\n# drained for scale-in (synchronous; the drain runs outside the watchdog's cgroup).\ntouch /run/openvidu-media-health.paused\nsystemctl stop openvidu-media-health 2>/dev/null || true\n\nif ! (set -o noclobber ; echo > /tmp/global.lock) ; then\n exit 1 # the global.lock already exists\nfi\n\n# Execute if docker is installed\nif [ -x \"$(command -v docker)\" ]; then\n\n echo \"Stopping media node services and waiting for termination...\"\n docker container kill --signal=SIGQUIT openvidu 2>/dev/null || true\n docker container kill --signal=SIGQUIT ingress 2>/dev/null || true\n docker container kill --signal=SIGQUIT egress 2>/dev/null || true\n for agent_container in $(docker ps --filter \"label=openvidu-agent=true\" --format '{{.Names}}' 2>/dev/null); do\n docker container kill --signal=SIGQUIT \"$agent_container\" 2>/dev/null || true\n done\n\n # If this node is being replaced because it is unhealthy, do not wait for sessions to end.\n if [ -f /run/openvidu-unhealthy ]; then\n docker ps -q | xargs -r docker kill >/dev/null 2>&1 || true\n fi\n\n # Wait for running containers to not be openvidu, ingress or egress\n while [ \"$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)\" -gt 0 ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)\" = \"true\" ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' ingress 2>/dev/null)\" = \"true\" ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' egress 2>/dev/null)\" = \"true\" ]; do\n echo \"Waiting for containers to stop...\"\n sleep 5\n done\nfi\n\naz login --identity --client-id ${mediaIdentityClientId}\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nSUBSCRIPTION_ID=${subscriptionId}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\nRESOURCE_ID=/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP_NAME/providers/Microsoft.Compute/virtualMachineScaleSets/$VM_SCALE_SET_NAME\n\nTIMESTAMP=$(date -u +\"%Y-%m-%dT%H:%M:%SZ\")\naz tag update --resource-id $RESOURCE_ID --operation replace --tags \"STATUS\"=\"HEALTHY\" \"InstanceDeleteTime\"=\"$TIMESTAMP\" \"storageAccount\"=\"${storageAccountName}\"\n\naz vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID\n", - "openviduMediaHealthScript": "#!/bin/bash\n# OpenVidu Media Node health watchdog.\n#\n# Asks the cloud to replace this Media Node with a new one, keeping the number of\n# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when:\n# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a\n# cool-down, so a persistent failure does not churn nodes and can be inspected,\n# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot,\n# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC\n# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC\n# while it answers an error or times out. Soft failures are not counted while the CPU\n# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in\n# the WARMUP_SEC after openvidu.service (re)started.\n# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage\n# fails the health check of every Media Node at once, and a new node could not bootstrap\n# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until\n# the node goes away.\n#\n# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS=\"ip:port ...\")\n# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)\n# Logs: journalctl -u openvidu-media-health\n#\n# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and\n# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed\n# by an opening brace, nor two opening braces in a row.\n\nSTATE_DIR=/var/lib/openvidu-media-health\nINSTALLED_MARKER=\"$STATE_DIR/installed\"\nBOOTSTRAP_FAILED_MARKER=\"$STATE_DIR/bootstrap-failed\"\nREPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh\nENV_FILE=/etc/openvidu/media-health.env\nPAUSE_FILE=/etc/openvidu/media-health.disabled\nDRAIN_PAUSE_FILE=/run/openvidu-media-health.paused\n\nlog() { echo \"[media-health] $*\"; }\n\n# Seconds since boot: monotonic, unaffected by clock steps\nuptime_sec() { cut -d. -f1 /proc/uptime; }\n\n# Busy CPU percentage since the previous call, in BUSY (0 on the first call)\ncpu_busy() {\n read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat\n total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal))\n idle=$((c_idle + c_iowait))\n BUSY=0\n if [ -n \"$PREV_TOTAL\" ] && [ \"$total\" -gt \"$PREV_TOTAL\" ]; then\n BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL)))\n fi\n PREV_TOTAL=$total\n PREV_IDLE=$idle\n}\n\n# openvidu.service is running and entered the active state less than WARMUP_SEC ago\n# (a stopped or failed openvidu.service is not starting: no warm-up for it)\nopenvidu_starting() {\n case \"$(systemctl show -p ActiveState --value openvidu 2>/dev/null)\" in\n active|activating|reloading) ;;\n *) return 1 ;;\n esac\n since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null)\n case \"$since\" in ''|0|*[!0-9]*) return 1 ;; esac\n [ $(($(uptime_sec) - since / 1000000)) -lt \"$WARMUP_SEC\" ]\n}\n\n# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare\n# TCP connect would also succeed against a wedged Redis.\nmaster_serving() {\n [ -n \"$MASTER_ADDRS\" ] || return 1\n for addr in $MASTER_ADDRS; do\n host=$(echo \"$addr\" | cut -d: -f1)\n port=$(echo \"$addr\" | cut -d: -f2)\n reply=$(timeout 5 bash -c 'exec 3<>\"/dev/tcp/$0/$1\" && printf \"PING\\r\\n\" >&3 && head -c 7 <&3' \"$host\" \"$port\" 2>/dev/null)\n case \"$reply\" in\n +PONG*|-NOAUTH*) return 0 ;;\n esac\n done\n return 1\n}\n\n# Request the replacement of this node ($1 = reason). Gated on a master answering, except\n# in the bootstrap phase before the master addresses are known.\nrequest_replacement() {\n now=$(uptime_sec)\n if [ -n \"$REQUESTED_AT\" ] && [ $((now - REQUESTED_AT)) -lt \"$REPLACE_RETRY_SEC\" ]; then\n return\n fi\n if [ -n \"$MASTER_ADDRS\" ] || [ -f \"$INSTALLED_MARKER\" ]; then\n if ! master_serving; then\n if [ -z \"$HOLD_LOGGED_AT\" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then\n log \"$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node\"\n HOLD_LOGGED_AT=$now\n fi\n return\n fi\n fi\n log \"$1: requesting the replacement of this node\"\n if \"$REPLACE_SCRIPT\" \"$1\"; then\n REQUESTED_AT=$now\n log \"replacement requested\"\n else\n log \"the replacement request failed, retrying in $INTERVAL_SEC s\"\n fi\n}\n\nmkdir -p \"$STATE_DIR\"\nREQUESTED_AT=\"\"\nHOLD_LOGGED_AT=\"\"\nFAILED_SEEN_AT=\"\"\nFAIL_SINCE=\"\"\nSATURATED=\"\"\nPREV_TOTAL=\"\"\nPREV_IDLE=\"\"\nlog \"started\"\n\nwhile true; do\n PROBE_URL=http://127.0.0.1:7880/\n INTERVAL_SEC=30\n HARD_UNHEALTHY_AFTER_SEC=300\n SOFT_UNHEALTHY_AFTER_SEC=600\n CPU_SATURATED_PCT=90\n WARMUP_SEC=900\n BOOTSTRAP_DEADLINE_SEC=5400\n BOOTSTRAP_FAILURE_COOLDOWN_SEC=600\n REPLACE_RETRY_SEC=600\n MASTER_ADDRS=\"\"\n [ -f \"$ENV_FILE\" ] && . \"$ENV_FILE\"\n NOW=$(uptime_sec)\n cpu_busy\n\n if [ -f \"$PAUSE_FILE\" ] || [ -f \"$DRAIN_PAUSE_FILE\" ]; then\n FAIL_SINCE=\"\"\n elif [ -f \"$BOOTSTRAP_FAILED_MARKER\" ]; then\n if [ -z \"$FAILED_SEEN_AT\" ]; then\n FAILED_SEEN_AT=$NOW\n log \"bootstrap failed ($(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s\"\n fi\n if [ $((NOW - FAILED_SEEN_AT)) -ge \"$BOOTSTRAP_FAILURE_COOLDOWN_SEC\" ]; then\n request_replacement \"bootstrap failed: $(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")\"\n fi\n elif [ ! -f \"$INSTALLED_MARKER\" ]; then\n if [ \"$NOW\" -ge \"$BOOTSTRAP_DEADLINE_SEC\" ]; then\n request_replacement \"bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot\"\n fi\n elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 \"$PROBE_URL\"; then\n [ -n \"$FAIL_SINCE\" ] && log \"LiveKit healthy again after $((NOW - FAIL_SINCE)) s\"\n FAIL_SINCE=\"\"\n SATURATED=\"\"\n else\n RC=$?\n case \"$RC\" in\n 7|52|56) KIND=hard ;;\n *) KIND=soft ;;\n esac\n if openvidu_starting; then\n FAIL_SINCE=\"\"\n elif [ \"$KIND\" = soft ] && [ \"$BUSY\" -ge \"$CPU_SATURATED_PCT\" ]; then\n [ -n \"$SATURATED\" ] || log \"LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted\"\n SATURATED=1\n FAIL_SINCE=\"\"\n else\n SATURATED=\"\"\n if [ -z \"$FAIL_SINCE\" ]; then\n FAIL_SINCE=$NOW\n log \"LiveKit health check failed (curl exit $RC)\"\n fi\n LIMIT=$SOFT_UNHEALTHY_AFTER_SEC\n [ \"$KIND\" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC\n if [ $((NOW - FAIL_SINCE)) -ge \"$LIMIT\" ]; then\n request_replacement \"LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)\"\n fi\n fi\n fi\n sleep \"$INTERVAL_SEC\"\ndone\n", + "openviduMediaHealthScript": "#!/bin/bash\n# OpenVidu Media Node health watchdog.\n#\n# Asks the cloud to replace this Media Node with a new one, keeping the number of\n# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when:\n# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a\n# cool-down, so a persistent failure does not churn nodes and can be inspected,\n# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot,\n# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC\n# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC\n# while it answers an error or times out. Soft failures are not counted while the CPU\n# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in\n# the WARMUP_SEC after openvidu.service (re)started.\n# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage\n# fails the health check of every Media Node at once, and a new node could not bootstrap\n# either; LiveKit failures are not counted until a master answers again. Once requested,\n# the replacement is requested again every REPLACE_RETRY_SEC until the node goes away.\n#\n# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS=\"ip:port ...\")\n# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)\n# Logs: journalctl -u openvidu-media-health\n#\n# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and\n# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed\n# by an opening brace, nor two opening braces in a row.\n\nSTATE_DIR=/var/lib/openvidu-media-health\nINSTALLED_MARKER=\"$STATE_DIR/installed\"\nBOOTSTRAP_FAILED_MARKER=\"$STATE_DIR/bootstrap-failed\"\nREPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh\nENV_FILE=/etc/openvidu/media-health.env\nPAUSE_FILE=/etc/openvidu/media-health.disabled\nDRAIN_PAUSE_FILE=/run/openvidu-media-health.paused\n\nlog() { echo \"[media-health] $*\"; }\n\n# Seconds since boot: monotonic, unaffected by clock steps\nuptime_sec() { cut -d. -f1 /proc/uptime; }\n\n# Busy CPU percentage since the previous call, in BUSY (0 on the first call)\ncpu_busy() {\n read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat\n total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal))\n idle=$((c_idle + c_iowait))\n BUSY=0\n if [ -n \"$PREV_TOTAL\" ] && [ \"$total\" -gt \"$PREV_TOTAL\" ]; then\n BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL)))\n fi\n PREV_TOTAL=$total\n PREV_IDLE=$idle\n}\n\n# openvidu.service is running and entered the active state less than WARMUP_SEC ago\n# (a stopped or failed openvidu.service is not starting: no warm-up for it)\nopenvidu_starting() {\n case \"$(systemctl show -p ActiveState --value openvidu 2>/dev/null)\" in\n active|activating|reloading) ;;\n *) return 1 ;;\n esac\n since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null)\n case \"$since\" in ''|0|*[!0-9]*) return 1 ;; esac\n [ $(($(uptime_sec) - since / 1000000)) -lt \"$WARMUP_SEC\" ]\n}\n\n# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare\n# TCP connect would also succeed against a wedged Redis.\nmaster_serving() {\n [ -n \"$MASTER_ADDRS\" ] || return 1\n for addr in $MASTER_ADDRS; do\n host=$(echo \"$addr\" | cut -d: -f1)\n port=$(echo \"$addr\" | cut -d: -f2)\n reply=$(timeout 5 bash -c 'exec 3<>\"/dev/tcp/$0/$1\" && printf \"PING\\r\\n\" >&3 && head -c 7 <&3' \"$host\" \"$port\" 2>/dev/null)\n case \"$reply\" in\n +PONG*|-NOAUTH*) return 0 ;;\n esac\n done\n return 1\n}\n\n# Request the replacement of this node ($1 = reason). Gated on a master answering, except\n# in the bootstrap phase before the master addresses are known.\nrequest_replacement() {\n now=$(uptime_sec)\n if [ -n \"$REQUESTED_AT\" ] && [ $((now - REQUESTED_AT)) -lt \"$REPLACE_RETRY_SEC\" ]; then\n return\n fi\n if [ -n \"$MASTER_ADDRS\" ] || [ -f \"$INSTALLED_MARKER\" ]; then\n if ! master_serving; then\n if [ -z \"$HOLD_LOGGED_AT\" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then\n log \"$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node\"\n HOLD_LOGGED_AT=$now\n fi\n return\n fi\n fi\n log \"$1: requesting the replacement of this node\"\n if \"$REPLACE_SCRIPT\" \"$1\"; then\n REQUESTED_AT=$now\n log \"replacement requested\"\n else\n log \"the replacement request failed, retrying in $INTERVAL_SEC s\"\n fi\n}\n\nmkdir -p \"$STATE_DIR\"\nREQUESTED_AT=\"\"\nHOLD_LOGGED_AT=\"\"\nFAILED_SEEN_AT=\"\"\nFAIL_SINCE=\"\"\nSATURATED=\"\"\nPREV_TOTAL=\"\"\nPREV_IDLE=\"\"\nlog \"started\"\n\nwhile true; do\n PROBE_URL=http://127.0.0.1:7880/\n INTERVAL_SEC=30\n HARD_UNHEALTHY_AFTER_SEC=300\n SOFT_UNHEALTHY_AFTER_SEC=600\n CPU_SATURATED_PCT=90\n WARMUP_SEC=900\n BOOTSTRAP_DEADLINE_SEC=5400\n BOOTSTRAP_FAILURE_COOLDOWN_SEC=600\n REPLACE_RETRY_SEC=600\n MASTER_ADDRS=\"\"\n [ -f \"$ENV_FILE\" ] && . \"$ENV_FILE\"\n NOW=$(uptime_sec)\n cpu_busy\n\n if [ -f \"$PAUSE_FILE\" ] || [ -f \"$DRAIN_PAUSE_FILE\" ]; then\n FAIL_SINCE=\"\"\n elif [ -f \"$BOOTSTRAP_FAILED_MARKER\" ]; then\n if [ -z \"$FAILED_SEEN_AT\" ]; then\n FAILED_SEEN_AT=$NOW\n log \"bootstrap failed ($(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s\"\n fi\n if [ $((NOW - FAILED_SEEN_AT)) -ge \"$BOOTSTRAP_FAILURE_COOLDOWN_SEC\" ]; then\n request_replacement \"bootstrap failed: $(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")\"\n fi\n elif [ ! -f \"$INSTALLED_MARKER\" ]; then\n if [ \"$NOW\" -ge \"$BOOTSTRAP_DEADLINE_SEC\" ]; then\n request_replacement \"bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot\"\n fi\n elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 \"$PROBE_URL\"; then\n [ -n \"$FAIL_SINCE\" ] && log \"LiveKit healthy again after $((NOW - FAIL_SINCE)) s\"\n FAIL_SINCE=\"\"\n SATURATED=\"\"\n else\n RC=$?\n case \"$RC\" in\n 7|52|56) KIND=hard ;;\n *) KIND=soft ;;\n esac\n if openvidu_starting; then\n FAIL_SINCE=\"\"\n elif ! master_serving; then\n # A master outage fails the health check of every Media Node at once: not counted,\n # so LiveKit gets the whole window to reconnect once a master answers again\n FAIL_SINCE=\"\"\n if [ -z \"$HOLD_LOGGED_AT\" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then\n log \"LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted\"\n HOLD_LOGGED_AT=$NOW\n fi\n elif [ \"$KIND\" = soft ] && [ \"$BUSY\" -ge \"$CPU_SATURATED_PCT\" ]; then\n [ -n \"$SATURATED\" ] || log \"LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted\"\n SATURATED=1\n FAIL_SINCE=\"\"\n else\n SATURATED=\"\"\n if [ -z \"$FAIL_SINCE\" ]; then\n FAIL_SINCE=$NOW\n log \"LiveKit health check failed (curl exit $RC)\"\n fi\n LIMIT=$SOFT_UNHEALTHY_AFTER_SEC\n [ \"$KIND\" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC\n if [ $((NOW - FAIL_SINCE)) -ge \"$LIMIT\" ]; then\n request_replacement \"LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)\"\n fi\n fi\n fi\n sleep \"$INTERVAL_SEC\"\ndone\n", "openviduMediaHealthUnit": "[Unit]\nDescription=OpenVidu Media Node health watchdog\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/local/bin/openvidu-media-health.sh\nRestart=always\nRestartSec=30\n\n[Install]\nWantedBy=multi-user.target\n", "media_replaceScriptMediaTemplate": "#!/bin/bash\n# openvidu-media-replace.sh \"\"\n# Requests the reimage of THIS Media Node, keeping the scale set capacity and the\n# scale-in protection: reimage provisions a new OS disk and the userData bootstrap runs\n# again on the same instance id. Called by the health watchdog; it never signals it.\n\nmkdir -p /var/lib/openvidu-media-health\necho \"$(date -u +%FT%TZ) $1\" >> /var/lib/openvidu-media-health/replace-requests.log\ntouch /run/openvidu-unhealthy\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nINSTANCE_RESOURCE_ID=$(curl -s -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo \"$INSTANCE_RESOURCE_ID\" | awk -F'/' '{print $NF}')\nif [ -z \"$INSTANCE_ID\" ]; then\n echo \"[media-replace] could not read this instance id from IMDS\"\n exit 1\nfi\n\nn=0\nwhile true; do\n n=$((n + 1))\n if az login --identity --client-id ${mediaIdentityClientId} >/dev/null 2>&1 && \\\n az vmss reimage --resource-group \"$RESOURCE_GROUP_NAME\" --name \"$VM_SCALE_SET_NAME\" --instance-ids \"$INSTANCE_ID\" --no-wait; then\n echo \"[media-replace] reimage requested for instance $INSTANCE_ID ($1)\"\n exit 0\n fi\n if [ \"$n\" -ge 5 ]; then\n echo \"[media-replace] reimage could not be requested after $n attempts\"\n exit 1\n fi\n echo \"[media-replace] reimage request failed (attempt $n), retrying in 12s\"\n sleep 12\ndone\n", "userDataMediaNodeTemplate": "#!/bin/bash -x\nset -eu -o pipefail\n\n# Mark this media node's bootstrap as failed so the health watchdog replaces it\nov_media_bootstrap_failed() {\n echo \"[OpenVidu] media node bootstrap failed: $1\"\n mkdir -p /var/lib/openvidu-media-health\n echo \"$1\" > /var/lib/openvidu-media-health/bootstrap-failed\n exit 1\n}\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# stop_media_nodes.sh\necho ${base64stop} | base64 -d > /usr/local/bin/stop_media_node.sh\nchmod +x /usr/local/bin/stop_media_node.sh\n\n# openvidu-media-health.sh (health watchdog), openvidu-media-replace.sh (reimage this node) and its unit\necho ${base64mediaHealth} | base64 -d > /usr/local/bin/openvidu-media-health.sh\nchmod +x /usr/local/bin/openvidu-media-health.sh\necho ${base64mediaReplace} | base64 -d > /usr/local/bin/openvidu-media-replace.sh\nchmod +x /usr/local/bin/openvidu-media-replace.sh\necho ${base64mediaHealthUnit} | base64 -d > /etc/systemd/system/openvidu-media-health.service\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y jq\n\n# Install azure cli\nAZURE_CLI_VERSION=2.90.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity --client-id ${mediaIdentityClientId}\n\n# Protect from scale in actions\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\naz vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-id $INSTANCE_ID --protect-from-scale-in true\n\nexport HOME=\"/root\"\n\n# Start the Media Node health watchdog before install.sh (az CLI is installed and the node\n# is scale-in protected): it backstops a hung or failed bootstrap and, once running, reimages\n# the node if LiveKit stays unhealthy. Write MASTER_ADDRS first so its master guard is armed.\nmkdir -p /etc/openvidu\necho 'MASTER_ADDRS=\"${privateIPMasterNode1}:7001 ${privateIPMasterNode2}:7001 ${privateIPMasterNode3}:7001 ${privateIPMasterNode4}:7001\"' > /etc/openvidu/media-health.env\nsystemctl daemon-reload\nsystemctl enable --now openvidu-media-health\n\n# Install OpenVidu\n/usr/local/bin/install.sh || ov_media_bootstrap_failed \"install.sh failed\"\n\n# Gate 2: wait for master readiness before starting\nWAIT_INTERVAL=5\nMAX_RETRIES=360\nRETRIES=0\nset +e\nwhile true; do\n FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv 2>/dev/null)\n if [ \"$FINISH_MASTER_NODE\" == \"true\" ]; then\n break\n fi\n RETRIES=$((RETRIES + 1))\n if [ $RETRIES -ge $MAX_RETRIES ]; then\n ov_media_bootstrap_failed \"timed out after 30 min waiting for FINISH-MASTER-NODE\"\n fi\n sleep $WAIT_INTERVAL\ndone\nset -e\n\n# Start OpenVidu\nsystemctl start openvidu || ov_media_bootstrap_failed \"systemctl start openvidu failed\"\n\n# Bootstrap finished: the watchdog switches from bootstrap-deadline mode to runtime health checks\nmkdir -p /var/lib/openvidu-media-health\ntouch /var/lib/openvidu-media-health/installed\n", diff --git a/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf b/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf index 11d7ee528..64f4f960c 100644 --- a/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf +++ b/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf @@ -1813,8 +1813,8 @@ EOF # the WARMUP_SEC after openvidu.service (re)started. # Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage # fails the health check of every Media Node at once, and a new node could not bootstrap -# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until -# the node goes away. +# either; LiveKit failures are not counted until a master answers again. Once requested, +# the replacement is requested again every REPLACE_RETRY_SEC until the node goes away. # # Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...") # Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused) @@ -1953,6 +1953,14 @@ while true; do esac if openvidu_starting; then FAIL_SINCE="" + elif ! master_serving; then + # A master outage fails the health check of every Media Node at once: not counted, + # so LiveKit gets the whole window to reconnect once a master answers again + FAIL_SINCE="" + if [ -z "$HOLD_LOGGED_AT" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then + log "LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted" + HOLD_LOGGED_AT=$NOW + fi elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted" SATURATED=1