diff --git a/openvidu-deployment/pro/elastic/aws/cf-openvidu-elastic.yaml b/openvidu-deployment/pro/elastic/aws/cf-openvidu-elastic.yaml index ac48b124c..7cb959deb 100644 --- a/openvidu-deployment/pro/elastic/aws/cf-openvidu-elastic.yaml +++ b/openvidu-deployment/pro/elastic/aws/cf-openvidu-elastic.yaml @@ -2173,8 +2173,8 @@ Resources: # the WARMUP_SEC after openvidu.service (re)started. # Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage # fails the health check of every Media Node at once, and a new node could not bootstrap - # either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until - # the node goes away. + # either; LiveKit failures are not counted until a master answers again. Once requested, + # the replacement is requested again every REPLACE_RETRY_SEC until the node goes away. # # Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...") # Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused) @@ -2313,6 +2313,14 @@ Resources: esac if openvidu_starting; then FAIL_SINCE="" + elif ! master_serving; then + # A master outage fails the health check of every Media Node at once: not counted, + # so LiveKit gets the whole window to reconnect once a master answers again + FAIL_SINCE="" + if [ -z "$HOLD_LOGGED_AT" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then + log "LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted" + HOLD_LOGGED_AT=$NOW + fi elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted" SATURATED=1 diff --git a/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.bicep b/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.bicep index f1342edfc..51854cfb0 100644 --- a/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.bicep +++ b/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.bicep @@ -1068,8 +1068,8 @@ var openviduMediaHealthScript = ''' # the WARMUP_SEC after openvidu.service (re)started. # Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage # fails the health check of every Media Node at once, and a new node could not bootstrap -# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until -# the node goes away. +# either; LiveKit failures are not counted until a master answers again. Once requested, +# the replacement is requested again every REPLACE_RETRY_SEC until the node goes away. # # Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...") # Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused) @@ -1208,6 +1208,14 @@ while true; do esac if openvidu_starting; then FAIL_SINCE="" + elif ! master_serving; then + # A master outage fails the health check of every Media Node at once: not counted, + # so LiveKit gets the whole window to reconnect once a master answers again + FAIL_SINCE="" + if [ -z "$HOLD_LOGGED_AT" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then + log "LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted" + HOLD_LOGGED_AT=$NOW + fi elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted" SATURATED=1 diff --git a/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.json b/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.json index 76dbf7b74..a7a0e09c2 100644 --- a/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.json +++ b/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.46.1.21595", - "templateHash": "1384799452684335174" + "templateHash": "3671752928188054764" } }, "parameters": { @@ -279,7 +279,7 @@ "storageAccountName": "[if(variables('isEmptyStorageAccountName'), uniqueString(resourceGroup().id), parameters('storageAccountName'))]" }, "stop_media_nodesScriptMediaTemplate": "#!/bin/bash\nset -e\n\n# Pause the health watchdog before draining so it never reimages a node that is being\n# drained for scale-in (synchronous; the drain runs outside the watchdog's cgroup).\ntouch /run/openvidu-media-health.paused\nsystemctl stop openvidu-media-health 2>/dev/null || true\n\nif ! (set -o noclobber ; echo > /tmp/global.lock) ; then\n exit 1 # the global.lock already exists\nfi\n\n# Execute if docker is installed\nif [ -x \"$(command -v docker)\" ]; then\n\n echo \"Stopping media node services and waiting for termination...\"\n docker container kill --signal=SIGQUIT openvidu 2>/dev/null || true\n docker container kill --signal=SIGQUIT ingress 2>/dev/null || true\n docker container kill --signal=SIGQUIT egress 2>/dev/null || true\n for agent_container in $(docker ps --filter \"label=openvidu-agent=true\" --format '{{.Names}}' 2>/dev/null); do\n docker container kill --signal=SIGQUIT \"$agent_container\" 2>/dev/null || true\n done\n\n # If this node is being replaced because it is unhealthy, do not wait for sessions to end.\n if [ -f /run/openvidu-unhealthy ]; then\n docker ps -q | xargs -r docker kill >/dev/null 2>&1 || true\n fi\n\n # Wait for running containers to not be openvidu, ingress, egress or an openvidu agent\n while [ \"$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)\" -gt 0 ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)\" = \"true\" ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' ingress 2>/dev/null)\" = \"true\" ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' egress 2>/dev/null)\" = \"true\" ]; do\n echo \"Waiting for containers to stop...\"\n sleep 5\n done\nfi\n\naz login --identity\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nSUBSCRIPTION_ID=${subscriptionId}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\nRESOURCE_ID=/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP_NAME/providers/Microsoft.Compute/virtualMachineScaleSets/$VM_SCALE_SET_NAME\nTIMESTAMP=$(date -u +\"%Y-%m-%dT%H:%M:%SZ\")\n\naz tag update --resource-id $RESOURCE_ID --operation replace --tags \"STATUS\"=\"HEALTHY\" \"InstanceDeleteTime\"=\"$TIMESTAMP\" \"storageAccount\"=\"${storageAccountName}\"\n\naz vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID\n", - "openviduMediaHealthScript": "#!/bin/bash\n# OpenVidu Media Node health watchdog.\n#\n# Asks the cloud to replace this Media Node with a new one, keeping the number of\n# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when:\n# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a\n# cool-down, so a persistent failure does not churn nodes and can be inspected,\n# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot,\n# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC\n# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC\n# while it answers an error or times out. Soft failures are not counted while the CPU\n# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in\n# the WARMUP_SEC after openvidu.service (re)started.\n# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage\n# fails the health check of every Media Node at once, and a new node could not bootstrap\n# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until\n# the node goes away.\n#\n# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS=\"ip:port ...\")\n# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)\n# Logs: journalctl -u openvidu-media-health\n#\n# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and\n# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed\n# by an opening brace, nor two opening braces in a row.\n\nSTATE_DIR=/var/lib/openvidu-media-health\nINSTALLED_MARKER=\"$STATE_DIR/installed\"\nBOOTSTRAP_FAILED_MARKER=\"$STATE_DIR/bootstrap-failed\"\nREPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh\nENV_FILE=/etc/openvidu/media-health.env\nPAUSE_FILE=/etc/openvidu/media-health.disabled\nDRAIN_PAUSE_FILE=/run/openvidu-media-health.paused\n\nlog() { echo \"[media-health] $*\"; }\n\n# Seconds since boot: monotonic, unaffected by clock steps\nuptime_sec() { cut -d. -f1 /proc/uptime; }\n\n# Busy CPU percentage since the previous call, in BUSY (0 on the first call)\ncpu_busy() {\n read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat\n total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal))\n idle=$((c_idle + c_iowait))\n BUSY=0\n if [ -n \"$PREV_TOTAL\" ] && [ \"$total\" -gt \"$PREV_TOTAL\" ]; then\n BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL)))\n fi\n PREV_TOTAL=$total\n PREV_IDLE=$idle\n}\n\n# openvidu.service is running and entered the active state less than WARMUP_SEC ago\n# (a stopped or failed openvidu.service is not starting: no warm-up for it)\nopenvidu_starting() {\n case \"$(systemctl show -p ActiveState --value openvidu 2>/dev/null)\" in\n active|activating|reloading) ;;\n *) return 1 ;;\n esac\n since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null)\n case \"$since\" in ''|0|*[!0-9]*) return 1 ;; esac\n [ $(($(uptime_sec) - since / 1000000)) -lt \"$WARMUP_SEC\" ]\n}\n\n# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare\n# TCP connect would also succeed against a wedged Redis.\nmaster_serving() {\n [ -n \"$MASTER_ADDRS\" ] || return 1\n for addr in $MASTER_ADDRS; do\n host=$(echo \"$addr\" | cut -d: -f1)\n port=$(echo \"$addr\" | cut -d: -f2)\n reply=$(timeout 5 bash -c 'exec 3<>\"/dev/tcp/$0/$1\" && printf \"PING\\r\\n\" >&3 && head -c 7 <&3' \"$host\" \"$port\" 2>/dev/null)\n case \"$reply\" in\n +PONG*|-NOAUTH*) return 0 ;;\n esac\n done\n return 1\n}\n\n# Request the replacement of this node ($1 = reason). Gated on a master answering, except\n# in the bootstrap phase before the master addresses are known.\nrequest_replacement() {\n now=$(uptime_sec)\n if [ -n \"$REQUESTED_AT\" ] && [ $((now - REQUESTED_AT)) -lt \"$REPLACE_RETRY_SEC\" ]; then\n return\n fi\n if [ -n \"$MASTER_ADDRS\" ] || [ -f \"$INSTALLED_MARKER\" ]; then\n if ! master_serving; then\n if [ -z \"$HOLD_LOGGED_AT\" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then\n log \"$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node\"\n HOLD_LOGGED_AT=$now\n fi\n return\n fi\n fi\n log \"$1: requesting the replacement of this node\"\n if \"$REPLACE_SCRIPT\" \"$1\"; then\n REQUESTED_AT=$now\n log \"replacement requested\"\n else\n log \"the replacement request failed, retrying in $INTERVAL_SEC s\"\n fi\n}\n\nmkdir -p \"$STATE_DIR\"\nREQUESTED_AT=\"\"\nHOLD_LOGGED_AT=\"\"\nFAILED_SEEN_AT=\"\"\nFAIL_SINCE=\"\"\nSATURATED=\"\"\nPREV_TOTAL=\"\"\nPREV_IDLE=\"\"\nlog \"started\"\n\nwhile true; do\n PROBE_URL=http://127.0.0.1:7880/\n INTERVAL_SEC=30\n HARD_UNHEALTHY_AFTER_SEC=300\n SOFT_UNHEALTHY_AFTER_SEC=600\n CPU_SATURATED_PCT=90\n WARMUP_SEC=900\n BOOTSTRAP_DEADLINE_SEC=5400\n BOOTSTRAP_FAILURE_COOLDOWN_SEC=600\n REPLACE_RETRY_SEC=600\n MASTER_ADDRS=\"\"\n [ -f \"$ENV_FILE\" ] && . \"$ENV_FILE\"\n NOW=$(uptime_sec)\n cpu_busy\n\n if [ -f \"$PAUSE_FILE\" ] || [ -f \"$DRAIN_PAUSE_FILE\" ]; then\n FAIL_SINCE=\"\"\n elif [ -f \"$BOOTSTRAP_FAILED_MARKER\" ]; then\n if [ -z \"$FAILED_SEEN_AT\" ]; then\n FAILED_SEEN_AT=$NOW\n log \"bootstrap failed ($(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s\"\n fi\n if [ $((NOW - FAILED_SEEN_AT)) -ge \"$BOOTSTRAP_FAILURE_COOLDOWN_SEC\" ]; then\n request_replacement \"bootstrap failed: $(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")\"\n fi\n elif [ ! -f \"$INSTALLED_MARKER\" ]; then\n if [ \"$NOW\" -ge \"$BOOTSTRAP_DEADLINE_SEC\" ]; then\n request_replacement \"bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot\"\n fi\n elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 \"$PROBE_URL\"; then\n [ -n \"$FAIL_SINCE\" ] && log \"LiveKit healthy again after $((NOW - FAIL_SINCE)) s\"\n FAIL_SINCE=\"\"\n SATURATED=\"\"\n else\n RC=$?\n case \"$RC\" in\n 7|52|56) KIND=hard ;;\n *) KIND=soft ;;\n esac\n if openvidu_starting; then\n FAIL_SINCE=\"\"\n elif [ \"$KIND\" = soft ] && [ \"$BUSY\" -ge \"$CPU_SATURATED_PCT\" ]; then\n [ -n \"$SATURATED\" ] || log \"LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted\"\n SATURATED=1\n FAIL_SINCE=\"\"\n else\n SATURATED=\"\"\n if [ -z \"$FAIL_SINCE\" ]; then\n FAIL_SINCE=$NOW\n log \"LiveKit health check failed (curl exit $RC)\"\n fi\n LIMIT=$SOFT_UNHEALTHY_AFTER_SEC\n [ \"$KIND\" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC\n if [ $((NOW - FAIL_SINCE)) -ge \"$LIMIT\" ]; then\n request_replacement \"LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)\"\n fi\n fi\n fi\n sleep \"$INTERVAL_SEC\"\ndone\n", + "openviduMediaHealthScript": "#!/bin/bash\n# OpenVidu Media Node health watchdog.\n#\n# Asks the cloud to replace this Media Node with a new one, keeping the number of\n# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when:\n# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a\n# cool-down, so a persistent failure does not churn nodes and can be inspected,\n# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot,\n# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC\n# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC\n# while it answers an error or times out. Soft failures are not counted while the CPU\n# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in\n# the WARMUP_SEC after openvidu.service (re)started.\n# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage\n# fails the health check of every Media Node at once, and a new node could not bootstrap\n# either; LiveKit failures are not counted until a master answers again. Once requested,\n# the replacement is requested again every REPLACE_RETRY_SEC until the node goes away.\n#\n# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS=\"ip:port ...\")\n# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)\n# Logs: journalctl -u openvidu-media-health\n#\n# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and\n# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed\n# by an opening brace, nor two opening braces in a row.\n\nSTATE_DIR=/var/lib/openvidu-media-health\nINSTALLED_MARKER=\"$STATE_DIR/installed\"\nBOOTSTRAP_FAILED_MARKER=\"$STATE_DIR/bootstrap-failed\"\nREPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh\nENV_FILE=/etc/openvidu/media-health.env\nPAUSE_FILE=/etc/openvidu/media-health.disabled\nDRAIN_PAUSE_FILE=/run/openvidu-media-health.paused\n\nlog() { echo \"[media-health] $*\"; }\n\n# Seconds since boot: monotonic, unaffected by clock steps\nuptime_sec() { cut -d. -f1 /proc/uptime; }\n\n# Busy CPU percentage since the previous call, in BUSY (0 on the first call)\ncpu_busy() {\n read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat\n total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal))\n idle=$((c_idle + c_iowait))\n BUSY=0\n if [ -n \"$PREV_TOTAL\" ] && [ \"$total\" -gt \"$PREV_TOTAL\" ]; then\n BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL)))\n fi\n PREV_TOTAL=$total\n PREV_IDLE=$idle\n}\n\n# openvidu.service is running and entered the active state less than WARMUP_SEC ago\n# (a stopped or failed openvidu.service is not starting: no warm-up for it)\nopenvidu_starting() {\n case \"$(systemctl show -p ActiveState --value openvidu 2>/dev/null)\" in\n active|activating|reloading) ;;\n *) return 1 ;;\n esac\n since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null)\n case \"$since\" in ''|0|*[!0-9]*) return 1 ;; esac\n [ $(($(uptime_sec) - since / 1000000)) -lt \"$WARMUP_SEC\" ]\n}\n\n# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare\n# TCP connect would also succeed against a wedged Redis.\nmaster_serving() {\n [ -n \"$MASTER_ADDRS\" ] || return 1\n for addr in $MASTER_ADDRS; do\n host=$(echo \"$addr\" | cut -d: -f1)\n port=$(echo \"$addr\" | cut -d: -f2)\n reply=$(timeout 5 bash -c 'exec 3<>\"/dev/tcp/$0/$1\" && printf \"PING\\r\\n\" >&3 && head -c 7 <&3' \"$host\" \"$port\" 2>/dev/null)\n case \"$reply\" in\n +PONG*|-NOAUTH*) return 0 ;;\n esac\n done\n return 1\n}\n\n# Request the replacement of this node ($1 = reason). Gated on a master answering, except\n# in the bootstrap phase before the master addresses are known.\nrequest_replacement() {\n now=$(uptime_sec)\n if [ -n \"$REQUESTED_AT\" ] && [ $((now - REQUESTED_AT)) -lt \"$REPLACE_RETRY_SEC\" ]; then\n return\n fi\n if [ -n \"$MASTER_ADDRS\" ] || [ -f \"$INSTALLED_MARKER\" ]; then\n if ! master_serving; then\n if [ -z \"$HOLD_LOGGED_AT\" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then\n log \"$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node\"\n HOLD_LOGGED_AT=$now\n fi\n return\n fi\n fi\n log \"$1: requesting the replacement of this node\"\n if \"$REPLACE_SCRIPT\" \"$1\"; then\n REQUESTED_AT=$now\n log \"replacement requested\"\n else\n log \"the replacement request failed, retrying in $INTERVAL_SEC s\"\n fi\n}\n\nmkdir -p \"$STATE_DIR\"\nREQUESTED_AT=\"\"\nHOLD_LOGGED_AT=\"\"\nFAILED_SEEN_AT=\"\"\nFAIL_SINCE=\"\"\nSATURATED=\"\"\nPREV_TOTAL=\"\"\nPREV_IDLE=\"\"\nlog \"started\"\n\nwhile true; do\n PROBE_URL=http://127.0.0.1:7880/\n INTERVAL_SEC=30\n HARD_UNHEALTHY_AFTER_SEC=300\n SOFT_UNHEALTHY_AFTER_SEC=600\n CPU_SATURATED_PCT=90\n WARMUP_SEC=900\n BOOTSTRAP_DEADLINE_SEC=5400\n BOOTSTRAP_FAILURE_COOLDOWN_SEC=600\n REPLACE_RETRY_SEC=600\n MASTER_ADDRS=\"\"\n [ -f \"$ENV_FILE\" ] && . \"$ENV_FILE\"\n NOW=$(uptime_sec)\n cpu_busy\n\n if [ -f \"$PAUSE_FILE\" ] || [ -f \"$DRAIN_PAUSE_FILE\" ]; then\n FAIL_SINCE=\"\"\n elif [ -f \"$BOOTSTRAP_FAILED_MARKER\" ]; then\n if [ -z \"$FAILED_SEEN_AT\" ]; then\n FAILED_SEEN_AT=$NOW\n log \"bootstrap failed ($(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s\"\n fi\n if [ $((NOW - FAILED_SEEN_AT)) -ge \"$BOOTSTRAP_FAILURE_COOLDOWN_SEC\" ]; then\n request_replacement \"bootstrap failed: $(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")\"\n fi\n elif [ ! -f \"$INSTALLED_MARKER\" ]; then\n if [ \"$NOW\" -ge \"$BOOTSTRAP_DEADLINE_SEC\" ]; then\n request_replacement \"bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot\"\n fi\n elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 \"$PROBE_URL\"; then\n [ -n \"$FAIL_SINCE\" ] && log \"LiveKit healthy again after $((NOW - FAIL_SINCE)) s\"\n FAIL_SINCE=\"\"\n SATURATED=\"\"\n else\n RC=$?\n case \"$RC\" in\n 7|52|56) KIND=hard ;;\n *) KIND=soft ;;\n esac\n if openvidu_starting; then\n FAIL_SINCE=\"\"\n elif ! master_serving; then\n # A master outage fails the health check of every Media Node at once: not counted,\n # so LiveKit gets the whole window to reconnect once a master answers again\n FAIL_SINCE=\"\"\n if [ -z \"$HOLD_LOGGED_AT\" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then\n log \"LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted\"\n HOLD_LOGGED_AT=$NOW\n fi\n elif [ \"$KIND\" = soft ] && [ \"$BUSY\" -ge \"$CPU_SATURATED_PCT\" ]; then\n [ -n \"$SATURATED\" ] || log \"LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted\"\n SATURATED=1\n FAIL_SINCE=\"\"\n else\n SATURATED=\"\"\n if [ -z \"$FAIL_SINCE\" ]; then\n FAIL_SINCE=$NOW\n log \"LiveKit health check failed (curl exit $RC)\"\n fi\n LIMIT=$SOFT_UNHEALTHY_AFTER_SEC\n [ \"$KIND\" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC\n if [ $((NOW - FAIL_SINCE)) -ge \"$LIMIT\" ]; then\n request_replacement \"LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)\"\n fi\n fi\n fi\n sleep \"$INTERVAL_SEC\"\ndone\n", "openviduMediaHealthUnit": "[Unit]\nDescription=OpenVidu Media Node health watchdog\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/local/bin/openvidu-media-health.sh\nRestart=always\nRestartSec=30\n\n[Install]\nWantedBy=multi-user.target\n", "media_replaceScriptMediaTemplate": "#!/bin/bash\n# openvidu-media-replace.sh \"\"\n# Requests the reimage of THIS Media Node, keeping the scale set capacity and the\n# scale-in protection: reimage provisions a new OS disk and the userData bootstrap runs\n# again on the same instance id. Called by the health watchdog; it never signals it.\n\nmkdir -p /var/lib/openvidu-media-health\necho \"$(date -u +%FT%TZ) $1\" >> /var/lib/openvidu-media-health/replace-requests.log\ntouch /run/openvidu-unhealthy\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nINSTANCE_RESOURCE_ID=$(curl -s -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo \"$INSTANCE_RESOURCE_ID\" | awk -F'/' '{print $NF}')\nif [ -z \"$INSTANCE_ID\" ]; then\n echo \"[media-replace] could not read this instance id from IMDS\"\n exit 1\nfi\n\nn=0\nwhile true; do\n n=$((n + 1))\n if az login --identity >/dev/null 2>&1 && \\\n az vmss reimage --resource-group \"$RESOURCE_GROUP_NAME\" --name \"$VM_SCALE_SET_NAME\" --instance-ids \"$INSTANCE_ID\" --no-wait; then\n echo \"[media-replace] reimage requested for instance $INSTANCE_ID ($1)\"\n exit 0\n fi\n if [ \"$n\" -ge 5 ]; then\n echo \"[media-replace] reimage could not be requested after $n attempts\"\n exit 1\n fi\n echo \"[media-replace] reimage request failed (attempt $n), retrying in 12s\"\n sleep 12\ndone\n", "userDataMediaNodeTemplate": "#!/bin/bash\nset -eu -o pipefail\n\n# Mark this media node's bootstrap as failed so the health watchdog replaces it\nov_media_bootstrap_failed() {\n echo \"[OpenVidu] media node bootstrap failed: $1\"\n mkdir -p /var/lib/openvidu-media-health\n echo \"$1\" > /var/lib/openvidu-media-health/bootstrap-failed\n exit 1\n}\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# stop_media_nodes.sh\necho ${base64stop} | base64 -d > /usr/local/bin/stop_media_node.sh\nchmod +x /usr/local/bin/stop_media_node.sh\n\n# openvidu-media-health.sh (health watchdog), openvidu-media-replace.sh (reimage this node) and its unit\necho ${base64mediaHealth} | base64 -d > /usr/local/bin/openvidu-media-health.sh\nchmod +x /usr/local/bin/openvidu-media-health.sh\necho ${base64mediaReplace} | base64 -d > /usr/local/bin/openvidu-media-replace.sh\nchmod +x /usr/local/bin/openvidu-media-replace.sh\necho ${base64mediaHealthUnit} | base64 -d > /etc/systemd/system/openvidu-media-health.service\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y jq\n\n# Install azure cli\nAZURE_CLI_VERSION=2.90.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity\n\n# Protect from scale in actions\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\naz vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-id $INSTANCE_ID --protect-from-scale-in true\n\nexport HOME=\"/root\"\n\n# Start the Media Node health watchdog before install.sh (az CLI is installed and the node\n# is scale-in protected): it backstops a hung or failed bootstrap and, once running, reimages\n# the node if LiveKit stays unhealthy. Write MASTER_ADDRS first so its master guard is armed.\nmkdir -p /etc/openvidu\necho 'MASTER_ADDRS=\"${privateIPMasterNode}:7000\"' > /etc/openvidu/media-health.env\nsystemctl daemon-reload\nsystemctl enable --now openvidu-media-health\n\n# Install OpenVidu\n/usr/local/bin/install.sh || ov_media_bootstrap_failed \"install.sh failed\"\n\n# Gate 2: wait for master readiness before starting\nWAIT_INTERVAL=5\nMAX_RETRIES=360\nRETRIES=0\nset +e\nwhile true; do\n FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv 2>/dev/null)\n if [ \"$FINISH_MASTER_NODE\" == \"true\" ]; then\n break\n fi\n RETRIES=$((RETRIES + 1))\n if [ $RETRIES -ge $MAX_RETRIES ]; then\n ov_media_bootstrap_failed \"timed out after 30 min waiting for FINISH-MASTER-NODE\"\n fi\n sleep $WAIT_INTERVAL\ndone\nset -e\n\n# Start OpenVidu\nsystemctl start openvidu || ov_media_bootstrap_failed \"systemctl start openvidu failed\"\n\n# Bootstrap finished: the watchdog switches from bootstrap-deadline mode to runtime health checks\nmkdir -p /var/lib/openvidu-media-health\ntouch /var/lib/openvidu-media-health/installed\n", diff --git a/openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf b/openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf index 3b566402d..73c6523d2 100644 --- a/openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf +++ b/openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf @@ -1429,8 +1429,8 @@ EOF # the WARMUP_SEC after openvidu.service (re)started. # Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage # fails the health check of every Media Node at once, and a new node could not bootstrap -# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until -# the node goes away. +# either; LiveKit failures are not counted until a master answers again. Once requested, +# the replacement is requested again every REPLACE_RETRY_SEC until the node goes away. # # Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...") # Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused) @@ -1569,6 +1569,14 @@ while true; do esac if openvidu_starting; then FAIL_SINCE="" + elif ! master_serving; then + # A master outage fails the health check of every Media Node at once: not counted, + # so LiveKit gets the whole window to reconnect once a master answers again + FAIL_SINCE="" + if [ -z "$HOLD_LOGGED_AT" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then + log "LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted" + HOLD_LOGGED_AT=$NOW + fi elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted" SATURATED=1 diff --git a/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml b/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml index fdfd0a2ef..8c94f6b8b 100644 --- a/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml +++ b/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml @@ -2816,8 +2816,8 @@ Resources: # the WARMUP_SEC after openvidu.service (re)started. # Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage # fails the health check of every Media Node at once, and a new node could not bootstrap - # either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until - # the node goes away. + # either; LiveKit failures are not counted until a master answers again. Once requested, + # the replacement is requested again every REPLACE_RETRY_SEC until the node goes away. # # Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...") # Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused) @@ -2956,6 +2956,14 @@ Resources: esac if openvidu_starting; then FAIL_SINCE="" + elif ! master_serving; then + # A master outage fails the health check of every Media Node at once: not counted, + # so LiveKit gets the whole window to reconnect once a master answers again + FAIL_SINCE="" + if [ -z "$HOLD_LOGGED_AT" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then + log "LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted" + HOLD_LOGGED_AT=$NOW + fi elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted" SATURATED=1 diff --git a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep index ec3b25fe4..e061ad34e 100644 --- a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep +++ b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep @@ -1461,8 +1461,8 @@ var openviduMediaHealthScript = ''' # the WARMUP_SEC after openvidu.service (re)started. # Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage # fails the health check of every Media Node at once, and a new node could not bootstrap -# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until -# the node goes away. +# either; LiveKit failures are not counted until a master answers again. Once requested, +# the replacement is requested again every REPLACE_RETRY_SEC until the node goes away. # # Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...") # Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused) @@ -1601,6 +1601,14 @@ while true; do esac if openvidu_starting; then FAIL_SINCE="" + elif ! master_serving; then + # A master outage fails the health check of every Media Node at once: not counted, + # so LiveKit gets the whole window to reconnect once a master answers again + FAIL_SINCE="" + if [ -z "$HOLD_LOGGED_AT" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then + log "LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted" + HOLD_LOGGED_AT=$NOW + fi elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted" SATURATED=1 diff --git a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json index b1591abba..7bc32a6da 100644 --- a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json +++ b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.46.1.21595", - "templateHash": "9885917927692102096" + "templateHash": "11373208048589082219" } }, "parameters": { @@ -249,7 +249,7 @@ "userDataTemplateMasterNode": "#!/bin/bash -x\nset -eu -o pipefail\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# after_install.sh\necho ${base64after_install} | base64 -d > /usr/local/bin/after_install.sh\nchmod +x /usr/local/bin/after_install.sh\n\n# get_public_ip.sh\necho ${base64get_public_ip} | base64 -d > /usr/local/bin/get_public_ip.sh\nchmod +x /usr/local/bin/get_public_ip.sh\n\n# update_config_from_secret.sh\necho ${base64update_config_from_secret} | base64 -d > /usr/local/bin/update_config_from_secret.sh\nchmod +x /usr/local/bin/update_config_from_secret.sh\n\n# update_secret_from_config.sh\necho ${base64update_secret_from_config} | base64 -d > /usr/local/bin/update_secret_from_config.sh\nchmod +x /usr/local/bin/update_secret_from_config.sh\n\n# get_value_from_config.sh\necho ${base64get_value_from_config} | base64 -d > /usr/local/bin/get_value_from_config.sh\nchmod +x /usr/local/bin/get_value_from_config.sh\n\n# store_secret.sh\necho ${base64store_secret} | base64 -d > /usr/local/bin/store_secret.sh\nchmod +x /usr/local/bin/store_secret.sh\n\n# check_app_ready.sh\necho ${base64check_app_ready} | base64 -d > /usr/local/bin/check_app_ready.sh\nchmod +x /usr/local/bin/check_app_ready.sh\n\n# restart.sh\necho ${base64restart} | base64 -d > /usr/local/bin/restart.sh\nchmod +x /usr/local/bin/restart.sh\n\necho ${base64config_blobStorage} | base64 -d > /usr/local/bin/config_blobStorage.sh\nchmod +x /usr/local/bin/config_blobStorage.sh\n\n# Install azure cli\nAZURE_CLI_VERSION=2.90.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity --client-id ${masterIdentityClientId} --allow-no-subscriptions\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\nexport HOME=\"/root\"\n\n# Install OpenVidu\n/usr/local/bin/install.sh || { echo \"[OpenVidu] error installing OpenVidu\"; exit 1; }\n\n#Config blob storage\n/usr/local/bin/config_blobStorage.sh || { echo \"[OpenVidu] error configuring Blob Storage\"; exit 1; }\n\n# Start OpenVidu\nsystemctl start openvidu || { echo \"[OpenVidu] error starting OpenVidu\"; exit 1; }\n\n# Update shared secret\n/usr/local/bin/after_install.sh || { echo \"[OpenVidu] error updating shared secret\"; exit 1; }\n\n# Launch on reboot\necho \"@reboot /usr/local/bin/restart.sh >> /var/log/openvidu-restart.log\" 2>&1 | crontab\n\n# check_app_ready.sh internally caps its wait at 1200s\n/usr/local/bin/check_app_ready.sh || { echo \"[OpenVidu] master node did not become healthy\"; exit 1; }\n\naz keyvault secret set --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --value \"true\"\n\nMASTER_NODE_NUM=${masterNodeNum}\nif [[ $MASTER_NODE_NUM -eq 4 ]]; then\n # Creating scale in lock\n set +e\n az storage blob upload --account-name ${storageAccountName} --container-name automation-locks --name lock.txt --file /dev/null --auth-mode key\n set -e\nfi\n", "installScriptTemplateMedia": "#!/bin/bash -x\nDOMAIN=\n\n# Install dependencies\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y \\\n curl \\\n unzip \\\n jq \\\n wget\n\n# Get own private IP\nPRIVATE_IP=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text\")\n\n# Gate 1: wait for master secrets and IPs before installing\nWAIT_INTERVAL=5\nMAX_RETRIES=360\nRETRIES=0\nset +e\nwhile true; do\n ALL_SECRETS_GENERATED=$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)\n MASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv 2>/dev/null)\n MASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv 2>/dev/null)\n MASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv 2>/dev/null)\n MASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv 2>/dev/null)\n\n if [ \"$ALL_SECRETS_GENERATED\" == \"true\" ] &&\n [ \"$MASTER_NODE_1_PRIVATE_IP\" != \"\" ] &&\n [ \"$MASTER_NODE_2_PRIVATE_IP\" != \"\" ] &&\n [ \"$MASTER_NODE_3_PRIVATE_IP\" != \"\" ] &&\n [ \"$MASTER_NODE_4_PRIVATE_IP\" != \"\" ]; then\n break\n fi\n\n RETRIES=$((RETRIES + 1))\n if [ $RETRIES -ge $MAX_RETRIES ]; then\n echo \"[OpenVidu] timed out after 30 min waiting for master nodes to generate secrets and publish their IPs\"\n exit 1\n fi\n\n sleep $WAIT_INTERVAL\ndone\nset -e\n\nMASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_PRIVATE_IP_LIST=\"$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP\"\nREDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name REDIS-PASSWORD --query value -o tsv)\nENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv)\nOPENVIDU_VERSION=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-VERSION --query value -o tsv)\n\n# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure\nINSTALLER_SCRIPT=\"/tmp/install_ov_media_node.sh\"\ncurl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o \"$INSTALLER_SCRIPT\" \"http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh\"\nif [ ! -s \"$INSTALLER_SCRIPT\" ]; then\n echo \"[OpenVidu] failed to download the media node installer script\"\n exit 1\nfi\n\n# Base command\nINSTALL_COMMAND=\"sh $INSTALLER_SCRIPT\"\n\n# Common arguments\nCOMMON_ARGS=(\n\"--no-tty\"\n\"--install\"\n\"--environment=azure\"\n\"--deployment-type='ha'\"\n\"--node-role='media-node'\"\n\"--master-node-private-ip-list=$MASTER_NODE_PRIVATE_IP_LIST\"\n\"--private-ip=$PRIVATE_IP\"\n\"--enabled-modules='$ENABLED_MODULES'\"\n\"--redis-password=$REDIS_PASSWORD\"\n)\n\n# Construct the final command with all arguments\nFINAL_COMMAND=\"$INSTALL_COMMAND $(printf \"%s \" \"${COMMON_ARGS[@]}\")\"\n\n# Install OpenVidu\nexec bash -c \"$FINAL_COMMAND\"\n", "stop_media_nodesScriptMediaTemplate": "#!/bin/bash\nset -e\n\n# Pause the health watchdog before draining so it never reimages a node that is being\n# drained for scale-in (synchronous; the drain runs outside the watchdog's cgroup).\ntouch /run/openvidu-media-health.paused\nsystemctl stop openvidu-media-health 2>/dev/null || true\n\nif ! (set -o noclobber ; echo > /tmp/global.lock) ; then\n exit 1 # the global.lock already exists\nfi\n\n# Execute if docker is installed\nif [ -x \"$(command -v docker)\" ]; then\n\n echo \"Stopping media node services and waiting for termination...\"\n docker container kill --signal=SIGQUIT openvidu 2>/dev/null || true\n docker container kill --signal=SIGQUIT ingress 2>/dev/null || true\n docker container kill --signal=SIGQUIT egress 2>/dev/null || true\n for agent_container in $(docker ps --filter \"label=openvidu-agent=true\" --format '{{.Names}}' 2>/dev/null); do\n docker container kill --signal=SIGQUIT \"$agent_container\" 2>/dev/null || true\n done\n\n # If this node is being replaced because it is unhealthy, do not wait for sessions to end.\n if [ -f /run/openvidu-unhealthy ]; then\n docker ps -q | xargs -r docker kill >/dev/null 2>&1 || true\n fi\n\n # Wait for running containers to not be openvidu, ingress or egress\n while [ \"$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)\" -gt 0 ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)\" = \"true\" ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' ingress 2>/dev/null)\" = \"true\" ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' egress 2>/dev/null)\" = \"true\" ]; do\n echo \"Waiting for containers to stop...\"\n sleep 5\n done\nfi\n\naz login --identity --client-id ${mediaIdentityClientId}\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nSUBSCRIPTION_ID=${subscriptionId}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\nRESOURCE_ID=/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP_NAME/providers/Microsoft.Compute/virtualMachineScaleSets/$VM_SCALE_SET_NAME\n\nTIMESTAMP=$(date -u +\"%Y-%m-%dT%H:%M:%SZ\")\naz tag update --resource-id $RESOURCE_ID --operation replace --tags \"STATUS\"=\"HEALTHY\" \"InstanceDeleteTime\"=\"$TIMESTAMP\" \"storageAccount\"=\"${storageAccountName}\"\n\naz vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID\n", - "openviduMediaHealthScript": "#!/bin/bash\n# OpenVidu Media Node health watchdog.\n#\n# Asks the cloud to replace this Media Node with a new one, keeping the number of\n# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when:\n# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a\n# cool-down, so a persistent failure does not churn nodes and can be inspected,\n# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot,\n# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC\n# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC\n# while it answers an error or times out. Soft failures are not counted while the CPU\n# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in\n# the WARMUP_SEC after openvidu.service (re)started.\n# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage\n# fails the health check of every Media Node at once, and a new node could not bootstrap\n# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until\n# the node goes away.\n#\n# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS=\"ip:port ...\")\n# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)\n# Logs: journalctl -u openvidu-media-health\n#\n# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and\n# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed\n# by an opening brace, nor two opening braces in a row.\n\nSTATE_DIR=/var/lib/openvidu-media-health\nINSTALLED_MARKER=\"$STATE_DIR/installed\"\nBOOTSTRAP_FAILED_MARKER=\"$STATE_DIR/bootstrap-failed\"\nREPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh\nENV_FILE=/etc/openvidu/media-health.env\nPAUSE_FILE=/etc/openvidu/media-health.disabled\nDRAIN_PAUSE_FILE=/run/openvidu-media-health.paused\n\nlog() { echo \"[media-health] $*\"; }\n\n# Seconds since boot: monotonic, unaffected by clock steps\nuptime_sec() { cut -d. -f1 /proc/uptime; }\n\n# Busy CPU percentage since the previous call, in BUSY (0 on the first call)\ncpu_busy() {\n read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat\n total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal))\n idle=$((c_idle + c_iowait))\n BUSY=0\n if [ -n \"$PREV_TOTAL\" ] && [ \"$total\" -gt \"$PREV_TOTAL\" ]; then\n BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL)))\n fi\n PREV_TOTAL=$total\n PREV_IDLE=$idle\n}\n\n# openvidu.service is running and entered the active state less than WARMUP_SEC ago\n# (a stopped or failed openvidu.service is not starting: no warm-up for it)\nopenvidu_starting() {\n case \"$(systemctl show -p ActiveState --value openvidu 2>/dev/null)\" in\n active|activating|reloading) ;;\n *) return 1 ;;\n esac\n since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null)\n case \"$since\" in ''|0|*[!0-9]*) return 1 ;; esac\n [ $(($(uptime_sec) - since / 1000000)) -lt \"$WARMUP_SEC\" ]\n}\n\n# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare\n# TCP connect would also succeed against a wedged Redis.\nmaster_serving() {\n [ -n \"$MASTER_ADDRS\" ] || return 1\n for addr in $MASTER_ADDRS; do\n host=$(echo \"$addr\" | cut -d: -f1)\n port=$(echo \"$addr\" | cut -d: -f2)\n reply=$(timeout 5 bash -c 'exec 3<>\"/dev/tcp/$0/$1\" && printf \"PING\\r\\n\" >&3 && head -c 7 <&3' \"$host\" \"$port\" 2>/dev/null)\n case \"$reply\" in\n +PONG*|-NOAUTH*) return 0 ;;\n esac\n done\n return 1\n}\n\n# Request the replacement of this node ($1 = reason). Gated on a master answering, except\n# in the bootstrap phase before the master addresses are known.\nrequest_replacement() {\n now=$(uptime_sec)\n if [ -n \"$REQUESTED_AT\" ] && [ $((now - REQUESTED_AT)) -lt \"$REPLACE_RETRY_SEC\" ]; then\n return\n fi\n if [ -n \"$MASTER_ADDRS\" ] || [ -f \"$INSTALLED_MARKER\" ]; then\n if ! master_serving; then\n if [ -z \"$HOLD_LOGGED_AT\" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then\n log \"$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node\"\n HOLD_LOGGED_AT=$now\n fi\n return\n fi\n fi\n log \"$1: requesting the replacement of this node\"\n if \"$REPLACE_SCRIPT\" \"$1\"; then\n REQUESTED_AT=$now\n log \"replacement requested\"\n else\n log \"the replacement request failed, retrying in $INTERVAL_SEC s\"\n fi\n}\n\nmkdir -p \"$STATE_DIR\"\nREQUESTED_AT=\"\"\nHOLD_LOGGED_AT=\"\"\nFAILED_SEEN_AT=\"\"\nFAIL_SINCE=\"\"\nSATURATED=\"\"\nPREV_TOTAL=\"\"\nPREV_IDLE=\"\"\nlog \"started\"\n\nwhile true; do\n PROBE_URL=http://127.0.0.1:7880/\n INTERVAL_SEC=30\n HARD_UNHEALTHY_AFTER_SEC=300\n SOFT_UNHEALTHY_AFTER_SEC=600\n CPU_SATURATED_PCT=90\n WARMUP_SEC=900\n BOOTSTRAP_DEADLINE_SEC=5400\n BOOTSTRAP_FAILURE_COOLDOWN_SEC=600\n REPLACE_RETRY_SEC=600\n MASTER_ADDRS=\"\"\n [ -f \"$ENV_FILE\" ] && . \"$ENV_FILE\"\n NOW=$(uptime_sec)\n cpu_busy\n\n if [ -f \"$PAUSE_FILE\" ] || [ -f \"$DRAIN_PAUSE_FILE\" ]; then\n FAIL_SINCE=\"\"\n elif [ -f \"$BOOTSTRAP_FAILED_MARKER\" ]; then\n if [ -z \"$FAILED_SEEN_AT\" ]; then\n FAILED_SEEN_AT=$NOW\n log \"bootstrap failed ($(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s\"\n fi\n if [ $((NOW - FAILED_SEEN_AT)) -ge \"$BOOTSTRAP_FAILURE_COOLDOWN_SEC\" ]; then\n request_replacement \"bootstrap failed: $(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")\"\n fi\n elif [ ! -f \"$INSTALLED_MARKER\" ]; then\n if [ \"$NOW\" -ge \"$BOOTSTRAP_DEADLINE_SEC\" ]; then\n request_replacement \"bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot\"\n fi\n elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 \"$PROBE_URL\"; then\n [ -n \"$FAIL_SINCE\" ] && log \"LiveKit healthy again after $((NOW - FAIL_SINCE)) s\"\n FAIL_SINCE=\"\"\n SATURATED=\"\"\n else\n RC=$?\n case \"$RC\" in\n 7|52|56) KIND=hard ;;\n *) KIND=soft ;;\n esac\n if openvidu_starting; then\n FAIL_SINCE=\"\"\n elif [ \"$KIND\" = soft ] && [ \"$BUSY\" -ge \"$CPU_SATURATED_PCT\" ]; then\n [ -n \"$SATURATED\" ] || log \"LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted\"\n SATURATED=1\n FAIL_SINCE=\"\"\n else\n SATURATED=\"\"\n if [ -z \"$FAIL_SINCE\" ]; then\n FAIL_SINCE=$NOW\n log \"LiveKit health check failed (curl exit $RC)\"\n fi\n LIMIT=$SOFT_UNHEALTHY_AFTER_SEC\n [ \"$KIND\" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC\n if [ $((NOW - FAIL_SINCE)) -ge \"$LIMIT\" ]; then\n request_replacement \"LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)\"\n fi\n fi\n fi\n sleep \"$INTERVAL_SEC\"\ndone\n", + "openviduMediaHealthScript": "#!/bin/bash\n# OpenVidu Media Node health watchdog.\n#\n# Asks the cloud to replace this Media Node with a new one, keeping the number of\n# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when:\n# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a\n# cool-down, so a persistent failure does not churn nodes and can be inspected,\n# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot,\n# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC\n# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC\n# while it answers an error or times out. Soft failures are not counted while the CPU\n# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in\n# the WARMUP_SEC after openvidu.service (re)started.\n# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage\n# fails the health check of every Media Node at once, and a new node could not bootstrap\n# either; LiveKit failures are not counted until a master answers again. Once requested,\n# the replacement is requested again every REPLACE_RETRY_SEC until the node goes away.\n#\n# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS=\"ip:port ...\")\n# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)\n# Logs: journalctl -u openvidu-media-health\n#\n# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and\n# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed\n# by an opening brace, nor two opening braces in a row.\n\nSTATE_DIR=/var/lib/openvidu-media-health\nINSTALLED_MARKER=\"$STATE_DIR/installed\"\nBOOTSTRAP_FAILED_MARKER=\"$STATE_DIR/bootstrap-failed\"\nREPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh\nENV_FILE=/etc/openvidu/media-health.env\nPAUSE_FILE=/etc/openvidu/media-health.disabled\nDRAIN_PAUSE_FILE=/run/openvidu-media-health.paused\n\nlog() { echo \"[media-health] $*\"; }\n\n# Seconds since boot: monotonic, unaffected by clock steps\nuptime_sec() { cut -d. -f1 /proc/uptime; }\n\n# Busy CPU percentage since the previous call, in BUSY (0 on the first call)\ncpu_busy() {\n read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat\n total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal))\n idle=$((c_idle + c_iowait))\n BUSY=0\n if [ -n \"$PREV_TOTAL\" ] && [ \"$total\" -gt \"$PREV_TOTAL\" ]; then\n BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL)))\n fi\n PREV_TOTAL=$total\n PREV_IDLE=$idle\n}\n\n# openvidu.service is running and entered the active state less than WARMUP_SEC ago\n# (a stopped or failed openvidu.service is not starting: no warm-up for it)\nopenvidu_starting() {\n case \"$(systemctl show -p ActiveState --value openvidu 2>/dev/null)\" in\n active|activating|reloading) ;;\n *) return 1 ;;\n esac\n since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null)\n case \"$since\" in ''|0|*[!0-9]*) return 1 ;; esac\n [ $(($(uptime_sec) - since / 1000000)) -lt \"$WARMUP_SEC\" ]\n}\n\n# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare\n# TCP connect would also succeed against a wedged Redis.\nmaster_serving() {\n [ -n \"$MASTER_ADDRS\" ] || return 1\n for addr in $MASTER_ADDRS; do\n host=$(echo \"$addr\" | cut -d: -f1)\n port=$(echo \"$addr\" | cut -d: -f2)\n reply=$(timeout 5 bash -c 'exec 3<>\"/dev/tcp/$0/$1\" && printf \"PING\\r\\n\" >&3 && head -c 7 <&3' \"$host\" \"$port\" 2>/dev/null)\n case \"$reply\" in\n +PONG*|-NOAUTH*) return 0 ;;\n esac\n done\n return 1\n}\n\n# Request the replacement of this node ($1 = reason). Gated on a master answering, except\n# in the bootstrap phase before the master addresses are known.\nrequest_replacement() {\n now=$(uptime_sec)\n if [ -n \"$REQUESTED_AT\" ] && [ $((now - REQUESTED_AT)) -lt \"$REPLACE_RETRY_SEC\" ]; then\n return\n fi\n if [ -n \"$MASTER_ADDRS\" ] || [ -f \"$INSTALLED_MARKER\" ]; then\n if ! master_serving; then\n if [ -z \"$HOLD_LOGGED_AT\" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then\n log \"$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node\"\n HOLD_LOGGED_AT=$now\n fi\n return\n fi\n fi\n log \"$1: requesting the replacement of this node\"\n if \"$REPLACE_SCRIPT\" \"$1\"; then\n REQUESTED_AT=$now\n log \"replacement requested\"\n else\n log \"the replacement request failed, retrying in $INTERVAL_SEC s\"\n fi\n}\n\nmkdir -p \"$STATE_DIR\"\nREQUESTED_AT=\"\"\nHOLD_LOGGED_AT=\"\"\nFAILED_SEEN_AT=\"\"\nFAIL_SINCE=\"\"\nSATURATED=\"\"\nPREV_TOTAL=\"\"\nPREV_IDLE=\"\"\nlog \"started\"\n\nwhile true; do\n PROBE_URL=http://127.0.0.1:7880/\n INTERVAL_SEC=30\n HARD_UNHEALTHY_AFTER_SEC=300\n SOFT_UNHEALTHY_AFTER_SEC=600\n CPU_SATURATED_PCT=90\n WARMUP_SEC=900\n BOOTSTRAP_DEADLINE_SEC=5400\n BOOTSTRAP_FAILURE_COOLDOWN_SEC=600\n REPLACE_RETRY_SEC=600\n MASTER_ADDRS=\"\"\n [ -f \"$ENV_FILE\" ] && . \"$ENV_FILE\"\n NOW=$(uptime_sec)\n cpu_busy\n\n if [ -f \"$PAUSE_FILE\" ] || [ -f \"$DRAIN_PAUSE_FILE\" ]; then\n FAIL_SINCE=\"\"\n elif [ -f \"$BOOTSTRAP_FAILED_MARKER\" ]; then\n if [ -z \"$FAILED_SEEN_AT\" ]; then\n FAILED_SEEN_AT=$NOW\n log \"bootstrap failed ($(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s\"\n fi\n if [ $((NOW - FAILED_SEEN_AT)) -ge \"$BOOTSTRAP_FAILURE_COOLDOWN_SEC\" ]; then\n request_replacement \"bootstrap failed: $(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")\"\n fi\n elif [ ! -f \"$INSTALLED_MARKER\" ]; then\n if [ \"$NOW\" -ge \"$BOOTSTRAP_DEADLINE_SEC\" ]; then\n request_replacement \"bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot\"\n fi\n elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 \"$PROBE_URL\"; then\n [ -n \"$FAIL_SINCE\" ] && log \"LiveKit healthy again after $((NOW - FAIL_SINCE)) s\"\n FAIL_SINCE=\"\"\n SATURATED=\"\"\n else\n RC=$?\n case \"$RC\" in\n 7|52|56) KIND=hard ;;\n *) KIND=soft ;;\n esac\n if openvidu_starting; then\n FAIL_SINCE=\"\"\n elif ! master_serving; then\n # A master outage fails the health check of every Media Node at once: not counted,\n # so LiveKit gets the whole window to reconnect once a master answers again\n FAIL_SINCE=\"\"\n if [ -z \"$HOLD_LOGGED_AT\" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then\n log \"LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted\"\n HOLD_LOGGED_AT=$NOW\n fi\n elif [ \"$KIND\" = soft ] && [ \"$BUSY\" -ge \"$CPU_SATURATED_PCT\" ]; then\n [ -n \"$SATURATED\" ] || log \"LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted\"\n SATURATED=1\n FAIL_SINCE=\"\"\n else\n SATURATED=\"\"\n if [ -z \"$FAIL_SINCE\" ]; then\n FAIL_SINCE=$NOW\n log \"LiveKit health check failed (curl exit $RC)\"\n fi\n LIMIT=$SOFT_UNHEALTHY_AFTER_SEC\n [ \"$KIND\" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC\n if [ $((NOW - FAIL_SINCE)) -ge \"$LIMIT\" ]; then\n request_replacement \"LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)\"\n fi\n fi\n fi\n sleep \"$INTERVAL_SEC\"\ndone\n", "openviduMediaHealthUnit": "[Unit]\nDescription=OpenVidu Media Node health watchdog\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/local/bin/openvidu-media-health.sh\nRestart=always\nRestartSec=30\n\n[Install]\nWantedBy=multi-user.target\n", "media_replaceScriptMediaTemplate": "#!/bin/bash\n# openvidu-media-replace.sh \"\"\n# Requests the reimage of THIS Media Node, keeping the scale set capacity and the\n# scale-in protection: reimage provisions a new OS disk and the userData bootstrap runs\n# again on the same instance id. Called by the health watchdog; it never signals it.\n\nmkdir -p /var/lib/openvidu-media-health\necho \"$(date -u +%FT%TZ) $1\" >> /var/lib/openvidu-media-health/replace-requests.log\ntouch /run/openvidu-unhealthy\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nINSTANCE_RESOURCE_ID=$(curl -s -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo \"$INSTANCE_RESOURCE_ID\" | awk -F'/' '{print $NF}')\nif [ -z \"$INSTANCE_ID\" ]; then\n echo \"[media-replace] could not read this instance id from IMDS\"\n exit 1\nfi\n\nn=0\nwhile true; do\n n=$((n + 1))\n if az login --identity --client-id ${mediaIdentityClientId} >/dev/null 2>&1 && \\\n az vmss reimage --resource-group \"$RESOURCE_GROUP_NAME\" --name \"$VM_SCALE_SET_NAME\" --instance-ids \"$INSTANCE_ID\" --no-wait; then\n echo \"[media-replace] reimage requested for instance $INSTANCE_ID ($1)\"\n exit 0\n fi\n if [ \"$n\" -ge 5 ]; then\n echo \"[media-replace] reimage could not be requested after $n attempts\"\n exit 1\n fi\n echo \"[media-replace] reimage request failed (attempt $n), retrying in 12s\"\n sleep 12\ndone\n", "userDataMediaNodeTemplate": "#!/bin/bash -x\nset -eu -o pipefail\n\n# Mark this media node's bootstrap as failed so the health watchdog replaces it\nov_media_bootstrap_failed() {\n echo \"[OpenVidu] media node bootstrap failed: $1\"\n mkdir -p /var/lib/openvidu-media-health\n echo \"$1\" > /var/lib/openvidu-media-health/bootstrap-failed\n exit 1\n}\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# stop_media_nodes.sh\necho ${base64stop} | base64 -d > /usr/local/bin/stop_media_node.sh\nchmod +x /usr/local/bin/stop_media_node.sh\n\n# openvidu-media-health.sh (health watchdog), openvidu-media-replace.sh (reimage this node) and its unit\necho ${base64mediaHealth} | base64 -d > /usr/local/bin/openvidu-media-health.sh\nchmod +x /usr/local/bin/openvidu-media-health.sh\necho ${base64mediaReplace} | base64 -d > /usr/local/bin/openvidu-media-replace.sh\nchmod +x /usr/local/bin/openvidu-media-replace.sh\necho ${base64mediaHealthUnit} | base64 -d > /etc/systemd/system/openvidu-media-health.service\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y jq\n\n# Install azure cli\nAZURE_CLI_VERSION=2.90.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity --client-id ${mediaIdentityClientId}\n\n# Protect from scale in actions\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\naz vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-id $INSTANCE_ID --protect-from-scale-in true\n\nexport HOME=\"/root\"\n\n# Start the Media Node health watchdog before install.sh (az CLI is installed and the node\n# is scale-in protected): it backstops a hung or failed bootstrap and, once running, reimages\n# the node if LiveKit stays unhealthy. Write MASTER_ADDRS first so its master guard is armed.\nmkdir -p /etc/openvidu\necho 'MASTER_ADDRS=\"${privateIPMasterNode1}:7001 ${privateIPMasterNode2}:7001 ${privateIPMasterNode3}:7001 ${privateIPMasterNode4}:7001\"' > /etc/openvidu/media-health.env\nsystemctl daemon-reload\nsystemctl enable --now openvidu-media-health\n\n# Install OpenVidu\n/usr/local/bin/install.sh || ov_media_bootstrap_failed \"install.sh failed\"\n\n# Gate 2: wait for master readiness before starting\nWAIT_INTERVAL=5\nMAX_RETRIES=360\nRETRIES=0\nset +e\nwhile true; do\n FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv 2>/dev/null)\n if [ \"$FINISH_MASTER_NODE\" == \"true\" ]; then\n break\n fi\n RETRIES=$((RETRIES + 1))\n if [ $RETRIES -ge $MAX_RETRIES ]; then\n ov_media_bootstrap_failed \"timed out after 30 min waiting for FINISH-MASTER-NODE\"\n fi\n sleep $WAIT_INTERVAL\ndone\nset -e\n\n# Start OpenVidu\nsystemctl start openvidu || ov_media_bootstrap_failed \"systemctl start openvidu failed\"\n\n# Bootstrap finished: the watchdog switches from bootstrap-deadline mode to runtime health checks\nmkdir -p /var/lib/openvidu-media-health\ntouch /var/lib/openvidu-media-health/installed\n", diff --git a/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf b/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf index 11d7ee528..64f4f960c 100644 --- a/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf +++ b/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf @@ -1813,8 +1813,8 @@ EOF # the WARMUP_SEC after openvidu.service (re)started. # Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage # fails the health check of every Media Node at once, and a new node could not bootstrap -# either. Once requested, the replacement is requested again every REPLACE_RETRY_SEC until -# the node goes away. +# either; LiveKit failures are not counted until a master answers again. Once requested, +# the replacement is requested again every REPLACE_RETRY_SEC until the node goes away. # # Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...") # Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused) @@ -1953,6 +1953,14 @@ while true; do esac if openvidu_starting; then FAIL_SINCE="" + elif ! master_serving; then + # A master outage fails the health check of every Media Node at once: not counted, + # so LiveKit gets the whole window to reconnect once a master answers again + FAIL_SINCE="" + if [ -z "$HOLD_LOGGED_AT" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then + log "LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted" + HOLD_LOGGED_AT=$NOW + fi elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then [ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted" SATURATED=1