diff --git a/openvidu-deployment/pro/elastic/aws/cf-openvidu-elastic.yaml b/openvidu-deployment/pro/elastic/aws/cf-openvidu-elastic.yaml index 7cb959deb..4b33f66dd 100644 --- a/openvidu-deployment/pro/elastic/aws/cf-openvidu-elastic.yaml +++ b/openvidu-deployment/pro/elastic/aws/cf-openvidu-elastic.yaml @@ -2353,6 +2353,9 @@ Resources: echo "$(date -u +%FT%TZ) $1" >> /var/lib/openvidu-media-health/replace-requests.log # Let stop_media_node.sh know this node is unhealthy, so it does not wait for sessions touch /run/openvidu-unhealthy + # Withdrawn if the request fails: the node then stays, and if it recovers + # a later scale-in must still wait for its sessions + trap '[ $? -eq 0 ] || rm -f /run/openvidu-unhealthy' EXIT # Token for IMDSv2 TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") diff --git a/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.bicep b/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.bicep index 51854cfb0..48084364d 100644 --- a/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.bicep +++ b/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.bicep @@ -1263,6 +1263,9 @@ var media_replaceScriptMediaTemplate = ''' mkdir -p /var/lib/openvidu-media-health echo "$(date -u +%FT%TZ) $1" >> /var/lib/openvidu-media-health/replace-requests.log touch /run/openvidu-unhealthy +# Withdrawn if the request fails: the node then stays, and if it recovers +# a later scale-in must still wait for its sessions +trap '[ $? -eq 0 ] || rm -f /run/openvidu-unhealthy' EXIT RESOURCE_GROUP_NAME=${resourceGroupName} VM_SCALE_SET_NAME=${vmScaleSetName} diff --git a/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.json b/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.json index a7a0e09c2..3731c2878 100644 --- a/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.json +++ b/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.46.1.21595", - "templateHash": "3671752928188054764" + "templateHash": "15778560378064529966" } }, "parameters": { @@ -281,7 +281,7 @@ "stop_media_nodesScriptMediaTemplate": "#!/bin/bash\nset -e\n\n# Pause the health watchdog before draining so it never reimages a node that is being\n# drained for scale-in (synchronous; the drain runs outside the watchdog's cgroup).\ntouch /run/openvidu-media-health.paused\nsystemctl stop openvidu-media-health 2>/dev/null || true\n\nif ! (set -o noclobber ; echo > /tmp/global.lock) ; then\n exit 1 # the global.lock already exists\nfi\n\n# Execute if docker is installed\nif [ -x \"$(command -v docker)\" ]; then\n\n echo \"Stopping media node services and waiting for termination...\"\n docker container kill --signal=SIGQUIT openvidu 2>/dev/null || true\n docker container kill --signal=SIGQUIT ingress 2>/dev/null || true\n docker container kill --signal=SIGQUIT egress 2>/dev/null || true\n for agent_container in $(docker ps --filter \"label=openvidu-agent=true\" --format '{{.Names}}' 2>/dev/null); do\n docker container kill --signal=SIGQUIT \"$agent_container\" 2>/dev/null || true\n done\n\n # If this node is being replaced because it is unhealthy, do not wait for sessions to end.\n if [ -f /run/openvidu-unhealthy ]; then\n docker ps -q | xargs -r docker kill >/dev/null 2>&1 || true\n fi\n\n # Wait for running containers to not be openvidu, ingress, egress or an openvidu agent\n while [ \"$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)\" -gt 0 ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)\" = \"true\" ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' ingress 2>/dev/null)\" = \"true\" ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' egress 2>/dev/null)\" = \"true\" ]; do\n echo \"Waiting for containers to stop...\"\n sleep 5\n done\nfi\n\naz login --identity\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nSUBSCRIPTION_ID=${subscriptionId}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\nRESOURCE_ID=/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP_NAME/providers/Microsoft.Compute/virtualMachineScaleSets/$VM_SCALE_SET_NAME\nTIMESTAMP=$(date -u +\"%Y-%m-%dT%H:%M:%SZ\")\n\naz tag update --resource-id $RESOURCE_ID --operation replace --tags \"STATUS\"=\"HEALTHY\" \"InstanceDeleteTime\"=\"$TIMESTAMP\" \"storageAccount\"=\"${storageAccountName}\"\n\naz vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID\n", "openviduMediaHealthScript": "#!/bin/bash\n# OpenVidu Media Node health watchdog.\n#\n# Asks the cloud to replace this Media Node with a new one, keeping the number of\n# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when:\n# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a\n# cool-down, so a persistent failure does not churn nodes and can be inspected,\n# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot,\n# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC\n# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC\n# while it answers an error or times out. Soft failures are not counted while the CPU\n# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in\n# the WARMUP_SEC after openvidu.service (re)started.\n# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage\n# fails the health check of every Media Node at once, and a new node could not bootstrap\n# either; LiveKit failures are not counted until a master answers again. Once requested,\n# the replacement is requested again every REPLACE_RETRY_SEC until the node goes away.\n#\n# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS=\"ip:port ...\")\n# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)\n# Logs: journalctl -u openvidu-media-health\n#\n# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and\n# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed\n# by an opening brace, nor two opening braces in a row.\n\nSTATE_DIR=/var/lib/openvidu-media-health\nINSTALLED_MARKER=\"$STATE_DIR/installed\"\nBOOTSTRAP_FAILED_MARKER=\"$STATE_DIR/bootstrap-failed\"\nREPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh\nENV_FILE=/etc/openvidu/media-health.env\nPAUSE_FILE=/etc/openvidu/media-health.disabled\nDRAIN_PAUSE_FILE=/run/openvidu-media-health.paused\n\nlog() { echo \"[media-health] $*\"; }\n\n# Seconds since boot: monotonic, unaffected by clock steps\nuptime_sec() { cut -d. -f1 /proc/uptime; }\n\n# Busy CPU percentage since the previous call, in BUSY (0 on the first call)\ncpu_busy() {\n read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat\n total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal))\n idle=$((c_idle + c_iowait))\n BUSY=0\n if [ -n \"$PREV_TOTAL\" ] && [ \"$total\" -gt \"$PREV_TOTAL\" ]; then\n BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL)))\n fi\n PREV_TOTAL=$total\n PREV_IDLE=$idle\n}\n\n# openvidu.service is running and entered the active state less than WARMUP_SEC ago\n# (a stopped or failed openvidu.service is not starting: no warm-up for it)\nopenvidu_starting() {\n case \"$(systemctl show -p ActiveState --value openvidu 2>/dev/null)\" in\n active|activating|reloading) ;;\n *) return 1 ;;\n esac\n since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null)\n case \"$since\" in ''|0|*[!0-9]*) return 1 ;; esac\n [ $(($(uptime_sec) - since / 1000000)) -lt \"$WARMUP_SEC\" ]\n}\n\n# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare\n# TCP connect would also succeed against a wedged Redis.\nmaster_serving() {\n [ -n \"$MASTER_ADDRS\" ] || return 1\n for addr in $MASTER_ADDRS; do\n host=$(echo \"$addr\" | cut -d: -f1)\n port=$(echo \"$addr\" | cut -d: -f2)\n reply=$(timeout 5 bash -c 'exec 3<>\"/dev/tcp/$0/$1\" && printf \"PING\\r\\n\" >&3 && head -c 7 <&3' \"$host\" \"$port\" 2>/dev/null)\n case \"$reply\" in\n +PONG*|-NOAUTH*) return 0 ;;\n esac\n done\n return 1\n}\n\n# Request the replacement of this node ($1 = reason). Gated on a master answering, except\n# in the bootstrap phase before the master addresses are known.\nrequest_replacement() {\n now=$(uptime_sec)\n if [ -n \"$REQUESTED_AT\" ] && [ $((now - REQUESTED_AT)) -lt \"$REPLACE_RETRY_SEC\" ]; then\n return\n fi\n if [ -n \"$MASTER_ADDRS\" ] || [ -f \"$INSTALLED_MARKER\" ]; then\n if ! master_serving; then\n if [ -z \"$HOLD_LOGGED_AT\" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then\n log \"$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node\"\n HOLD_LOGGED_AT=$now\n fi\n return\n fi\n fi\n log \"$1: requesting the replacement of this node\"\n if \"$REPLACE_SCRIPT\" \"$1\"; then\n REQUESTED_AT=$now\n log \"replacement requested\"\n else\n log \"the replacement request failed, retrying in $INTERVAL_SEC s\"\n fi\n}\n\nmkdir -p \"$STATE_DIR\"\nREQUESTED_AT=\"\"\nHOLD_LOGGED_AT=\"\"\nFAILED_SEEN_AT=\"\"\nFAIL_SINCE=\"\"\nSATURATED=\"\"\nPREV_TOTAL=\"\"\nPREV_IDLE=\"\"\nlog \"started\"\n\nwhile true; do\n PROBE_URL=http://127.0.0.1:7880/\n INTERVAL_SEC=30\n HARD_UNHEALTHY_AFTER_SEC=300\n SOFT_UNHEALTHY_AFTER_SEC=600\n CPU_SATURATED_PCT=90\n WARMUP_SEC=900\n BOOTSTRAP_DEADLINE_SEC=5400\n BOOTSTRAP_FAILURE_COOLDOWN_SEC=600\n REPLACE_RETRY_SEC=600\n MASTER_ADDRS=\"\"\n [ -f \"$ENV_FILE\" ] && . \"$ENV_FILE\"\n NOW=$(uptime_sec)\n cpu_busy\n\n if [ -f \"$PAUSE_FILE\" ] || [ -f \"$DRAIN_PAUSE_FILE\" ]; then\n FAIL_SINCE=\"\"\n elif [ -f \"$BOOTSTRAP_FAILED_MARKER\" ]; then\n if [ -z \"$FAILED_SEEN_AT\" ]; then\n FAILED_SEEN_AT=$NOW\n log \"bootstrap failed ($(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s\"\n fi\n if [ $((NOW - FAILED_SEEN_AT)) -ge \"$BOOTSTRAP_FAILURE_COOLDOWN_SEC\" ]; then\n request_replacement \"bootstrap failed: $(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")\"\n fi\n elif [ ! -f \"$INSTALLED_MARKER\" ]; then\n if [ \"$NOW\" -ge \"$BOOTSTRAP_DEADLINE_SEC\" ]; then\n request_replacement \"bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot\"\n fi\n elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 \"$PROBE_URL\"; then\n [ -n \"$FAIL_SINCE\" ] && log \"LiveKit healthy again after $((NOW - FAIL_SINCE)) s\"\n FAIL_SINCE=\"\"\n SATURATED=\"\"\n else\n RC=$?\n case \"$RC\" in\n 7|52|56) KIND=hard ;;\n *) KIND=soft ;;\n esac\n if openvidu_starting; then\n FAIL_SINCE=\"\"\n elif ! master_serving; then\n # A master outage fails the health check of every Media Node at once: not counted,\n # so LiveKit gets the whole window to reconnect once a master answers again\n FAIL_SINCE=\"\"\n if [ -z \"$HOLD_LOGGED_AT\" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then\n log \"LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted\"\n HOLD_LOGGED_AT=$NOW\n fi\n elif [ \"$KIND\" = soft ] && [ \"$BUSY\" -ge \"$CPU_SATURATED_PCT\" ]; then\n [ -n \"$SATURATED\" ] || log \"LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted\"\n SATURATED=1\n FAIL_SINCE=\"\"\n else\n SATURATED=\"\"\n if [ -z \"$FAIL_SINCE\" ]; then\n FAIL_SINCE=$NOW\n log \"LiveKit health check failed (curl exit $RC)\"\n fi\n LIMIT=$SOFT_UNHEALTHY_AFTER_SEC\n [ \"$KIND\" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC\n if [ $((NOW - FAIL_SINCE)) -ge \"$LIMIT\" ]; then\n request_replacement \"LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)\"\n fi\n fi\n fi\n sleep \"$INTERVAL_SEC\"\ndone\n", "openviduMediaHealthUnit": "[Unit]\nDescription=OpenVidu Media Node health watchdog\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/local/bin/openvidu-media-health.sh\nRestart=always\nRestartSec=30\n\n[Install]\nWantedBy=multi-user.target\n", - "media_replaceScriptMediaTemplate": "#!/bin/bash\n# openvidu-media-replace.sh \"\"\n# Requests the reimage of THIS Media Node, keeping the scale set capacity and the\n# scale-in protection: reimage provisions a new OS disk and the userData bootstrap runs\n# again on the same instance id. Called by the health watchdog; it never signals it.\n\nmkdir -p /var/lib/openvidu-media-health\necho \"$(date -u +%FT%TZ) $1\" >> /var/lib/openvidu-media-health/replace-requests.log\ntouch /run/openvidu-unhealthy\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nINSTANCE_RESOURCE_ID=$(curl -s -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo \"$INSTANCE_RESOURCE_ID\" | awk -F'/' '{print $NF}')\nif [ -z \"$INSTANCE_ID\" ]; then\n echo \"[media-replace] could not read this instance id from IMDS\"\n exit 1\nfi\n\nn=0\nwhile true; do\n n=$((n + 1))\n if az login --identity >/dev/null 2>&1 && \\\n az vmss reimage --resource-group \"$RESOURCE_GROUP_NAME\" --name \"$VM_SCALE_SET_NAME\" --instance-ids \"$INSTANCE_ID\" --no-wait; then\n echo \"[media-replace] reimage requested for instance $INSTANCE_ID ($1)\"\n exit 0\n fi\n if [ \"$n\" -ge 5 ]; then\n echo \"[media-replace] reimage could not be requested after $n attempts\"\n exit 1\n fi\n echo \"[media-replace] reimage request failed (attempt $n), retrying in 12s\"\n sleep 12\ndone\n", + "media_replaceScriptMediaTemplate": "#!/bin/bash\n# openvidu-media-replace.sh \"\"\n# Requests the reimage of THIS Media Node, keeping the scale set capacity and the\n# scale-in protection: reimage provisions a new OS disk and the userData bootstrap runs\n# again on the same instance id. Called by the health watchdog; it never signals it.\n\nmkdir -p /var/lib/openvidu-media-health\necho \"$(date -u +%FT%TZ) $1\" >> /var/lib/openvidu-media-health/replace-requests.log\ntouch /run/openvidu-unhealthy\n# Withdrawn if the request fails: the node then stays, and if it recovers\n# a later scale-in must still wait for its sessions\ntrap '[ $? -eq 0 ] || rm -f /run/openvidu-unhealthy' EXIT\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nINSTANCE_RESOURCE_ID=$(curl -s -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo \"$INSTANCE_RESOURCE_ID\" | awk -F'/' '{print $NF}')\nif [ -z \"$INSTANCE_ID\" ]; then\n echo \"[media-replace] could not read this instance id from IMDS\"\n exit 1\nfi\n\nn=0\nwhile true; do\n n=$((n + 1))\n if az login --identity >/dev/null 2>&1 && \\\n az vmss reimage --resource-group \"$RESOURCE_GROUP_NAME\" --name \"$VM_SCALE_SET_NAME\" --instance-ids \"$INSTANCE_ID\" --no-wait; then\n echo \"[media-replace] reimage requested for instance $INSTANCE_ID ($1)\"\n exit 0\n fi\n if [ \"$n\" -ge 5 ]; then\n echo \"[media-replace] reimage could not be requested after $n attempts\"\n exit 1\n fi\n echo \"[media-replace] reimage request failed (attempt $n), retrying in 12s\"\n sleep 12\ndone\n", "userDataMediaNodeTemplate": "#!/bin/bash\nset -eu -o pipefail\n\n# Mark this media node's bootstrap as failed so the health watchdog replaces it\nov_media_bootstrap_failed() {\n echo \"[OpenVidu] media node bootstrap failed: $1\"\n mkdir -p /var/lib/openvidu-media-health\n echo \"$1\" > /var/lib/openvidu-media-health/bootstrap-failed\n exit 1\n}\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# stop_media_nodes.sh\necho ${base64stop} | base64 -d > /usr/local/bin/stop_media_node.sh\nchmod +x /usr/local/bin/stop_media_node.sh\n\n# openvidu-media-health.sh (health watchdog), openvidu-media-replace.sh (reimage this node) and its unit\necho ${base64mediaHealth} | base64 -d > /usr/local/bin/openvidu-media-health.sh\nchmod +x /usr/local/bin/openvidu-media-health.sh\necho ${base64mediaReplace} | base64 -d > /usr/local/bin/openvidu-media-replace.sh\nchmod +x /usr/local/bin/openvidu-media-replace.sh\necho ${base64mediaHealthUnit} | base64 -d > /etc/systemd/system/openvidu-media-health.service\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y jq\n\n# Install azure cli\nAZURE_CLI_VERSION=2.90.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity\n\n# Protect from scale in actions\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\naz vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-id $INSTANCE_ID --protect-from-scale-in true\n\nexport HOME=\"/root\"\n\n# Start the Media Node health watchdog before install.sh (az CLI is installed and the node\n# is scale-in protected): it backstops a hung or failed bootstrap and, once running, reimages\n# the node if LiveKit stays unhealthy. Write MASTER_ADDRS first so its master guard is armed.\nmkdir -p /etc/openvidu\necho 'MASTER_ADDRS=\"${privateIPMasterNode}:7000\"' > /etc/openvidu/media-health.env\nsystemctl daemon-reload\nsystemctl enable --now openvidu-media-health\n\n# Install OpenVidu\n/usr/local/bin/install.sh || ov_media_bootstrap_failed \"install.sh failed\"\n\n# Gate 2: wait for master readiness before starting\nWAIT_INTERVAL=5\nMAX_RETRIES=360\nRETRIES=0\nset +e\nwhile true; do\n FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv 2>/dev/null)\n if [ \"$FINISH_MASTER_NODE\" == \"true\" ]; then\n break\n fi\n RETRIES=$((RETRIES + 1))\n if [ $RETRIES -ge $MAX_RETRIES ]; then\n ov_media_bootstrap_failed \"timed out after 30 min waiting for FINISH-MASTER-NODE\"\n fi\n sleep $WAIT_INTERVAL\ndone\nset -e\n\n# Start OpenVidu\nsystemctl start openvidu || ov_media_bootstrap_failed \"systemctl start openvidu failed\"\n\n# Bootstrap finished: the watchdog switches from bootstrap-deadline mode to runtime health checks\nmkdir -p /var/lib/openvidu-media-health\ntouch /var/lib/openvidu-media-health/installed\n", "stop_media_nodesScriptMedia": "[reduce(items(variables('stopMediaNodeParams')), createObject('value', variables('stop_media_nodesScriptMediaTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value]", "base64stopMediaNode": "[base64(variables('stop_media_nodesScriptMedia'))]", diff --git a/openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf b/openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf index 73c6523d2..521e45d0b 100644 --- a/openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf +++ b/openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf @@ -1610,6 +1610,9 @@ mkdir -p /var/lib/openvidu-media-health echo "$(date -u +%FT%TZ) $REASON" >> /var/lib/openvidu-media-health/replace-requests.log # Tell the drain script this node is being replaced so it skips waiting for sessions. touch /run/openvidu-unhealthy +# Withdrawn if the request fails: the node then stays, and if it recovers +# a later scale-in must still wait for its sessions +trap '[ $? -eq 0 ] || rm -f /run/openvidu-unhealthy' EXIT META="http://metadata.google.internal/computeMetadata/v1" meta() { curl -s -H "Metadata-Flavor: Google" "$META/$1"; } diff --git a/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml b/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml index ee6cd5df4..4016a2926 100644 --- a/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml +++ b/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml @@ -2996,6 +2996,9 @@ Resources: echo "$(date -u +%FT%TZ) $1" >> /var/lib/openvidu-media-health/replace-requests.log # Let stop_media_node.sh know this node is unhealthy, so it does not wait for sessions touch /run/openvidu-unhealthy + # Withdrawn if the request fails: the node then stays, and if it recovers + # a later scale-in must still wait for its sessions + trap '[ $? -eq 0 ] || rm -f /run/openvidu-unhealthy' EXIT # Token for IMDSv2 TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") diff --git a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep index e061ad34e..f0f0558f5 100644 --- a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep +++ b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep @@ -1656,6 +1656,9 @@ var media_replaceScriptMediaTemplate = ''' mkdir -p /var/lib/openvidu-media-health echo "$(date -u +%FT%TZ) $1" >> /var/lib/openvidu-media-health/replace-requests.log touch /run/openvidu-unhealthy +# Withdrawn if the request fails: the node then stays, and if it recovers +# a later scale-in must still wait for its sessions +trap '[ $? -eq 0 ] || rm -f /run/openvidu-unhealthy' EXIT RESOURCE_GROUP_NAME=${resourceGroupName} VM_SCALE_SET_NAME=${vmScaleSetName} diff --git a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json index 7bc32a6da..e63fdc843 100644 --- a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json +++ b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.46.1.21595", - "templateHash": "11373208048589082219" + "templateHash": "3526845124636083473" } }, "parameters": { @@ -251,7 +251,7 @@ "stop_media_nodesScriptMediaTemplate": "#!/bin/bash\nset -e\n\n# Pause the health watchdog before draining so it never reimages a node that is being\n# drained for scale-in (synchronous; the drain runs outside the watchdog's cgroup).\ntouch /run/openvidu-media-health.paused\nsystemctl stop openvidu-media-health 2>/dev/null || true\n\nif ! (set -o noclobber ; echo > /tmp/global.lock) ; then\n exit 1 # the global.lock already exists\nfi\n\n# Execute if docker is installed\nif [ -x \"$(command -v docker)\" ]; then\n\n echo \"Stopping media node services and waiting for termination...\"\n docker container kill --signal=SIGQUIT openvidu 2>/dev/null || true\n docker container kill --signal=SIGQUIT ingress 2>/dev/null || true\n docker container kill --signal=SIGQUIT egress 2>/dev/null || true\n for agent_container in $(docker ps --filter \"label=openvidu-agent=true\" --format '{{.Names}}' 2>/dev/null); do\n docker container kill --signal=SIGQUIT \"$agent_container\" 2>/dev/null || true\n done\n\n # If this node is being replaced because it is unhealthy, do not wait for sessions to end.\n if [ -f /run/openvidu-unhealthy ]; then\n docker ps -q | xargs -r docker kill >/dev/null 2>&1 || true\n fi\n\n # Wait for running containers to not be openvidu, ingress or egress\n while [ \"$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)\" -gt 0 ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)\" = \"true\" ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' ingress 2>/dev/null)\" = \"true\" ] || \\\n [ \"$(docker inspect -f '{{.State.Running}}' egress 2>/dev/null)\" = \"true\" ]; do\n echo \"Waiting for containers to stop...\"\n sleep 5\n done\nfi\n\naz login --identity --client-id ${mediaIdentityClientId}\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nSUBSCRIPTION_ID=${subscriptionId}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\nRESOURCE_ID=/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP_NAME/providers/Microsoft.Compute/virtualMachineScaleSets/$VM_SCALE_SET_NAME\n\nTIMESTAMP=$(date -u +\"%Y-%m-%dT%H:%M:%SZ\")\naz tag update --resource-id $RESOURCE_ID --operation replace --tags \"STATUS\"=\"HEALTHY\" \"InstanceDeleteTime\"=\"$TIMESTAMP\" \"storageAccount\"=\"${storageAccountName}\"\n\naz vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID\n", "openviduMediaHealthScript": "#!/bin/bash\n# OpenVidu Media Node health watchdog.\n#\n# Asks the cloud to replace this Media Node with a new one, keeping the number of\n# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when:\n# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a\n# cool-down, so a persistent failure does not churn nodes and can be inspected,\n# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot,\n# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC\n# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC\n# while it answers an error or times out. Soft failures are not counted while the CPU\n# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in\n# the WARMUP_SEC after openvidu.service (re)started.\n# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage\n# fails the health check of every Media Node at once, and a new node could not bootstrap\n# either; LiveKit failures are not counted until a master answers again. Once requested,\n# the replacement is requested again every REPLACE_RETRY_SEC until the node goes away.\n#\n# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS=\"ip:port ...\")\n# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)\n# Logs: journalctl -u openvidu-media-health\n#\n# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and\n# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed\n# by an opening brace, nor two opening braces in a row.\n\nSTATE_DIR=/var/lib/openvidu-media-health\nINSTALLED_MARKER=\"$STATE_DIR/installed\"\nBOOTSTRAP_FAILED_MARKER=\"$STATE_DIR/bootstrap-failed\"\nREPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh\nENV_FILE=/etc/openvidu/media-health.env\nPAUSE_FILE=/etc/openvidu/media-health.disabled\nDRAIN_PAUSE_FILE=/run/openvidu-media-health.paused\n\nlog() { echo \"[media-health] $*\"; }\n\n# Seconds since boot: monotonic, unaffected by clock steps\nuptime_sec() { cut -d. -f1 /proc/uptime; }\n\n# Busy CPU percentage since the previous call, in BUSY (0 on the first call)\ncpu_busy() {\n read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat\n total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal))\n idle=$((c_idle + c_iowait))\n BUSY=0\n if [ -n \"$PREV_TOTAL\" ] && [ \"$total\" -gt \"$PREV_TOTAL\" ]; then\n BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL)))\n fi\n PREV_TOTAL=$total\n PREV_IDLE=$idle\n}\n\n# openvidu.service is running and entered the active state less than WARMUP_SEC ago\n# (a stopped or failed openvidu.service is not starting: no warm-up for it)\nopenvidu_starting() {\n case \"$(systemctl show -p ActiveState --value openvidu 2>/dev/null)\" in\n active|activating|reloading) ;;\n *) return 1 ;;\n esac\n since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null)\n case \"$since\" in ''|0|*[!0-9]*) return 1 ;; esac\n [ $(($(uptime_sec) - since / 1000000)) -lt \"$WARMUP_SEC\" ]\n}\n\n# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare\n# TCP connect would also succeed against a wedged Redis.\nmaster_serving() {\n [ -n \"$MASTER_ADDRS\" ] || return 1\n for addr in $MASTER_ADDRS; do\n host=$(echo \"$addr\" | cut -d: -f1)\n port=$(echo \"$addr\" | cut -d: -f2)\n reply=$(timeout 5 bash -c 'exec 3<>\"/dev/tcp/$0/$1\" && printf \"PING\\r\\n\" >&3 && head -c 7 <&3' \"$host\" \"$port\" 2>/dev/null)\n case \"$reply\" in\n +PONG*|-NOAUTH*) return 0 ;;\n esac\n done\n return 1\n}\n\n# Request the replacement of this node ($1 = reason). Gated on a master answering, except\n# in the bootstrap phase before the master addresses are known.\nrequest_replacement() {\n now=$(uptime_sec)\n if [ -n \"$REQUESTED_AT\" ] && [ $((now - REQUESTED_AT)) -lt \"$REPLACE_RETRY_SEC\" ]; then\n return\n fi\n if [ -n \"$MASTER_ADDRS\" ] || [ -f \"$INSTALLED_MARKER\" ]; then\n if ! master_serving; then\n if [ -z \"$HOLD_LOGGED_AT\" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then\n log \"$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node\"\n HOLD_LOGGED_AT=$now\n fi\n return\n fi\n fi\n log \"$1: requesting the replacement of this node\"\n if \"$REPLACE_SCRIPT\" \"$1\"; then\n REQUESTED_AT=$now\n log \"replacement requested\"\n else\n log \"the replacement request failed, retrying in $INTERVAL_SEC s\"\n fi\n}\n\nmkdir -p \"$STATE_DIR\"\nREQUESTED_AT=\"\"\nHOLD_LOGGED_AT=\"\"\nFAILED_SEEN_AT=\"\"\nFAIL_SINCE=\"\"\nSATURATED=\"\"\nPREV_TOTAL=\"\"\nPREV_IDLE=\"\"\nlog \"started\"\n\nwhile true; do\n PROBE_URL=http://127.0.0.1:7880/\n INTERVAL_SEC=30\n HARD_UNHEALTHY_AFTER_SEC=300\n SOFT_UNHEALTHY_AFTER_SEC=600\n CPU_SATURATED_PCT=90\n WARMUP_SEC=900\n BOOTSTRAP_DEADLINE_SEC=5400\n BOOTSTRAP_FAILURE_COOLDOWN_SEC=600\n REPLACE_RETRY_SEC=600\n MASTER_ADDRS=\"\"\n [ -f \"$ENV_FILE\" ] && . \"$ENV_FILE\"\n NOW=$(uptime_sec)\n cpu_busy\n\n if [ -f \"$PAUSE_FILE\" ] || [ -f \"$DRAIN_PAUSE_FILE\" ]; then\n FAIL_SINCE=\"\"\n elif [ -f \"$BOOTSTRAP_FAILED_MARKER\" ]; then\n if [ -z \"$FAILED_SEEN_AT\" ]; then\n FAILED_SEEN_AT=$NOW\n log \"bootstrap failed ($(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s\"\n fi\n if [ $((NOW - FAILED_SEEN_AT)) -ge \"$BOOTSTRAP_FAILURE_COOLDOWN_SEC\" ]; then\n request_replacement \"bootstrap failed: $(head -c 300 \"$BOOTSTRAP_FAILED_MARKER\")\"\n fi\n elif [ ! -f \"$INSTALLED_MARKER\" ]; then\n if [ \"$NOW\" -ge \"$BOOTSTRAP_DEADLINE_SEC\" ]; then\n request_replacement \"bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot\"\n fi\n elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 \"$PROBE_URL\"; then\n [ -n \"$FAIL_SINCE\" ] && log \"LiveKit healthy again after $((NOW - FAIL_SINCE)) s\"\n FAIL_SINCE=\"\"\n SATURATED=\"\"\n else\n RC=$?\n case \"$RC\" in\n 7|52|56) KIND=hard ;;\n *) KIND=soft ;;\n esac\n if openvidu_starting; then\n FAIL_SINCE=\"\"\n elif ! master_serving; then\n # A master outage fails the health check of every Media Node at once: not counted,\n # so LiveKit gets the whole window to reconnect once a master answers again\n FAIL_SINCE=\"\"\n if [ -z \"$HOLD_LOGGED_AT\" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then\n log \"LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted\"\n HOLD_LOGGED_AT=$NOW\n fi\n elif [ \"$KIND\" = soft ] && [ \"$BUSY\" -ge \"$CPU_SATURATED_PCT\" ]; then\n [ -n \"$SATURATED\" ] || log \"LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted\"\n SATURATED=1\n FAIL_SINCE=\"\"\n else\n SATURATED=\"\"\n if [ -z \"$FAIL_SINCE\" ]; then\n FAIL_SINCE=$NOW\n log \"LiveKit health check failed (curl exit $RC)\"\n fi\n LIMIT=$SOFT_UNHEALTHY_AFTER_SEC\n [ \"$KIND\" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC\n if [ $((NOW - FAIL_SINCE)) -ge \"$LIMIT\" ]; then\n request_replacement \"LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)\"\n fi\n fi\n fi\n sleep \"$INTERVAL_SEC\"\ndone\n", "openviduMediaHealthUnit": "[Unit]\nDescription=OpenVidu Media Node health watchdog\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/local/bin/openvidu-media-health.sh\nRestart=always\nRestartSec=30\n\n[Install]\nWantedBy=multi-user.target\n", - "media_replaceScriptMediaTemplate": "#!/bin/bash\n# openvidu-media-replace.sh \"\"\n# Requests the reimage of THIS Media Node, keeping the scale set capacity and the\n# scale-in protection: reimage provisions a new OS disk and the userData bootstrap runs\n# again on the same instance id. Called by the health watchdog; it never signals it.\n\nmkdir -p /var/lib/openvidu-media-health\necho \"$(date -u +%FT%TZ) $1\" >> /var/lib/openvidu-media-health/replace-requests.log\ntouch /run/openvidu-unhealthy\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nINSTANCE_RESOURCE_ID=$(curl -s -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo \"$INSTANCE_RESOURCE_ID\" | awk -F'/' '{print $NF}')\nif [ -z \"$INSTANCE_ID\" ]; then\n echo \"[media-replace] could not read this instance id from IMDS\"\n exit 1\nfi\n\nn=0\nwhile true; do\n n=$((n + 1))\n if az login --identity --client-id ${mediaIdentityClientId} >/dev/null 2>&1 && \\\n az vmss reimage --resource-group \"$RESOURCE_GROUP_NAME\" --name \"$VM_SCALE_SET_NAME\" --instance-ids \"$INSTANCE_ID\" --no-wait; then\n echo \"[media-replace] reimage requested for instance $INSTANCE_ID ($1)\"\n exit 0\n fi\n if [ \"$n\" -ge 5 ]; then\n echo \"[media-replace] reimage could not be requested after $n attempts\"\n exit 1\n fi\n echo \"[media-replace] reimage request failed (attempt $n), retrying in 12s\"\n sleep 12\ndone\n", + "media_replaceScriptMediaTemplate": "#!/bin/bash\n# openvidu-media-replace.sh \"\"\n# Requests the reimage of THIS Media Node, keeping the scale set capacity and the\n# scale-in protection: reimage provisions a new OS disk and the userData bootstrap runs\n# again on the same instance id. Called by the health watchdog; it never signals it.\n\nmkdir -p /var/lib/openvidu-media-health\necho \"$(date -u +%FT%TZ) $1\" >> /var/lib/openvidu-media-health/replace-requests.log\ntouch /run/openvidu-unhealthy\n# Withdrawn if the request fails: the node then stays, and if it recovers\n# a later scale-in must still wait for its sessions\ntrap '[ $? -eq 0 ] || rm -f /run/openvidu-unhealthy' EXIT\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nINSTANCE_RESOURCE_ID=$(curl -s -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo \"$INSTANCE_RESOURCE_ID\" | awk -F'/' '{print $NF}')\nif [ -z \"$INSTANCE_ID\" ]; then\n echo \"[media-replace] could not read this instance id from IMDS\"\n exit 1\nfi\n\nn=0\nwhile true; do\n n=$((n + 1))\n if az login --identity --client-id ${mediaIdentityClientId} >/dev/null 2>&1 && \\\n az vmss reimage --resource-group \"$RESOURCE_GROUP_NAME\" --name \"$VM_SCALE_SET_NAME\" --instance-ids \"$INSTANCE_ID\" --no-wait; then\n echo \"[media-replace] reimage requested for instance $INSTANCE_ID ($1)\"\n exit 0\n fi\n if [ \"$n\" -ge 5 ]; then\n echo \"[media-replace] reimage could not be requested after $n attempts\"\n exit 1\n fi\n echo \"[media-replace] reimage request failed (attempt $n), retrying in 12s\"\n sleep 12\ndone\n", "userDataMediaNodeTemplate": "#!/bin/bash -x\nset -eu -o pipefail\n\n# Mark this media node's bootstrap as failed so the health watchdog replaces it\nov_media_bootstrap_failed() {\n echo \"[OpenVidu] media node bootstrap failed: $1\"\n mkdir -p /var/lib/openvidu-media-health\n echo \"$1\" > /var/lib/openvidu-media-health/bootstrap-failed\n exit 1\n}\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# stop_media_nodes.sh\necho ${base64stop} | base64 -d > /usr/local/bin/stop_media_node.sh\nchmod +x /usr/local/bin/stop_media_node.sh\n\n# openvidu-media-health.sh (health watchdog), openvidu-media-replace.sh (reimage this node) and its unit\necho ${base64mediaHealth} | base64 -d > /usr/local/bin/openvidu-media-health.sh\nchmod +x /usr/local/bin/openvidu-media-health.sh\necho ${base64mediaReplace} | base64 -d > /usr/local/bin/openvidu-media-replace.sh\nchmod +x /usr/local/bin/openvidu-media-replace.sh\necho ${base64mediaHealthUnit} | base64 -d > /etc/systemd/system/openvidu-media-health.service\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y jq\n\n# Install azure cli\nAZURE_CLI_VERSION=2.90.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity --client-id ${mediaIdentityClientId}\n\n# Protect from scale in actions\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\naz vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-id $INSTANCE_ID --protect-from-scale-in true\n\nexport HOME=\"/root\"\n\n# Start the Media Node health watchdog before install.sh (az CLI is installed and the node\n# is scale-in protected): it backstops a hung or failed bootstrap and, once running, reimages\n# the node if LiveKit stays unhealthy. Write MASTER_ADDRS first so its master guard is armed.\nmkdir -p /etc/openvidu\necho 'MASTER_ADDRS=\"${privateIPMasterNode1}:7001 ${privateIPMasterNode2}:7001 ${privateIPMasterNode3}:7001 ${privateIPMasterNode4}:7001\"' > /etc/openvidu/media-health.env\nsystemctl daemon-reload\nsystemctl enable --now openvidu-media-health\n\n# Install OpenVidu\n/usr/local/bin/install.sh || ov_media_bootstrap_failed \"install.sh failed\"\n\n# Gate 2: wait for master readiness before starting\nWAIT_INTERVAL=5\nMAX_RETRIES=360\nRETRIES=0\nset +e\nwhile true; do\n FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv 2>/dev/null)\n if [ \"$FINISH_MASTER_NODE\" == \"true\" ]; then\n break\n fi\n RETRIES=$((RETRIES + 1))\n if [ $RETRIES -ge $MAX_RETRIES ]; then\n ov_media_bootstrap_failed \"timed out after 30 min waiting for FINISH-MASTER-NODE\"\n fi\n sleep $WAIT_INTERVAL\ndone\nset -e\n\n# Start OpenVidu\nsystemctl start openvidu || ov_media_bootstrap_failed \"systemctl start openvidu failed\"\n\n# Bootstrap finished: the watchdog switches from bootstrap-deadline mode to runtime health checks\nmkdir -p /var/lib/openvidu-media-health\ntouch /var/lib/openvidu-media-health/installed\n", "base64openviduMediaHealth": "[base64(variables('openviduMediaHealthScript'))]", "base64openviduMediaHealthUnit": "[base64(variables('openviduMediaHealthUnit'))]", diff --git a/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf b/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf index 64f4f960c..67dbdbe38 100644 --- a/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf +++ b/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf @@ -1994,6 +1994,9 @@ mkdir -p /var/lib/openvidu-media-health echo "$(date -u +%FT%TZ) $REASON" >> /var/lib/openvidu-media-health/replace-requests.log # Tell the drain script this node is being replaced so it skips waiting for sessions. touch /run/openvidu-unhealthy +# Withdrawn if the request fails: the node then stays, and if it recovers +# a later scale-in must still wait for its sessions +trap '[ $? -eq 0 ] || rm -f /run/openvidu-unhealthy' EXIT META="http://metadata.google.internal/computeMetadata/v1" meta() { curl -s -H "Metadata-Flavor: Google" "$META/$1"; }