From 1794e7e16b80496125fd39e0a44122fc6d6e9484 Mon Sep 17 00:00:00 2001 From: Piwccle Date: Fri, 24 Jul 2026 13:42:09 +0200 Subject: [PATCH 1/6] openvidu-deployment: optimize GCP HA deployment time Parallel master node creation (drop chained depends_on), static internal IPs for masters passed via metadata (removes the runtime IP exchange through Secret Manager and its unbounded wait), bounded timeouts on ALL_SECRETS_GENERATED/check_app_ready polls, robust installer fetch (curl --retry to file instead of sh <(curl)), media node reboot guard, apt cleanup and depends_on on enabled APIs. Validated with ov-cloud-tester (sc-deploy-destroy, ha, dev): PASS. deploy 1m54s (baseline 3m30s), ready 6m26s, destroy 3m13s. --- .../ha/gcp/DOC-CHANGES-ha-optimizations.md | 144 ++++++++++ .../pro/ha/gcp/tf-gpc-openvidu-ha.tf | 246 +++++++++++------- 2 files changed, 299 insertions(+), 91 deletions(-) create mode 100644 openvidu-deployment/pro/ha/gcp/DOC-CHANGES-ha-optimizations.md diff --git a/openvidu-deployment/pro/ha/gcp/DOC-CHANGES-ha-optimizations.md b/openvidu-deployment/pro/ha/gcp/DOC-CHANGES-ha-optimizations.md new file mode 100644 index 000000000..e73e2c6f8 --- /dev/null +++ b/openvidu-deployment/pro/ha/gcp/DOC-CHANGES-ha-optimizations.md @@ -0,0 +1,144 @@ +# Cambios de documentación para `ha-optimizations` (despliegue HA GCP) + +Instrucciones para el Claude que trabaje en el repo de docs +`/home/sergio/Escritorio/openvidu/openvidu.io` (rama `next`, ya actualizada). + +Estas instrucciones derivan de los cambios aplicados en +`openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf` en la rama `ha-optimizations` +(paralelización de los 4 master nodes, IPs internas estáticas para los masters, +timeouts en las esperas, fetch robusto del instalador y `depends_on` hacia las APIs). + +RESUMEN: el ÚNICO cambio de contenido necesario son las dos cifras de tiempo de +despliegue en el `install.md` del HA de GCP. Todo lo demás (parámetros públicos, +capturas, otros clouds, otros deployment types, ficheros de config on-node) se +mantiene igual. Léelo entero antes de tocar nada. + +--- + +## (a) Cifras de tiempo a actualizar — OBLIGATORIO + +Fichero: `docs/docs/self-hosting/ha/gcp/install.md` + +Hay DOS ocurrencias de "7 to 12 minutes" en este fichero (aprox. líneas 249 y 308). +Localízalas por el texto literal, no confíes solo en el número de línea (puede haberse +desplazado). Sustituye la cifra por el placeholder y deja una nota para rellenarla con +la medición real de `ov-cloud-tester` tras estos cambios (el despliegue ahora es más +rápido: masters en paralelo y sin el handshake de IPs por Secret Manager). + +### Ocurrencia 1 (sección "## Deploying the stack", ~línea 249) + +Texto actual literal: + +``` +When you are satisfied with your input values, click _"Continue"_ and then _"Create deployment"_. The deployment will be validated and all resources will be created. Wait around 7 to 12 minutes for the nodes to install OpenVidu. +``` + +Texto propuesto (rellenar `X`/`Y` con la medición de `ov-cloud-tester`): + +``` +When you are satisfied with your input values, click _"Continue"_ and then _"Create deployment"_. The deployment will be validated and all resources will be created. Wait around X to Y minutes for the nodes to install OpenVidu. +``` + +### Ocurrencia 2 (sección "## Checking...", ~línea 308) + +Texto actual literal: + +``` +When your Google Cloud Platform deployment reaches the **`Active`** state, it means that all resources have been created. You will need to wait about 7 to 12 minutes for the instances to install OpenVidu, as mentioned before. After this time, try connecting to the deployment URL. If it doesn't work, we recommend checking the previous section. Once everything is ready, you can check the [Administration](./admin.md) section to learn how to manage your deployment. +``` + +Texto propuesto (rellenar `X`/`Y` con la medición de `ov-cloud-tester`): + +``` +When your Google Cloud Platform deployment reaches the **`Active`** state, it means that all resources have been created. You will need to wait about X to Y minutes for the instances to install OpenVidu, as mentioned before. After this time, try connecting to the deployment URL. If it doesn't work, we recommend checking the previous section. Once everything is ready, you can check the [Administration](./admin.md) section to learn how to manage your deployment. +``` + +> Placeholder a rellenar: `X to Y minutes — rellenar con la medición de ov-cloud-tester`. +> Mantén las dos ocurrencias con la MISMA cifra (la segunda dice "as mentioned before"). + +### NO tocar las cifras de otros ficheros + +Estos cambios afectan SOLO al HA de GCP. NO modifiques las cifras "7 to 12 minutes" +que aparecen en otros documentos (son otros clouds / otros deployment types y están +fuera del alcance de este trabajo): + +- `docs/docs/self-hosting/elastic/gcp/install.md` (líneas ~236 y ~295) +- `docs/docs/self-hosting/elastic/azure/install.md` (líneas ~95 y ~149) +- Cualquier otro `install.md` de la doc. + +--- + +## (b) Parámetros públicos de la plantilla GCP: SIN CAMBIOS — confirmación + +NO hay que tocar tablas de parámetros ni capturas de la sección de parámetros. + +Motivo: las nuevas IPs estáticas de los master nodes son **internas** +(`google_compute_address` con `address_type = "INTERNAL"`, reservadas dentro de la +subred `default` de la región) y se asignan vía `network_interface.network_ip`. No se +ha añadido, eliminado ni renombrado ninguna variable de entrada ni salida de Terraform +(`variables.tf` y `output.tf` intactos). Por tanto: + +- La tabla "Mandatory Parameters" y "Optional Parameters" de + `docs/docs/self-hosting/ha/gcp/install.md` (aprox. líneas 95–230) **se queda igual**. +- No hay parámetros nuevos que el usuario deba introducir en Infrastructure Manager. +- Las capturas del formulario de parámetros **no cambian**. + +--- + +## (c) Otras afirmaciones de la doc GCP HA que puedan quedar obsoletas + +Se ha revisado toda la doc del HA de GCP (`docs/docs/self-hosting/ha/gcp/*.md`) y los +includes compartidos. Conclusiones: + +### c.1 — Secretos `MASTER_NODE_{1..4}_PRIVATE_IP` de Secret Manager (ELIMINADOS) + +En `ha-optimizations` se han eliminado del despliegue los 4 secretos de Secret Manager +`MASTER_NODE_1_PRIVATE_IP` … `MASTER_NODE_4_PRIVATE_IP` (ya no se generan: las IPs de +los masters son ahora estáticas y se pasan por metadata). La doc del HA de GCP **no los +menciona por nombre en el texto**, así que no hay texto que corregir. + +- Único impacto posible: la captura del Secret Manager en + `install.md` (sección "Check deployment outputs in GCP Secret Manager"), + imagen `assets/images/platform/self-hosting/shared/gcp/secrets-manager.png`. + Esa captura PODRÍA mostrar los 4 secretos de IP que ya no existirán. Es puramente + cosmético (el texto solo dice "you will see all the secrets by their name"). Acción + OPCIONAL: si se regenera la captura en algún momento, hacerlo tras un despliegue con + `ha-optimizations`. NO es bloqueante para este cambio. + +### c.2 — `backup-and-restore.md`: NO TOCAR + +`docs/docs/self-hosting/how-to-guides/backup-and-restore.md` (aprox. líneas 676–692) +documenta variables `MASTER_NODE_1_PRIVATE_IP` … `MASTER_NODE_4_PRIVATE_IP` dentro del +fichero de configuración on-node `/opt/openvidu/config/node/master-node.env`. + +IMPORTANTE: **esas NO son los secretos eliminados**. Son variables del fichero de +configuración que el instalador sigue escribiendo a partir del flag +`--master-node-private-ip-list` (que se sigue pasando, ahora derivado de las IPs +estáticas). Ese flujo no cambia. **No modifiques `backup-and-restore.md`.** + +### c.3 — `upgrade.md`: NO TOCAR + +`docs/docs/self-hosting/ha/gcp/upgrade.md` (línea ~91) menciona `store_secret.sh` y el +secreto `OPENVIDU_VERSION`. Ni el script `store_secret.sh` ni el secreto +`OPENVIDU_VERSION` se han tocado. **No requiere cambios.** + +### c.4 — Flujo interno (ALL_SECRETS_GENERATED, handshake de IPs): no documentado + +El mecanismo interno de coordinación (líder master-1 genera secretos y marca +`ALL_SECRETS_GENERATED`, polling, etc.) no está documentado públicamente, y su cambio +(eliminación del intercambio de IPs por Secret Manager) no altera nada de cara al +usuario. No hay texto que actualizar. + +--- + +## Checklist de aplicación + +- [ ] En `docs/docs/self-hosting/ha/gcp/install.md`, sustituir "7 to 12 minutes" por + "X to Y minutes" en las DOS ocurrencias (~249 y ~308), con la misma cifra en ambas. +- [ ] Medir el tiempo real de despliegue con `ov-cloud-tester` sobre `ha-optimizations` + y rellenar `X`/`Y`. +- [ ] Verificar que NO se han tocado tablas de parámetros ni capturas. +- [ ] Verificar que NO se han tocado `backup-and-restore.md`, `upgrade.md`, ni las + cifras de tiempo de elastic/azure/otros clouds. +- [ ] (Opcional) Regenerar `secrets-manager.png` si se desea reflejar la ausencia de los + 4 secretos de IP. diff --git a/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf b/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf index 781fc7472..a01c72442 100644 --- a/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf +++ b/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf @@ -20,14 +20,15 @@ resource "google_secret_manager_secret" "openvidu_shared_info" { "OPENVIDU_PRO_LICENSE", "OPENVIDU_RTC_ENGINE", "REDIS_PASSWORD", "MONGO_ADMIN_USERNAME", "MONGO_ADMIN_PASSWORD", "MONGO_REPLICA_SET_KEY", "MINIO_ACCESS_KEY", "MINIO_SECRET_KEY", "DASHBOARD_ADMIN_USERNAME", "DASHBOARD_ADMIN_PASSWORD", "GRAFANA_ADMIN_USERNAME", - "GRAFANA_ADMIN_PASSWORD", "ENABLED_MODULES", "OPENVIDU_VERSION", "ALL_SECRETS_GENERATED", - "MASTER_NODE_1_PRIVATE_IP", "MASTER_NODE_2_PRIVATE_IP", "MASTER_NODE_3_PRIVATE_IP", "MASTER_NODE_4_PRIVATE_IP" + "GRAFANA_ADMIN_PASSWORD", "ENABLED_MODULES", "OPENVIDU_VERSION", "ALL_SECRETS_GENERATED" ]) secret_id = each.key replication { auto {} } + + depends_on = [google_project_service.secretmanager_api] } # GCS buckets for HA deployment @@ -37,6 +38,8 @@ resource "google_storage_bucket" "appdata_bucket" { location = var.region force_destroy = true uniform_bucket_level_access = true + + depends_on = [google_project_service.storage_api] } resource "google_storage_bucket" "clusterdata_bucket" { @@ -45,12 +48,16 @@ resource "google_storage_bucket" "clusterdata_bucket" { location = var.region force_destroy = true uniform_bucket_level_access = true + + depends_on = [google_project_service.storage_api] } # Service account for the instances resource "google_service_account" "service_account" { account_id = lower("${substr(var.stackName, 0, 12)}-sa") display_name = "OpenVidu instance service account" + + depends_on = [google_project_service.iam_api] } # IAM bindings for the service account @@ -58,6 +65,8 @@ resource "google_project_iam_member" "iam_project_role" { project = var.projectId role = "roles/owner" member = "serviceAccount:${google_service_account.service_account.email}" + + depends_on = [google_project_service.cloudresourcemanager_api] } # External SSH access to Master Nodes @@ -72,6 +81,8 @@ resource "google_compute_firewall" "external_master_ssh" { source_ranges = ["0.0.0.0/0"] target_tags = [lower("${var.stackName}-master-node")] + + depends_on = [google_project_service.compute_api] } # External access to Media Nodes (SSH and media traffic) @@ -90,6 +101,8 @@ resource "google_compute_firewall" "external_media_access" { source_ranges = ["0.0.0.0/0"] target_tags = [lower("${var.stackName}-media-node")] + + depends_on = [google_project_service.compute_api] } # Load Balancer health checks and HTTP traffic to Master Nodes @@ -104,6 +117,8 @@ resource "google_compute_firewall" "lb_to_master_http" { source_ranges = ["0.0.0.0/0"] target_tags = [lower("${var.stackName}-master-node")] + + depends_on = [google_project_service.compute_api] } # Master node internal services communication @@ -129,6 +144,8 @@ resource "google_compute_firewall" "master_to_master_internal" { source_tags = [lower("${var.stackName}-master-node")] target_tags = [lower("${var.stackName}-master-node")] + + depends_on = [google_project_service.compute_api] } # Media Nodes to Master Nodes communication @@ -153,6 +170,8 @@ resource "google_compute_firewall" "media_to_master_services" { source_tags = [lower("${var.stackName}-media-node")] target_tags = [lower("${var.stackName}-master-node")] + + depends_on = [google_project_service.compute_api] } # Master Nodes to Media Nodes communication @@ -171,6 +190,8 @@ resource "google_compute_firewall" "master_to_media_services" { source_tags = [lower("${var.stackName}-master-node")] target_tags = [lower("${var.stackName}-media-node")] + + depends_on = [google_project_service.compute_api] } @@ -179,6 +200,8 @@ resource "google_compute_address" "nlb_ip" { count = var.publicIpAddress == "" ? 1 : 0 name = lower("${var.stackName}-nlb-ip") region = var.region + + depends_on = [google_project_service.compute_api] } # Data source for existing IP address when publicIpAddress is provided @@ -206,6 +229,8 @@ resource "google_compute_region_health_check" "tcp_health_check" { timeout_sec = 5 healthy_threshold = 3 unhealthy_threshold = 4 + + depends_on = [google_project_service.compute_api] } # Regional backend service for the TCP NLB @@ -289,6 +314,34 @@ locals { is_c4a_instance = startswith(var.masterNodesInstanceType, "c4a-") } +# Default subnetwork the master instances live in (they use network = "default" with no +# explicit subnetwork, so GCP places them in the auto-mode default subnetwork of the region) +data "google_compute_subnetwork" "default" { + name = "default" + region = var.region + + depends_on = [google_project_service.compute_api] +} + +# Static internal IPs for the master nodes. Reserving them up front lets the media instance +# template and every master receive the full list without waiting for the master instances to +# boot, which removes the runtime IP-exchange handshake through Secret Manager. +resource "google_compute_address" "master_internal_ip" { + count = 4 + name = lower("${var.stackName}-master-node-${count.index + 1}-internal-ip") + address_type = "INTERNAL" + region = var.region + subnetwork = data.google_compute_subnetwork.default.id + + depends_on = [google_project_service.compute_api] +} + +locals { + # Master node private IPs come from the reserved static internal addresses (index 0 -> master + # node 1, ...). Comma-separated to match the installer's --master-node-private-ip-list format. + master_node_private_ip_list = join(",", google_compute_address.master_internal_ip[*].address) +} + # Master Node 1 resource "google_compute_instance" "openvidu_master_node_1" { name = lower("${var.stackName}-master-node-1") @@ -305,13 +358,15 @@ resource "google_compute_instance" "openvidu_master_node_1" { } network_interface { - network = "default" + network = "default" + network_ip = google_compute_address.master_internal_ip[0].address access_config {} } metadata = { stackName = var.stackName masterNodeNum = "1" + masterNodePrivateIPList = local.master_node_private_ip_list domainName = var.domainName certificateType = var.certificateType ownPublicCertificate = var.ownPublicCertificate @@ -355,13 +410,15 @@ resource "google_compute_instance" "openvidu_master_node_2" { } network_interface { - network = "default" + network = "default" + network_ip = google_compute_address.master_internal_ip[1].address access_config {} } metadata = { stackName = var.stackName masterNodeNum = "2" + masterNodePrivateIPList = local.master_node_private_ip_list domainName = var.domainName certificateType = var.certificateType ownPublicCertificate = var.ownPublicCertificate @@ -387,8 +444,6 @@ resource "google_compute_instance" "openvidu_master_node_2" { node-type = "master" node-num = "2" } - - depends_on = [google_compute_instance.openvidu_master_node_1] } # Master Node 3 @@ -407,13 +462,15 @@ resource "google_compute_instance" "openvidu_master_node_3" { } network_interface { - network = "default" + network = "default" + network_ip = google_compute_address.master_internal_ip[2].address access_config {} } metadata = { stackName = var.stackName masterNodeNum = "3" + masterNodePrivateIPList = local.master_node_private_ip_list domainName = var.domainName certificateType = var.certificateType ownPublicCertificate = var.ownPublicCertificate @@ -439,8 +496,6 @@ resource "google_compute_instance" "openvidu_master_node_3" { node-type = "master" node-num = "3" } - - depends_on = [google_compute_instance.openvidu_master_node_2] } # Master Node 4 @@ -459,13 +514,15 @@ resource "google_compute_instance" "openvidu_master_node_4" { } network_interface { - network = "default" + network = "default" + network_ip = google_compute_address.master_internal_ip[3].address access_config {} } metadata = { stackName = var.stackName masterNodeNum = "4" + masterNodePrivateIPList = local.master_node_private_ip_list domainName = var.domainName certificateType = var.certificateType ownPublicCertificate = var.ownPublicCertificate @@ -491,8 +548,6 @@ resource "google_compute_instance" "openvidu_master_node_4" { node-type = "master" node-num = "4" } - - depends_on = [google_compute_instance.openvidu_master_node_3] } # ------------------------- scale in resources ------------------------- @@ -800,6 +855,12 @@ resource "google_cloudfunctions2_function" "scalein_function" { } service_account_email = google_service_account.service_account.email } + + depends_on = [ + google_project_service.cloudfunctions_api, + google_project_service.cloudbuild_api, + google_project_service.run_api + ] } # Cloud Scheduler to trigger the function every 5 minutes @@ -829,6 +890,8 @@ resource "google_cloud_scheduler_job" "scale_scheduler" { service_account_email = google_service_account.service_account.email } } + + depends_on = [google_project_service.cloudscheduler_api] } locals { @@ -858,7 +921,7 @@ resource "google_compute_instance_template" "media_node_template" { metadata = { stackName = var.stackName - masterNodePrivateIPList = "${google_compute_instance.openvidu_master_node_1.network_interface[0].network_ip},${google_compute_instance.openvidu_master_node_2.network_interface[0].network_ip},${google_compute_instance.openvidu_master_node_3.network_interface[0].network_ip},${google_compute_instance.openvidu_master_node_4.network_interface[0].network_ip}" + masterNodePrivateIPList = local.master_node_private_ip_list bucketAppDataName = local.isEmptyAppData ? google_storage_bucket.appdata_bucket[0].name : var.GCSAppDataBucketName bucketClusterDataName = local.isEmptyClusterData ? google_storage_bucket.clusterdata_bucket[0].name : var.GCSClusterDataBucketName region = var.region @@ -880,13 +943,6 @@ resource "google_compute_instance_template" "media_node_template" { lifecycle { create_before_destroy = true } - - depends_on = [ - google_compute_instance.openvidu_master_node_1, - google_compute_instance.openvidu_master_node_2, - google_compute_instance.openvidu_master_node_3, - google_compute_instance.openvidu_master_node_4 - ] } # Managed Instance Group for Media Nodes @@ -904,13 +960,6 @@ resource "google_compute_region_instance_group_manager" "media_node_group" { name = "http" port = 7880 } - - depends_on = [ - google_compute_instance.openvidu_master_node_1, - google_compute_instance.openvidu_master_node_2, - google_compute_instance.openvidu_master_node_3, - google_compute_instance.openvidu_master_node_4 - ] } # Autoscaler for Media Nodes @@ -969,11 +1018,8 @@ get_meta() { curl -s -H "Metadata-Flavor: Google" "$${METADATA_URL}/$1"; } # Get master node number from metadata MASTER_NODE_NUM=$(get_meta "instance/attributes/masterNodeNum") -# Get own private IP -PRIVATE_IP=$(get_meta "instance/network-interfaces/0/ip") - -# Store current private IP -PRIVATE_IP="$(/usr/local/bin/store_secret.sh save MASTER_NODE_$${MASTER_NODE_NUM}_PRIVATE_IP $PRIVATE_IP)" +# Get the list of master node private IPs (static internal IPs passed via metadata) +MASTER_NODE_PRIVATE_IP_LIST=$(get_meta "instance/attributes/masterNodePrivateIPList") # Check if secrets have been generated ALL_SECRETS_GENERATED=$(gcloud secrets versions access latest --secret=ALL_SECRETS_GENERATED 2>/dev/null || echo "false") @@ -1033,33 +1079,20 @@ if [[ $MASTER_NODE_NUM -eq 1 ]] && [[ "$ALL_SECRETS_GENERATED" == "false" ]]; th ALL_SECRETS_GENERATED="$(/usr/local/bin/store_secret.sh save ALL_SECRETS_GENERATED "true")" fi -# Wait for all master nodes to store their private IPs -while true; do - MASTER_NODE_1_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_1_PRIVATE_IP 2>/dev/null || echo "") - MASTER_NODE_2_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_2_PRIVATE_IP 2>/dev/null || echo "") - MASTER_NODE_3_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_3_PRIVATE_IP 2>/dev/null || echo "") - MASTER_NODE_4_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_4_PRIVATE_IP 2>/dev/null || echo "") - - # Check if all master nodes have stored their private IPs - if [[ "$MASTER_NODE_1_PRIVATE_IP" != "" ]] && - [[ "$MASTER_NODE_2_PRIVATE_IP" != "" ]] && - [[ "$MASTER_NODE_3_PRIVATE_IP" != "" ]] && - [[ "$MASTER_NODE_4_PRIVATE_IP" != "" ]]; then - break +# Wait for master-1 to finish generating all shared secrets before reading them. +# Bounded to 360 iterations x 5s = 30 minutes. +i=0 +while ! gcloud secrets versions access latest --secret=ALL_SECRETS_GENERATED 2>/dev/null | grep -q "true"; do + i=$((i + 1)) + if [ "$i" -ge 360 ]; then + echo "Timed out after 30 minutes waiting for master-1 to finish generating secrets" >&2 + exit 1 fi + echo "Waiting for master-1 to finish generating secrets..." sleep 5 done -# Wait for master-1 to finish generating all shared secrets before reading them -while ! gcloud secrets versions access latest --secret=ALL_SECRETS_GENERATED 2>/dev/null | grep -q "true"; do echo "Waiting for master-1 to finish generating secrets..."; sleep 5; done - # Fetch all values from Secret Manager -MASTER_NODE_1_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_1_PRIVATE_IP) -MASTER_NODE_2_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_2_PRIVATE_IP) -MASTER_NODE_3_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_3_PRIVATE_IP) -MASTER_NODE_4_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_4_PRIVATE_IP) -MASTER_NODE_PRIVATE_IP_LIST="$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP" - DOMAIN=$(gcloud secrets versions access latest --secret=DOMAIN_NAME) OPENVIDU_PRO_LICENSE=$(gcloud secrets versions access latest --secret=OPENVIDU_PRO_LICENSE) OPENVIDU_RTC_ENGINE=$(gcloud secrets versions access latest --secret=OPENVIDU_RTC_ENGINE) @@ -1082,8 +1115,15 @@ LIVEKIT_API_KEY=$(gcloud secrets versions access latest --secret=LIVEKIT_API_KEY LIVEKIT_API_SECRET=$(gcloud secrets versions access latest --secret=LIVEKIT_API_SECRET) ENABLED_MODULES=$(gcloud secrets versions access latest --secret=ENABLED_MODULES) -# Build install command -INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh)" +# Build install command. Download the installer to a file first: process substitution +# (sh <(curl ...)) would silently run an empty script and exit 0 on a transient curl failure. +INSTALLER_SCRIPT="/tmp/install_ov_master_node.sh" +curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh" +if [ ! -s "$INSTALLER_SCRIPT" ]; then + echo "Downloaded OpenVidu master node installer is empty or missing" >&2 + exit 1 +fi +INSTALL_COMMAND="sh $INSTALLER_SCRIPT" # Common arguments COMMON_ARGS=( @@ -1434,11 +1474,18 @@ EOF check_app_ready_script = <<-EOF #!/bin/bash +# Bounded to 240 iterations x 5s = 20 minutes. +i=0 while true; do HTTP_STATUS=$(curl -Ik http://localhost:7880/health/caddy 2>/dev/null | head -n1 | awk '{print $2}') if [ "$HTTP_STATUS" == "200" ]; then break fi + i=$((i + 1)) + if [ "$i" -ge 240 ]; then + echo "Timed out after 20 minutes waiting for OpenVidu to become ready" >&2 + exit 1 + fi sleep 5 done EOF @@ -1513,8 +1560,6 @@ CONFIG_S3_EOF echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout - apt-get update && apt-get install -y - GCLOUD_VERSION=573.0.0 # Install google cli if ! command -v gcloud >/dev/null 2>&1; then @@ -1578,8 +1623,15 @@ MASTER_NODE_PRIVATE_IP_LIST=$(get_meta "instance/attributes/masterNodePrivateIPL STACK_NAME=$(get_meta "instance/attributes/stackName") PRIVATE_IP=$(get_meta "instance/network-interfaces/0/ip") -# Wait for master nodes to be ready by checking secrets +# Wait for master nodes to be ready by checking secrets. +# Bounded to 180 iterations x 10s = 30 minutes. +i=0 while ! gcloud secrets versions access latest --secret=ALL_SECRETS_GENERATED 2>/dev/null | grep -q "true"; do + i=$((i + 1)) + if [ "$i" -ge 180 ]; then + echo "Timed out after 30 minutes waiting for master nodes to initialize secrets" >&2 + exit 1 + fi echo "Waiting for master nodes to initialize secrets..." sleep 10 done @@ -1597,8 +1649,16 @@ if [[ "$OPENVIDU_VERSION" == "none" ]]; then exit 1 fi -# Build install command for media node -INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh)" +# Build install command for media node. Download the installer to a file first: process +# substitution (sh <(curl ...)) would silently run an empty script and exit 0 on a transient +# curl failure. +INSTALLER_SCRIPT="/tmp/install_ov_media_node.sh" +curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh" +if [ ! -s "$INSTALLER_SCRIPT" ]; then + echo "Downloaded OpenVidu media node installer is empty or missing" >&2 + exit 1 +fi +INSTALL_COMMAND="sh $INSTALLER_SCRIPT" # Media node arguments COMMON_ARGS=( @@ -1698,51 +1758,55 @@ EOF #!/bin/bash -x set -eu -o pipefail -# Create scripts -cat > /usr/local/bin/install.sh << 'INSTALL_EOF' +# Check if installation already completed +if [ -f /usr/local/bin/openvidu_install_counter.txt ]; then + # Launch on reboot + systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; } +else + # Create scripts + cat > /usr/local/bin/install.sh << 'INSTALL_EOF' ${local.install_script_media} INSTALL_EOF -chmod +x /usr/local/bin/install.sh + chmod +x /usr/local/bin/install.sh -cat > /usr/local/bin/graceful_shutdown.sh << 'GRACEFUL_SHUTDOWN_EOF' + cat > /usr/local/bin/graceful_shutdown.sh << 'GRACEFUL_SHUTDOWN_EOF' ${local.graceful_shutdown_script} GRACEFUL_SHUTDOWN_EOF -chmod +x /usr/local/bin/graceful_shutdown.sh + chmod +x /usr/local/bin/graceful_shutdown.sh -echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout + echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout -apt-get update && apt-get install -y + GCLOUD_VERSION=573.0.0 + # Install google cli + if ! command -v gcloud >/dev/null 2>&1; then + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + apt-get update && apt-get install -y google-cloud-cli=$${GCLOUD_VERSION}-0 + fi -GCLOUD_VERSION=573.0.0 -# Install google cli -if ! command -v gcloud >/dev/null 2>&1; then - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg - echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list - apt-get update && apt-get install -y google-cloud-cli=$${GCLOUD_VERSION}-0 -fi + # Authenticate with gcloud using instance service account + gcloud auth activate-service-account --key-file=/dev/null 2>/dev/null || true + gcloud config set account $(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email" -H "Metadata-Flavor: Google") + gcloud config set project $(curl -s "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google") -# Authenticate with gcloud using instance service account -gcloud auth activate-service-account --key-file=/dev/null 2>/dev/null || true -gcloud config set account $(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email" -H "Metadata-Flavor: Google") -gcloud config set project $(curl -s "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google") + export HOME="/root" -export HOME="/root" + # Install OpenVidu Media Node + /usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu Media Node"; exit 1; } -# Install OpenVidu Media Node -/usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu Media Node"; exit 1; } + # Mark installation as complete + echo "installation_complete" > /usr/local/bin/openvidu_install_counter.txt -# Mark installation as complete -echo "installation_complete" > /usr/local/bin/openvidu_install_counter.txt + # Start OpenVidu + systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; } -# Start OpenVidu -systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; } - -# Add cron job to check if instance is abandoned every minute -cat > /usr/local/bin/check_abandoned.sh << 'CHECK_ABANDONED_EOF' + # Add cron job to check if instance is abandoned every minute + cat > /usr/local/bin/check_abandoned.sh << 'CHECK_ABANDONED_EOF' ${local.crontab_job_media} CHECK_ABANDONED_EOF -chmod +x /usr/local/bin/check_abandoned.sh + chmod +x /usr/local/bin/check_abandoned.sh -echo "*/1 * * * * /usr/local/bin/check_abandoned.sh > /var/log/openvidu-abandoned-check.log 2>&1" | crontab - + echo "*/1 * * * * /usr/local/bin/check_abandoned.sh > /var/log/openvidu-abandoned-check.log 2>&1" | crontab - +fi EOF } From 0a587eb33ee9fd77e4010f8e05ebd000d3795320 Mon Sep 17 00:00:00 2001 From: Piwccle Date: Fri, 24 Jul 2026 14:13:50 +0200 Subject: [PATCH 2/6] openvidu-deployment: translate GCP HA DOC-CHANGES to English --- .../ha/gcp/DOC-CHANGES-ha-optimizations.md | 173 +++++++++--------- 1 file changed, 87 insertions(+), 86 deletions(-) diff --git a/openvidu-deployment/pro/ha/gcp/DOC-CHANGES-ha-optimizations.md b/openvidu-deployment/pro/ha/gcp/DOC-CHANGES-ha-optimizations.md index e73e2c6f8..e9dd0d897 100644 --- a/openvidu-deployment/pro/ha/gcp/DOC-CHANGES-ha-optimizations.md +++ b/openvidu-deployment/pro/ha/gcp/DOC-CHANGES-ha-optimizations.md @@ -1,144 +1,145 @@ -# Cambios de documentación para `ha-optimizations` (despliegue HA GCP) +# Documentation changes for `ha-optimizations` (GCP HA deployment) -Instrucciones para el Claude que trabaje en el repo de docs -`/home/sergio/Escritorio/openvidu/openvidu.io` (rama `next`, ya actualizada). +Instructions for the Claude working on the docs repo +`/home/sergio/Escritorio/openvidu/openvidu.io` (branch `next`, already up to date). -Estas instrucciones derivan de los cambios aplicados en -`openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf` en la rama `ha-optimizations` -(paralelización de los 4 master nodes, IPs internas estáticas para los masters, -timeouts en las esperas, fetch robusto del instalador y `depends_on` hacia las APIs). +These instructions derive from the changes applied in +`openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf` on the `ha-optimizations` branch +(parallelization of the 4 master nodes, static internal IPs for the masters, +wait timeouts, robust installer fetch, and `depends_on` on the APIs). -RESUMEN: el ÚNICO cambio de contenido necesario son las dos cifras de tiempo de -despliegue en el `install.md` del HA de GCP. Todo lo demás (parámetros públicos, -capturas, otros clouds, otros deployment types, ficheros de config on-node) se -mantiene igual. Léelo entero antes de tocar nada. +SUMMARY: the ONLY content change required is the two deployment time figures in +GCP HA's `install.md`. Everything else (public parameters, screenshots, other clouds, +other deployment types, on-node config files) stays the same. Read it in full before +touching anything. --- -## (a) Cifras de tiempo a actualizar — OBLIGATORIO +## (a) Time figures to update — MANDATORY -Fichero: `docs/docs/self-hosting/ha/gcp/install.md` +File: `docs/docs/self-hosting/ha/gcp/install.md` -Hay DOS ocurrencias de "7 to 12 minutes" en este fichero (aprox. líneas 249 y 308). -Localízalas por el texto literal, no confíes solo en el número de línea (puede haberse -desplazado). Sustituye la cifra por el placeholder y deja una nota para rellenarla con -la medición real de `ov-cloud-tester` tras estos cambios (el despliegue ahora es más -rápido: masters en paralelo y sin el handshake de IPs por Secret Manager). +There are TWO occurrences of "7 to 12 minutes" in this file (approx. lines 249 and 308). +Locate them by the literal text; do not rely only on the line number (it may have +shifted). Replace the figure with the placeholder and leave a note to fill it in with +the actual measurement from `ov-cloud-tester` after these changes (the deployment is now +faster: masters run in parallel and there is no more IP handshake via Secret Manager). -### Ocurrencia 1 (sección "## Deploying the stack", ~línea 249) +### Occurrence 1 (section "## Deploying the stack", ~line 249) -Texto actual literal: +Current literal text: ``` When you are satisfied with your input values, click _"Continue"_ and then _"Create deployment"_. The deployment will be validated and all resources will be created. Wait around 7 to 12 minutes for the nodes to install OpenVidu. ``` -Texto propuesto (rellenar `X`/`Y` con la medición de `ov-cloud-tester`): +Proposed text (fill in `X`/`Y` with the measurement from `ov-cloud-tester`): ``` When you are satisfied with your input values, click _"Continue"_ and then _"Create deployment"_. The deployment will be validated and all resources will be created. Wait around X to Y minutes for the nodes to install OpenVidu. ``` -### Ocurrencia 2 (sección "## Checking...", ~línea 308) +### Occurrence 2 (section "## Checking...", ~line 308) -Texto actual literal: +Current literal text: ``` When your Google Cloud Platform deployment reaches the **`Active`** state, it means that all resources have been created. You will need to wait about 7 to 12 minutes for the instances to install OpenVidu, as mentioned before. After this time, try connecting to the deployment URL. If it doesn't work, we recommend checking the previous section. Once everything is ready, you can check the [Administration](./admin.md) section to learn how to manage your deployment. ``` -Texto propuesto (rellenar `X`/`Y` con la medición de `ov-cloud-tester`): +Proposed text (fill in `X`/`Y` with the measurement from `ov-cloud-tester`): ``` When your Google Cloud Platform deployment reaches the **`Active`** state, it means that all resources have been created. You will need to wait about X to Y minutes for the instances to install OpenVidu, as mentioned before. After this time, try connecting to the deployment URL. If it doesn't work, we recommend checking the previous section. Once everything is ready, you can check the [Administration](./admin.md) section to learn how to manage your deployment. ``` -> Placeholder a rellenar: `X to Y minutes — rellenar con la medición de ov-cloud-tester`. -> Mantén las dos ocurrencias con la MISMA cifra (la segunda dice "as mentioned before"). +> Placeholder to fill in: `X to Y minutes — fill in with the measurement from ov-cloud-tester`. +> Keep both occurrences with the SAME figure (the second one says "as mentioned before"). -### NO tocar las cifras de otros ficheros +### Do NOT touch the figures in other files -Estos cambios afectan SOLO al HA de GCP. NO modifiques las cifras "7 to 12 minutes" -que aparecen en otros documentos (son otros clouds / otros deployment types y están -fuera del alcance de este trabajo): +These changes affect ONLY GCP HA. Do NOT modify the "7 to 12 minutes" figures +that appear in other documents (they belong to other clouds / other deployment types and +are out of scope for this work): -- `docs/docs/self-hosting/elastic/gcp/install.md` (líneas ~236 y ~295) -- `docs/docs/self-hosting/elastic/azure/install.md` (líneas ~95 y ~149) -- Cualquier otro `install.md` de la doc. +- `docs/docs/self-hosting/elastic/gcp/install.md` (lines ~236 and ~295) +- `docs/docs/self-hosting/elastic/azure/install.md` (lines ~95 and ~149) +- Any other `install.md` in the docs. --- -## (b) Parámetros públicos de la plantilla GCP: SIN CAMBIOS — confirmación +## (b) GCP template public parameters: NO CHANGES — confirmation -NO hay que tocar tablas de parámetros ni capturas de la sección de parámetros. +Do not touch parameter tables or screenshots in the parameters section. -Motivo: las nuevas IPs estáticas de los master nodes son **internas** -(`google_compute_address` con `address_type = "INTERNAL"`, reservadas dentro de la -subred `default` de la región) y se asignan vía `network_interface.network_ip`. No se -ha añadido, eliminado ni renombrado ninguna variable de entrada ni salida de Terraform -(`variables.tf` y `output.tf` intactos). Por tanto: +Reason: the new static IPs of the master nodes are **internal** +(`google_compute_address` with `address_type = "INTERNAL"`, reserved within the +region's `default` subnet) and are assigned via `network_interface.network_ip`. No +Terraform input or output variable has been added, removed, or renamed +(`variables.tf` and `output.tf` unchanged). Therefore: -- La tabla "Mandatory Parameters" y "Optional Parameters" de - `docs/docs/self-hosting/ha/gcp/install.md` (aprox. líneas 95–230) **se queda igual**. -- No hay parámetros nuevos que el usuario deba introducir en Infrastructure Manager. -- Las capturas del formulario de parámetros **no cambian**. +- The "Mandatory Parameters" and "Optional Parameters" table in + `docs/docs/self-hosting/ha/gcp/install.md` (approx. lines 95–230) **stays the same**. +- There are no new parameters the user needs to enter in Infrastructure Manager. +- The parameter form screenshots **do not change**. --- -## (c) Otras afirmaciones de la doc GCP HA que puedan quedar obsoletas +## (c) Other statements in the GCP HA docs that may become outdated -Se ha revisado toda la doc del HA de GCP (`docs/docs/self-hosting/ha/gcp/*.md`) y los -includes compartidos. Conclusiones: +The entire GCP HA documentation (`docs/docs/self-hosting/ha/gcp/*.md`) and the shared +includes have been reviewed. Conclusions: -### c.1 — Secretos `MASTER_NODE_{1..4}_PRIVATE_IP` de Secret Manager (ELIMINADOS) +### c.1 — Secret Manager secrets `MASTER_NODE_{1..4}_PRIVATE_IP` (REMOVED) -En `ha-optimizations` se han eliminado del despliegue los 4 secretos de Secret Manager -`MASTER_NODE_1_PRIVATE_IP` … `MASTER_NODE_4_PRIVATE_IP` (ya no se generan: las IPs de -los masters son ahora estáticas y se pasan por metadata). La doc del HA de GCP **no los -menciona por nombre en el texto**, así que no hay texto que corregir. +In `ha-optimizations`, the 4 Secret Manager secrets +`MASTER_NODE_1_PRIVATE_IP` … `MASTER_NODE_4_PRIVATE_IP` have been removed from the +deployment (they are no longer generated: the masters' IPs are now static and are +passed via metadata). The GCP HA docs **do not mention them by name in the text**, so +there is no text to fix. -- Único impacto posible: la captura del Secret Manager en - `install.md` (sección "Check deployment outputs in GCP Secret Manager"), - imagen `assets/images/platform/self-hosting/shared/gcp/secrets-manager.png`. - Esa captura PODRÍA mostrar los 4 secretos de IP que ya no existirán. Es puramente - cosmético (el texto solo dice "you will see all the secrets by their name"). Acción - OPCIONAL: si se regenera la captura en algún momento, hacerlo tras un despliegue con - `ha-optimizations`. NO es bloqueante para este cambio. +- Only possible impact: the Secret Manager screenshot in + `install.md` (section "Check deployment outputs in GCP Secret Manager"), + image `assets/images/platform/self-hosting/shared/gcp/secrets-manager.png`. + That screenshot COULD show the 4 IP secrets that will no longer exist. It is purely + cosmetic (the text only says "you will see all the secrets by their name"). OPTIONAL + action: if the screenshot is regenerated at some point, do it after a deployment with + `ha-optimizations`. This is NOT blocking for this change. -### c.2 — `backup-and-restore.md`: NO TOCAR +### c.2 — `backup-and-restore.md`: DO NOT TOUCH -`docs/docs/self-hosting/how-to-guides/backup-and-restore.md` (aprox. líneas 676–692) -documenta variables `MASTER_NODE_1_PRIVATE_IP` … `MASTER_NODE_4_PRIVATE_IP` dentro del -fichero de configuración on-node `/opt/openvidu/config/node/master-node.env`. +`docs/docs/self-hosting/how-to-guides/backup-and-restore.md` (approx. lines 676–692) +documents the variables `MASTER_NODE_1_PRIVATE_IP` … `MASTER_NODE_4_PRIVATE_IP` inside +the on-node configuration file `/opt/openvidu/config/node/master-node.env`. -IMPORTANTE: **esas NO son los secretos eliminados**. Son variables del fichero de -configuración que el instalador sigue escribiendo a partir del flag -`--master-node-private-ip-list` (que se sigue pasando, ahora derivado de las IPs -estáticas). Ese flujo no cambia. **No modifiques `backup-and-restore.md`.** +IMPORTANT: **these are NOT the removed secrets**. They are configuration file +variables that the installer still writes based on the +`--master-node-private-ip-list` flag (which is still passed, now derived from the +static IPs). That flow does not change. **Do not modify `backup-and-restore.md`.** -### c.3 — `upgrade.md`: NO TOCAR +### c.3 — `upgrade.md`: DO NOT TOUCH -`docs/docs/self-hosting/ha/gcp/upgrade.md` (línea ~91) menciona `store_secret.sh` y el -secreto `OPENVIDU_VERSION`. Ni el script `store_secret.sh` ni el secreto -`OPENVIDU_VERSION` se han tocado. **No requiere cambios.** +`docs/docs/self-hosting/ha/gcp/upgrade.md` (line ~91) mentions `store_secret.sh` and +the `OPENVIDU_VERSION` secret. Neither the `store_secret.sh` script nor the +`OPENVIDU_VERSION` secret has been touched. **No changes required.** -### c.4 — Flujo interno (ALL_SECRETS_GENERATED, handshake de IPs): no documentado +### c.4 — Internal flow (ALL_SECRETS_GENERATED, IP handshake): not documented -El mecanismo interno de coordinación (líder master-1 genera secretos y marca -`ALL_SECRETS_GENERATED`, polling, etc.) no está documentado públicamente, y su cambio -(eliminación del intercambio de IPs por Secret Manager) no altera nada de cara al -usuario. No hay texto que actualizar. +The internal coordination mechanism (master-1 leader generates secrets and marks +`ALL_SECRETS_GENERATED`, polling, etc.) is not publicly documented, and its change +(removal of the IP exchange via Secret Manager) does not change anything user-facing. +There is no text to update. --- -## Checklist de aplicación +## Implementation checklist -- [ ] En `docs/docs/self-hosting/ha/gcp/install.md`, sustituir "7 to 12 minutes" por - "X to Y minutes" en las DOS ocurrencias (~249 y ~308), con la misma cifra en ambas. -- [ ] Medir el tiempo real de despliegue con `ov-cloud-tester` sobre `ha-optimizations` - y rellenar `X`/`Y`. -- [ ] Verificar que NO se han tocado tablas de parámetros ni capturas. -- [ ] Verificar que NO se han tocado `backup-and-restore.md`, `upgrade.md`, ni las - cifras de tiempo de elastic/azure/otros clouds. -- [ ] (Opcional) Regenerar `secrets-manager.png` si se desea reflejar la ausencia de los - 4 secretos de IP. +- [ ] In `docs/docs/self-hosting/ha/gcp/install.md`, replace "7 to 12 minutes" with + "X to Y minutes" in the TWO occurrences (~249 and ~308), using the same figure in both. +- [ ] Measure the actual deployment time with `ov-cloud-tester` on `ha-optimizations` + and fill in `X`/`Y`. +- [ ] Verify that NO parameter tables or screenshots have been touched. +- [ ] Verify that `backup-and-restore.md`, `upgrade.md`, and the elastic/azure/other + clouds time figures have NOT been touched. +- [ ] (Optional) Regenerate `secrets-manager.png` if you want to reflect the absence of + the 4 IP secrets. From 196b3f767f03be5f3eb67c80e0f04b9b5b58e5fa Mon Sep 17 00:00:00 2001 From: Piwccle Date: Fri, 24 Jul 2026 14:31:46 +0200 Subject: [PATCH 3/6] openvidu-deployment: optimize Azure HA deployment time Parallel master VM creation (drop chained dependsOn), publish FINISH-MASTER-NODE from the first healthy master after its health gate instead of unconditionally from master 4 (removes the SPOF and the fixed sleep 150), overlap the heavy media node install with the masters (gate 1: secrets + 4 IPs before installing; gate 2: FINISH before service start, 5s polling instead of 1s), bounded timeouts on every Key Vault poll, Key Vault/storage wait margins 100s -> 300s, robust installer fetch (curl --retry to file instead of sh <(curl)), apt cleanup. Fix createUiDefinition.json: the portal 'Container Name' field emitted 'containerName' but the template parameter is 'appDataContainerName', so the user-provided value never reached the template. ARM JSON recompiled with bicep 0.45.15 (committed JSON was 0.44.1; codegen noise verified to be metadata-only against the pristine bicep). Validated with ov-cloud-tester (sc-deploy-destroy, ha, dev): PASS. deploy 2m11s, ready 10m21s, destroy 7m14s. --- .../ha/azure/DOC-CHANGES-ha-optimizations.md | 181 ++++++++++++++++++ .../pro/ha/azure/cf-openvidu-ha.bicep | 125 ++++++++---- .../pro/ha/azure/cf-openvidu-ha.json | 19 +- .../pro/ha/azure/createUiDefinition.json | 2 +- 4 files changed, 282 insertions(+), 45 deletions(-) create mode 100644 openvidu-deployment/pro/ha/azure/DOC-CHANGES-ha-optimizations.md diff --git a/openvidu-deployment/pro/ha/azure/DOC-CHANGES-ha-optimizations.md b/openvidu-deployment/pro/ha/azure/DOC-CHANGES-ha-optimizations.md new file mode 100644 index 000000000..0f2bbb933 --- /dev/null +++ b/openvidu-deployment/pro/ha/azure/DOC-CHANGES-ha-optimizations.md @@ -0,0 +1,181 @@ +# Documentation changes for `ha-optimizations` (Azure HA deployment) + +Instructions for the Claude working on the docs repo +`/home/sergio/Escritorio/openvidu/openvidu.io` (branch `next`, already up to date). + +These instructions derive from the changes applied in +`openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep`, +`openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json` (recompiled) and +`openvidu-deployment/pro/ha/azure/createUiDefinition.json` on the `ha-optimizations` +branch. Functional summary of what changed in the deployment: + +- The 4 master VMs are now created in parallel (the `dependsOn` chain + MasterNode2→1, 3→2, 4→3 was removed; coordination stays data-driven via Key Vault). +- Media Nodes now install (Docker + image pulls) **in parallel** with the masters, + instead of waiting idle until all masters finished before starting to install. +- The master-node-4 single point of failure was removed: readiness + (`FINISH-MASTER-NODE`) is now published by the **first healthy master**, right after + its own health check, and a fixed `sleep 150` was removed. +- Wait timeouts hardened (Key Vault availability and the blob-storage config retry + raised from 100s to 300s; the previously unbounded polls are now bounded to 30 min), + and the installer download is now retried and validated (`curl --retry 8 ... -o file` + + non-empty check) instead of the fragile `sh <(curl ...)`. +- Bug fix in `createUiDefinition.json`: the portal now maps the "Container Name" field + to the correct template parameter (`appDataContainerName`) — see section (b). + +SUMMARY: the ONLY content change required in the docs is the two deployment time +figures in Azure HA's `install.md`. Everything else (public parameters, screenshots, +the "Container Name" field description, other clouds, other deployment types) stays the +same. Read this file in full before touching anything. + +--- + +## (a) Time figures to update — MANDATORY + +File: `docs/docs/self-hosting/ha/azure/install.md` + +There are TWO deployment-time figures in this file. Locate each by its literal text +(line numbers below are approximate and may have shifted). The deployment is now +faster (masters run in parallel and the media install overlaps with the masters' +install, plus the fixed 150s sleep is gone), so both figures must be re-measured. + +### Occurrence 1 (section "## Deploying the stack", ~line 97) + +Current literal text: + +``` +Whenever you are satisfied with your Template parameters, just click on _"Next"_ to trigger the validation process. If correct, click on _"Create"_ to start the deployment process (which will take about 10 to 15 minutes). +``` + +Proposed text (fill in `X`/`Y` with the measurement from `ov-cloud-tester`): + +``` +Whenever you are satisfied with your Template parameters, just click on _"Next"_ to trigger the validation process. If correct, click on _"Create"_ to start the deployment process (which will take about X to Y minutes). +``` + +### Occurrence 2 (section "## Configuration and administration", ~line 181) + +Current literal text: + +``` +When your Azure stack reaches the **`Succeeded`** status, it means that all resources have been created. You will need to wait about 5 to 10 minutes for the instance to install OpenVidu. After this time, try connecting to the deployment URL. If it doesn't work, we recommend checking the previous section. Once everything is ready, you can check the [Administration](./admin.md) section to learn how to manage your deployment. +``` + +Proposed text (fill in `X`/`Y` with the measurement from `ov-cloud-tester`): + +``` +When your Azure stack reaches the **`Succeeded`** status, it means that all resources have been created. You will need to wait about X to Y minutes for the instance to install OpenVidu. After this time, try connecting to the deployment URL. If it doesn't work, we recommend checking the previous section. Once everything is ready, you can check the [Administration](./admin.md) section to learn how to manage your deployment. +``` + +> Placeholder to fill in: `X to Y minutes — fill in with the measurement from ov-cloud-tester`. +> The two figures are independent (occurrence 1 = full stack creation time; occurrence 2 +> = extra wait after `Succeeded` for OpenVidu to install). Measure and fill each on its own. + +### Do NOT touch the same figures in other files + +- `"10 to 15 minutes"` appears ONLY in `docs/docs/self-hosting/ha/azure/install.md`, so + there is no ambiguity for occurrence 1. +- `"5 to 10 minutes"` also appears in OTHER documents that are OUT OF SCOPE here (they + belong to other deployment types / clouds). Do NOT modify them: + - `docs/docs/self-hosting/single-node-pro/azure/install.md` + - `docs/docs/self-hosting/single-node/azure/install.md` + - `docs/docs/self-hosting/single-node-pro/gcp/install.md` + - `docs/docs/self-hosting/single-node/gcp/install.md` + - `shared/self-hosting/gcp/deploying-stack.md` + Only change the occurrence inside `docs/docs/self-hosting/ha/azure/install.md`. + +--- + +## (b) "Container Name" field fix — NO doc change needed + +The bug: in `createUiDefinition.json` the portal emitted the output key `containerName`, +but the ARM/Bicep template parameter is `appDataContainerName`. Because the key did not +match, the value typed by the user in the "Container Name" field never reached the +template — a custom container name was silently ignored and the deployment always fell +back to the default `openvidu-appdata`. The fix renames only that output **key** to +`appDataContainerName` (the value still comes from the same UI field +`steps('parameters STORAGE').containerName`). + +Impact on the docs: **NONE.** Explicit reasoning so you can confirm and move on: + +- The visible UI element name and its label are unchanged — the form still shows a field + labeled **"Container Name"**. Only a hidden output-mapping key changed. +- The shared snippet `shared/self-hosting/azure/storage-account.md` (included by the + Azure HA `install.md` via `--8<--`) describes that field as: + *"**Container Name** is the name that you desire for the container ... If you leave it + blank it will create the container with name `openvidu-appdata`."* That description + now matches reality in BOTH cases (blank → `openvidu-appdata`, as before; non-blank → + the value is finally honored). The text was already written as if the field worked, so + no wording needs to change. +- The screenshot `assets/images/platform/self-hosting/shared/azure/storageaccount.png` + (referenced only from `shared/self-hosting/azure/storage-account.md`) shows the same + unchanged form, so it does NOT need to be regenerated. + +Action: leave `shared/self-hosting/azure/storage-account.md` and its screenshot as they +are. + +--- + +## (c) Public template parameters / outputs: NO CHANGES — confirmation + +Do not touch parameter tables or screenshots in the parameters section. + +Reason: no public parameter was added, removed or renamed. `appDataContainerName` and +`storageAccountName` already existed as template parameters; the createUiDefinition fix +only corrects an existing field mapping. The Bicep template has no `outputs` section, and +none was added. Therefore: + +- Any "Parameters" tables/screenshots in `docs/docs/self-hosting/ha/azure/install.md` + and in the shared Azure snippets **stay the same**. +- There are no new fields the user must fill in the Azure portal form. +- The parameter form screenshots **do not change**. + +--- + +## (d) Other statements in the Azure HA docs that may become outdated + +The Azure HA documentation (`docs/docs/self-hosting/ha/azure/*.md`) and the shared Azure +includes (`shared/self-hosting/azure/*.md`) were reviewed against the internal changes. +Conclusions: + +### d.1 — Master node ordering / parallelization: not documented + +The docs never state that master nodes are created sequentially or "one by one", nor do +they describe the `dependsOn` chain. Removing it changes nothing user-facing. No text to +update. + +### d.2 — `FINISH-MASTER-NODE` / master-4 SPOF / scale-in lock (`lock.txt`): not documented + +The internal readiness handshake (`FINISH-MASTER-NODE`), the fact that it used to be +written only by master-4, and the `automation-locks/lock.txt` scale-in lock blob are +NOT mentioned anywhere in the docs. The scale-in behavior described in +`shared/self-hosting/azure/scale-in-config.md` (the graceful drain and its "up to 5 +minutes" / Azure "15 minutes" figures) is about Azure's termination timing and is +UNCHANGED by this work — **do not touch it**. + +### d.3 — Internal coordination (`ALL-SECRETS-GENERATED`, Key Vault polling): not documented + +The master-1-as-leader secrets generation, the Key Vault polling, and the new bounded +timeouts are internal and not publicly documented. Nothing user-facing changes. No text +to update. + +### d.4 — `admin.md`, `upgrade.md`, `index.md` (Azure HA): DO NOT TOUCH + +Reviewed; none of them reference the internals that changed. No changes required. + +--- + +## Implementation checklist + +- [ ] In `docs/docs/self-hosting/ha/azure/install.md`, replace `"10 to 15 minutes"` + (~line 97) and `"5 to 10 minutes"` (~line 181) with the re-measured + `"X to Y minutes"` figures, located by their literal text. +- [ ] Measure the actual deployment times with `ov-cloud-tester` on `ha-optimizations` + and fill in `X`/`Y` for each occurrence independently. +- [ ] Confirm NO parameter tables or screenshots were touched (section c). +- [ ] Confirm `shared/self-hosting/azure/storage-account.md` and `storageaccount.png` + were left unchanged (section b). +- [ ] Confirm the `"5 to 10 minutes"` figures in single-node / single-node-pro / GCP + docs and `shared/self-hosting/gcp/deploying-stack.md` were NOT touched (section a). +- [ ] Confirm `scale-in-config.md`, `admin.md`, `upgrade.md`, `index.md` were NOT touched + (section d). diff --git a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep index ebe27892b..1c659b6b7 100644 --- a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep +++ b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep @@ -304,7 +304,7 @@ apt-get update && apt-get install -y \ # Wait for the keyvault availability -MAX_WAIT=100 +MAX_WAIT=300 WAIT_INTERVAL=1 ELAPSED_TIME=0 set +e @@ -397,6 +397,10 @@ if [[ $MASTER_NODE_NUM -eq 1 ]] && [[ "$ALL_SECRETS_GENERATED" == "" || "$ALL_SE ALL_SECRETS_GENERATED="$(/usr/local/bin/store_secret.sh save ALL-SECRETS-GENERATED "true")" fi +# Wait (bounded: 360 x 5s = 30 min) until all 4 master nodes have published their private IPs +IP_WAIT_MAX_RETRIES=360 +IP_WAIT_INTERVAL=5 +IP_WAIT_RETRIES=0 while true; do MASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv) || true MASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv) || true @@ -409,11 +413,26 @@ while true; do [[ "$MASTER_NODE_4_PRIVATE_IP" != "" ]]; then break fi - sleep 5 + IP_WAIT_RETRIES=$((IP_WAIT_RETRIES + 1)) + if [ $IP_WAIT_RETRIES -ge $IP_WAIT_MAX_RETRIES ]; then + echo "[OpenVidu] timed out after 30 min waiting for the 4 master nodes to publish their private IPs" + exit 1 + fi + sleep $IP_WAIT_INTERVAL done -# Wait until master-node-1 has generated all shared secrets before fetching them -while [[ "$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)" != "true" ]]; do sleep 5; done +# Wait (bounded: 360 x 5s = 30 min) until master-node-1 has generated all shared secrets before fetching them +SECRETS_WAIT_MAX_RETRIES=360 +SECRETS_WAIT_INTERVAL=5 +SECRETS_WAIT_RETRIES=0 +while [[ "$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)" != "true" ]]; do + SECRETS_WAIT_RETRIES=$((SECRETS_WAIT_RETRIES + 1)) + if [ $SECRETS_WAIT_RETRIES -ge $SECRETS_WAIT_MAX_RETRIES ]; then + echo "[OpenVidu] timed out after 30 min waiting for ALL-SECRETS-GENERATED to become true" + exit 1 + fi + sleep $SECRETS_WAIT_INTERVAL +done # Fetch the values in the keyvault @@ -448,8 +467,16 @@ fi ENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv) +# Download the installer script (retry transient curl failures; fail if empty/missing) +INSTALLER_SCRIPT="/tmp/install_ov_master_node.sh" +curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh" +if [ ! -s "$INSTALLER_SCRIPT" ]; then + echo "[OpenVidu] failed to download the master node installer script" + exit 1 +fi + # Base command -INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh)" +INSTALL_COMMAND="sh $INSTALLER_SCRIPT" # Common arguments COMMON_ARGS=( @@ -811,7 +838,7 @@ INSTALL_DIR="/opt/openvidu" CLUSTER_CONFIG_DIR="${INSTALL_DIR}/config/cluster" # Retry login + storage key fetch to allow the Contributor role assignment to propagate -MAX_WAIT=100 +MAX_WAIT=300 WAIT_INTERVAL=1 ELAPSED_TIME=0 set +e @@ -1043,8 +1070,6 @@ az login --identity --allow-no-subscriptions echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout -apt-get update && apt-get install -y - export HOME="/root" # Install OpenVidu @@ -1062,20 +1087,21 @@ systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; # Launch on reboot echo "@reboot /usr/local/bin/restart.sh >> /var/log/openvidu-restart.log" 2>&1 | crontab +# Wait until this master node is healthy (check_app_ready.sh caps at 1200s). +# On failure exit WITHOUT publishing readiness, so media nodes keep waiting for another master. +/usr/local/bin/check_app_ready.sh || { echo "[OpenVidu] master node did not become healthy"; exit 1; } + +# Publish readiness so media nodes can start. Every healthy master writes the same value, so the +# first healthy master unblocks media and a failed master can no longer leave this unset (no SPOF). +az keyvault secret set --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --value "true" + MASTER_NODE_NUM=${masterNodeNum} if [[ $MASTER_NODE_NUM -eq 4 ]]; then # Creating scale in lock set +e az storage blob upload --account-name ${storageAccountName} --container-name automation-locks --name lock.txt --file /dev/null --auth-mode key set -e - - #Finish all the nodes - az keyvault secret set --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --value "true" fi - -# Wait for the app -sleep 150 -/usr/local/bin/check_app_ready.sh ''' var userDataMasterNode1 = reduce( @@ -1168,7 +1194,6 @@ resource openviduMasterNode2 'Microsoft.Compute/virtualMachines@2023-09-01' = { } userData: base64(userDataMasterNode2) } - dependsOn: [openviduMasterNode1] } resource openviduMasterNode3 'Microsoft.Compute/virtualMachines@2023-09-01' = { @@ -1203,7 +1228,6 @@ resource openviduMasterNode3 'Microsoft.Compute/virtualMachines@2023-09-01' = { } userData: base64(userDataMasterNode3) } - dependsOn: [openviduMasterNode2] } resource openviduMasterNode4 'Microsoft.Compute/virtualMachines@2023-09-01' = { @@ -1238,7 +1262,6 @@ resource openviduMasterNode4 'Microsoft.Compute/virtualMachines@2023-09-01' = { } userData: base64(userDataMasterNode4) } - dependsOn: [openviduMasterNode3] } /*------------------------------------------- MEDIA NODES -------------------------------------------*/ @@ -1272,23 +1295,31 @@ apt-get update && apt-get install -y \ # Get own private IP PRIVATE_IP=$(curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text") -WAIT_INTERVAL=1 -MAX_WAIT=10000 -ELAPSED_TIME=0 +# Gate 1: wait (bounded: 360 x 5s = 30 min) until master nodes generated the shared secrets and +# published their 4 private IPs. This lets the heavy media installer run in parallel with the masters. +WAIT_INTERVAL=5 +MAX_RETRIES=360 +RETRIES=0 set +e while true; do - # get secret value - FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv) + ALL_SECRETS_GENERATED=$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null) + MASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv 2>/dev/null) + MASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv 2>/dev/null) + MASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv 2>/dev/null) + MASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv 2>/dev/null) - # Check if all master nodes finished - if [ "$FINISH_MASTER_NODE" == "true" ]; then + # Break once secrets are generated and all 4 master IPs are published + if [ "$ALL_SECRETS_GENERATED" == "true" ] && + [ "$MASTER_NODE_1_PRIVATE_IP" != "" ] && + [ "$MASTER_NODE_2_PRIVATE_IP" != "" ] && + [ "$MASTER_NODE_3_PRIVATE_IP" != "" ] && + [ "$MASTER_NODE_4_PRIVATE_IP" != "" ]; then break fi - ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL)) - - # Check if the maximum waiting time has been reached - if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then + RETRIES=$((RETRIES + 1)) + if [ $RETRIES -ge $MAX_RETRIES ]; then + echo "[OpenVidu] timed out after 30 min waiting for master nodes to generate secrets and publish their IPs" exit 1 fi @@ -1305,8 +1336,16 @@ REDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name RED ENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv) OPENVIDU_VERSION=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-VERSION --query value -o tsv) +# Download the installer script (retry transient curl failures; fail if empty/missing) +INSTALLER_SCRIPT="/tmp/install_ov_media_node.sh" +curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh" +if [ ! -s "$INSTALLER_SCRIPT" ]; then + echo "[OpenVidu] failed to download the media node installer script" + exit 1 +fi + # Base command -INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh)" +INSTALL_COMMAND="sh $INSTALLER_SCRIPT" # Common arguments COMMON_ARGS=( @@ -1413,8 +1452,7 @@ chmod +x /usr/local/bin/delete_media_node.sh echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout -apt-get update && apt-get install -y -apt-get install -y jq +apt-get update && apt-get install -y jq # Install azure cli AZURE_CLI_VERSION=2.87.0 @@ -1435,9 +1473,29 @@ az vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME - export HOME="/root" -# Install OpenVidu +# Install OpenVidu (heavy: docker + image pulls). Overlaps with the masters' own installs. /usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu"; /usr/local/bin/delete_media_node.sh; } +# Gate 2: wait (bounded: 360 x 5s = 30 min) for the master nodes to publish readiness before starting +WAIT_INTERVAL=5 +MAX_RETRIES=360 +RETRIES=0 +set +e +while true; do + FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv 2>/dev/null) + if [ "$FINISH_MASTER_NODE" == "true" ]; then + break + fi + RETRIES=$((RETRIES + 1)) + if [ $RETRIES -ge $MAX_RETRIES ]; then + echo "[OpenVidu] timed out after 30 min waiting for FINISH-MASTER-NODE" + /usr/local/bin/delete_media_node.sh + exit 1 + fi + sleep $WAIT_INTERVAL +done +set -e + # Start OpenVidu systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; /usr/local/bin/delete_media_node.sh; } ''' @@ -1470,6 +1528,7 @@ var userDataParamsMedia = { base64delete_mediaNode: base64delete_mediaNode_ScriptMedia resourceGroupName: resourceGroup().name vmScaleSetName: '${stackName}-mediaNodeScaleSet' + keyVaultName: keyVaultName } var userDataMediaNode = reduce( diff --git a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json index 0f97b212c..b0a5992b2 100644 --- a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json +++ b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json @@ -4,8 +4,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.44.1.10279", - "templateHash": "8155530769596354082" + "version": "0.45.15.27210", + "templateHash": "16682765069403816529" } }, "parameters": { @@ -289,7 +289,7 @@ "masterNodeNum": "4", "additionalInstallFlags": "[parameters('additionalInstallFlags')]" }, - "installScriptTemplateMaster": "#!/bin/bash -x\nset -e\nOPENVIDU_VERSION=main\nDOMAIN=\n\n# Assume azure cli is installed\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y \\\n curl \\\n unzip \\\n jq \\\n wget\n\n\n# Wait for the keyvault availability\nMAX_WAIT=100\nWAIT_INTERVAL=1\nELAPSED_TIME=0\nset +e\nwhile true; do\n # Check keyvault availability\n az keyvault secret list --vault-name ${keyVaultName} \n\n # If it is available, exit the loop\n if [ $? -eq 0 ]; then\n break\n fi\n\n # If not, wait and check again incrementing the time\n ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL))\n\n # If exceeded the maximum time, exit with error\n if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then\n exit 1\n fi\n\n # Esperar antes de la próxima comprobación\n sleep $WAIT_INTERVAL\ndone\nset -e\n\nMASTER_NODE_NUM=${masterNodeNum}\n\n# Get own private IP\nPRIVATE_IP=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text\")\n\n# Store current private IP\nPRIVATE_IP=\"$(/usr/local/bin/store_secret.sh save MASTER-NODE-${masterNodeNum}-PRIVATE-IP $PRIVATE_IP)\"\n\n\nif [[ $MASTER_NODE_NUM -eq 1 ]] && [[ \"$ALL_SECRETS_GENERATED\" == \"\" || \"$ALL_SECRETS_GENERATED\" == \"false\" ]]; then\n\n # Configure Domain name\n if [[ \"${domainName}\" == '' ]]; then\n # Get public IP using the get_public_ip.sh script\n PUBLIC_IP=$(/usr/local/bin/get_public_ip.sh 2>/dev/null)\n if [[ $? -ne 0 || -z \"${PUBLIC_IP}\" ]]; then\n echo \"Could not determine public IP.\"\n exit 1\n fi\n\n DOMAIN=\"$PUBLIC_IP\"\n else\n DOMAIN=${domainName}\n fi\n DOMAIN=\"$(/usr/local/bin/store_secret.sh save DOMAIN-NAME \"${DOMAIN}\")\"\n\n # Publish service URLs early (idempotent; after_install re-writes byte-identical values)\n OPENVIDU_URL=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-URL \"https://${DOMAIN}/\")\"\n LIVEKIT_URL=\"$(/usr/local/bin/store_secret.sh save LIVEKIT-URL \"wss://${DOMAIN}/\")\"\n DASHBOARD_URL=\"$(/usr/local/bin/store_secret.sh save DASHBOARD-URL \"https://${DOMAIN}/dashboard/\")\"\n GRAFANA_URL=\"$(/usr/local/bin/store_secret.sh save GRAFANA-URL \"https://${DOMAIN}/grafana/\")\"\n MINIO_URL=\"$(/usr/local/bin/store_secret.sh save MINIO-URL \"https://${DOMAIN}/minio-console/\")\"\n\n # Meet initial admin user and password\n MEET_INITIAL_ADMIN_USER=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-ADMIN-USER \"admin\")\"\n if [[ \"${initialMeetAdminPassword}\" != '' ]]; then\n MEET_INITIAL_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-ADMIN-PASSWORD \"${initialMeetAdminPassword}\")\"\n else\n MEET_INITIAL_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate MEET-INITIAL-ADMIN-PASSWORD)\"\n fi\n if [[ \"${initialMeetApiKey}\" != '' ]]; then\n MEET_INITIAL_API_KEY=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-API-KEY \"${initialMeetApiKey}\")\"\n else\n MEET_INITIAL_API_KEY=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-API-KEY \"\")\"\n fi\n\n # Store usernames and generate random passwords\n OPENVIDU_PRO_LICENSE=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-PRO-LICENSE \"${openviduLicense}\")\"\n OPENVIDU_RTC_ENGINE=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-RTC-ENGINE \"${rtcEngine}\")\"\n REDIS_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate REDIS-PASSWORD)\"\n MONGO_ADMIN_USERNAME=\"$(/usr/local/bin/store_secret.sh save MONGO-ADMIN-USERNAME \"mongoadmin\")\"\n MONGO_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate MONGO-ADMIN-PASSWORD)\"\n MONGO_REPLICA_SET_KEY=\"$(/usr/local/bin/store_secret.sh generate MONGO-REPLICA-SET-KEY)\"\n MINIO_ACCESS_KEY=\"$(/usr/local/bin/store_secret.sh save MINIO-ACCESS-KEY \"minioadmin\")\"\n MINIO_SECRET_KEY=\"$(/usr/local/bin/store_secret.sh generate MINIO-SECRET-KEY)\"\n DASHBOARD_ADMIN_USERNAME=\"$(/usr/local/bin/store_secret.sh save DASHBOARD-ADMIN-USERNAME \"dashboardadmin\")\"\n DASHBOARD_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate DASHBOARD-ADMIN-PASSWORD)\"\n GRAFANA_ADMIN_USERNAME=\"$(/usr/local/bin/store_secret.sh save GRAFANA-ADMIN-USERNAME \"grafanaadmin\")\"\n GRAFANA_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate GRAFANA-ADMIN-PASSWORD)\"\n MEET_INITIAL_ADMIN_USER=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-ADMIN-USER \"admin\")\"\n LIVEKIT_API_KEY=\"$(/usr/local/bin/store_secret.sh generate LIVEKIT-API-KEY \"API\" 12)\"\n LIVEKIT_API_SECRET=\"$(/usr/local/bin/store_secret.sh generate LIVEKIT-API-SECRET)\"\n OPENVIDU_VERSION=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-VERSION \"${OPENVIDU_VERSION}\")\"\n ENABLED_MODULES=\"$(/usr/local/bin/store_secret.sh save ENABLED-MODULES \"observability,openviduMeet,v2compatibility\")\"\n ALL_SECRETS_GENERATED=\"$(/usr/local/bin/store_secret.sh save ALL-SECRETS-GENERATED \"true\")\"\nfi\n\nwhile true; do\n MASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv) || true\n MASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv) || true\n MASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv) || true\n MASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv) || true\n # Check if all master nodes have stored their private IPs\n if [[ \"$MASTER_NODE_1_PRIVATE_IP\" != \"\" ]] &&\n [[ \"$MASTER_NODE_2_PRIVATE_IP\" != \"\" ]] &&\n [[ \"$MASTER_NODE_3_PRIVATE_IP\" != \"\" ]] &&\n [[ \"$MASTER_NODE_4_PRIVATE_IP\" != \"\" ]]; then\n break\n fi\n sleep 5\ndone\n\n# Wait until master-node-1 has generated all shared secrets before fetching them\nwhile [[ \"$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)\" != \"true\" ]]; do sleep 5; done\n\n\n# Fetch the values in the keyvault\nMASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_PRIVATE_IP_LIST=\"$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP\"\n\nDOMAIN=$(az keyvault secret show --vault-name ${keyVaultName} --name DOMAIN-NAME --query value -o tsv)\nOPENVIDU_RTC_ENGINE=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-RTC-ENGINE --query value -o tsv)\nOPENVIDU_PRO_LICENSE=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-PRO-LICENSE --query value -o tsv)\nREDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name REDIS-PASSWORD --query value -o tsv)\nMONGO_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-ADMIN-USERNAME --query value -o tsv)\nMONGO_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-ADMIN-PASSWORD --query value -o tsv)\nMONGO_REPLICA_SET_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-REPLICA-SET-KEY --query value -o tsv)\nDASHBOARD_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-USERNAME --query value -o tsv)\nDASHBOARD_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-PASSWORD --query value -o tsv)\nMINIO_ACCESS_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MINIO-ACCESS-KEY --query value -o tsv)\nMINIO_SECRET_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MINIO-SECRET-KEY --query value -o tsv)\nGRAFANA_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name GRAFANA-ADMIN-USERNAME --query value -o tsv)\nGRAFANA_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name GRAFANA-ADMIN-PASSWORD --query value -o tsv)\nLIVEKIT_API_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name LIVEKIT-API-KEY --query value -o tsv)\nLIVEKIT_API_SECRET=$(az keyvault secret show --vault-name ${keyVaultName} --name LIVEKIT-API-SECRET --query value -o tsv)\nMEET_INITIAL_ADMIN_USER=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-USER --query value -o tsv)\nMEET_INITIAL_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-PASSWORD --query value -o tsv)\nif [[ \"${initialMeetApiKey}\" != '' ]]; then\n MEET_INITIAL_API_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-API-KEY --query value -o tsv)\nelse\n MEET_INITIAL_API_KEY=\"\"\nfi\nENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv)\n\n\n# Base command\nINSTALL_COMMAND=\"sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh)\"\n\n# Common arguments\nCOMMON_ARGS=(\n \"--no-tty\"\n \"--install\"\n \"--environment=azure\"\n \"--deployment-type='ha'\"\n \"--node-role='master-node'\"\n \"--external-load-balancer\"\n \"--internal-tls-termination\"\n \"--master-node-private-ip-list='$MASTER_NODE_PRIVATE_IP_LIST'\"\n \"--openvidu-pro-license='$OPENVIDU_PRO_LICENSE'\"\n \"--domain-name='$DOMAIN'\"\n \"--enabled-modules='$ENABLED_MODULES'\"\n \"--rtc-engine=$OPENVIDU_RTC_ENGINE\"\n \"--redis-password=$REDIS_PASSWORD\"\n \"--mongo-admin-user=$MONGO_ADMIN_USERNAME\"\n \"--mongo-admin-password=$MONGO_ADMIN_PASSWORD\"\n \"--mongo-replica-set-key=$MONGO_REPLICA_SET_KEY\"\n \"--minio-access-key=$MINIO_ACCESS_KEY\"\n \"--minio-secret-key=$MINIO_SECRET_KEY\"\n \"--dashboard-admin-user=$DASHBOARD_ADMIN_USERNAME\"\n \"--dashboard-admin-password=$DASHBOARD_ADMIN_PASSWORD\"\n \"--grafana-admin-user=$GRAFANA_ADMIN_USERNAME\"\n \"--grafana-admin-password=$GRAFANA_ADMIN_PASSWORD\"\n \"--meet-initial-admin-password=$MEET_INITIAL_ADMIN_PASSWORD\"\n \"--meet-initial-api-key=$MEET_INITIAL_API_KEY\"\n \"--livekit-api-key=$LIVEKIT_API_KEY\"\n \"--livekit-api-secret=$LIVEKIT_API_SECRET\"\n)\n\n# Include additional installer flags provided by the user\nif [[ \"${additionalInstallFlags}\" != \"\" ]]; then\n IFS=',' read -ra EXTRA_FLAGS <<< \"${additionalInstallFlags}\"\n for extra_flag in \"${EXTRA_FLAGS[@]}\"; do\n # Trim whitespace around each flag\n extra_flag=\"$(echo -e \"${extra_flag}\" | sed -e 's/^[ \\t]*//' -e 's/[ \\t]*$//')\"\n if [[ \"$extra_flag\" != \"\" ]]; then\n COMMON_ARGS+=(\"$extra_flag\")\n fi\n done\nfi\n\n# Certificate arguments\nif [[ \"${certificateType}\" == \"selfsigned\" ]]; then\n CERT_ARGS=(\n \"--certificate-type=selfsigned\"\n )\nelif [[ \"${certificateType}\" == \"letsencrypt\" ]]; then\n CERT_ARGS=(\n \"--certificate-type=letsencrypt\"\n )\nelse\n # Use base64 encoded certificates directly\n OWN_CERT_CRT=${ownPublicCertificate}\n OWN_CERT_KEY=${ownPrivateCertificate}\n\n CERT_ARGS=(\n \"--certificate-type=owncert\"\n \"--owncert-public-key=$OWN_CERT_CRT\"\n \"--owncert-private-key=$OWN_CERT_KEY\"\n )\nfi\n\n# Construct the final command\nFINAL_COMMAND=\"$INSTALL_COMMAND $(printf \"%s \" \"${COMMON_ARGS[@]}\") $(printf \"%s \" \"${CERT_ARGS[@]}\")\"\n\n# Install OpenVidu\nexec bash -c \"$FINAL_COMMAND\"\n", + "installScriptTemplateMaster": "#!/bin/bash -x\nset -e\nOPENVIDU_VERSION=main\nDOMAIN=\n\n# Assume azure cli is installed\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y \\\n curl \\\n unzip \\\n jq \\\n wget\n\n\n# Wait for the keyvault availability\nMAX_WAIT=300\nWAIT_INTERVAL=1\nELAPSED_TIME=0\nset +e\nwhile true; do\n # Check keyvault availability\n az keyvault secret list --vault-name ${keyVaultName} \n\n # If it is available, exit the loop\n if [ $? -eq 0 ]; then\n break\n fi\n\n # If not, wait and check again incrementing the time\n ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL))\n\n # If exceeded the maximum time, exit with error\n if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then\n exit 1\n fi\n\n # Esperar antes de la próxima comprobación\n sleep $WAIT_INTERVAL\ndone\nset -e\n\nMASTER_NODE_NUM=${masterNodeNum}\n\n# Get own private IP\nPRIVATE_IP=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text\")\n\n# Store current private IP\nPRIVATE_IP=\"$(/usr/local/bin/store_secret.sh save MASTER-NODE-${masterNodeNum}-PRIVATE-IP $PRIVATE_IP)\"\n\n\nif [[ $MASTER_NODE_NUM -eq 1 ]] && [[ \"$ALL_SECRETS_GENERATED\" == \"\" || \"$ALL_SECRETS_GENERATED\" == \"false\" ]]; then\n\n # Configure Domain name\n if [[ \"${domainName}\" == '' ]]; then\n # Get public IP using the get_public_ip.sh script\n PUBLIC_IP=$(/usr/local/bin/get_public_ip.sh 2>/dev/null)\n if [[ $? -ne 0 || -z \"${PUBLIC_IP}\" ]]; then\n echo \"Could not determine public IP.\"\n exit 1\n fi\n\n DOMAIN=\"$PUBLIC_IP\"\n else\n DOMAIN=${domainName}\n fi\n DOMAIN=\"$(/usr/local/bin/store_secret.sh save DOMAIN-NAME \"${DOMAIN}\")\"\n\n # Publish service URLs early (idempotent; after_install re-writes byte-identical values)\n OPENVIDU_URL=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-URL \"https://${DOMAIN}/\")\"\n LIVEKIT_URL=\"$(/usr/local/bin/store_secret.sh save LIVEKIT-URL \"wss://${DOMAIN}/\")\"\n DASHBOARD_URL=\"$(/usr/local/bin/store_secret.sh save DASHBOARD-URL \"https://${DOMAIN}/dashboard/\")\"\n GRAFANA_URL=\"$(/usr/local/bin/store_secret.sh save GRAFANA-URL \"https://${DOMAIN}/grafana/\")\"\n MINIO_URL=\"$(/usr/local/bin/store_secret.sh save MINIO-URL \"https://${DOMAIN}/minio-console/\")\"\n\n # Meet initial admin user and password\n MEET_INITIAL_ADMIN_USER=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-ADMIN-USER \"admin\")\"\n if [[ \"${initialMeetAdminPassword}\" != '' ]]; then\n MEET_INITIAL_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-ADMIN-PASSWORD \"${initialMeetAdminPassword}\")\"\n else\n MEET_INITIAL_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate MEET-INITIAL-ADMIN-PASSWORD)\"\n fi\n if [[ \"${initialMeetApiKey}\" != '' ]]; then\n MEET_INITIAL_API_KEY=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-API-KEY \"${initialMeetApiKey}\")\"\n else\n MEET_INITIAL_API_KEY=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-API-KEY \"\")\"\n fi\n\n # Store usernames and generate random passwords\n OPENVIDU_PRO_LICENSE=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-PRO-LICENSE \"${openviduLicense}\")\"\n OPENVIDU_RTC_ENGINE=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-RTC-ENGINE \"${rtcEngine}\")\"\n REDIS_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate REDIS-PASSWORD)\"\n MONGO_ADMIN_USERNAME=\"$(/usr/local/bin/store_secret.sh save MONGO-ADMIN-USERNAME \"mongoadmin\")\"\n MONGO_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate MONGO-ADMIN-PASSWORD)\"\n MONGO_REPLICA_SET_KEY=\"$(/usr/local/bin/store_secret.sh generate MONGO-REPLICA-SET-KEY)\"\n MINIO_ACCESS_KEY=\"$(/usr/local/bin/store_secret.sh save MINIO-ACCESS-KEY \"minioadmin\")\"\n MINIO_SECRET_KEY=\"$(/usr/local/bin/store_secret.sh generate MINIO-SECRET-KEY)\"\n DASHBOARD_ADMIN_USERNAME=\"$(/usr/local/bin/store_secret.sh save DASHBOARD-ADMIN-USERNAME \"dashboardadmin\")\"\n DASHBOARD_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate DASHBOARD-ADMIN-PASSWORD)\"\n GRAFANA_ADMIN_USERNAME=\"$(/usr/local/bin/store_secret.sh save GRAFANA-ADMIN-USERNAME \"grafanaadmin\")\"\n GRAFANA_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate GRAFANA-ADMIN-PASSWORD)\"\n MEET_INITIAL_ADMIN_USER=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-ADMIN-USER \"admin\")\"\n LIVEKIT_API_KEY=\"$(/usr/local/bin/store_secret.sh generate LIVEKIT-API-KEY \"API\" 12)\"\n LIVEKIT_API_SECRET=\"$(/usr/local/bin/store_secret.sh generate LIVEKIT-API-SECRET)\"\n OPENVIDU_VERSION=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-VERSION \"${OPENVIDU_VERSION}\")\"\n ENABLED_MODULES=\"$(/usr/local/bin/store_secret.sh save ENABLED-MODULES \"observability,openviduMeet,v2compatibility\")\"\n ALL_SECRETS_GENERATED=\"$(/usr/local/bin/store_secret.sh save ALL-SECRETS-GENERATED \"true\")\"\nfi\n\n# Wait (bounded: 360 x 5s = 30 min) until all 4 master nodes have published their private IPs\nIP_WAIT_MAX_RETRIES=360\nIP_WAIT_INTERVAL=5\nIP_WAIT_RETRIES=0\nwhile true; do\n MASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv) || true\n MASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv) || true\n MASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv) || true\n MASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv) || true\n # Check if all master nodes have stored their private IPs\n if [[ \"$MASTER_NODE_1_PRIVATE_IP\" != \"\" ]] &&\n [[ \"$MASTER_NODE_2_PRIVATE_IP\" != \"\" ]] &&\n [[ \"$MASTER_NODE_3_PRIVATE_IP\" != \"\" ]] &&\n [[ \"$MASTER_NODE_4_PRIVATE_IP\" != \"\" ]]; then\n break\n fi\n IP_WAIT_RETRIES=$((IP_WAIT_RETRIES + 1))\n if [ $IP_WAIT_RETRIES -ge $IP_WAIT_MAX_RETRIES ]; then\n echo \"[OpenVidu] timed out after 30 min waiting for the 4 master nodes to publish their private IPs\"\n exit 1\n fi\n sleep $IP_WAIT_INTERVAL\ndone\n\n# Wait (bounded: 360 x 5s = 30 min) until master-node-1 has generated all shared secrets before fetching them\nSECRETS_WAIT_MAX_RETRIES=360\nSECRETS_WAIT_INTERVAL=5\nSECRETS_WAIT_RETRIES=0\nwhile [[ \"$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)\" != \"true\" ]]; do\n SECRETS_WAIT_RETRIES=$((SECRETS_WAIT_RETRIES + 1))\n if [ $SECRETS_WAIT_RETRIES -ge $SECRETS_WAIT_MAX_RETRIES ]; then\n echo \"[OpenVidu] timed out after 30 min waiting for ALL-SECRETS-GENERATED to become true\"\n exit 1\n fi\n sleep $SECRETS_WAIT_INTERVAL\ndone\n\n\n# Fetch the values in the keyvault\nMASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_PRIVATE_IP_LIST=\"$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP\"\n\nDOMAIN=$(az keyvault secret show --vault-name ${keyVaultName} --name DOMAIN-NAME --query value -o tsv)\nOPENVIDU_RTC_ENGINE=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-RTC-ENGINE --query value -o tsv)\nOPENVIDU_PRO_LICENSE=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-PRO-LICENSE --query value -o tsv)\nREDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name REDIS-PASSWORD --query value -o tsv)\nMONGO_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-ADMIN-USERNAME --query value -o tsv)\nMONGO_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-ADMIN-PASSWORD --query value -o tsv)\nMONGO_REPLICA_SET_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-REPLICA-SET-KEY --query value -o tsv)\nDASHBOARD_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-USERNAME --query value -o tsv)\nDASHBOARD_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-PASSWORD --query value -o tsv)\nMINIO_ACCESS_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MINIO-ACCESS-KEY --query value -o tsv)\nMINIO_SECRET_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MINIO-SECRET-KEY --query value -o tsv)\nGRAFANA_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name GRAFANA-ADMIN-USERNAME --query value -o tsv)\nGRAFANA_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name GRAFANA-ADMIN-PASSWORD --query value -o tsv)\nLIVEKIT_API_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name LIVEKIT-API-KEY --query value -o tsv)\nLIVEKIT_API_SECRET=$(az keyvault secret show --vault-name ${keyVaultName} --name LIVEKIT-API-SECRET --query value -o tsv)\nMEET_INITIAL_ADMIN_USER=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-USER --query value -o tsv)\nMEET_INITIAL_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-PASSWORD --query value -o tsv)\nif [[ \"${initialMeetApiKey}\" != '' ]]; then\n MEET_INITIAL_API_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-API-KEY --query value -o tsv)\nelse\n MEET_INITIAL_API_KEY=\"\"\nfi\nENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv)\n\n\n# Download the installer script (retry transient curl failures; fail if empty/missing)\nINSTALLER_SCRIPT=\"/tmp/install_ov_master_node.sh\"\ncurl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o \"$INSTALLER_SCRIPT\" \"http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh\"\nif [ ! -s \"$INSTALLER_SCRIPT\" ]; then\n echo \"[OpenVidu] failed to download the master node installer script\"\n exit 1\nfi\n\n# Base command\nINSTALL_COMMAND=\"sh $INSTALLER_SCRIPT\"\n\n# Common arguments\nCOMMON_ARGS=(\n \"--no-tty\"\n \"--install\"\n \"--environment=azure\"\n \"--deployment-type='ha'\"\n \"--node-role='master-node'\"\n \"--external-load-balancer\"\n \"--internal-tls-termination\"\n \"--master-node-private-ip-list='$MASTER_NODE_PRIVATE_IP_LIST'\"\n \"--openvidu-pro-license='$OPENVIDU_PRO_LICENSE'\"\n \"--domain-name='$DOMAIN'\"\n \"--enabled-modules='$ENABLED_MODULES'\"\n \"--rtc-engine=$OPENVIDU_RTC_ENGINE\"\n \"--redis-password=$REDIS_PASSWORD\"\n \"--mongo-admin-user=$MONGO_ADMIN_USERNAME\"\n \"--mongo-admin-password=$MONGO_ADMIN_PASSWORD\"\n \"--mongo-replica-set-key=$MONGO_REPLICA_SET_KEY\"\n \"--minio-access-key=$MINIO_ACCESS_KEY\"\n \"--minio-secret-key=$MINIO_SECRET_KEY\"\n \"--dashboard-admin-user=$DASHBOARD_ADMIN_USERNAME\"\n \"--dashboard-admin-password=$DASHBOARD_ADMIN_PASSWORD\"\n \"--grafana-admin-user=$GRAFANA_ADMIN_USERNAME\"\n \"--grafana-admin-password=$GRAFANA_ADMIN_PASSWORD\"\n \"--meet-initial-admin-password=$MEET_INITIAL_ADMIN_PASSWORD\"\n \"--meet-initial-api-key=$MEET_INITIAL_API_KEY\"\n \"--livekit-api-key=$LIVEKIT_API_KEY\"\n \"--livekit-api-secret=$LIVEKIT_API_SECRET\"\n)\n\n# Include additional installer flags provided by the user\nif [[ \"${additionalInstallFlags}\" != \"\" ]]; then\n IFS=',' read -ra EXTRA_FLAGS <<< \"${additionalInstallFlags}\"\n for extra_flag in \"${EXTRA_FLAGS[@]}\"; do\n # Trim whitespace around each flag\n extra_flag=\"$(echo -e \"${extra_flag}\" | sed -e 's/^[ \\t]*//' -e 's/[ \\t]*$//')\"\n if [[ \"$extra_flag\" != \"\" ]]; then\n COMMON_ARGS+=(\"$extra_flag\")\n fi\n done\nfi\n\n# Certificate arguments\nif [[ \"${certificateType}\" == \"selfsigned\" ]]; then\n CERT_ARGS=(\n \"--certificate-type=selfsigned\"\n )\nelif [[ \"${certificateType}\" == \"letsencrypt\" ]]; then\n CERT_ARGS=(\n \"--certificate-type=letsencrypt\"\n )\nelse\n # Use base64 encoded certificates directly\n OWN_CERT_CRT=${ownPublicCertificate}\n OWN_CERT_KEY=${ownPrivateCertificate}\n\n CERT_ARGS=(\n \"--certificate-type=owncert\"\n \"--owncert-public-key=$OWN_CERT_CRT\"\n \"--owncert-private-key=$OWN_CERT_KEY\"\n )\nfi\n\n# Construct the final command\nFINAL_COMMAND=\"$INSTALL_COMMAND $(printf \"%s \" \"${COMMON_ARGS[@]}\") $(printf \"%s \" \"${CERT_ARGS[@]}\")\"\n\n# Install OpenVidu\nexec bash -c \"$FINAL_COMMAND\"\n", "after_installScriptTemplateMaster": "#!/bin/bash\nset -e\n\naz login --identity --allow-no-subscriptions > /dev/null\n\n# Generate URLs\nDOMAIN=$(az keyvault secret show --vault-name ${keyVaultName} --name DOMAIN-NAME --query value -o tsv)\nOPENVIDU_URL=\"https://${DOMAIN}/\"\nLIVEKIT_URL=\"wss://${DOMAIN}/\"\nDASHBOARD_URL=\"https://${DOMAIN}/dashboard/\"\nGRAFANA_URL=\"https://${DOMAIN}/grafana/\"\nMINIO_URL=\"https://${DOMAIN}/minio-console/\"\n\n# Update shared secret\naz keyvault secret set --vault-name ${keyVaultName} --name DOMAIN-NAME --value $DOMAIN\naz keyvault secret set --vault-name ${keyVaultName} --name OPENVIDU-URL --value $OPENVIDU_URL\naz keyvault secret set --vault-name ${keyVaultName} --name LIVEKIT-URL --value $LIVEKIT_URL\naz keyvault secret set --vault-name ${keyVaultName} --name DASHBOARD-URL --value $DASHBOARD_URL\naz keyvault secret set --vault-name ${keyVaultName} --name GRAFANA-URL --value $GRAFANA_URL\naz keyvault secret set --vault-name ${keyVaultName} --name MINIO-URL --value $MINIO_URL\n\naz keyvault secret show --vault-name ${keyVaultName} --name MINIO-URL\n\nif [[ $? -ne 0 ]]; then\n echo \"Error updating keyvault\"\nfi\n", "update_config_from_secretScriptTemplateMaster": "#!/bin/bash\nset -e\n\naz login --identity --allow-no-subscriptions > /dev/null\n\n# Installation directory\nINSTALL_DIR=\"/opt/openvidu\"\nCLUSTER_CONFIG_DIR=\"${INSTALL_DIR}/config/cluster\"\nMASTER_NODE_CONFIG_DIR=\"${INSTALL_DIR}/config/node\"\n\n# Replace DOMAIN_NAME\nexport DOMAIN=$(az keyvault secret show --vault-name ${keyVaultName} --name DOMAIN-NAME --query value -o tsv)\nif [[ -n \"$DOMAIN\" ]]; then\n sed -i \"s/DOMAIN_NAME=.*/DOMAIN_NAME=$DOMAIN/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nelse\n exit 1\nfi\n\n# Get the rest of the values\nexport REDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name REDIS-PASSWORD --query value -o tsv)\nexport OPENVIDU_RTC_ENGINE=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-RTC-ENGINE --query value -o tsv)\nexport OPENVIDU_PRO_LICENSE=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-PRO-LICENSE --query value -o tsv)\nexport MONGO_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-ADMIN-USERNAME --query value -o tsv)\nexport MONGO_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-ADMIN-PASSWORD --query value -o tsv)\nexport MONGO_REPLICA_SET_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-REPLICA-SET-KEY --query value -o tsv)\nexport DASHBOARD_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-USERNAME --query value -o tsv)\nexport DASHBOARD_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-PASSWORD --query value -o tsv)\nexport MINIO_ACCESS_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MINIO-ACCESS-KEY --query value -o tsv)\nexport MINIO_SECRET_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MINIO-SECRET-KEY --query value -o tsv)\nexport GRAFANA_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name GRAFANA-ADMIN-USERNAME --query value -o tsv)\nexport GRAFANA_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name GRAFANA-ADMIN-PASSWORD --query value -o tsv)\nexport LIVEKIT_API_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name LIVEKIT-API-KEY --query value -o tsv)\nexport LIVEKIT_API_SECRET=$(az keyvault secret show --vault-name ${keyVaultName} --name LIVEKIT-API-SECRET --query value -o tsv)\nexport MEET_INITIAL_ADMIN_USER=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-USER --query value -o tsv)\nexport MEET_INITIAL_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-PASSWORD --query value -o tsv)\nif [[ \"${initialMeetApiKey}\" != '' ]]; then\n export MEET_INITIAL_API_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-API-KEY --query value -o tsv)\nfi\nexport ENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv)\n\n# Replace rest of the values\nsed -i \"s/REDIS_PASSWORD=.*/REDIS_PASSWORD=$REDIS_PASSWORD/\" \"${MASTER_NODE_CONFIG_DIR}/master_node.env\"\nsed -i \"s/OPENVIDU_RTC_ENGINE=.*/OPENVIDU_RTC_ENGINE=$OPENVIDU_RTC_ENGINE/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/OPENVIDU_PRO_LICENSE=.*/OPENVIDU_PRO_LICENSE=$OPENVIDU_PRO_LICENSE/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/MONGO_ADMIN_USERNAME=.*/MONGO_ADMIN_USERNAME=$MONGO_ADMIN_USERNAME/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/MONGO_ADMIN_PASSWORD=.*/MONGO_ADMIN_PASSWORD=$MONGO_ADMIN_PASSWORD/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/MONGO_REPLICA_SET_KEY=.*/MONGO_REPLICA_SET_KEY=$MONGO_REPLICA_SET_KEY/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/DASHBOARD_ADMIN_USERNAME=.*/DASHBOARD_ADMIN_USERNAME=$DASHBOARD_ADMIN_USERNAME/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/DASHBOARD_ADMIN_PASSWORD=.*/DASHBOARD_ADMIN_PASSWORD=$DASHBOARD_ADMIN_PASSWORD/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/MINIO_ACCESS_KEY=.*/MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/MINIO_SECRET_KEY=.*/MINIO_SECRET_KEY=$MINIO_SECRET_KEY/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/GRAFANA_ADMIN_USERNAME=.*/GRAFANA_ADMIN_USERNAME=$GRAFANA_ADMIN_USERNAME/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/GRAFANA_ADMIN_PASSWORD=.*/GRAFANA_ADMIN_PASSWORD=$GRAFANA_ADMIN_PASSWORD/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/LIVEKIT_API_KEY=.*/LIVEKIT_API_KEY=$LIVEKIT_API_KEY/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/LIVEKIT_API_SECRET=.*/LIVEKIT_API_SECRET=$LIVEKIT_API_SECRET/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/MEET_INITIAL_ADMIN_USER=.*/MEET_INITIAL_ADMIN_USER=$MEET_INITIAL_ADMIN_USER/\" \"${CLUSTER_CONFIG_DIR}/master_node/meet.env\"\nsed -i \"s/MEET_INITIAL_ADMIN_PASSWORD=.*/MEET_INITIAL_ADMIN_PASSWORD=$MEET_INITIAL_ADMIN_PASSWORD/\" \"${CLUSTER_CONFIG_DIR}/master_node/meet.env\"\nif [[ \"${initialMeetApiKey}\" != '' ]]; then\n sed -i \"s/MEET_INITIAL_API_KEY=.*/MEET_INITIAL_API_KEY=$MEET_INITIAL_API_KEY/\" \"${CLUSTER_CONFIG_DIR}/master_node/meet.env\"\nfi\nsed -i \"s/ENABLED_MODULES=.*/ENABLED_MODULES=$ENABLED_MODULES/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\n\n# Update URLs in secret\nOPENVIDU_URL=\"https://${DOMAIN}/\"\nLIVEKIT_URL=\"wss://${DOMAIN}/\"\nDASHBOARD_URL=\"https://${DOMAIN}/dashboard/\"\nGRAFANA_URL=\"https://${DOMAIN}/grafana/\"\nMINIO_URL=\"https://${DOMAIN}/minio-console/\"\n\n# Update shared secret\naz keyvault secret set --vault-name ${keyVaultName} --name DOMAIN-NAME --value $DOMAIN\naz keyvault secret set --vault-name ${keyVaultName} --name OPENVIDU-URL --value $OPENVIDU_URL\naz keyvault secret set --vault-name ${keyVaultName} --name LIVEKIT-URL --value $LIVEKIT_URL\naz keyvault secret set --vault-name ${keyVaultName} --name DASHBOARD-URL --value $DASHBOARD_URL\naz keyvault secret set --vault-name ${keyVaultName} --name GRAFANA-URL --value $GRAFANA_URL\naz keyvault secret set --vault-name ${keyVaultName} --name MINIO-URL --value $MINIO_URL\n", "update_secret_from_configScriptTemplateMaster": "#!/bin/bash\nset -e\n\naz login --identity --allow-no-subscriptions > /dev/null\n\n# Installation directory\nINSTALL_DIR=\"/opt/openvidu\"\nCLUSTER_CONFIG_DIR=\"${INSTALL_DIR}/config/cluster\"\nMASTER_NODE_CONFIG_DIR=\"${INSTALL_DIR}/config/node\"\n\n# Get current values of the config\nREDIS_PASSWORD=\"$(/usr/local/bin/get_value_from_config.sh REDIS_PASSWORD \"${MASTER_NODE_CONFIG_DIR}/master_node.env\")\"\nDOMAIN_NAME=\"$(/usr/local/bin/get_value_from_config.sh DOMAIN_NAME \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nOPENVIDU_RTC_ENGINE=\"$(/usr/local/bin/get_value_from_config.sh OPENVIDU_RTC_ENGINE \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nOPENVIDU_PRO_LICENSE=\"$(/usr/local/bin/get_value_from_config.sh OPENVIDU_PRO_LICENSE \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nMONGO_ADMIN_USERNAME=\"$(/usr/local/bin/get_value_from_config.sh MONGO_ADMIN_USERNAME \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nMONGO_ADMIN_PASSWORD=\"$(/usr/local/bin/get_value_from_config.sh MONGO_ADMIN_PASSWORD \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nMONGO_REPLICA_SET_KEY=\"$(/usr/local/bin/get_value_from_config.sh MONGO_REPLICA_SET_KEY \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nMINIO_ACCESS_KEY=\"$(/usr/local/bin/get_value_from_config.sh MINIO_ACCESS_KEY \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nMINIO_SECRET_KEY=\"$(/usr/local/bin/get_value_from_config.sh MINIO_SECRET_KEY \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nDASHBOARD_ADMIN_USERNAME=\"$(/usr/local/bin/get_value_from_config.sh DASHBOARD_ADMIN_USERNAME \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nDASHBOARD_ADMIN_PASSWORD=\"$(/usr/local/bin/get_value_from_config.sh DASHBOARD_ADMIN_PASSWORD \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nGRAFANA_ADMIN_USERNAME=\"$(/usr/local/bin/get_value_from_config.sh GRAFANA_ADMIN_USERNAME \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nGRAFANA_ADMIN_PASSWORD=\"$(/usr/local/bin/get_value_from_config.sh GRAFANA_ADMIN_PASSWORD \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nLIVEKIT_API_KEY=\"$(/usr/local/bin/get_value_from_config.sh LIVEKIT_API_KEY \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nLIVEKIT_API_SECRET=\"$(/usr/local/bin/get_value_from_config.sh LIVEKIT_API_SECRET \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nMEET_INITIAL_ADMIN_USER=\"$(/usr/local/bin/get_value_from_config.sh MEET_INITIAL_ADMIN_USER \"${CLUSTER_CONFIG_DIR}/master_node/meet.env\")\"\nMEET_INITIAL_ADMIN_PASSWORD=\"$(/usr/local/bin/get_value_from_config.sh MEET_INITIAL_ADMIN_PASSWORD \"${CLUSTER_CONFIG_DIR}/master_node/meet.env\")\"\nif [[ \"${initialMeetApiKey}\" != '' ]]; then\n MEET_INITIAL_API_KEY=\"$(/usr/local/bin/get_value_from_config.sh MEET_INITIAL_API_KEY \"${CLUSTER_CONFIG_DIR}/master_node/meet.env\")\"\nfi\nENABLED_MODULES=\"$(/usr/local/bin/get_value_from_config.sh ENABLED_MODULES \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\n\n# Update shared secret\naz keyvault secret set --vault-name ${keyVaultName} --name REDIS-PASSWORD --value $REDIS_PASSWORD\naz keyvault secret set --vault-name ${keyVaultName} --name DOMAIN-NAME --value $DOMAIN_NAME\naz keyvault secret set --vault-name ${keyVaultName} --name OPENVIDU-RTC-ENGINE --value $OPENVIDU_RTC_ENGINE\naz keyvault secret set --vault-name ${keyVaultName} --name OPENVIDU-PRO-LICENSE --value $OPENVIDU_PRO_LICENSE\naz keyvault secret set --vault-name ${keyVaultName} --name MONGO-ADMIN-USERNAME --value $MONGO_ADMIN_USERNAME\naz keyvault secret set --vault-name ${keyVaultName} --name MONGO-ADMIN-PASSWORD --value $MONGO_ADMIN_PASSWORD\naz keyvault secret set --vault-name ${keyVaultName} --name MONGO-REPLICA-SET-KEY --value $MONGO_REPLICA_SET_KEY\naz keyvault secret set --vault-name ${keyVaultName} --name MINIO-ACCESS-KEY --value $MINIO_ACCESS_KEY\naz keyvault secret set --vault-name ${keyVaultName} --name MINIO-SECRET-KEY --value $MINIO_SECRET_KEY\naz keyvault secret set --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-USERNAME --value $DASHBOARD_ADMIN_USERNAME\naz keyvault secret set --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-PASSWORD --value $DASHBOARD_ADMIN_PASSWORD\naz keyvault secret set --vault-name ${keyVaultName} --name GRAFANA-ADMIN-USERNAME --value $GRAFANA_ADMIN_USERNAME\naz keyvault secret set --vault-name ${keyVaultName} --name GRAFANA-ADMIN-PASSWORD --value $GRAFANA_ADMIN_PASSWORD\naz keyvault secret set --vault-name ${keyVaultName} --name LIVEKIT-API-KEY --value $LIVEKIT_API_KEY\naz keyvault secret set --vault-name ${keyVaultName} --name LIVEKIT-API-SECRET --value $LIVEKIT_API_SECRET\naz keyvault secret set --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-USER --value $MEET_INITIAL_ADMIN_USER\naz keyvault secret set --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-PASSWORD --value $MEET_INITIAL_ADMIN_PASSWORD\nif [[ \"${initialMeetApiKey}\" != '' ]]; then\n az keyvault secret set --vault-name ${keyVaultName} --name MEET-INITIAL-API-KEY --value $MEET_INITIAL_API_KEY\nfi\naz keyvault secret set --vault-name ${keyVaultName} --name ENABLED-MODULES --value $ENABLED_MODULES\n", @@ -298,7 +298,7 @@ "get_public_ip": "#!/bin/bash\naz login --identity --allow-no-subscriptions > /dev/null\n\naz network public-ip show \\\n --id ${publicIPId} \\\n --query \"ipAddress\" -o tsv\n", "check_app_readyScriptMaster": "#!/bin/bash\nset -e\nMAX_WAIT=1200\nWAIT_INTERVAL=5\nELAPSED_TIME=0\nwhile true; do\n HTTP_STATUS=$(curl -Ik http://localhost:7880/health/caddy | head -n1 | awk '{print $2}')\n if [ \"$HTTP_STATUS\" = \"200\" ]; then\n break\n fi\n ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL))\n if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then\n exit 1\n fi\n sleep 5\ndone\n", "restartScriptMaster": "#!/bin/bash\nset -e\n# Stop all services\nsystemctl stop openvidu\n\n# Update config from secret\n/usr/local/bin/update_config_from_secret.sh\n\n# Start all services\nsystemctl start openvidu\n", - "config_blobStorageTemplate": "#!/bin/bash\nset -e\n\n# Install dir and config dir\nINSTALL_DIR=\"/opt/openvidu\"\nCLUSTER_CONFIG_DIR=\"${INSTALL_DIR}/config/cluster\"\n\n# Retry login + storage key fetch to allow the Contributor role assignment to propagate\nMAX_WAIT=100\nWAIT_INTERVAL=1\nELAPSED_TIME=0\nset +e\nwhile true; do\n az login --identity\n\n # Config azure blob storage\n AZURE_ACCOUNT_NAME=\"${storageAccountName}\"\n AZURE_ACCOUNT_KEY=$(az storage account keys list --account-name ${storageAccountName} --query '[0].value' -o tsv)\n\n # If the key was fetched successfully, exit the loop\n if [ $? -eq 0 ]; then\n break\n fi\n\n # If not, wait and check again incrementing the time\n ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL))\n\n # If exceeded the maximum time, exit with error\n if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then\n exit 1\n fi\n\n # Esperar antes de la próxima comprobación\n sleep $WAIT_INTERVAL\ndone\nset -e\nAZURE_CONTAINER_NAME=\"${storageAccountContainerName}\"\n\nsed -i \"s|AZURE_ACCOUNT_NAME=.*|AZURE_ACCOUNT_NAME=$AZURE_ACCOUNT_NAME|\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s|AZURE_ACCOUNT_KEY=.*|AZURE_ACCOUNT_KEY=$AZURE_ACCOUNT_KEY|\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s|AZURE_CONTAINER_NAME=.*|AZURE_CONTAINER_NAME=$AZURE_CONTAINER_NAME|\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\n", + "config_blobStorageTemplate": "#!/bin/bash\nset -e\n\n# Install dir and config dir\nINSTALL_DIR=\"/opt/openvidu\"\nCLUSTER_CONFIG_DIR=\"${INSTALL_DIR}/config/cluster\"\n\n# Retry login + storage key fetch to allow the Contributor role assignment to propagate\nMAX_WAIT=300\nWAIT_INTERVAL=1\nELAPSED_TIME=0\nset +e\nwhile true; do\n az login --identity\n\n # Config azure blob storage\n AZURE_ACCOUNT_NAME=\"${storageAccountName}\"\n AZURE_ACCOUNT_KEY=$(az storage account keys list --account-name ${storageAccountName} --query '[0].value' -o tsv)\n\n # If the key was fetched successfully, exit the loop\n if [ $? -eq 0 ]; then\n break\n fi\n\n # If not, wait and check again incrementing the time\n ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL))\n\n # If exceeded the maximum time, exit with error\n if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then\n exit 1\n fi\n\n # Esperar antes de la próxima comprobación\n sleep $WAIT_INTERVAL\ndone\nset -e\nAZURE_CONTAINER_NAME=\"${storageAccountContainerName}\"\n\nsed -i \"s|AZURE_ACCOUNT_NAME=.*|AZURE_ACCOUNT_NAME=$AZURE_ACCOUNT_NAME|\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s|AZURE_ACCOUNT_KEY=.*|AZURE_ACCOUNT_KEY=$AZURE_ACCOUNT_KEY|\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s|AZURE_CONTAINER_NAME=.*|AZURE_CONTAINER_NAME=$AZURE_CONTAINER_NAME|\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\n", "installScriptMaster1": "[reduce(items(variables('stringInterpolationParamsMaster1')), createObject('value', variables('installScriptTemplateMaster')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value]", "installScriptMaster2": "[reduce(items(variables('stringInterpolationParamsMaster2')), createObject('value', variables('installScriptTemplateMaster')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value]", "installScriptMaster3": "[reduce(items(variables('stringInterpolationParamsMaster3')), createObject('value', variables('installScriptTemplateMaster')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value]", @@ -320,8 +320,8 @@ "base64store_secretMaster": "[base64(variables('store_secretScriptMaster'))]", "base64check_app_readyMaster": "[base64(variables('check_app_readyScriptMaster'))]", "base64restartMaster": "[base64(variables('restartScriptMaster'))]", - "userDataTemplateMasterNode": "#!/bin/bash -x\nset -eu -o pipefail\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# after_install.sh\necho ${base64after_install} | base64 -d > /usr/local/bin/after_install.sh\nchmod +x /usr/local/bin/after_install.sh\n\n# get_public_ip.sh\necho ${base64get_public_ip} | base64 -d > /usr/local/bin/get_public_ip.sh\nchmod +x /usr/local/bin/get_public_ip.sh\n\n# update_config_from_secret.sh\necho ${base64update_config_from_secret} | base64 -d > /usr/local/bin/update_config_from_secret.sh\nchmod +x /usr/local/bin/update_config_from_secret.sh\n\n# update_secret_from_config.sh\necho ${base64update_secret_from_config} | base64 -d > /usr/local/bin/update_secret_from_config.sh\nchmod +x /usr/local/bin/update_secret_from_config.sh\n\n# get_value_from_config.sh\necho ${base64get_value_from_config} | base64 -d > /usr/local/bin/get_value_from_config.sh\nchmod +x /usr/local/bin/get_value_from_config.sh\n\n# store_secret.sh\necho ${base64store_secret} | base64 -d > /usr/local/bin/store_secret.sh\nchmod +x /usr/local/bin/store_secret.sh\n\n# check_app_ready.sh\necho ${base64check_app_ready} | base64 -d > /usr/local/bin/check_app_ready.sh\nchmod +x /usr/local/bin/check_app_ready.sh\n\n# restart.sh\necho ${base64restart} | base64 -d > /usr/local/bin/restart.sh\nchmod +x /usr/local/bin/restart.sh\n\necho ${base64config_blobStorage} | base64 -d > /usr/local/bin/config_blobStorage.sh\nchmod +x /usr/local/bin/config_blobStorage.sh\n\n# Install azure cli\nAZURE_CLI_VERSION=2.87.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity --allow-no-subscriptions\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y\n\nexport HOME=\"/root\"\n\n# Install OpenVidu\n/usr/local/bin/install.sh || { echo \"[OpenVidu] error installing OpenVidu\"; exit 1; }\n\n#Config blob storage\n/usr/local/bin/config_blobStorage.sh || { echo \"[OpenVidu] error configuring Blob Storage\"; exit 1; }\n\n# Start OpenVidu\nsystemctl start openvidu || { echo \"[OpenVidu] error starting OpenVidu\"; exit 1; }\n\n# Update shared secret\n/usr/local/bin/after_install.sh || { echo \"[OpenVidu] error updating shared secret\"; exit 1; }\n\n# Launch on reboot\necho \"@reboot /usr/local/bin/restart.sh >> /var/log/openvidu-restart.log\" 2>&1 | crontab\n\nMASTER_NODE_NUM=${masterNodeNum}\nif [[ $MASTER_NODE_NUM -eq 4 ]]; then\n # Creating scale in lock\n set +e\n az storage blob upload --account-name ${storageAccountName} --container-name automation-locks --name lock.txt --file /dev/null --auth-mode key\n set -e\n\n #Finish all the nodes\n az keyvault secret set --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --value \"true\"\nfi\n\n# Wait for the app\nsleep 150\n/usr/local/bin/check_app_ready.sh\n", - "installScriptTemplateMedia": "#!/bin/bash -x\nDOMAIN=\n\n# Install dependencies\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y \\\n curl \\\n unzip \\\n jq \\\n wget\n\n# Get own private IP\nPRIVATE_IP=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text\")\n\nWAIT_INTERVAL=1\nMAX_WAIT=10000\nELAPSED_TIME=0\nset +e\nwhile true; do\n # get secret value\n FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv)\n\n # Check if all master nodes finished\n if [ \"$FINISH_MASTER_NODE\" == \"true\" ]; then\n break\n fi\n\n ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL))\n\n # Check if the maximum waiting time has been reached\n if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then\n exit 1\n fi\n\n sleep $WAIT_INTERVAL\ndone\nset -e\n\nMASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_PRIVATE_IP_LIST=\"$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP\"\nREDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name REDIS-PASSWORD --query value -o tsv)\nENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv)\nOPENVIDU_VERSION=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-VERSION --query value -o tsv)\n\n# Base command\nINSTALL_COMMAND=\"sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh)\"\n\n# Common arguments\nCOMMON_ARGS=(\n\"--no-tty\"\n\"--install\"\n\"--environment=azure\"\n\"--deployment-type='ha'\"\n\"--node-role='media-node'\"\n\"--master-node-private-ip-list=$MASTER_NODE_PRIVATE_IP_LIST\"\n\"--private-ip=$PRIVATE_IP\"\n\"--enabled-modules='$ENABLED_MODULES'\"\n\"--redis-password=$REDIS_PASSWORD\"\n)\n\n# Construct the final command with all arguments\nFINAL_COMMAND=\"$INSTALL_COMMAND $(printf \"%s \" \"${COMMON_ARGS[@]}\")\"\n\n# Install OpenVidu\nexec bash -c \"$FINAL_COMMAND\"\n", + "userDataTemplateMasterNode": "#!/bin/bash -x\nset -eu -o pipefail\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# after_install.sh\necho ${base64after_install} | base64 -d > /usr/local/bin/after_install.sh\nchmod +x /usr/local/bin/after_install.sh\n\n# get_public_ip.sh\necho ${base64get_public_ip} | base64 -d > /usr/local/bin/get_public_ip.sh\nchmod +x /usr/local/bin/get_public_ip.sh\n\n# update_config_from_secret.sh\necho ${base64update_config_from_secret} | base64 -d > /usr/local/bin/update_config_from_secret.sh\nchmod +x /usr/local/bin/update_config_from_secret.sh\n\n# update_secret_from_config.sh\necho ${base64update_secret_from_config} | base64 -d > /usr/local/bin/update_secret_from_config.sh\nchmod +x /usr/local/bin/update_secret_from_config.sh\n\n# get_value_from_config.sh\necho ${base64get_value_from_config} | base64 -d > /usr/local/bin/get_value_from_config.sh\nchmod +x /usr/local/bin/get_value_from_config.sh\n\n# store_secret.sh\necho ${base64store_secret} | base64 -d > /usr/local/bin/store_secret.sh\nchmod +x /usr/local/bin/store_secret.sh\n\n# check_app_ready.sh\necho ${base64check_app_ready} | base64 -d > /usr/local/bin/check_app_ready.sh\nchmod +x /usr/local/bin/check_app_ready.sh\n\n# restart.sh\necho ${base64restart} | base64 -d > /usr/local/bin/restart.sh\nchmod +x /usr/local/bin/restart.sh\n\necho ${base64config_blobStorage} | base64 -d > /usr/local/bin/config_blobStorage.sh\nchmod +x /usr/local/bin/config_blobStorage.sh\n\n# Install azure cli\nAZURE_CLI_VERSION=2.87.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity --allow-no-subscriptions\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\nexport HOME=\"/root\"\n\n# Install OpenVidu\n/usr/local/bin/install.sh || { echo \"[OpenVidu] error installing OpenVidu\"; exit 1; }\n\n#Config blob storage\n/usr/local/bin/config_blobStorage.sh || { echo \"[OpenVidu] error configuring Blob Storage\"; exit 1; }\n\n# Start OpenVidu\nsystemctl start openvidu || { echo \"[OpenVidu] error starting OpenVidu\"; exit 1; }\n\n# Update shared secret\n/usr/local/bin/after_install.sh || { echo \"[OpenVidu] error updating shared secret\"; exit 1; }\n\n# Launch on reboot\necho \"@reboot /usr/local/bin/restart.sh >> /var/log/openvidu-restart.log\" 2>&1 | crontab\n\n# Wait until this master node is healthy (check_app_ready.sh caps at 1200s).\n# On failure exit WITHOUT publishing readiness, so media nodes keep waiting for another master.\n/usr/local/bin/check_app_ready.sh || { echo \"[OpenVidu] master node did not become healthy\"; exit 1; }\n\n# Publish readiness so media nodes can start. Every healthy master writes the same value, so the\n# first healthy master unblocks media and a failed master can no longer leave this unset (no SPOF).\naz keyvault secret set --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --value \"true\"\n\nMASTER_NODE_NUM=${masterNodeNum}\nif [[ $MASTER_NODE_NUM -eq 4 ]]; then\n # Creating scale in lock\n set +e\n az storage blob upload --account-name ${storageAccountName} --container-name automation-locks --name lock.txt --file /dev/null --auth-mode key\n set -e\nfi\n", + "installScriptTemplateMedia": "#!/bin/bash -x\nDOMAIN=\n\n# Install dependencies\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y \\\n curl \\\n unzip \\\n jq \\\n wget\n\n# Get own private IP\nPRIVATE_IP=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text\")\n\n# Gate 1: wait (bounded: 360 x 5s = 30 min) until master nodes generated the shared secrets and\n# published their 4 private IPs. This lets the heavy media installer run in parallel with the masters.\nWAIT_INTERVAL=5\nMAX_RETRIES=360\nRETRIES=0\nset +e\nwhile true; do\n ALL_SECRETS_GENERATED=$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)\n MASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv 2>/dev/null)\n MASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv 2>/dev/null)\n MASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv 2>/dev/null)\n MASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv 2>/dev/null)\n\n # Break once secrets are generated and all 4 master IPs are published\n if [ \"$ALL_SECRETS_GENERATED\" == \"true\" ] &&\n [ \"$MASTER_NODE_1_PRIVATE_IP\" != \"\" ] &&\n [ \"$MASTER_NODE_2_PRIVATE_IP\" != \"\" ] &&\n [ \"$MASTER_NODE_3_PRIVATE_IP\" != \"\" ] &&\n [ \"$MASTER_NODE_4_PRIVATE_IP\" != \"\" ]; then\n break\n fi\n\n RETRIES=$((RETRIES + 1))\n if [ $RETRIES -ge $MAX_RETRIES ]; then\n echo \"[OpenVidu] timed out after 30 min waiting for master nodes to generate secrets and publish their IPs\"\n exit 1\n fi\n\n sleep $WAIT_INTERVAL\ndone\nset -e\n\nMASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_PRIVATE_IP_LIST=\"$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP\"\nREDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name REDIS-PASSWORD --query value -o tsv)\nENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv)\nOPENVIDU_VERSION=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-VERSION --query value -o tsv)\n\n# Download the installer script (retry transient curl failures; fail if empty/missing)\nINSTALLER_SCRIPT=\"/tmp/install_ov_media_node.sh\"\ncurl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o \"$INSTALLER_SCRIPT\" \"http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh\"\nif [ ! -s \"$INSTALLER_SCRIPT\" ]; then\n echo \"[OpenVidu] failed to download the media node installer script\"\n exit 1\nfi\n\n# Base command\nINSTALL_COMMAND=\"sh $INSTALLER_SCRIPT\"\n\n# Common arguments\nCOMMON_ARGS=(\n\"--no-tty\"\n\"--install\"\n\"--environment=azure\"\n\"--deployment-type='ha'\"\n\"--node-role='media-node'\"\n\"--master-node-private-ip-list=$MASTER_NODE_PRIVATE_IP_LIST\"\n\"--private-ip=$PRIVATE_IP\"\n\"--enabled-modules='$ENABLED_MODULES'\"\n\"--redis-password=$REDIS_PASSWORD\"\n)\n\n# Construct the final command with all arguments\nFINAL_COMMAND=\"$INSTALL_COMMAND $(printf \"%s \" \"${COMMON_ARGS[@]}\")\"\n\n# Install OpenVidu\nexec bash -c \"$FINAL_COMMAND\"\n", "stopMediaNodeParams": { "subscriptionId": "[subscription().subscriptionId]", "resourceGroupName": "[resourceGroup().name]", @@ -330,7 +330,7 @@ }, "stop_media_nodesScriptMediaTemplate": "#!/bin/bash\nset -e\n\nif ! (set -o noclobber ; echo > /tmp/global.lock) ; then\n exit 1 # the global.lock already exists\nfi\n\n# Execute if docker is installed\nif [ -x \"$(command -v docker)\" ]; then\n\n echo \"Stopping media node services and waiting for termination...\"\n docker container kill --signal=SIGQUIT openvidu || true\n docker container kill --signal=SIGQUIT ingress || true\n docker container kill --signal=SIGQUIT egress || true\n for agent_container in $(docker ps --filter \"label=openvidu-agent=true\" --format '{{.Names}}'); do\n docker container kill --signal=SIGQUIT \"$agent_container\"\n done\n\n # Wait for running containers to not be openvidu, ingress or egress\n while [ $(docker ps --filter \"label=openvidu-agent=true\" -q | wc -l) -gt 0 ] || \\\n [ $(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null) == \"true\" ] || \\\n [ $(docker inspect -f '{{.State.Running}}' ingress 2>/dev/null) == \"true\" ] || \\\n [ $(docker inspect -f '{{.State.Running}}' egress 2>/dev/null) == \"true\" ]; do\n echo \"Waiting for containers to stop...\"\n sleep 5\n done\nfi\n\naz login --identity\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nSUBSCRIPTION_ID=${subscriptionId}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\nRESOURCE_ID=/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP_NAME/providers/Microsoft.Compute/virtualMachineScaleSets/$VM_SCALE_SET_NAME\n\nTIMESTAMP=$(date -u +\"%Y-%m-%dT%H:%M:%SZ\")\naz tag update --resource-id $RESOURCE_ID --operation replace --tags \"STATUS\"=\"HEALTHY\" \"InstanceDeleteTime\"=\"$TIMESTAMP\" \"storageAccount\"=\"${storageAccountName}\"\n\naz vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID\n", "delete_mediaNode_ScriptMediaTemplate": "#!/bin/bash\nset -e\n\naz login --identity\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\n\n\naz vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID\n", - "userDataMediaNodeTemplate": "#!/bin/bash -x\nset -eu -o pipefail\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# stop_media_nodes.sh\necho ${base64stop} | base64 -d > /usr/local/bin/stop_media_node.sh\nchmod +x /usr/local/bin/stop_media_node.sh\n\n# delete_media_node.sh\necho ${base64delete_mediaNode} | base64 -d > /usr/local/bin/delete_media_node.sh\nchmod +x /usr/local/bin/delete_media_node.sh\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y \napt-get install -y jq\n\n# Install azure cli\nAZURE_CLI_VERSION=2.87.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity\n\n# Protect from scale in actions\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\naz vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-id $INSTANCE_ID --protect-from-scale-in true\n\nexport HOME=\"/root\"\n\n# Install OpenVidu\n/usr/local/bin/install.sh || { echo \"[OpenVidu] error installing OpenVidu\"; /usr/local/bin/delete_media_node.sh; }\n\n# Start OpenVidu\nsystemctl start openvidu || { echo \"[OpenVidu] error starting OpenVidu\"; /usr/local/bin/delete_media_node.sh; }\n", + "userDataMediaNodeTemplate": "#!/bin/bash -x\nset -eu -o pipefail\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# stop_media_nodes.sh\necho ${base64stop} | base64 -d > /usr/local/bin/stop_media_node.sh\nchmod +x /usr/local/bin/stop_media_node.sh\n\n# delete_media_node.sh\necho ${base64delete_mediaNode} | base64 -d > /usr/local/bin/delete_media_node.sh\nchmod +x /usr/local/bin/delete_media_node.sh\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y jq\n\n# Install azure cli\nAZURE_CLI_VERSION=2.87.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity\n\n# Protect from scale in actions\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\naz vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-id $INSTANCE_ID --protect-from-scale-in true\n\nexport HOME=\"/root\"\n\n# Install OpenVidu (heavy: docker + image pulls). Overlaps with the masters' own installs.\n/usr/local/bin/install.sh || { echo \"[OpenVidu] error installing OpenVidu\"; /usr/local/bin/delete_media_node.sh; }\n\n# Gate 2: wait (bounded: 360 x 5s = 30 min) for the master nodes to publish readiness before starting\nWAIT_INTERVAL=5\nMAX_RETRIES=360\nRETRIES=0\nset +e\nwhile true; do\n FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv 2>/dev/null)\n if [ \"$FINISH_MASTER_NODE\" == \"true\" ]; then\n break\n fi\n RETRIES=$((RETRIES + 1))\n if [ $RETRIES -ge $MAX_RETRIES ]; then\n echo \"[OpenVidu] timed out after 30 min waiting for FINISH-MASTER-NODE\"\n /usr/local/bin/delete_media_node.sh\n exit 1\n fi\n sleep $WAIT_INTERVAL\ndone\nset -e\n\n# Start OpenVidu\nsystemctl start openvidu || { echo \"[OpenVidu] error starting OpenVidu\"; /usr/local/bin/delete_media_node.sh; }\n", "stop_media_nodesScriptMedia": "[reduce(items(variables('stopMediaNodeParams')), createObject('value', variables('stop_media_nodesScriptMediaTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value]", "delete_mediaNode_ScriptMedia": "[reduce(items(variables('stopMediaNodeParams')), createObject('value', variables('delete_mediaNode_ScriptMediaTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value]", "base64stopMediaNode": "[base64(variables('stop_media_nodesScriptMedia'))]", @@ -531,7 +531,6 @@ }, "dependsOn": [ "[resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface2', parameters('stackName')))]", - "[resourceId('Microsoft.Compute/virtualMachines', format('{0}-VM-MasterNode1', parameters('stackName')))]", "[resourceId('Microsoft.Network/publicIPAddresses', parameters('publicIpAddressObject').name)]", "[resourceId('Microsoft.Storage/storageAccounts', uniqueString(resourceGroup().id))]" ] @@ -574,7 +573,6 @@ }, "dependsOn": [ "[resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface3', parameters('stackName')))]", - "[resourceId('Microsoft.Compute/virtualMachines', format('{0}-VM-MasterNode2', parameters('stackName')))]", "[resourceId('Microsoft.Network/publicIPAddresses', parameters('publicIpAddressObject').name)]", "[resourceId('Microsoft.Storage/storageAccounts', uniqueString(resourceGroup().id))]" ] @@ -617,7 +615,6 @@ }, "dependsOn": [ "[resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface4', parameters('stackName')))]", - "[resourceId('Microsoft.Compute/virtualMachines', format('{0}-VM-MasterNode3', parameters('stackName')))]", "[resourceId('Microsoft.Network/publicIPAddresses', parameters('publicIpAddressObject').name)]", "[resourceId('Microsoft.Storage/storageAccounts', uniqueString(resourceGroup().id))]" ] @@ -696,7 +693,7 @@ } ] }, - "userData": "[base64(reduce(items(createObject('base64install', base64(reduce(items(createObject('privateIPMasterNode1', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface1', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'privateIPMasterNode2', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface2', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'privateIPMasterNode3', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface3', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'privateIPMasterNode4', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface4', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'keyVaultName', variables('keyVaultName'))), createObject('value', variables('installScriptTemplateMedia')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value), 'base64stop', variables('base64stopMediaNode'), 'base64delete_mediaNode', variables('base64delete_mediaNode_ScriptMedia'), 'resourceGroupName', resourceGroup().name, 'vmScaleSetName', format('{0}-mediaNodeScaleSet', parameters('stackName')))), createObject('value', variables('userDataMediaNodeTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value)]" + "userData": "[base64(reduce(items(createObject('base64install', base64(reduce(items(createObject('privateIPMasterNode1', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface1', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'privateIPMasterNode2', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface2', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'privateIPMasterNode3', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface3', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'privateIPMasterNode4', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface4', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'keyVaultName', variables('keyVaultName'))), createObject('value', variables('installScriptTemplateMedia')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value), 'base64stop', variables('base64stopMediaNode'), 'base64delete_mediaNode', variables('base64delete_mediaNode_ScriptMedia'), 'resourceGroupName', resourceGroup().name, 'vmScaleSetName', format('{0}-mediaNodeScaleSet', parameters('stackName')), 'keyVaultName', variables('keyVaultName'))), createObject('value', variables('userDataMediaNodeTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value)]" } }, "dependsOn": [ diff --git a/openvidu-deployment/pro/ha/azure/createUiDefinition.json b/openvidu-deployment/pro/ha/azure/createUiDefinition.json index 3e7d2db87..6e12791db 100644 --- a/openvidu-deployment/pro/ha/azure/createUiDefinition.json +++ b/openvidu-deployment/pro/ha/azure/createUiDefinition.json @@ -553,7 +553,7 @@ "datetime": "[steps('parameters SCALING').datetime]", "automationAccountName": "[steps('parameters SCALING').automationAccountName]", "storageAccountName": "[steps('parameters STORAGE').storageAccountName]", - "containerName": "[steps('parameters STORAGE').containerName]", + "appDataContainerName": "[steps('parameters STORAGE').containerName]", "additionalInstallFlags": "[steps('FLAGS').additionalInstallFlags]" } } From b8fdde9751c4c5cc129cc3f6d8354983ca999efb Mon Sep 17 00:00:00 2001 From: Piwccle Date: Fri, 24 Jul 2026 17:11:58 +0200 Subject: [PATCH 4/6] openvidu-deployment: shorten comments in GCP/Azure HA templates --- .../pro/ha/azure/cf-openvidu-ha.bicep | 20 +++++++------------ .../pro/ha/azure/cf-openvidu-ha.json | 10 +++++----- .../pro/ha/gcp/tf-gpc-openvidu-ha.tf | 20 ++++--------------- 3 files changed, 16 insertions(+), 34 deletions(-) diff --git a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep index 1c659b6b7..0d771c174 100644 --- a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep +++ b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep @@ -397,7 +397,6 @@ if [[ $MASTER_NODE_NUM -eq 1 ]] && [[ "$ALL_SECRETS_GENERATED" == "" || "$ALL_SE ALL_SECRETS_GENERATED="$(/usr/local/bin/store_secret.sh save ALL-SECRETS-GENERATED "true")" fi -# Wait (bounded: 360 x 5s = 30 min) until all 4 master nodes have published their private IPs IP_WAIT_MAX_RETRIES=360 IP_WAIT_INTERVAL=5 IP_WAIT_RETRIES=0 @@ -421,7 +420,7 @@ while true; do sleep $IP_WAIT_INTERVAL done -# Wait (bounded: 360 x 5s = 30 min) until master-node-1 has generated all shared secrets before fetching them +# Wait until master-node-1 has generated all shared secrets before fetching them SECRETS_WAIT_MAX_RETRIES=360 SECRETS_WAIT_INTERVAL=5 SECRETS_WAIT_RETRIES=0 @@ -467,7 +466,7 @@ fi ENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv) -# Download the installer script (retry transient curl failures; fail if empty/missing) +# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure INSTALLER_SCRIPT="/tmp/install_ov_master_node.sh" curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh" if [ ! -s "$INSTALLER_SCRIPT" ]; then @@ -1087,12 +1086,9 @@ systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; # Launch on reboot echo "@reboot /usr/local/bin/restart.sh >> /var/log/openvidu-restart.log" 2>&1 | crontab -# Wait until this master node is healthy (check_app_ready.sh caps at 1200s). -# On failure exit WITHOUT publishing readiness, so media nodes keep waiting for another master. +# check_app_ready.sh internally caps its wait at 1200s /usr/local/bin/check_app_ready.sh || { echo "[OpenVidu] master node did not become healthy"; exit 1; } -# Publish readiness so media nodes can start. Every healthy master writes the same value, so the -# first healthy master unblocks media and a failed master can no longer leave this unset (no SPOF). az keyvault secret set --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --value "true" MASTER_NODE_NUM=${masterNodeNum} @@ -1295,8 +1291,7 @@ apt-get update && apt-get install -y \ # Get own private IP PRIVATE_IP=$(curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text") -# Gate 1: wait (bounded: 360 x 5s = 30 min) until master nodes generated the shared secrets and -# published their 4 private IPs. This lets the heavy media installer run in parallel with the masters. +# Gate 1: wait for master secrets and IPs before installing WAIT_INTERVAL=5 MAX_RETRIES=360 RETRIES=0 @@ -1308,7 +1303,6 @@ while true; do MASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv 2>/dev/null) MASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv 2>/dev/null) - # Break once secrets are generated and all 4 master IPs are published if [ "$ALL_SECRETS_GENERATED" == "true" ] && [ "$MASTER_NODE_1_PRIVATE_IP" != "" ] && [ "$MASTER_NODE_2_PRIVATE_IP" != "" ] && @@ -1336,7 +1330,7 @@ REDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name RED ENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv) OPENVIDU_VERSION=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-VERSION --query value -o tsv) -# Download the installer script (retry transient curl failures; fail if empty/missing) +# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure INSTALLER_SCRIPT="/tmp/install_ov_media_node.sh" curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh" if [ ! -s "$INSTALLER_SCRIPT" ]; then @@ -1473,10 +1467,10 @@ az vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME - export HOME="/root" -# Install OpenVidu (heavy: docker + image pulls). Overlaps with the masters' own installs. +# Install OpenVidu /usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu"; /usr/local/bin/delete_media_node.sh; } -# Gate 2: wait (bounded: 360 x 5s = 30 min) for the master nodes to publish readiness before starting +# Gate 2: wait for master readiness before starting WAIT_INTERVAL=5 MAX_RETRIES=360 RETRIES=0 diff --git a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json index b0a5992b2..0a8750e2f 100644 --- a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json +++ b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.45.15.27210", - "templateHash": "16682765069403816529" + "templateHash": "10036394790552381773" } }, "parameters": { @@ -289,7 +289,7 @@ "masterNodeNum": "4", "additionalInstallFlags": "[parameters('additionalInstallFlags')]" }, - "installScriptTemplateMaster": "#!/bin/bash -x\nset -e\nOPENVIDU_VERSION=main\nDOMAIN=\n\n# Assume azure cli is installed\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y \\\n curl \\\n unzip \\\n jq \\\n wget\n\n\n# Wait for the keyvault availability\nMAX_WAIT=300\nWAIT_INTERVAL=1\nELAPSED_TIME=0\nset +e\nwhile true; do\n # Check keyvault availability\n az keyvault secret list --vault-name ${keyVaultName} \n\n # If it is available, exit the loop\n if [ $? -eq 0 ]; then\n break\n fi\n\n # If not, wait and check again incrementing the time\n ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL))\n\n # If exceeded the maximum time, exit with error\n if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then\n exit 1\n fi\n\n # Esperar antes de la próxima comprobación\n sleep $WAIT_INTERVAL\ndone\nset -e\n\nMASTER_NODE_NUM=${masterNodeNum}\n\n# Get own private IP\nPRIVATE_IP=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text\")\n\n# Store current private IP\nPRIVATE_IP=\"$(/usr/local/bin/store_secret.sh save MASTER-NODE-${masterNodeNum}-PRIVATE-IP $PRIVATE_IP)\"\n\n\nif [[ $MASTER_NODE_NUM -eq 1 ]] && [[ \"$ALL_SECRETS_GENERATED\" == \"\" || \"$ALL_SECRETS_GENERATED\" == \"false\" ]]; then\n\n # Configure Domain name\n if [[ \"${domainName}\" == '' ]]; then\n # Get public IP using the get_public_ip.sh script\n PUBLIC_IP=$(/usr/local/bin/get_public_ip.sh 2>/dev/null)\n if [[ $? -ne 0 || -z \"${PUBLIC_IP}\" ]]; then\n echo \"Could not determine public IP.\"\n exit 1\n fi\n\n DOMAIN=\"$PUBLIC_IP\"\n else\n DOMAIN=${domainName}\n fi\n DOMAIN=\"$(/usr/local/bin/store_secret.sh save DOMAIN-NAME \"${DOMAIN}\")\"\n\n # Publish service URLs early (idempotent; after_install re-writes byte-identical values)\n OPENVIDU_URL=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-URL \"https://${DOMAIN}/\")\"\n LIVEKIT_URL=\"$(/usr/local/bin/store_secret.sh save LIVEKIT-URL \"wss://${DOMAIN}/\")\"\n DASHBOARD_URL=\"$(/usr/local/bin/store_secret.sh save DASHBOARD-URL \"https://${DOMAIN}/dashboard/\")\"\n GRAFANA_URL=\"$(/usr/local/bin/store_secret.sh save GRAFANA-URL \"https://${DOMAIN}/grafana/\")\"\n MINIO_URL=\"$(/usr/local/bin/store_secret.sh save MINIO-URL \"https://${DOMAIN}/minio-console/\")\"\n\n # Meet initial admin user and password\n MEET_INITIAL_ADMIN_USER=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-ADMIN-USER \"admin\")\"\n if [[ \"${initialMeetAdminPassword}\" != '' ]]; then\n MEET_INITIAL_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-ADMIN-PASSWORD \"${initialMeetAdminPassword}\")\"\n else\n MEET_INITIAL_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate MEET-INITIAL-ADMIN-PASSWORD)\"\n fi\n if [[ \"${initialMeetApiKey}\" != '' ]]; then\n MEET_INITIAL_API_KEY=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-API-KEY \"${initialMeetApiKey}\")\"\n else\n MEET_INITIAL_API_KEY=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-API-KEY \"\")\"\n fi\n\n # Store usernames and generate random passwords\n OPENVIDU_PRO_LICENSE=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-PRO-LICENSE \"${openviduLicense}\")\"\n OPENVIDU_RTC_ENGINE=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-RTC-ENGINE \"${rtcEngine}\")\"\n REDIS_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate REDIS-PASSWORD)\"\n MONGO_ADMIN_USERNAME=\"$(/usr/local/bin/store_secret.sh save MONGO-ADMIN-USERNAME \"mongoadmin\")\"\n MONGO_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate MONGO-ADMIN-PASSWORD)\"\n MONGO_REPLICA_SET_KEY=\"$(/usr/local/bin/store_secret.sh generate MONGO-REPLICA-SET-KEY)\"\n MINIO_ACCESS_KEY=\"$(/usr/local/bin/store_secret.sh save MINIO-ACCESS-KEY \"minioadmin\")\"\n MINIO_SECRET_KEY=\"$(/usr/local/bin/store_secret.sh generate MINIO-SECRET-KEY)\"\n DASHBOARD_ADMIN_USERNAME=\"$(/usr/local/bin/store_secret.sh save DASHBOARD-ADMIN-USERNAME \"dashboardadmin\")\"\n DASHBOARD_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate DASHBOARD-ADMIN-PASSWORD)\"\n GRAFANA_ADMIN_USERNAME=\"$(/usr/local/bin/store_secret.sh save GRAFANA-ADMIN-USERNAME \"grafanaadmin\")\"\n GRAFANA_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate GRAFANA-ADMIN-PASSWORD)\"\n MEET_INITIAL_ADMIN_USER=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-ADMIN-USER \"admin\")\"\n LIVEKIT_API_KEY=\"$(/usr/local/bin/store_secret.sh generate LIVEKIT-API-KEY \"API\" 12)\"\n LIVEKIT_API_SECRET=\"$(/usr/local/bin/store_secret.sh generate LIVEKIT-API-SECRET)\"\n OPENVIDU_VERSION=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-VERSION \"${OPENVIDU_VERSION}\")\"\n ENABLED_MODULES=\"$(/usr/local/bin/store_secret.sh save ENABLED-MODULES \"observability,openviduMeet,v2compatibility\")\"\n ALL_SECRETS_GENERATED=\"$(/usr/local/bin/store_secret.sh save ALL-SECRETS-GENERATED \"true\")\"\nfi\n\n# Wait (bounded: 360 x 5s = 30 min) until all 4 master nodes have published their private IPs\nIP_WAIT_MAX_RETRIES=360\nIP_WAIT_INTERVAL=5\nIP_WAIT_RETRIES=0\nwhile true; do\n MASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv) || true\n MASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv) || true\n MASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv) || true\n MASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv) || true\n # Check if all master nodes have stored their private IPs\n if [[ \"$MASTER_NODE_1_PRIVATE_IP\" != \"\" ]] &&\n [[ \"$MASTER_NODE_2_PRIVATE_IP\" != \"\" ]] &&\n [[ \"$MASTER_NODE_3_PRIVATE_IP\" != \"\" ]] &&\n [[ \"$MASTER_NODE_4_PRIVATE_IP\" != \"\" ]]; then\n break\n fi\n IP_WAIT_RETRIES=$((IP_WAIT_RETRIES + 1))\n if [ $IP_WAIT_RETRIES -ge $IP_WAIT_MAX_RETRIES ]; then\n echo \"[OpenVidu] timed out after 30 min waiting for the 4 master nodes to publish their private IPs\"\n exit 1\n fi\n sleep $IP_WAIT_INTERVAL\ndone\n\n# Wait (bounded: 360 x 5s = 30 min) until master-node-1 has generated all shared secrets before fetching them\nSECRETS_WAIT_MAX_RETRIES=360\nSECRETS_WAIT_INTERVAL=5\nSECRETS_WAIT_RETRIES=0\nwhile [[ \"$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)\" != \"true\" ]]; do\n SECRETS_WAIT_RETRIES=$((SECRETS_WAIT_RETRIES + 1))\n if [ $SECRETS_WAIT_RETRIES -ge $SECRETS_WAIT_MAX_RETRIES ]; then\n echo \"[OpenVidu] timed out after 30 min waiting for ALL-SECRETS-GENERATED to become true\"\n exit 1\n fi\n sleep $SECRETS_WAIT_INTERVAL\ndone\n\n\n# Fetch the values in the keyvault\nMASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_PRIVATE_IP_LIST=\"$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP\"\n\nDOMAIN=$(az keyvault secret show --vault-name ${keyVaultName} --name DOMAIN-NAME --query value -o tsv)\nOPENVIDU_RTC_ENGINE=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-RTC-ENGINE --query value -o tsv)\nOPENVIDU_PRO_LICENSE=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-PRO-LICENSE --query value -o tsv)\nREDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name REDIS-PASSWORD --query value -o tsv)\nMONGO_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-ADMIN-USERNAME --query value -o tsv)\nMONGO_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-ADMIN-PASSWORD --query value -o tsv)\nMONGO_REPLICA_SET_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-REPLICA-SET-KEY --query value -o tsv)\nDASHBOARD_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-USERNAME --query value -o tsv)\nDASHBOARD_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-PASSWORD --query value -o tsv)\nMINIO_ACCESS_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MINIO-ACCESS-KEY --query value -o tsv)\nMINIO_SECRET_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MINIO-SECRET-KEY --query value -o tsv)\nGRAFANA_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name GRAFANA-ADMIN-USERNAME --query value -o tsv)\nGRAFANA_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name GRAFANA-ADMIN-PASSWORD --query value -o tsv)\nLIVEKIT_API_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name LIVEKIT-API-KEY --query value -o tsv)\nLIVEKIT_API_SECRET=$(az keyvault secret show --vault-name ${keyVaultName} --name LIVEKIT-API-SECRET --query value -o tsv)\nMEET_INITIAL_ADMIN_USER=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-USER --query value -o tsv)\nMEET_INITIAL_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-PASSWORD --query value -o tsv)\nif [[ \"${initialMeetApiKey}\" != '' ]]; then\n MEET_INITIAL_API_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-API-KEY --query value -o tsv)\nelse\n MEET_INITIAL_API_KEY=\"\"\nfi\nENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv)\n\n\n# Download the installer script (retry transient curl failures; fail if empty/missing)\nINSTALLER_SCRIPT=\"/tmp/install_ov_master_node.sh\"\ncurl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o \"$INSTALLER_SCRIPT\" \"http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh\"\nif [ ! -s \"$INSTALLER_SCRIPT\" ]; then\n echo \"[OpenVidu] failed to download the master node installer script\"\n exit 1\nfi\n\n# Base command\nINSTALL_COMMAND=\"sh $INSTALLER_SCRIPT\"\n\n# Common arguments\nCOMMON_ARGS=(\n \"--no-tty\"\n \"--install\"\n \"--environment=azure\"\n \"--deployment-type='ha'\"\n \"--node-role='master-node'\"\n \"--external-load-balancer\"\n \"--internal-tls-termination\"\n \"--master-node-private-ip-list='$MASTER_NODE_PRIVATE_IP_LIST'\"\n \"--openvidu-pro-license='$OPENVIDU_PRO_LICENSE'\"\n \"--domain-name='$DOMAIN'\"\n \"--enabled-modules='$ENABLED_MODULES'\"\n \"--rtc-engine=$OPENVIDU_RTC_ENGINE\"\n \"--redis-password=$REDIS_PASSWORD\"\n \"--mongo-admin-user=$MONGO_ADMIN_USERNAME\"\n \"--mongo-admin-password=$MONGO_ADMIN_PASSWORD\"\n \"--mongo-replica-set-key=$MONGO_REPLICA_SET_KEY\"\n \"--minio-access-key=$MINIO_ACCESS_KEY\"\n \"--minio-secret-key=$MINIO_SECRET_KEY\"\n \"--dashboard-admin-user=$DASHBOARD_ADMIN_USERNAME\"\n \"--dashboard-admin-password=$DASHBOARD_ADMIN_PASSWORD\"\n \"--grafana-admin-user=$GRAFANA_ADMIN_USERNAME\"\n \"--grafana-admin-password=$GRAFANA_ADMIN_PASSWORD\"\n \"--meet-initial-admin-password=$MEET_INITIAL_ADMIN_PASSWORD\"\n \"--meet-initial-api-key=$MEET_INITIAL_API_KEY\"\n \"--livekit-api-key=$LIVEKIT_API_KEY\"\n \"--livekit-api-secret=$LIVEKIT_API_SECRET\"\n)\n\n# Include additional installer flags provided by the user\nif [[ \"${additionalInstallFlags}\" != \"\" ]]; then\n IFS=',' read -ra EXTRA_FLAGS <<< \"${additionalInstallFlags}\"\n for extra_flag in \"${EXTRA_FLAGS[@]}\"; do\n # Trim whitespace around each flag\n extra_flag=\"$(echo -e \"${extra_flag}\" | sed -e 's/^[ \\t]*//' -e 's/[ \\t]*$//')\"\n if [[ \"$extra_flag\" != \"\" ]]; then\n COMMON_ARGS+=(\"$extra_flag\")\n fi\n done\nfi\n\n# Certificate arguments\nif [[ \"${certificateType}\" == \"selfsigned\" ]]; then\n CERT_ARGS=(\n \"--certificate-type=selfsigned\"\n )\nelif [[ \"${certificateType}\" == \"letsencrypt\" ]]; then\n CERT_ARGS=(\n \"--certificate-type=letsencrypt\"\n )\nelse\n # Use base64 encoded certificates directly\n OWN_CERT_CRT=${ownPublicCertificate}\n OWN_CERT_KEY=${ownPrivateCertificate}\n\n CERT_ARGS=(\n \"--certificate-type=owncert\"\n \"--owncert-public-key=$OWN_CERT_CRT\"\n \"--owncert-private-key=$OWN_CERT_KEY\"\n )\nfi\n\n# Construct the final command\nFINAL_COMMAND=\"$INSTALL_COMMAND $(printf \"%s \" \"${COMMON_ARGS[@]}\") $(printf \"%s \" \"${CERT_ARGS[@]}\")\"\n\n# Install OpenVidu\nexec bash -c \"$FINAL_COMMAND\"\n", + "installScriptTemplateMaster": "#!/bin/bash -x\nset -e\nOPENVIDU_VERSION=main\nDOMAIN=\n\n# Assume azure cli is installed\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y \\\n curl \\\n unzip \\\n jq \\\n wget\n\n\n# Wait for the keyvault availability\nMAX_WAIT=300\nWAIT_INTERVAL=1\nELAPSED_TIME=0\nset +e\nwhile true; do\n # Check keyvault availability\n az keyvault secret list --vault-name ${keyVaultName} \n\n # If it is available, exit the loop\n if [ $? -eq 0 ]; then\n break\n fi\n\n # If not, wait and check again incrementing the time\n ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL))\n\n # If exceeded the maximum time, exit with error\n if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then\n exit 1\n fi\n\n # Esperar antes de la próxima comprobación\n sleep $WAIT_INTERVAL\ndone\nset -e\n\nMASTER_NODE_NUM=${masterNodeNum}\n\n# Get own private IP\nPRIVATE_IP=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text\")\n\n# Store current private IP\nPRIVATE_IP=\"$(/usr/local/bin/store_secret.sh save MASTER-NODE-${masterNodeNum}-PRIVATE-IP $PRIVATE_IP)\"\n\n\nif [[ $MASTER_NODE_NUM -eq 1 ]] && [[ \"$ALL_SECRETS_GENERATED\" == \"\" || \"$ALL_SECRETS_GENERATED\" == \"false\" ]]; then\n\n # Configure Domain name\n if [[ \"${domainName}\" == '' ]]; then\n # Get public IP using the get_public_ip.sh script\n PUBLIC_IP=$(/usr/local/bin/get_public_ip.sh 2>/dev/null)\n if [[ $? -ne 0 || -z \"${PUBLIC_IP}\" ]]; then\n echo \"Could not determine public IP.\"\n exit 1\n fi\n\n DOMAIN=\"$PUBLIC_IP\"\n else\n DOMAIN=${domainName}\n fi\n DOMAIN=\"$(/usr/local/bin/store_secret.sh save DOMAIN-NAME \"${DOMAIN}\")\"\n\n # Publish service URLs early (idempotent; after_install re-writes byte-identical values)\n OPENVIDU_URL=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-URL \"https://${DOMAIN}/\")\"\n LIVEKIT_URL=\"$(/usr/local/bin/store_secret.sh save LIVEKIT-URL \"wss://${DOMAIN}/\")\"\n DASHBOARD_URL=\"$(/usr/local/bin/store_secret.sh save DASHBOARD-URL \"https://${DOMAIN}/dashboard/\")\"\n GRAFANA_URL=\"$(/usr/local/bin/store_secret.sh save GRAFANA-URL \"https://${DOMAIN}/grafana/\")\"\n MINIO_URL=\"$(/usr/local/bin/store_secret.sh save MINIO-URL \"https://${DOMAIN}/minio-console/\")\"\n\n # Meet initial admin user and password\n MEET_INITIAL_ADMIN_USER=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-ADMIN-USER \"admin\")\"\n if [[ \"${initialMeetAdminPassword}\" != '' ]]; then\n MEET_INITIAL_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-ADMIN-PASSWORD \"${initialMeetAdminPassword}\")\"\n else\n MEET_INITIAL_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate MEET-INITIAL-ADMIN-PASSWORD)\"\n fi\n if [[ \"${initialMeetApiKey}\" != '' ]]; then\n MEET_INITIAL_API_KEY=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-API-KEY \"${initialMeetApiKey}\")\"\n else\n MEET_INITIAL_API_KEY=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-API-KEY \"\")\"\n fi\n\n # Store usernames and generate random passwords\n OPENVIDU_PRO_LICENSE=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-PRO-LICENSE \"${openviduLicense}\")\"\n OPENVIDU_RTC_ENGINE=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-RTC-ENGINE \"${rtcEngine}\")\"\n REDIS_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate REDIS-PASSWORD)\"\n MONGO_ADMIN_USERNAME=\"$(/usr/local/bin/store_secret.sh save MONGO-ADMIN-USERNAME \"mongoadmin\")\"\n MONGO_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate MONGO-ADMIN-PASSWORD)\"\n MONGO_REPLICA_SET_KEY=\"$(/usr/local/bin/store_secret.sh generate MONGO-REPLICA-SET-KEY)\"\n MINIO_ACCESS_KEY=\"$(/usr/local/bin/store_secret.sh save MINIO-ACCESS-KEY \"minioadmin\")\"\n MINIO_SECRET_KEY=\"$(/usr/local/bin/store_secret.sh generate MINIO-SECRET-KEY)\"\n DASHBOARD_ADMIN_USERNAME=\"$(/usr/local/bin/store_secret.sh save DASHBOARD-ADMIN-USERNAME \"dashboardadmin\")\"\n DASHBOARD_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate DASHBOARD-ADMIN-PASSWORD)\"\n GRAFANA_ADMIN_USERNAME=\"$(/usr/local/bin/store_secret.sh save GRAFANA-ADMIN-USERNAME \"grafanaadmin\")\"\n GRAFANA_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate GRAFANA-ADMIN-PASSWORD)\"\n MEET_INITIAL_ADMIN_USER=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-ADMIN-USER \"admin\")\"\n LIVEKIT_API_KEY=\"$(/usr/local/bin/store_secret.sh generate LIVEKIT-API-KEY \"API\" 12)\"\n LIVEKIT_API_SECRET=\"$(/usr/local/bin/store_secret.sh generate LIVEKIT-API-SECRET)\"\n OPENVIDU_VERSION=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-VERSION \"${OPENVIDU_VERSION}\")\"\n ENABLED_MODULES=\"$(/usr/local/bin/store_secret.sh save ENABLED-MODULES \"observability,openviduMeet,v2compatibility\")\"\n ALL_SECRETS_GENERATED=\"$(/usr/local/bin/store_secret.sh save ALL-SECRETS-GENERATED \"true\")\"\nfi\n\nIP_WAIT_MAX_RETRIES=360\nIP_WAIT_INTERVAL=5\nIP_WAIT_RETRIES=0\nwhile true; do\n MASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv) || true\n MASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv) || true\n MASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv) || true\n MASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv) || true\n # Check if all master nodes have stored their private IPs\n if [[ \"$MASTER_NODE_1_PRIVATE_IP\" != \"\" ]] &&\n [[ \"$MASTER_NODE_2_PRIVATE_IP\" != \"\" ]] &&\n [[ \"$MASTER_NODE_3_PRIVATE_IP\" != \"\" ]] &&\n [[ \"$MASTER_NODE_4_PRIVATE_IP\" != \"\" ]]; then\n break\n fi\n IP_WAIT_RETRIES=$((IP_WAIT_RETRIES + 1))\n if [ $IP_WAIT_RETRIES -ge $IP_WAIT_MAX_RETRIES ]; then\n echo \"[OpenVidu] timed out after 30 min waiting for the 4 master nodes to publish their private IPs\"\n exit 1\n fi\n sleep $IP_WAIT_INTERVAL\ndone\n\n# Wait until master-node-1 has generated all shared secrets before fetching them\nSECRETS_WAIT_MAX_RETRIES=360\nSECRETS_WAIT_INTERVAL=5\nSECRETS_WAIT_RETRIES=0\nwhile [[ \"$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)\" != \"true\" ]]; do\n SECRETS_WAIT_RETRIES=$((SECRETS_WAIT_RETRIES + 1))\n if [ $SECRETS_WAIT_RETRIES -ge $SECRETS_WAIT_MAX_RETRIES ]; then\n echo \"[OpenVidu] timed out after 30 min waiting for ALL-SECRETS-GENERATED to become true\"\n exit 1\n fi\n sleep $SECRETS_WAIT_INTERVAL\ndone\n\n\n# Fetch the values in the keyvault\nMASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_PRIVATE_IP_LIST=\"$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP\"\n\nDOMAIN=$(az keyvault secret show --vault-name ${keyVaultName} --name DOMAIN-NAME --query value -o tsv)\nOPENVIDU_RTC_ENGINE=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-RTC-ENGINE --query value -o tsv)\nOPENVIDU_PRO_LICENSE=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-PRO-LICENSE --query value -o tsv)\nREDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name REDIS-PASSWORD --query value -o tsv)\nMONGO_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-ADMIN-USERNAME --query value -o tsv)\nMONGO_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-ADMIN-PASSWORD --query value -o tsv)\nMONGO_REPLICA_SET_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-REPLICA-SET-KEY --query value -o tsv)\nDASHBOARD_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-USERNAME --query value -o tsv)\nDASHBOARD_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-PASSWORD --query value -o tsv)\nMINIO_ACCESS_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MINIO-ACCESS-KEY --query value -o tsv)\nMINIO_SECRET_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MINIO-SECRET-KEY --query value -o tsv)\nGRAFANA_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name GRAFANA-ADMIN-USERNAME --query value -o tsv)\nGRAFANA_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name GRAFANA-ADMIN-PASSWORD --query value -o tsv)\nLIVEKIT_API_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name LIVEKIT-API-KEY --query value -o tsv)\nLIVEKIT_API_SECRET=$(az keyvault secret show --vault-name ${keyVaultName} --name LIVEKIT-API-SECRET --query value -o tsv)\nMEET_INITIAL_ADMIN_USER=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-USER --query value -o tsv)\nMEET_INITIAL_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-PASSWORD --query value -o tsv)\nif [[ \"${initialMeetApiKey}\" != '' ]]; then\n MEET_INITIAL_API_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-API-KEY --query value -o tsv)\nelse\n MEET_INITIAL_API_KEY=\"\"\nfi\nENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv)\n\n\n# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure\nINSTALLER_SCRIPT=\"/tmp/install_ov_master_node.sh\"\ncurl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o \"$INSTALLER_SCRIPT\" \"http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh\"\nif [ ! -s \"$INSTALLER_SCRIPT\" ]; then\n echo \"[OpenVidu] failed to download the master node installer script\"\n exit 1\nfi\n\n# Base command\nINSTALL_COMMAND=\"sh $INSTALLER_SCRIPT\"\n\n# Common arguments\nCOMMON_ARGS=(\n \"--no-tty\"\n \"--install\"\n \"--environment=azure\"\n \"--deployment-type='ha'\"\n \"--node-role='master-node'\"\n \"--external-load-balancer\"\n \"--internal-tls-termination\"\n \"--master-node-private-ip-list='$MASTER_NODE_PRIVATE_IP_LIST'\"\n \"--openvidu-pro-license='$OPENVIDU_PRO_LICENSE'\"\n \"--domain-name='$DOMAIN'\"\n \"--enabled-modules='$ENABLED_MODULES'\"\n \"--rtc-engine=$OPENVIDU_RTC_ENGINE\"\n \"--redis-password=$REDIS_PASSWORD\"\n \"--mongo-admin-user=$MONGO_ADMIN_USERNAME\"\n \"--mongo-admin-password=$MONGO_ADMIN_PASSWORD\"\n \"--mongo-replica-set-key=$MONGO_REPLICA_SET_KEY\"\n \"--minio-access-key=$MINIO_ACCESS_KEY\"\n \"--minio-secret-key=$MINIO_SECRET_KEY\"\n \"--dashboard-admin-user=$DASHBOARD_ADMIN_USERNAME\"\n \"--dashboard-admin-password=$DASHBOARD_ADMIN_PASSWORD\"\n \"--grafana-admin-user=$GRAFANA_ADMIN_USERNAME\"\n \"--grafana-admin-password=$GRAFANA_ADMIN_PASSWORD\"\n \"--meet-initial-admin-password=$MEET_INITIAL_ADMIN_PASSWORD\"\n \"--meet-initial-api-key=$MEET_INITIAL_API_KEY\"\n \"--livekit-api-key=$LIVEKIT_API_KEY\"\n \"--livekit-api-secret=$LIVEKIT_API_SECRET\"\n)\n\n# Include additional installer flags provided by the user\nif [[ \"${additionalInstallFlags}\" != \"\" ]]; then\n IFS=',' read -ra EXTRA_FLAGS <<< \"${additionalInstallFlags}\"\n for extra_flag in \"${EXTRA_FLAGS[@]}\"; do\n # Trim whitespace around each flag\n extra_flag=\"$(echo -e \"${extra_flag}\" | sed -e 's/^[ \\t]*//' -e 's/[ \\t]*$//')\"\n if [[ \"$extra_flag\" != \"\" ]]; then\n COMMON_ARGS+=(\"$extra_flag\")\n fi\n done\nfi\n\n# Certificate arguments\nif [[ \"${certificateType}\" == \"selfsigned\" ]]; then\n CERT_ARGS=(\n \"--certificate-type=selfsigned\"\n )\nelif [[ \"${certificateType}\" == \"letsencrypt\" ]]; then\n CERT_ARGS=(\n \"--certificate-type=letsencrypt\"\n )\nelse\n # Use base64 encoded certificates directly\n OWN_CERT_CRT=${ownPublicCertificate}\n OWN_CERT_KEY=${ownPrivateCertificate}\n\n CERT_ARGS=(\n \"--certificate-type=owncert\"\n \"--owncert-public-key=$OWN_CERT_CRT\"\n \"--owncert-private-key=$OWN_CERT_KEY\"\n )\nfi\n\n# Construct the final command\nFINAL_COMMAND=\"$INSTALL_COMMAND $(printf \"%s \" \"${COMMON_ARGS[@]}\") $(printf \"%s \" \"${CERT_ARGS[@]}\")\"\n\n# Install OpenVidu\nexec bash -c \"$FINAL_COMMAND\"\n", "after_installScriptTemplateMaster": "#!/bin/bash\nset -e\n\naz login --identity --allow-no-subscriptions > /dev/null\n\n# Generate URLs\nDOMAIN=$(az keyvault secret show --vault-name ${keyVaultName} --name DOMAIN-NAME --query value -o tsv)\nOPENVIDU_URL=\"https://${DOMAIN}/\"\nLIVEKIT_URL=\"wss://${DOMAIN}/\"\nDASHBOARD_URL=\"https://${DOMAIN}/dashboard/\"\nGRAFANA_URL=\"https://${DOMAIN}/grafana/\"\nMINIO_URL=\"https://${DOMAIN}/minio-console/\"\n\n# Update shared secret\naz keyvault secret set --vault-name ${keyVaultName} --name DOMAIN-NAME --value $DOMAIN\naz keyvault secret set --vault-name ${keyVaultName} --name OPENVIDU-URL --value $OPENVIDU_URL\naz keyvault secret set --vault-name ${keyVaultName} --name LIVEKIT-URL --value $LIVEKIT_URL\naz keyvault secret set --vault-name ${keyVaultName} --name DASHBOARD-URL --value $DASHBOARD_URL\naz keyvault secret set --vault-name ${keyVaultName} --name GRAFANA-URL --value $GRAFANA_URL\naz keyvault secret set --vault-name ${keyVaultName} --name MINIO-URL --value $MINIO_URL\n\naz keyvault secret show --vault-name ${keyVaultName} --name MINIO-URL\n\nif [[ $? -ne 0 ]]; then\n echo \"Error updating keyvault\"\nfi\n", "update_config_from_secretScriptTemplateMaster": "#!/bin/bash\nset -e\n\naz login --identity --allow-no-subscriptions > /dev/null\n\n# Installation directory\nINSTALL_DIR=\"/opt/openvidu\"\nCLUSTER_CONFIG_DIR=\"${INSTALL_DIR}/config/cluster\"\nMASTER_NODE_CONFIG_DIR=\"${INSTALL_DIR}/config/node\"\n\n# Replace DOMAIN_NAME\nexport DOMAIN=$(az keyvault secret show --vault-name ${keyVaultName} --name DOMAIN-NAME --query value -o tsv)\nif [[ -n \"$DOMAIN\" ]]; then\n sed -i \"s/DOMAIN_NAME=.*/DOMAIN_NAME=$DOMAIN/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nelse\n exit 1\nfi\n\n# Get the rest of the values\nexport REDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name REDIS-PASSWORD --query value -o tsv)\nexport OPENVIDU_RTC_ENGINE=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-RTC-ENGINE --query value -o tsv)\nexport OPENVIDU_PRO_LICENSE=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-PRO-LICENSE --query value -o tsv)\nexport MONGO_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-ADMIN-USERNAME --query value -o tsv)\nexport MONGO_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-ADMIN-PASSWORD --query value -o tsv)\nexport MONGO_REPLICA_SET_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-REPLICA-SET-KEY --query value -o tsv)\nexport DASHBOARD_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-USERNAME --query value -o tsv)\nexport DASHBOARD_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-PASSWORD --query value -o tsv)\nexport MINIO_ACCESS_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MINIO-ACCESS-KEY --query value -o tsv)\nexport MINIO_SECRET_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MINIO-SECRET-KEY --query value -o tsv)\nexport GRAFANA_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name GRAFANA-ADMIN-USERNAME --query value -o tsv)\nexport GRAFANA_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name GRAFANA-ADMIN-PASSWORD --query value -o tsv)\nexport LIVEKIT_API_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name LIVEKIT-API-KEY --query value -o tsv)\nexport LIVEKIT_API_SECRET=$(az keyvault secret show --vault-name ${keyVaultName} --name LIVEKIT-API-SECRET --query value -o tsv)\nexport MEET_INITIAL_ADMIN_USER=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-USER --query value -o tsv)\nexport MEET_INITIAL_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-PASSWORD --query value -o tsv)\nif [[ \"${initialMeetApiKey}\" != '' ]]; then\n export MEET_INITIAL_API_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-API-KEY --query value -o tsv)\nfi\nexport ENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv)\n\n# Replace rest of the values\nsed -i \"s/REDIS_PASSWORD=.*/REDIS_PASSWORD=$REDIS_PASSWORD/\" \"${MASTER_NODE_CONFIG_DIR}/master_node.env\"\nsed -i \"s/OPENVIDU_RTC_ENGINE=.*/OPENVIDU_RTC_ENGINE=$OPENVIDU_RTC_ENGINE/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/OPENVIDU_PRO_LICENSE=.*/OPENVIDU_PRO_LICENSE=$OPENVIDU_PRO_LICENSE/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/MONGO_ADMIN_USERNAME=.*/MONGO_ADMIN_USERNAME=$MONGO_ADMIN_USERNAME/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/MONGO_ADMIN_PASSWORD=.*/MONGO_ADMIN_PASSWORD=$MONGO_ADMIN_PASSWORD/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/MONGO_REPLICA_SET_KEY=.*/MONGO_REPLICA_SET_KEY=$MONGO_REPLICA_SET_KEY/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/DASHBOARD_ADMIN_USERNAME=.*/DASHBOARD_ADMIN_USERNAME=$DASHBOARD_ADMIN_USERNAME/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/DASHBOARD_ADMIN_PASSWORD=.*/DASHBOARD_ADMIN_PASSWORD=$DASHBOARD_ADMIN_PASSWORD/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/MINIO_ACCESS_KEY=.*/MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/MINIO_SECRET_KEY=.*/MINIO_SECRET_KEY=$MINIO_SECRET_KEY/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/GRAFANA_ADMIN_USERNAME=.*/GRAFANA_ADMIN_USERNAME=$GRAFANA_ADMIN_USERNAME/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/GRAFANA_ADMIN_PASSWORD=.*/GRAFANA_ADMIN_PASSWORD=$GRAFANA_ADMIN_PASSWORD/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/LIVEKIT_API_KEY=.*/LIVEKIT_API_KEY=$LIVEKIT_API_KEY/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/LIVEKIT_API_SECRET=.*/LIVEKIT_API_SECRET=$LIVEKIT_API_SECRET/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/MEET_INITIAL_ADMIN_USER=.*/MEET_INITIAL_ADMIN_USER=$MEET_INITIAL_ADMIN_USER/\" \"${CLUSTER_CONFIG_DIR}/master_node/meet.env\"\nsed -i \"s/MEET_INITIAL_ADMIN_PASSWORD=.*/MEET_INITIAL_ADMIN_PASSWORD=$MEET_INITIAL_ADMIN_PASSWORD/\" \"${CLUSTER_CONFIG_DIR}/master_node/meet.env\"\nif [[ \"${initialMeetApiKey}\" != '' ]]; then\n sed -i \"s/MEET_INITIAL_API_KEY=.*/MEET_INITIAL_API_KEY=$MEET_INITIAL_API_KEY/\" \"${CLUSTER_CONFIG_DIR}/master_node/meet.env\"\nfi\nsed -i \"s/ENABLED_MODULES=.*/ENABLED_MODULES=$ENABLED_MODULES/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\n\n# Update URLs in secret\nOPENVIDU_URL=\"https://${DOMAIN}/\"\nLIVEKIT_URL=\"wss://${DOMAIN}/\"\nDASHBOARD_URL=\"https://${DOMAIN}/dashboard/\"\nGRAFANA_URL=\"https://${DOMAIN}/grafana/\"\nMINIO_URL=\"https://${DOMAIN}/minio-console/\"\n\n# Update shared secret\naz keyvault secret set --vault-name ${keyVaultName} --name DOMAIN-NAME --value $DOMAIN\naz keyvault secret set --vault-name ${keyVaultName} --name OPENVIDU-URL --value $OPENVIDU_URL\naz keyvault secret set --vault-name ${keyVaultName} --name LIVEKIT-URL --value $LIVEKIT_URL\naz keyvault secret set --vault-name ${keyVaultName} --name DASHBOARD-URL --value $DASHBOARD_URL\naz keyvault secret set --vault-name ${keyVaultName} --name GRAFANA-URL --value $GRAFANA_URL\naz keyvault secret set --vault-name ${keyVaultName} --name MINIO-URL --value $MINIO_URL\n", "update_secret_from_configScriptTemplateMaster": "#!/bin/bash\nset -e\n\naz login --identity --allow-no-subscriptions > /dev/null\n\n# Installation directory\nINSTALL_DIR=\"/opt/openvidu\"\nCLUSTER_CONFIG_DIR=\"${INSTALL_DIR}/config/cluster\"\nMASTER_NODE_CONFIG_DIR=\"${INSTALL_DIR}/config/node\"\n\n# Get current values of the config\nREDIS_PASSWORD=\"$(/usr/local/bin/get_value_from_config.sh REDIS_PASSWORD \"${MASTER_NODE_CONFIG_DIR}/master_node.env\")\"\nDOMAIN_NAME=\"$(/usr/local/bin/get_value_from_config.sh DOMAIN_NAME \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nOPENVIDU_RTC_ENGINE=\"$(/usr/local/bin/get_value_from_config.sh OPENVIDU_RTC_ENGINE \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nOPENVIDU_PRO_LICENSE=\"$(/usr/local/bin/get_value_from_config.sh OPENVIDU_PRO_LICENSE \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nMONGO_ADMIN_USERNAME=\"$(/usr/local/bin/get_value_from_config.sh MONGO_ADMIN_USERNAME \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nMONGO_ADMIN_PASSWORD=\"$(/usr/local/bin/get_value_from_config.sh MONGO_ADMIN_PASSWORD \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nMONGO_REPLICA_SET_KEY=\"$(/usr/local/bin/get_value_from_config.sh MONGO_REPLICA_SET_KEY \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nMINIO_ACCESS_KEY=\"$(/usr/local/bin/get_value_from_config.sh MINIO_ACCESS_KEY \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nMINIO_SECRET_KEY=\"$(/usr/local/bin/get_value_from_config.sh MINIO_SECRET_KEY \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nDASHBOARD_ADMIN_USERNAME=\"$(/usr/local/bin/get_value_from_config.sh DASHBOARD_ADMIN_USERNAME \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nDASHBOARD_ADMIN_PASSWORD=\"$(/usr/local/bin/get_value_from_config.sh DASHBOARD_ADMIN_PASSWORD \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nGRAFANA_ADMIN_USERNAME=\"$(/usr/local/bin/get_value_from_config.sh GRAFANA_ADMIN_USERNAME \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nGRAFANA_ADMIN_PASSWORD=\"$(/usr/local/bin/get_value_from_config.sh GRAFANA_ADMIN_PASSWORD \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nLIVEKIT_API_KEY=\"$(/usr/local/bin/get_value_from_config.sh LIVEKIT_API_KEY \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nLIVEKIT_API_SECRET=\"$(/usr/local/bin/get_value_from_config.sh LIVEKIT_API_SECRET \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nMEET_INITIAL_ADMIN_USER=\"$(/usr/local/bin/get_value_from_config.sh MEET_INITIAL_ADMIN_USER \"${CLUSTER_CONFIG_DIR}/master_node/meet.env\")\"\nMEET_INITIAL_ADMIN_PASSWORD=\"$(/usr/local/bin/get_value_from_config.sh MEET_INITIAL_ADMIN_PASSWORD \"${CLUSTER_CONFIG_DIR}/master_node/meet.env\")\"\nif [[ \"${initialMeetApiKey}\" != '' ]]; then\n MEET_INITIAL_API_KEY=\"$(/usr/local/bin/get_value_from_config.sh MEET_INITIAL_API_KEY \"${CLUSTER_CONFIG_DIR}/master_node/meet.env\")\"\nfi\nENABLED_MODULES=\"$(/usr/local/bin/get_value_from_config.sh ENABLED_MODULES \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\n\n# Update shared secret\naz keyvault secret set --vault-name ${keyVaultName} --name REDIS-PASSWORD --value $REDIS_PASSWORD\naz keyvault secret set --vault-name ${keyVaultName} --name DOMAIN-NAME --value $DOMAIN_NAME\naz keyvault secret set --vault-name ${keyVaultName} --name OPENVIDU-RTC-ENGINE --value $OPENVIDU_RTC_ENGINE\naz keyvault secret set --vault-name ${keyVaultName} --name OPENVIDU-PRO-LICENSE --value $OPENVIDU_PRO_LICENSE\naz keyvault secret set --vault-name ${keyVaultName} --name MONGO-ADMIN-USERNAME --value $MONGO_ADMIN_USERNAME\naz keyvault secret set --vault-name ${keyVaultName} --name MONGO-ADMIN-PASSWORD --value $MONGO_ADMIN_PASSWORD\naz keyvault secret set --vault-name ${keyVaultName} --name MONGO-REPLICA-SET-KEY --value $MONGO_REPLICA_SET_KEY\naz keyvault secret set --vault-name ${keyVaultName} --name MINIO-ACCESS-KEY --value $MINIO_ACCESS_KEY\naz keyvault secret set --vault-name ${keyVaultName} --name MINIO-SECRET-KEY --value $MINIO_SECRET_KEY\naz keyvault secret set --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-USERNAME --value $DASHBOARD_ADMIN_USERNAME\naz keyvault secret set --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-PASSWORD --value $DASHBOARD_ADMIN_PASSWORD\naz keyvault secret set --vault-name ${keyVaultName} --name GRAFANA-ADMIN-USERNAME --value $GRAFANA_ADMIN_USERNAME\naz keyvault secret set --vault-name ${keyVaultName} --name GRAFANA-ADMIN-PASSWORD --value $GRAFANA_ADMIN_PASSWORD\naz keyvault secret set --vault-name ${keyVaultName} --name LIVEKIT-API-KEY --value $LIVEKIT_API_KEY\naz keyvault secret set --vault-name ${keyVaultName} --name LIVEKIT-API-SECRET --value $LIVEKIT_API_SECRET\naz keyvault secret set --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-USER --value $MEET_INITIAL_ADMIN_USER\naz keyvault secret set --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-PASSWORD --value $MEET_INITIAL_ADMIN_PASSWORD\nif [[ \"${initialMeetApiKey}\" != '' ]]; then\n az keyvault secret set --vault-name ${keyVaultName} --name MEET-INITIAL-API-KEY --value $MEET_INITIAL_API_KEY\nfi\naz keyvault secret set --vault-name ${keyVaultName} --name ENABLED-MODULES --value $ENABLED_MODULES\n", @@ -320,8 +320,8 @@ "base64store_secretMaster": "[base64(variables('store_secretScriptMaster'))]", "base64check_app_readyMaster": "[base64(variables('check_app_readyScriptMaster'))]", "base64restartMaster": "[base64(variables('restartScriptMaster'))]", - "userDataTemplateMasterNode": "#!/bin/bash -x\nset -eu -o pipefail\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# after_install.sh\necho ${base64after_install} | base64 -d > /usr/local/bin/after_install.sh\nchmod +x /usr/local/bin/after_install.sh\n\n# get_public_ip.sh\necho ${base64get_public_ip} | base64 -d > /usr/local/bin/get_public_ip.sh\nchmod +x /usr/local/bin/get_public_ip.sh\n\n# update_config_from_secret.sh\necho ${base64update_config_from_secret} | base64 -d > /usr/local/bin/update_config_from_secret.sh\nchmod +x /usr/local/bin/update_config_from_secret.sh\n\n# update_secret_from_config.sh\necho ${base64update_secret_from_config} | base64 -d > /usr/local/bin/update_secret_from_config.sh\nchmod +x /usr/local/bin/update_secret_from_config.sh\n\n# get_value_from_config.sh\necho ${base64get_value_from_config} | base64 -d > /usr/local/bin/get_value_from_config.sh\nchmod +x /usr/local/bin/get_value_from_config.sh\n\n# store_secret.sh\necho ${base64store_secret} | base64 -d > /usr/local/bin/store_secret.sh\nchmod +x /usr/local/bin/store_secret.sh\n\n# check_app_ready.sh\necho ${base64check_app_ready} | base64 -d > /usr/local/bin/check_app_ready.sh\nchmod +x /usr/local/bin/check_app_ready.sh\n\n# restart.sh\necho ${base64restart} | base64 -d > /usr/local/bin/restart.sh\nchmod +x /usr/local/bin/restart.sh\n\necho ${base64config_blobStorage} | base64 -d > /usr/local/bin/config_blobStorage.sh\nchmod +x /usr/local/bin/config_blobStorage.sh\n\n# Install azure cli\nAZURE_CLI_VERSION=2.87.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity --allow-no-subscriptions\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\nexport HOME=\"/root\"\n\n# Install OpenVidu\n/usr/local/bin/install.sh || { echo \"[OpenVidu] error installing OpenVidu\"; exit 1; }\n\n#Config blob storage\n/usr/local/bin/config_blobStorage.sh || { echo \"[OpenVidu] error configuring Blob Storage\"; exit 1; }\n\n# Start OpenVidu\nsystemctl start openvidu || { echo \"[OpenVidu] error starting OpenVidu\"; exit 1; }\n\n# Update shared secret\n/usr/local/bin/after_install.sh || { echo \"[OpenVidu] error updating shared secret\"; exit 1; }\n\n# Launch on reboot\necho \"@reboot /usr/local/bin/restart.sh >> /var/log/openvidu-restart.log\" 2>&1 | crontab\n\n# Wait until this master node is healthy (check_app_ready.sh caps at 1200s).\n# On failure exit WITHOUT publishing readiness, so media nodes keep waiting for another master.\n/usr/local/bin/check_app_ready.sh || { echo \"[OpenVidu] master node did not become healthy\"; exit 1; }\n\n# Publish readiness so media nodes can start. Every healthy master writes the same value, so the\n# first healthy master unblocks media and a failed master can no longer leave this unset (no SPOF).\naz keyvault secret set --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --value \"true\"\n\nMASTER_NODE_NUM=${masterNodeNum}\nif [[ $MASTER_NODE_NUM -eq 4 ]]; then\n # Creating scale in lock\n set +e\n az storage blob upload --account-name ${storageAccountName} --container-name automation-locks --name lock.txt --file /dev/null --auth-mode key\n set -e\nfi\n", - "installScriptTemplateMedia": "#!/bin/bash -x\nDOMAIN=\n\n# Install dependencies\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y \\\n curl \\\n unzip \\\n jq \\\n wget\n\n# Get own private IP\nPRIVATE_IP=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text\")\n\n# Gate 1: wait (bounded: 360 x 5s = 30 min) until master nodes generated the shared secrets and\n# published their 4 private IPs. This lets the heavy media installer run in parallel with the masters.\nWAIT_INTERVAL=5\nMAX_RETRIES=360\nRETRIES=0\nset +e\nwhile true; do\n ALL_SECRETS_GENERATED=$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)\n MASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv 2>/dev/null)\n MASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv 2>/dev/null)\n MASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv 2>/dev/null)\n MASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv 2>/dev/null)\n\n # Break once secrets are generated and all 4 master IPs are published\n if [ \"$ALL_SECRETS_GENERATED\" == \"true\" ] &&\n [ \"$MASTER_NODE_1_PRIVATE_IP\" != \"\" ] &&\n [ \"$MASTER_NODE_2_PRIVATE_IP\" != \"\" ] &&\n [ \"$MASTER_NODE_3_PRIVATE_IP\" != \"\" ] &&\n [ \"$MASTER_NODE_4_PRIVATE_IP\" != \"\" ]; then\n break\n fi\n\n RETRIES=$((RETRIES + 1))\n if [ $RETRIES -ge $MAX_RETRIES ]; then\n echo \"[OpenVidu] timed out after 30 min waiting for master nodes to generate secrets and publish their IPs\"\n exit 1\n fi\n\n sleep $WAIT_INTERVAL\ndone\nset -e\n\nMASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_PRIVATE_IP_LIST=\"$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP\"\nREDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name REDIS-PASSWORD --query value -o tsv)\nENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv)\nOPENVIDU_VERSION=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-VERSION --query value -o tsv)\n\n# Download the installer script (retry transient curl failures; fail if empty/missing)\nINSTALLER_SCRIPT=\"/tmp/install_ov_media_node.sh\"\ncurl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o \"$INSTALLER_SCRIPT\" \"http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh\"\nif [ ! -s \"$INSTALLER_SCRIPT\" ]; then\n echo \"[OpenVidu] failed to download the media node installer script\"\n exit 1\nfi\n\n# Base command\nINSTALL_COMMAND=\"sh $INSTALLER_SCRIPT\"\n\n# Common arguments\nCOMMON_ARGS=(\n\"--no-tty\"\n\"--install\"\n\"--environment=azure\"\n\"--deployment-type='ha'\"\n\"--node-role='media-node'\"\n\"--master-node-private-ip-list=$MASTER_NODE_PRIVATE_IP_LIST\"\n\"--private-ip=$PRIVATE_IP\"\n\"--enabled-modules='$ENABLED_MODULES'\"\n\"--redis-password=$REDIS_PASSWORD\"\n)\n\n# Construct the final command with all arguments\nFINAL_COMMAND=\"$INSTALL_COMMAND $(printf \"%s \" \"${COMMON_ARGS[@]}\")\"\n\n# Install OpenVidu\nexec bash -c \"$FINAL_COMMAND\"\n", + "userDataTemplateMasterNode": "#!/bin/bash -x\nset -eu -o pipefail\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# after_install.sh\necho ${base64after_install} | base64 -d > /usr/local/bin/after_install.sh\nchmod +x /usr/local/bin/after_install.sh\n\n# get_public_ip.sh\necho ${base64get_public_ip} | base64 -d > /usr/local/bin/get_public_ip.sh\nchmod +x /usr/local/bin/get_public_ip.sh\n\n# update_config_from_secret.sh\necho ${base64update_config_from_secret} | base64 -d > /usr/local/bin/update_config_from_secret.sh\nchmod +x /usr/local/bin/update_config_from_secret.sh\n\n# update_secret_from_config.sh\necho ${base64update_secret_from_config} | base64 -d > /usr/local/bin/update_secret_from_config.sh\nchmod +x /usr/local/bin/update_secret_from_config.sh\n\n# get_value_from_config.sh\necho ${base64get_value_from_config} | base64 -d > /usr/local/bin/get_value_from_config.sh\nchmod +x /usr/local/bin/get_value_from_config.sh\n\n# store_secret.sh\necho ${base64store_secret} | base64 -d > /usr/local/bin/store_secret.sh\nchmod +x /usr/local/bin/store_secret.sh\n\n# check_app_ready.sh\necho ${base64check_app_ready} | base64 -d > /usr/local/bin/check_app_ready.sh\nchmod +x /usr/local/bin/check_app_ready.sh\n\n# restart.sh\necho ${base64restart} | base64 -d > /usr/local/bin/restart.sh\nchmod +x /usr/local/bin/restart.sh\n\necho ${base64config_blobStorage} | base64 -d > /usr/local/bin/config_blobStorage.sh\nchmod +x /usr/local/bin/config_blobStorage.sh\n\n# Install azure cli\nAZURE_CLI_VERSION=2.87.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity --allow-no-subscriptions\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\nexport HOME=\"/root\"\n\n# Install OpenVidu\n/usr/local/bin/install.sh || { echo \"[OpenVidu] error installing OpenVidu\"; exit 1; }\n\n#Config blob storage\n/usr/local/bin/config_blobStorage.sh || { echo \"[OpenVidu] error configuring Blob Storage\"; exit 1; }\n\n# Start OpenVidu\nsystemctl start openvidu || { echo \"[OpenVidu] error starting OpenVidu\"; exit 1; }\n\n# Update shared secret\n/usr/local/bin/after_install.sh || { echo \"[OpenVidu] error updating shared secret\"; exit 1; }\n\n# Launch on reboot\necho \"@reboot /usr/local/bin/restart.sh >> /var/log/openvidu-restart.log\" 2>&1 | crontab\n\n# check_app_ready.sh internally caps its wait at 1200s\n/usr/local/bin/check_app_ready.sh || { echo \"[OpenVidu] master node did not become healthy\"; exit 1; }\n\naz keyvault secret set --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --value \"true\"\n\nMASTER_NODE_NUM=${masterNodeNum}\nif [[ $MASTER_NODE_NUM -eq 4 ]]; then\n # Creating scale in lock\n set +e\n az storage blob upload --account-name ${storageAccountName} --container-name automation-locks --name lock.txt --file /dev/null --auth-mode key\n set -e\nfi\n", + "installScriptTemplateMedia": "#!/bin/bash -x\nDOMAIN=\n\n# Install dependencies\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y \\\n curl \\\n unzip \\\n jq \\\n wget\n\n# Get own private IP\nPRIVATE_IP=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text\")\n\n# Gate 1: wait for master secrets and IPs before installing\nWAIT_INTERVAL=5\nMAX_RETRIES=360\nRETRIES=0\nset +e\nwhile true; do\n ALL_SECRETS_GENERATED=$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)\n MASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv 2>/dev/null)\n MASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv 2>/dev/null)\n MASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv 2>/dev/null)\n MASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv 2>/dev/null)\n\n if [ \"$ALL_SECRETS_GENERATED\" == \"true\" ] &&\n [ \"$MASTER_NODE_1_PRIVATE_IP\" != \"\" ] &&\n [ \"$MASTER_NODE_2_PRIVATE_IP\" != \"\" ] &&\n [ \"$MASTER_NODE_3_PRIVATE_IP\" != \"\" ] &&\n [ \"$MASTER_NODE_4_PRIVATE_IP\" != \"\" ]; then\n break\n fi\n\n RETRIES=$((RETRIES + 1))\n if [ $RETRIES -ge $MAX_RETRIES ]; then\n echo \"[OpenVidu] timed out after 30 min waiting for master nodes to generate secrets and publish their IPs\"\n exit 1\n fi\n\n sleep $WAIT_INTERVAL\ndone\nset -e\n\nMASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_PRIVATE_IP_LIST=\"$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP\"\nREDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name REDIS-PASSWORD --query value -o tsv)\nENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv)\nOPENVIDU_VERSION=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-VERSION --query value -o tsv)\n\n# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure\nINSTALLER_SCRIPT=\"/tmp/install_ov_media_node.sh\"\ncurl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o \"$INSTALLER_SCRIPT\" \"http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh\"\nif [ ! -s \"$INSTALLER_SCRIPT\" ]; then\n echo \"[OpenVidu] failed to download the media node installer script\"\n exit 1\nfi\n\n# Base command\nINSTALL_COMMAND=\"sh $INSTALLER_SCRIPT\"\n\n# Common arguments\nCOMMON_ARGS=(\n\"--no-tty\"\n\"--install\"\n\"--environment=azure\"\n\"--deployment-type='ha'\"\n\"--node-role='media-node'\"\n\"--master-node-private-ip-list=$MASTER_NODE_PRIVATE_IP_LIST\"\n\"--private-ip=$PRIVATE_IP\"\n\"--enabled-modules='$ENABLED_MODULES'\"\n\"--redis-password=$REDIS_PASSWORD\"\n)\n\n# Construct the final command with all arguments\nFINAL_COMMAND=\"$INSTALL_COMMAND $(printf \"%s \" \"${COMMON_ARGS[@]}\")\"\n\n# Install OpenVidu\nexec bash -c \"$FINAL_COMMAND\"\n", "stopMediaNodeParams": { "subscriptionId": "[subscription().subscriptionId]", "resourceGroupName": "[resourceGroup().name]", @@ -330,7 +330,7 @@ }, "stop_media_nodesScriptMediaTemplate": "#!/bin/bash\nset -e\n\nif ! (set -o noclobber ; echo > /tmp/global.lock) ; then\n exit 1 # the global.lock already exists\nfi\n\n# Execute if docker is installed\nif [ -x \"$(command -v docker)\" ]; then\n\n echo \"Stopping media node services and waiting for termination...\"\n docker container kill --signal=SIGQUIT openvidu || true\n docker container kill --signal=SIGQUIT ingress || true\n docker container kill --signal=SIGQUIT egress || true\n for agent_container in $(docker ps --filter \"label=openvidu-agent=true\" --format '{{.Names}}'); do\n docker container kill --signal=SIGQUIT \"$agent_container\"\n done\n\n # Wait for running containers to not be openvidu, ingress or egress\n while [ $(docker ps --filter \"label=openvidu-agent=true\" -q | wc -l) -gt 0 ] || \\\n [ $(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null) == \"true\" ] || \\\n [ $(docker inspect -f '{{.State.Running}}' ingress 2>/dev/null) == \"true\" ] || \\\n [ $(docker inspect -f '{{.State.Running}}' egress 2>/dev/null) == \"true\" ]; do\n echo \"Waiting for containers to stop...\"\n sleep 5\n done\nfi\n\naz login --identity\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nSUBSCRIPTION_ID=${subscriptionId}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\nRESOURCE_ID=/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP_NAME/providers/Microsoft.Compute/virtualMachineScaleSets/$VM_SCALE_SET_NAME\n\nTIMESTAMP=$(date -u +\"%Y-%m-%dT%H:%M:%SZ\")\naz tag update --resource-id $RESOURCE_ID --operation replace --tags \"STATUS\"=\"HEALTHY\" \"InstanceDeleteTime\"=\"$TIMESTAMP\" \"storageAccount\"=\"${storageAccountName}\"\n\naz vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID\n", "delete_mediaNode_ScriptMediaTemplate": "#!/bin/bash\nset -e\n\naz login --identity\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\n\n\naz vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID\n", - "userDataMediaNodeTemplate": "#!/bin/bash -x\nset -eu -o pipefail\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# stop_media_nodes.sh\necho ${base64stop} | base64 -d > /usr/local/bin/stop_media_node.sh\nchmod +x /usr/local/bin/stop_media_node.sh\n\n# delete_media_node.sh\necho ${base64delete_mediaNode} | base64 -d > /usr/local/bin/delete_media_node.sh\nchmod +x /usr/local/bin/delete_media_node.sh\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y jq\n\n# Install azure cli\nAZURE_CLI_VERSION=2.87.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity\n\n# Protect from scale in actions\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\naz vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-id $INSTANCE_ID --protect-from-scale-in true\n\nexport HOME=\"/root\"\n\n# Install OpenVidu (heavy: docker + image pulls). Overlaps with the masters' own installs.\n/usr/local/bin/install.sh || { echo \"[OpenVidu] error installing OpenVidu\"; /usr/local/bin/delete_media_node.sh; }\n\n# Gate 2: wait (bounded: 360 x 5s = 30 min) for the master nodes to publish readiness before starting\nWAIT_INTERVAL=5\nMAX_RETRIES=360\nRETRIES=0\nset +e\nwhile true; do\n FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv 2>/dev/null)\n if [ \"$FINISH_MASTER_NODE\" == \"true\" ]; then\n break\n fi\n RETRIES=$((RETRIES + 1))\n if [ $RETRIES -ge $MAX_RETRIES ]; then\n echo \"[OpenVidu] timed out after 30 min waiting for FINISH-MASTER-NODE\"\n /usr/local/bin/delete_media_node.sh\n exit 1\n fi\n sleep $WAIT_INTERVAL\ndone\nset -e\n\n# Start OpenVidu\nsystemctl start openvidu || { echo \"[OpenVidu] error starting OpenVidu\"; /usr/local/bin/delete_media_node.sh; }\n", + "userDataMediaNodeTemplate": "#!/bin/bash -x\nset -eu -o pipefail\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# stop_media_nodes.sh\necho ${base64stop} | base64 -d > /usr/local/bin/stop_media_node.sh\nchmod +x /usr/local/bin/stop_media_node.sh\n\n# delete_media_node.sh\necho ${base64delete_mediaNode} | base64 -d > /usr/local/bin/delete_media_node.sh\nchmod +x /usr/local/bin/delete_media_node.sh\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y jq\n\n# Install azure cli\nAZURE_CLI_VERSION=2.87.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity\n\n# Protect from scale in actions\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\naz vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-id $INSTANCE_ID --protect-from-scale-in true\n\nexport HOME=\"/root\"\n\n# Install OpenVidu\n/usr/local/bin/install.sh || { echo \"[OpenVidu] error installing OpenVidu\"; /usr/local/bin/delete_media_node.sh; }\n\n# Gate 2: wait for master readiness before starting\nWAIT_INTERVAL=5\nMAX_RETRIES=360\nRETRIES=0\nset +e\nwhile true; do\n FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv 2>/dev/null)\n if [ \"$FINISH_MASTER_NODE\" == \"true\" ]; then\n break\n fi\n RETRIES=$((RETRIES + 1))\n if [ $RETRIES -ge $MAX_RETRIES ]; then\n echo \"[OpenVidu] timed out after 30 min waiting for FINISH-MASTER-NODE\"\n /usr/local/bin/delete_media_node.sh\n exit 1\n fi\n sleep $WAIT_INTERVAL\ndone\nset -e\n\n# Start OpenVidu\nsystemctl start openvidu || { echo \"[OpenVidu] error starting OpenVidu\"; /usr/local/bin/delete_media_node.sh; }\n", "stop_media_nodesScriptMedia": "[reduce(items(variables('stopMediaNodeParams')), createObject('value', variables('stop_media_nodesScriptMediaTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value]", "delete_mediaNode_ScriptMedia": "[reduce(items(variables('stopMediaNodeParams')), createObject('value', variables('delete_mediaNode_ScriptMediaTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value]", "base64stopMediaNode": "[base64(variables('stop_media_nodesScriptMedia'))]", diff --git a/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf b/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf index a01c72442..027e89a71 100644 --- a/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf +++ b/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf @@ -314,8 +314,7 @@ locals { is_c4a_instance = startswith(var.masterNodesInstanceType, "c4a-") } -# Default subnetwork the master instances live in (they use network = "default" with no -# explicit subnetwork, so GCP places them in the auto-mode default subnetwork of the region) +# Must match the subnetwork the master instances implicitly use (network = "default", no explicit subnetwork) data "google_compute_subnetwork" "default" { name = "default" region = var.region @@ -323,9 +322,6 @@ data "google_compute_subnetwork" "default" { depends_on = [google_project_service.compute_api] } -# Static internal IPs for the master nodes. Reserving them up front lets the media instance -# template and every master receive the full list without waiting for the master instances to -# boot, which removes the runtime IP-exchange handshake through Secret Manager. resource "google_compute_address" "master_internal_ip" { count = 4 name = lower("${var.stackName}-master-node-${count.index + 1}-internal-ip") @@ -337,8 +333,7 @@ resource "google_compute_address" "master_internal_ip" { } locals { - # Master node private IPs come from the reserved static internal addresses (index 0 -> master - # node 1, ...). Comma-separated to match the installer's --master-node-private-ip-list format. + # Comma-separated: must match the installer's --master-node-private-ip-list format master_node_private_ip_list = join(",", google_compute_address.master_internal_ip[*].address) } @@ -1018,7 +1013,6 @@ get_meta() { curl -s -H "Metadata-Flavor: Google" "$${METADATA_URL}/$1"; } # Get master node number from metadata MASTER_NODE_NUM=$(get_meta "instance/attributes/masterNodeNum") -# Get the list of master node private IPs (static internal IPs passed via metadata) MASTER_NODE_PRIVATE_IP_LIST=$(get_meta "instance/attributes/masterNodePrivateIPList") # Check if secrets have been generated @@ -1080,7 +1074,6 @@ if [[ $MASTER_NODE_NUM -eq 1 ]] && [[ "$ALL_SECRETS_GENERATED" == "false" ]]; th fi # Wait for master-1 to finish generating all shared secrets before reading them. -# Bounded to 360 iterations x 5s = 30 minutes. i=0 while ! gcloud secrets versions access latest --secret=ALL_SECRETS_GENERATED 2>/dev/null | grep -q "true"; do i=$((i + 1)) @@ -1115,8 +1108,7 @@ LIVEKIT_API_KEY=$(gcloud secrets versions access latest --secret=LIVEKIT_API_KEY LIVEKIT_API_SECRET=$(gcloud secrets versions access latest --secret=LIVEKIT_API_SECRET) ENABLED_MODULES=$(gcloud secrets versions access latest --secret=ENABLED_MODULES) -# Build install command. Download the installer to a file first: process substitution -# (sh <(curl ...)) would silently run an empty script and exit 0 on a transient curl failure. +# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure INSTALLER_SCRIPT="/tmp/install_ov_master_node.sh" curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh" if [ ! -s "$INSTALLER_SCRIPT" ]; then @@ -1474,7 +1466,6 @@ EOF check_app_ready_script = <<-EOF #!/bin/bash -# Bounded to 240 iterations x 5s = 20 minutes. i=0 while true; do HTTP_STATUS=$(curl -Ik http://localhost:7880/health/caddy 2>/dev/null | head -n1 | awk '{print $2}') @@ -1624,7 +1615,6 @@ STACK_NAME=$(get_meta "instance/attributes/stackName") PRIVATE_IP=$(get_meta "instance/network-interfaces/0/ip") # Wait for master nodes to be ready by checking secrets. -# Bounded to 180 iterations x 10s = 30 minutes. i=0 while ! gcloud secrets versions access latest --secret=ALL_SECRETS_GENERATED 2>/dev/null | grep -q "true"; do i=$((i + 1)) @@ -1649,9 +1639,7 @@ if [[ "$OPENVIDU_VERSION" == "none" ]]; then exit 1 fi -# Build install command for media node. Download the installer to a file first: process -# substitution (sh <(curl ...)) would silently run an empty script and exit 0 on a transient -# curl failure. +# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure INSTALLER_SCRIPT="/tmp/install_ov_media_node.sh" curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh" if [ ! -s "$INSTALLER_SCRIPT" ]; then From 4d6b7acd7cc5e17090316c9b6c09241ed4e7d5dc Mon Sep 17 00:00:00 2001 From: Piwccle Date: Fri, 24 Jul 2026 17:11:58 +0200 Subject: [PATCH 5/6] openvidu-deployment: optimize AWS HA deployment time Parallel master creation: drop the MasterNodeN -> WaitConditionN-1 DependsOn chain. Each master gets a fixed MASTER_NODE_NUM via UserData (replaces subnet-matching detection) and publishes its private IP to its own SSM parameter (atomic per-key writes; the 4 parameters are stack resources), removing the IP exchange through the shared Secrets Manager JSON and its lost-update race. after_install.sh is now leader-only, so the shared JSON keeps a single writer with parallel masters. Ungate media nodes and the NLB from WaitCondition4: media wait for secrets + the 4 SSM IPs (bounded polls) before the heavy install, and for a healthy master (:7880/health/caddy) before starting the service. Bounded timeout on the previously infinite IP wait, robust installer fetch (curl --retry to file instead of sh <(curl)), SSM read/write IAM statements, MEET_INITIAL_API_KEY dead-code fix, SubnetProcessor Lambda cleanup (unused boto3 client and ec2:DescribeSubnets permission). Validated with ov-cloud-tester (sc-deploy-destroy, ha, dev): PASS. deploy 5m24s (baseline ~12m), ready 21m14s (wait-ready dominated by external DNS propagation), destroy 7m39s. --- .../ha/aws/DOC-CHANGES-ha-optimizations.md | 195 +++++++++++++ .../pro/ha/aws/cf-openvidu-ha.yaml | 259 ++++++++++++------ 2 files changed, 364 insertions(+), 90 deletions(-) create mode 100644 openvidu-deployment/pro/ha/aws/DOC-CHANGES-ha-optimizations.md diff --git a/openvidu-deployment/pro/ha/aws/DOC-CHANGES-ha-optimizations.md b/openvidu-deployment/pro/ha/aws/DOC-CHANGES-ha-optimizations.md new file mode 100644 index 000000000..ef4c9cc5d --- /dev/null +++ b/openvidu-deployment/pro/ha/aws/DOC-CHANGES-ha-optimizations.md @@ -0,0 +1,195 @@ +# Documentation changes for `ha-optimizations` (AWS HA deployment) + +Instructions for the Claude working on the docs repo +`/home/sergio/Escritorio/openvidu/openvidu.io` (branch `next`, already up to date). + +These instructions derive from the changes applied in +`openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml` on the `ha-optimizations` branch. +Functional summary of what changed in the deployment: + +- The 4 master EC2 instances are now created **in parallel** (the `DependsOn` chain + MasterNode2→WaitCondition1, 3→2, 4→3 was removed; coordination stays data-driven). +- Each master now publishes its private IP to its **own** SSM Parameter Store parameter + (`/openvidu//master-node-N-private-ip`), an atomic per-key write, instead + of a racy read-modify-write into the single shared Secrets Manager JSON. The 4 + `MASTER_NODE_*_PRIVATE_IP` keys were removed from that JSON. +- Media Nodes now install (Docker + image pulls) **in parallel** with the masters. They + are no longer gated behind `MasterNodesWaitCondition4`; instead they poll for the + shared secrets and the master IPs, and wait for at least one healthy master before + starting. The Network Load Balancer is likewise no longer gated behind + `MasterNodesWaitCondition4`. +- Hardening: the previously unbounded "wait for the 4 master IPs" loop is now bounded + (30 min), and the installer download is retried and validated + (`curl --retry 8 ... -o file` + non-empty check) instead of the fragile + `sh <(curl ...)`. + +SUMMARY: **no doc content change is strictly required.** The AWS HA docs publish no +deployment-time figure and describe no internals that changed. The only *optional* +edits are (a) adding a deployment-time figure and (c.1) regenerating one screenshot. +Everything else (public parameters, the single Output, screenshots, other clouds, other +deployment types, on-node config files) stays the same. Read this file in full before +touching anything. + +--- + +## (a) Deployment-time figure — OPTIONAL (none exists today) + +Unlike the GCP and Azure HA `install.md` files, the **AWS HA `install.md` currently +publishes NO deployment-time figure at all** (no "wait about N minutes" sentence). So +there is nothing to update here. This is only a suggestion in case you want to add one +now that the deployment is faster (masters in parallel + media install overlapping the +masters' install + no IP handshake through Secrets Manager). + +File: `docs/docs/self-hosting/ha/aws/install.md` + +Two natural, OPTIONAL insertion points (locate by literal text; line numbers approximate): + +### Option 1 — end of "## Deploying the stack" (~line 155) + +Current literal text (the paragraph that ends the section): + +``` +When you are ready with your CloudFormation parameters, just click on _"Next"_, specify in _"Stack failure options"_ the option _"Preserve successfully provisioned resources"_ to be able to troubleshoot the deployment in case of error, click on _"Next"_ again, and finally _"Submit"_. +``` + +Proposed (append one sentence; fill in `X`/`Y` from `ov-cloud-tester`): + +``` +When you are ready with your CloudFormation parameters, just click on _"Next"_, specify in _"Stack failure options"_ the option _"Preserve successfully provisioned resources"_ to be able to troubleshoot the deployment in case of error, click on _"Next"_ again, and finally _"Submit"_. The stack will take about X to Y minutes to create all resources. +``` + +### Option 2 — "## Configuration and administration" (~line 190) + +Current literal text: + +``` +When your CloudFormation stack reaches the **`CREATE_COMPLETE`** status, your OpenVidu High Availability deployment is ready to use. You can check the [Administration](./admin.md) section to learn how to manage your deployment. +``` + +Proposed (fill in `X`/`Y` from `ov-cloud-tester`): + +``` +When your CloudFormation stack reaches the **`CREATE_COMPLETE`** status (about X to Y minutes), your OpenVidu High Availability deployment is ready to use. You can check the [Administration](./admin.md) section to learn how to manage your deployment. +``` + +> Placeholder to fill in: `X to Y minutes — fill with the ov-cloud-tester measurement`. +> If you add a figure in BOTH options, keep them consistent. Do NOT invent a number — +> leave the `X to Y` placeholder until the real measurement is available. +> This is entirely optional; skipping section (a) leaves the docs correct. + +--- + +## (b) Public template parameters / outputs: NO CHANGES — confirmation + +Do not touch parameter tables or screenshots in the parameters section. + +Reason: **no CloudFormation `Parameters` were added, removed or renamed, and the single +`Outputs` entry (`ServicesAndCredentials`) is unchanged.** The new per-master IPs live in +SSM Parameter Store parameters that are created internally by the template; they are not +template parameters and require no user input. Therefore: + +- The parameter sections of `docs/docs/self-hosting/ha/aws/install.md` + ("## CloudFormation Parameters" and its subsections, ~lines 52–137) **stay the same**. +- There are no new fields the user must fill in the CloudFormation form. +- The parameter form screenshots **do not change**. +- The `ServicesAndCredentials` output and the "Configure your application to use the + deployment" section (~lines 163–180) are unaffected. + +--- + +## (c) Other statements in the AWS HA docs that may become outdated + +The AWS HA documentation (`docs/docs/self-hosting/ha/aws/*.md`) and the shared AWS +includes (`shared/self-hosting/aws/*.md`) were reviewed against the internal changes. +Conclusions: + +### c.1 — Secrets Manager contents: `MASTER_NODE_{1..4}_PRIVATE_IP` removed from the JSON + +The 4 keys `MASTER_NODE_1_PRIVATE_IP` … `MASTER_NODE_4_PRIVATE_IP` were removed from the +`OpenViduSharedInfo` Secrets Manager JSON (the masters' IPs now live in SSM Parameter +Store). The AWS HA docs **do not mention these keys by name in any text**, so there is no +prose to fix. + +- `install.md` (~line 165–177) and `admin.md` (~line 226–238) describe the + `ServicesAndCredentials` secret only generically ("contains all URLs and credentials", + "get the JSON with all the information"). The "most relevant" values are listed via the + shared includes `shared/self-hosting/aws/credentials-general.md` and + `credentials-v2compatibility.md`, neither of which lists the IP keys. No change. +- Only possible impact: the secret screenshot + `assets/images/platform/self-hosting/ha/aws/2-secrets.png` (shown in `install.md`) + COULD still display the 4 removed IP keys. This is purely cosmetic. OPTIONAL action: + if/when the screenshot is regenerated, do it against an `ha-optimizations` deployment. + This is NOT blocking. + +### c.2 — `upgrade.md`: DO NOT TOUCH + +`docs/docs/self-hosting/ha/aws/upgrade.md` (~lines 82–92) documents +`/usr/local/bin/store_secret.sh save OPENVIDU_VERSION ""` and the +`OPENVIDU_VERSION` Secrets Manager secret. **Neither `store_secret.sh` nor the +`OPENVIDU_VERSION` secret was changed** by this work. No changes required. + +### c.3 — `backup-and-restore.md`: DO NOT TOUCH + +`docs/docs/self-hosting/how-to-guides/backup-and-restore.md` (~lines 676–692) documents +`MASTER_NODE_1_PRIVATE_IP` … `MASTER_NODE_4_PRIVATE_IP` inside the **on-node** +configuration file `/opt/openvidu/config/node/master-node.env`. + +IMPORTANT: **these are NOT the removed Secrets Manager keys.** They are config-file +variables that the installer still writes from the `--master-node-private-ip-list` flag +(which is still passed — now built from the SSM parameters instead of the shared JSON). +That on-node flow is unchanged. **Do not modify `backup-and-restore.md`.** + +### c.4 — Master boot order / parallelization: not documented + +The AWS HA docs never state that master nodes are created sequentially / "one by one", +nor do they describe the `DependsOn` chain, `MasterNodesWaitCondition*`, or the fact that +the Load Balancer / Media Nodes used to wait for `MasterNodesWaitCondition4`. Removing +that chain and un-gating the NLB and Media Nodes changes nothing user-facing. No text to +update. + +### c.5 — Internal coordination (`ALL_SECRETS_GENERATED`, SSM IP handshake, health gates): not documented + +The master-1-as-leader secret generation, the `ALL_SECRETS_GENERATED` flag, the new SSM +per-master IP handshake, the bounded wait loops, and the media "wait for a healthy +master" gate are all internal and not publicly documented. Nothing user-facing changes. +No text to update. `admin.md` and `index.md` were also reviewed: no references to the +changed internals. No changes required. + +--- + +## (d) New resources created by the stack: 4 SSM parameters + +The stack now creates **4 SSM Parameter Store parameters** as CloudFormation resources: + +``` +/openvidu//master-node-1-private-ip +/openvidu//master-node-2-private-ip +/openvidu//master-node-3-private-ip +/openvidu//master-node-4-private-ip +``` + +They are visible in the AWS Systems Manager → Parameter Store console and are deleted +with the stack. They are harmless and require no user interaction. + +The AWS HA docs do **not** enumerate the resources the stack creates anywhere, so there +is nothing to update. This note exists only so that, if such an enumeration is ever added +(or if a reader asks about unfamiliar parameters in the console), the 4 parameters are +accounted for and expected. + +--- + +## Implementation checklist + +- [ ] (OPTIONAL, section a) In `docs/docs/self-hosting/ha/aws/install.md`, add a + deployment-time sentence with the `X to Y minutes` placeholder at one of the two + insertion points; measure with `ov-cloud-tester` on `ha-optimizations` and fill in + `X`/`Y`. Skipping this leaves the docs correct. +- [ ] Confirm NO parameter tables or screenshots were touched (section b). +- [ ] Confirm `upgrade.md` (`store_secret.sh` / `OPENVIDU_VERSION`) was NOT touched (c.2). +- [ ] Confirm `backup-and-restore.md` (on-node `master-node.env` IP vars) was NOT touched + (c.3). +- [ ] (OPTIONAL, section c.1) Regenerate `2-secrets.png` against an `ha-optimizations` + deployment if you want it to stop showing the 4 removed `MASTER_NODE_*_PRIVATE_IP` + keys. Not blocking. +- [ ] No action needed for the 4 new SSM parameters unless a resource enumeration is + added to the docs (section d). diff --git a/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml b/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml index 5f7a91afc..b9b259b41 100644 --- a/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml +++ b/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml @@ -1242,14 +1242,39 @@ Resources: "GRAFANA_ADMIN_USERNAME": "none", "GRAFANA_ADMIN_PASSWORD": "none", "ENABLED_MODULES": "none", - "MASTER_NODE_1_PRIVATE_IP": "none", - "MASTER_NODE_2_PRIVATE_IP": "none", - "MASTER_NODE_3_PRIVATE_IP": "none", - "MASTER_NODE_4_PRIVATE_IP": "none", "OPENVIDU_VERSION": "none", "ALL_SECRETS_GENERATED": "false" } + # One SSM parameter per master: atomic per-key writes + MasterNode1PrivateIpParameter: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub /openvidu/${AWS::StackName}/master-node-1-private-ip + Type: String + Value: "none" + + MasterNode2PrivateIpParameter: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub /openvidu/${AWS::StackName}/master-node-2-private-ip + Type: String + Value: "none" + + MasterNode3PrivateIpParameter: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub /openvidu/${AWS::StackName}/master-node-3-private-ip + Type: String + Value: "none" + + MasterNode4PrivateIpParameter: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub /openvidu/${AWS::StackName}/master-node-4-private-ip + Type: String + Value: "none" + S3AppDataBucketResource: Type: 'AWS::S3::Bucket' Properties: @@ -1281,8 +1306,7 @@ Resources: Condition: CreateClusterDataBucket # ------------------------- - # Preprocess subnets to allocate Volumes and ENIs across Availability Zones - # For OpenVidu Master Nodes + # Normalize the master node subnet list to exactly 4 entries, one per OpenVidu Master Node # ------------------------- SubnetProcessorFunction: Type: AWS::Lambda::Function @@ -1293,13 +1317,11 @@ Resources: Code: ZipFile: | import cfnresponse - import boto3 def lambda_handler(event, context): try: # Process event data subnets = event['ResourceProperties']['Subnets'] - ec2 = boto3.client('ec2') # Ensure we have at least four subnets by cycling through the available subnets subnets = (subnets * 4)[:4] # Repeat the list to have at least 4 elements and then take the first 4 @@ -1355,10 +1377,6 @@ Resources: - logs:CreateLogStream - logs:PutLogEvents Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/SubnetProcessor-${AWS::Region}-${AWS::StackName}:*' - - Effect: Allow - Action: - - ec2:DescribeSubnets - Resource: '*' OpenViduMasterNodeRole: Type: AWS::IAM::Role @@ -1390,6 +1408,11 @@ Resources: - secretsmanager:GetSecretValue - secretsmanager:UpdateSecret Resource: !Ref OpenViduSharedInfo + - Effect: Allow + Action: + - ssm:GetParameter + - ssm:PutParameter + Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/openvidu/${AWS::StackName}/* - Fn::If: - CreateRecordingsBucket - Effect: Allow @@ -1474,6 +1497,10 @@ Resources: Action: - secretsmanager:GetSecretValue Resource: !Ref OpenViduSharedInfo + - Effect: Allow + Action: + - ssm:GetParameter + Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/openvidu/${AWS::StackName}/* - Effect: Allow Action: - autoscaling:SetInstanceHealth @@ -1526,8 +1553,6 @@ Resources: OpenViduMediaInstanceProfile: Type: AWS::IAM::InstanceProfile - DependsOn: - - MasterNodesWaitCondition4 Properties: InstanceProfileName: !Sub OpenViduMediaInstanceProfile-${AWS::Region}-${AWS::StackName} Roles: @@ -1566,42 +1591,15 @@ Resources: rm -rf awscliv2.zip aws fi - # Singal to notify instance is waiting + # Signal to notify instance is waiting SIGNAL_NAME="$1" + # Exported earlier in this instance's UserData + MASTER_NODE_NUM="${!MASTER_NODE_NUM}" + # Token for IMDSv2 TOKEN="$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")" - # Subnets IDs - SUBNETS=( - "${SubnetProcessor.Subnet1}" - "${SubnetProcessor.Subnet2}" - "${SubnetProcessor.Subnet3}" - "${SubnetProcessor.Subnet4}" - ) - - MAC_ADDRESS="$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254//latest/meta-data/mac)" - SUBNET_ID="$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s "http://169.254.169.254/latest/meta-data/network/interfaces/macs/$MAC_ADDRESS/subnet-id")" - - # Check master node number - MASTER_NODE_NUM=1 - for subnet in "${!SUBNETS[@]}"; do - if [[ "$subnet" == "$SUBNET_ID" ]]; then - - SHARED_SECRET=$(aws secretsmanager get-secret-value \ - --region ${AWS::Region} \ - --secret-id openvidu-ha-${AWS::Region}-${AWS::StackName} \ - --query SecretString --output text || echo 'none') - - # Check if current master node is reacheable with ping - ACUTAL_MASTER_NODE_IP=$(echo "$SHARED_SECRET" | jq -r ".MASTER_NODE_${!MASTER_NODE_NUM}_PRIVATE_IP") - if [[ "$ACUTAL_MASTER_NODE_IP" == "none" ]]; then - break - fi - fi - MASTER_NODE_NUM=$((MASTER_NODE_NUM + 1)) - done - # Get own private IP PRIVATE_IP="$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/local-ipv4)" @@ -1610,8 +1608,12 @@ Resources: exit 1 fi - # Store current private IP - /usr/local/bin/store_secret.sh save MASTER_NODE_${!MASTER_NODE_NUM}_PRIVATE_IP "${!PRIVATE_IP}" + aws ssm put-parameter \ + --region ${AWS::Region} \ + --name "/openvidu/${AWS::StackName}/master-node-${!MASTER_NODE_NUM}-private-ip" \ + --value "${!PRIVATE_IP}" \ + --type String \ + --overwrite > /dev/null SHARED_SECRET=$(aws secretsmanager get-secret-value \ --region ${AWS::Region} \ @@ -1690,39 +1692,39 @@ Resources: # sending the signal call cfn-signal -e $? --stack ${AWS::StackId} --resource "$SIGNAL_NAME" --region ${AWS::Region} - # Wait for all master nodes to store their private IPs + IP_WAIT_ATTEMPTS=0 + IP_WAIT_MAX=360 while true; do - SHARED_SECRET=$(aws secretsmanager get-secret-value \ - --region ${AWS::Region} \ - --secret-id openvidu-ha-${AWS::Region}-${AWS::StackName} \ - --query SecretString --output text || echo 'none') + MASTER_NODE_1_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-1-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + MASTER_NODE_2_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-2-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + MASTER_NODE_3_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-3-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + MASTER_NODE_4_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-4-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') - MASTER_NODE_1_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_1_PRIVATE_IP') - MASTER_NODE_2_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_2_PRIVATE_IP') - MASTER_NODE_3_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_3_PRIVATE_IP') - MASTER_NODE_4_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_4_PRIVATE_IP') - - # Check if all master nodes have stored their private IPs - if [[ "$MASTER_NODE_1_PRIVATE_IP" != "none" ]] && - [[ "$MASTER_NODE_2_PRIVATE_IP" != "none" ]] && - [[ "$MASTER_NODE_3_PRIVATE_IP" != "none" ]] && - [[ "$MASTER_NODE_4_PRIVATE_IP" != "none" ]]; then + # Check if all master nodes have published their private IPs + if [[ "$MASTER_NODE_1_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_1_PRIVATE_IP" ]] && + [[ "$MASTER_NODE_2_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_2_PRIVATE_IP" ]] && + [[ "$MASTER_NODE_3_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_3_PRIVATE_IP" ]] && + [[ "$MASTER_NODE_4_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_4_PRIVATE_IP" ]]; then break fi + + IP_WAIT_ATTEMPTS=$((IP_WAIT_ATTEMPTS + 1)) + if [[ $IP_WAIT_ATTEMPTS -ge $IP_WAIT_MAX ]]; then + echo "Error: timed out after 30 minutes waiting for all master nodes to publish their private IPs to SSM" + exit 1 + fi sleep 5 done + # Comma-separated 1-4: must match installer's --master-node-private-ip-list format + MASTER_NODE_PRIVATE_IP_LIST="$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP" + + # Re-read: other (non-IP) values still come from the shared secret SHARED_SECRET=$(aws secretsmanager get-secret-value \ --region ${AWS::Region} \ --secret-id openvidu-ha-${AWS::Region}-${AWS::StackName} \ --query SecretString --output text) - MASTER_NODE_1_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_1_PRIVATE_IP') - MASTER_NODE_2_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_2_PRIVATE_IP') - MASTER_NODE_3_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_3_PRIVATE_IP') - MASTER_NODE_4_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_4_PRIVATE_IP') - MASTER_NODE_PRIVATE_IP_LIST="$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP" - DOMAIN=$(echo "$SHARED_SECRET" | jq -r '.DOMAIN_NAME') OPENVIDU_PRO_LICENSE=$(echo "$SHARED_SECRET" | jq -r '.OPENVIDU_PRO_LICENSE') OPENVIDU_RTC_ENGINE=$(echo "$SHARED_SECRET" | jq -r '.OPENVIDU_RTC_ENGINE') @@ -1738,18 +1740,22 @@ Resources: GRAFANA_ADMIN_PASSWORD=$(echo "$SHARED_SECRET" | jq -r '.GRAFANA_ADMIN_PASSWORD') MEET_INITIAL_ADMIN_USER=$(echo "$SHARED_SECRET" | jq -r '.MEET_INITIAL_ADMIN_USER') MEET_INITIAL_ADMIN_PASSWORD=$(echo "$SHARED_SECRET" | jq -r '.MEET_INITIAL_ADMIN_PASSWORD') + MEET_INITIAL_API_KEY="" if [[ "${InitialMeetApiKey}" != '' ]]; then MEET_INITIAL_API_KEY=$(echo "$SHARED_SECRET" | jq -r '.MEET_INITIAL_API_KEY') - else - MEET_INITIAL_API_KEY="" fi - MEET_INITIAL_API_KEY=$(echo "$SHARED_SECRET" | jq -r '.MEET_INITIAL_API_KEY') LIVEKIT_API_KEY=$(echo "$SHARED_SECRET" | jq -r '.LIVEKIT_API_KEY') LIVEKIT_API_SECRET=$(echo "$SHARED_SECRET" | jq -r '.LIVEKIT_API_SECRET') ENABLED_MODULES=$(echo "$SHARED_SECRET" | jq -r '.ENABLED_MODULES') - # Base command - INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh)" + # Download first: sh <(curl ...) would silently run an empty script on a transient curl failure + INSTALLER_SCRIPT="/tmp/install_ov_master_node.sh" + curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh" + if [ ! -s "$INSTALLER_SCRIPT" ]; then + echo "Downloaded OpenVidu master node installer is empty or missing" >&2 + exit 1 + fi + INSTALL_COMMAND="sh $INSTALLER_SCRIPT" # Common arguments COMMON_ARGS=( @@ -1837,6 +1843,10 @@ Resources: content: !Sub | #!/bin/bash set -e + # Only master 1 writes the shared secret, to avoid a lost-update race + if [[ "${!MASTER_NODE_NUM:-}" != "1" ]]; then + exit 0 + fi # Get current shared secret SHARED_SECRET=$(aws secretsmanager get-secret-value \ --region ${AWS::Region} \ @@ -2103,6 +2113,8 @@ Resources: OpenViduMasterNode1: Type: AWS::EC2::Instance + # Depends on its own SSM parameter so its put-parameter can't race the parameter's creation + DependsOn: MasterNode1PrivateIpParameter Properties: LaunchTemplate: LaunchTemplateId: !Ref OpenViduMasterLaunchTemplate @@ -2133,6 +2145,8 @@ Resources: cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource OpenViduMasterLaunchTemplate + export MASTER_NODE_NUM=1 + # Install OpenVidu /usr/local/bin/install.sh "MasterNodesWaitCondition1" || { echo "[OpenVidu] error installing OpenVidu"; exit 1; } @@ -2179,7 +2193,7 @@ Resources: OpenViduMasterNode2: Type: AWS::EC2::Instance - DependsOn: MasterNodesWaitCondition1 + DependsOn: MasterNode2PrivateIpParameter Properties: LaunchTemplate: LaunchTemplateId: !Ref OpenViduMasterLaunchTemplate @@ -2210,6 +2224,8 @@ Resources: cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource OpenViduMasterLaunchTemplate + export MASTER_NODE_NUM=2 + # Install OpenVidu /usr/local/bin/install.sh "MasterNodesWaitCondition2" || { echo "[OpenVidu] error installing OpenVidu"; exit 1; } @@ -2256,7 +2272,7 @@ Resources: OpenViduMasterNode3: Type: AWS::EC2::Instance - DependsOn: MasterNodesWaitCondition2 + DependsOn: MasterNode3PrivateIpParameter Properties: LaunchTemplate: LaunchTemplateId: !Ref OpenViduMasterLaunchTemplate @@ -2287,6 +2303,8 @@ Resources: cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource OpenViduMasterLaunchTemplate + export MASTER_NODE_NUM=3 + # Install OpenVidu /usr/local/bin/install.sh "MasterNodesWaitCondition3" || { echo "[OpenVidu] error installing OpenVidu"; exit 1; } @@ -2333,7 +2351,7 @@ Resources: OpenViduMasterNode4: Type: AWS::EC2::Instance - DependsOn: MasterNodesWaitCondition3 + DependsOn: MasterNode4PrivateIpParameter Properties: LaunchTemplate: LaunchTemplateId: !Ref OpenViduMasterLaunchTemplate @@ -2364,6 +2382,8 @@ Resources: cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource OpenViduMasterLaunchTemplate + export MASTER_NODE_NUM=4 + # Install OpenVidu /usr/local/bin/install.sh "MasterNodesWaitCondition4" || { echo "[OpenVidu] error installing OpenVidu"; exit 1; } @@ -2453,6 +2473,25 @@ Resources: exit 1 fi + SECRETS_WAIT_ATTEMPTS=0 + SECRETS_WAIT_MAX=360 + while true; do + ALL_SECRETS_GENERATED=$(echo "$SHARED_SECRET" | jq -r '.ALL_SECRETS_GENERATED') + if [[ "$ALL_SECRETS_GENERATED" == "true" ]]; then + break + fi + SECRETS_WAIT_ATTEMPTS=$((SECRETS_WAIT_ATTEMPTS + 1)) + if [[ $SECRETS_WAIT_ATTEMPTS -ge $SECRETS_WAIT_MAX ]]; then + echo "Error: timed out after 30 minutes waiting for shared secrets to be generated" + exit 1 + fi + sleep 5 + SHARED_SECRET="$(aws secretsmanager get-secret-value \ + --region ${AWS::Region} \ + --secret-id openvidu-ha-${AWS::Region}-${AWS::StackName} \ + --query SecretString --output text || echo 'none')" + done + # Get OpenVidu Media Nodes version to deploy OPENVIDU_VERSION=$(echo "$SHARED_SECRET" | jq -r '.OPENVIDU_VERSION') if [[ "$OPENVIDU_VERSION" == "none" ]]; then @@ -2460,20 +2499,39 @@ Resources: exit 1 fi - ALL_SECRETS_GENERATED=$(echo "$SHARED_SECRET" | jq -r '.ALL_SECRETS_GENERATED') - if [[ "$ALL_SECRETS_GENERATED" == "none" ]]; then - echo "Error: Secrets not generated" - exit 1 - fi - MASTER_NODE_1_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_1_PRIVATE_IP') - MASTER_NODE_2_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_2_PRIVATE_IP') - MASTER_NODE_3_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_3_PRIVATE_IP') - MASTER_NODE_4_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_4_PRIVATE_IP') + IP_WAIT_ATTEMPTS=0 + IP_WAIT_MAX=360 + while true; do + MASTER_NODE_1_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-1-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + MASTER_NODE_2_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-2-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + MASTER_NODE_3_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-3-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + MASTER_NODE_4_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-4-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + + if [[ "$MASTER_NODE_1_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_1_PRIVATE_IP" ]] && + [[ "$MASTER_NODE_2_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_2_PRIVATE_IP" ]] && + [[ "$MASTER_NODE_3_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_3_PRIVATE_IP" ]] && + [[ "$MASTER_NODE_4_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_4_PRIVATE_IP" ]]; then + break + fi + + IP_WAIT_ATTEMPTS=$((IP_WAIT_ATTEMPTS + 1)) + if [[ $IP_WAIT_ATTEMPTS -ge $IP_WAIT_MAX ]]; then + echo "Error: timed out after 30 minutes waiting for all master nodes to publish their private IPs to SSM" + exit 1 + fi + sleep 5 + done MASTER_NODE_PRIVATE_IP_LIST="$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP" REDIS_PASSWORD=$(echo "$SHARED_SECRET" | jq -r '.REDIS_PASSWORD') - # Base command - INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh)" + # Download first: sh <(curl ...) would silently run an empty script on a transient curl failure + INSTALLER_SCRIPT="/tmp/install_ov_media_node.sh" + curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh" + if [ ! -s "$INSTALLER_SCRIPT" ]; then + echo "Downloaded OpenVidu media node installer is empty or missing" >&2 + exit 1 + fi + INSTALL_COMMAND="sh $INSTALLER_SCRIPT" # Common arguments COMMON_ARGS=( @@ -2609,13 +2667,36 @@ Resources: python3 -m pip install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-${!CFN_BOOTSTRAP_VERSION}.tar.gz fi - cfn-init --region ${AWS::Region} --stack ${AWS::StackId} --resource OpenViduMediaNodeLaunchTemplate + cfn-init --region ${AWS::Region} --stack ${AWS::StackName} --resource OpenViduMediaNodeLaunchTemplate export HOME="/root" # Install OpenVidu /usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu"; /usr/local/bin/set_as_unhealthy.sh; exit 1; } + MASTER_IP_1=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-1-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + MASTER_IP_2=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-2-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + MASTER_IP_3=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-3-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + MASTER_IP_4=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-4-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + MASTER_HEALTHY=false + for i in $(seq 1 360); do + for MASTER_IP in "$MASTER_IP_1" "$MASTER_IP_2" "$MASTER_IP_3" "$MASTER_IP_4"; do + if [ "$MASTER_IP" != "none" ] && [ -n "$MASTER_IP" ] && curl -sf "http://$MASTER_IP:7880/health/caddy" >/dev/null 2>&1; then + MASTER_HEALTHY=true + break + fi + done + if [ "$MASTER_HEALTHY" = "true" ]; then + break + fi + sleep 5 + done + if [ "$MASTER_HEALTHY" != "true" ]; then + echo "[OpenVidu] no master node became healthy after 30 minutes" + /usr/local/bin/set_as_unhealthy.sh + exit 1 + fi + # Start OpenVidu systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; /usr/local/bin/set_as_unhealthy.sh; exit 1; } @@ -3191,8 +3272,6 @@ Resources: LoadBalancer: Type: AWS::ElasticLoadBalancingV2::LoadBalancer - DependsOn: - - MasterNodesWaitCondition4 Properties: Name: Fn::Join: From 31a8488f23ef71922c4fd828714c5008e21dcd7b Mon Sep 17 00:00:00 2001 From: Piwccle Date: Sat, 25 Jul 2026 00:46:02 +0200 Subject: [PATCH 6/6] openvidu-deployment: fix eventually-consistent secret reads in AWS HA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GetSecretValue is eventually consistent: with masters booting in parallel, the final re-read of the shared JSON could return a stale pre-generation version (ALL_SECRETS_GENERATED already observed true but values still 'none'), so the installer aborted with e.g. 'mongo-admin-password: must have at least 10 characters'. The old serialized boot masked this window (minutes between generation and re-reads); parallel boot exposed it intermittently (1 of 3 tries). Fix: read-until-valid — retry the re-read (5s, 5 min cap) until the same snapshot has ALL_SECRETS_GENERATED=true AND every always-generated value != 'none'; the media gate now validates OPENVIDU_VERSION and REDIS_PASSWORD content too, not just the flag. Validated with ov-cloud-tester (sc-deploy-destroy, ha, dev, --tries 3): 3/3 PASS, deploy mean 5m19s (spread 76ms), ready mean 21m24s. Control run without the fix reproduced the race (1/3 failed). --- .../pro/ha/aws/cf-openvidu-ha.yaml | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml b/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml index b9b259b41..cce197135 100644 --- a/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml +++ b/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml @@ -1719,11 +1719,24 @@ Resources: # Comma-separated 1-4: must match installer's --master-node-private-ip-list format MASTER_NODE_PRIVATE_IP_LIST="$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP" - # Re-read: other (non-IP) values still come from the shared secret - SHARED_SECRET=$(aws secretsmanager get-secret-value \ - --region ${AWS::Region} \ - --secret-id openvidu-ha-${AWS::Region}-${AWS::StackName} \ - --query SecretString --output text) + # Re-read: other (non-IP) values still come from the shared secret. + # GetSecretValue is eventually consistent: retry until a read returns the generated values + SECRET_READ_ATTEMPTS=0 + while true; do + SHARED_SECRET=$(aws secretsmanager get-secret-value \ + --region ${AWS::Region} \ + --secret-id openvidu-ha-${AWS::Region}-${AWS::StackName} \ + --query SecretString --output text) + if echo "$SHARED_SECRET" | jq -e '(.ALL_SECRETS_GENERATED == "true") and ([.DOMAIN_NAME, .OPENVIDU_VERSION, .REDIS_PASSWORD, .MONGO_ADMIN_PASSWORD, .MONGO_REPLICA_SET_KEY, .MINIO_SECRET_KEY, .DASHBOARD_ADMIN_PASSWORD, .GRAFANA_ADMIN_PASSWORD, .LIVEKIT_API_KEY, .LIVEKIT_API_SECRET] | all(. != "none"))' > /dev/null; then + break + fi + SECRET_READ_ATTEMPTS=$((SECRET_READ_ATTEMPTS + 1)) + if [[ $SECRET_READ_ATTEMPTS -ge 60 ]]; then + echo "Error: shared secret still incomplete after 5 minutes of stale reads" + exit 1 + fi + sleep 5 + done DOMAIN=$(echo "$SHARED_SECRET" | jq -r '.DOMAIN_NAME') OPENVIDU_PRO_LICENSE=$(echo "$SHARED_SECRET" | jq -r '.OPENVIDU_PRO_LICENSE') @@ -2476,8 +2489,8 @@ Resources: SECRETS_WAIT_ATTEMPTS=0 SECRETS_WAIT_MAX=360 while true; do - ALL_SECRETS_GENERATED=$(echo "$SHARED_SECRET" | jq -r '.ALL_SECRETS_GENERATED') - if [[ "$ALL_SECRETS_GENERATED" == "true" ]]; then + # Validate content, not just the flag: stale eventually-consistent reads can return pre-generation values + if echo "$SHARED_SECRET" | jq -e '(.ALL_SECRETS_GENERATED == "true") and (.OPENVIDU_VERSION != "none") and (.REDIS_PASSWORD != "none")' > /dev/null; then break fi SECRETS_WAIT_ATTEMPTS=$((SECRETS_WAIT_ATTEMPTS + 1))