ambiser
/
openvidu
mirror of https://github.com/OpenVidu/openvidu.git
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add early publishing of derived URLs to GCP Secret Manager 

- Introduced early publishing of derived URLs (OPENVIDU_URL, LIVEKIT_URL, DASHBOARD_URL, GRAFANA_URL, MINIO_URL) to prevent race conditions during phase-2.
- Added a wait loop to ensure all secrets are generated before proceeding with fetching shared secrets.
pull/876/merge
Piwccle 2026-07-08 17:32:27 +02:00
parent 2bea0979a7
commit d4186f227c
4 changed files with 176 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml
View File

@ -1622,6 +1622,13 @@ Resources:
# If the private IP is the same as the first master node, generate the secrets
if [[ $MASTER_NODE_NUM -eq 1 ]] && [[ "$ALL_SECRETS_GENERATED" == "false" ]]; then
DOMAIN="$(/usr/local/bin/store_secret.sh save DOMAIN_NAME "${DomainName}")"
# Publish access URLs early so other nodes can read them during phase 2
# (idempotent: after_install rewrites these same byte-identical values later)
/usr/local/bin/store_secret.sh save OPENVIDU_URL "https://${!DOMAIN}/"
/usr/local/bin/store_secret.sh save LIVEKIT_URL "wss://${!DOMAIN}/"
/usr/local/bin/store_secret.sh save DASHBOARD_URL "https://${!DOMAIN}/dashboard/"
/usr/local/bin/store_secret.sh save GRAFANA_URL "https://${!DOMAIN}/grafana/"
/usr/local/bin/store_secret.sh save MINIO_URL "https://${!DOMAIN}/minio-console/"
OPENVIDU_PRO_LICENSE="$(/usr/local/bin/store_secret.sh save OPENVIDU_PRO_LICENSE "${OpenViduLicense}")"
OPENVIDU_RTC_ENGINE="$(/usr/local/bin/store_secret.sh save OPENVIDU_RTC_ENGINE "${RTCEngine}")"
# Store version so media nodes can use it to install the same version
@ -1657,17 +1664,28 @@ Resources:
ALL_SECRETS_GENERATED="$(/usr/local/bin/store_secret.sh save ALL_SECRETS_GENERATED "true")"
fi
# Fetch the shared secret again
# Fetch the shared secret again, waiting until master-1 has generated all secrets.
# Bounded (up to 900s at 5s): a genuinely stuck master-1 still surfaces after the deadline.
SECRETS_WAIT_ATTEMPTS=0
SECRETS_WAIT_MAX=180
while true; do
SHARED_SECRET=$(aws secretsmanager get-secret-value \
--region ${AWS::Region} \
--secret-id openvidu-ha-${AWS::Region}-${AWS::StackName} \
--query SecretString --output text)
--query SecretString --output text || echo 'none')
ALL_SECRETS_GENERATED=$(echo "$SHARED_SECRET" | jq -r '.ALL_SECRETS_GENERATED')
if [[ "${!ALL_SECRETS_GENERATED}" == "false" ]]; then
if [[ "${!ALL_SECRETS_GENERATED}" == "true" ]]; then
break
fi
SECRETS_WAIT_ATTEMPTS=$((SECRETS_WAIT_ATTEMPTS + 1))
if [[ $SECRETS_WAIT_ATTEMPTS -ge $SECRETS_WAIT_MAX ]]; then
echo "Error: Secrets not generated"
exit 1
fi
sleep 5
done
# sending the signal call
cfn-signal -e $? --stack ${AWS::StackId} --resource "$SIGNAL_NAME" --region ${AWS::Region}
@ -2124,6 +2142,27 @@ Resources:
# Start OpenVidu
systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; }
# Local readiness gate: wait up to 300s for Caddy health, restart once if it does not converge
OPENVIDU_READY=false
for i in $(seq 1 60); do
if curl -fsS http://127.0.0.1:7880/health/caddy >/dev/null 2>&1; then
OPENVIDU_READY=true
break
fi
sleep 5
done
if [ "$OPENVIDU_READY" != "true" ]; then
echo "[OpenVidu] not healthy after 300s, restarting once"
systemctl restart openvidu || true
for i in $(seq 1 60); do
if curl -fsS http://127.0.0.1:7880/health/caddy >/dev/null 2>&1; then
OPENVIDU_READY=true
break
fi
sleep 5
done
fi
# Update shared secret
/usr/local/bin/after_install.sh || { echo "[OpenVidu] error updating shared secret"; exit 1; }
@ -2180,6 +2219,27 @@ Resources:
# Start OpenVidu
systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; }
# Local readiness gate: wait up to 300s for Caddy health, restart once if it does not converge
OPENVIDU_READY=false
for i in $(seq 1 60); do
if curl -fsS http://127.0.0.1:7880/health/caddy >/dev/null 2>&1; then
OPENVIDU_READY=true
break
fi
sleep 5
done
if [ "$OPENVIDU_READY" != "true" ]; then
echo "[OpenVidu] not healthy after 300s, restarting once"
systemctl restart openvidu || true
for i in $(seq 1 60); do
if curl -fsS http://127.0.0.1:7880/health/caddy >/dev/null 2>&1; then
OPENVIDU_READY=true
break
fi
sleep 5
done
fi
# Update shared secret
/usr/local/bin/after_install.sh || { echo "[OpenVidu] error updating shared secret"; exit 1; }
@ -2236,6 +2296,27 @@ Resources:
# Start OpenVidu
systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; }
# Local readiness gate: wait up to 300s for Caddy health, restart once if it does not converge
OPENVIDU_READY=false
for i in $(seq 1 60); do
if curl -fsS http://127.0.0.1:7880/health/caddy >/dev/null 2>&1; then
OPENVIDU_READY=true
break
fi
sleep 5
done
if [ "$OPENVIDU_READY" != "true" ]; then
echo "[OpenVidu] not healthy after 300s, restarting once"
systemctl restart openvidu || true
for i in $(seq 1 60); do
if curl -fsS http://127.0.0.1:7880/health/caddy >/dev/null 2>&1; then
OPENVIDU_READY=true
break
fi
sleep 5
done
fi
# Update shared secret
/usr/local/bin/after_install.sh || { echo "[OpenVidu] error updating shared secret"; exit 1; }
@ -2292,6 +2373,27 @@ Resources:
# Start OpenVidu
systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; }
# Local readiness gate: wait up to 300s for Caddy health, restart once if it does not converge
OPENVIDU_READY=false
for i in $(seq 1 60); do
if curl -fsS http://127.0.0.1:7880/health/caddy >/dev/null 2>&1; then
OPENVIDU_READY=true
break
fi
sleep 5
done
if [ "$OPENVIDU_READY" != "true" ]; then
echo "[OpenVidu] not healthy after 300s, restarting once"
systemctl restart openvidu || true
for i in $(seq 1 60); do
if curl -fsS http://127.0.0.1:7880/health/caddy >/dev/null 2>&1; then
OPENVIDU_READY=true
break
fi
sleep 5
done
fi
# Update shared secret
/usr/local/bin/after_install.sh || { echo "[OpenVidu] error updating shared secret"; exit 1; }

51
openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep
View File

@ -356,6 +356,13 @@ if [[ $MASTER_NODE_NUM -eq 1 ]] && [[ "$ALL_SECRETS_GENERATED" == "" || "$ALL_SE
fi
DOMAIN="$(/usr/local/bin/store_secret.sh save DOMAIN-NAME "${DOMAIN}")"
# Publish service URLs early (idempotent; after_install re-writes byte-identical values)
OPENVIDU_URL="$(/usr/local/bin/store_secret.sh save OPENVIDU-URL "https://${DOMAIN}/")"
LIVEKIT_URL="$(/usr/local/bin/store_secret.sh save LIVEKIT-URL "wss://${DOMAIN}/")"
DASHBOARD_URL="$(/usr/local/bin/store_secret.sh save DASHBOARD-URL "https://${DOMAIN}/dashboard/")"
GRAFANA_URL="$(/usr/local/bin/store_secret.sh save GRAFANA-URL "https://${DOMAIN}/grafana/")"
MINIO_URL="$(/usr/local/bin/store_secret.sh save MINIO-URL "https://${DOMAIN}/minio-console/")"
# Meet initial admin user and password
MEET_INITIAL_ADMIN_USER="$(/usr/local/bin/store_secret.sh save MEET-INITIAL-ADMIN-USER "admin")"
if [[ "${initialMeetAdminPassword}" != '' ]]; then
@ -405,6 +412,9 @@ while true; do
sleep 5
done
# Wait until master-node-1 has generated all shared secrets before fetching them
while [[ "$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)" != "true" ]]; do sleep 5; done
# Fetch the values in the keyvault
MASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv)
@ -763,11 +773,18 @@ az network public-ip show \
var check_app_readyScriptMaster = '''
#!/bin/bash
set -e
MAX_WAIT=1200
WAIT_INTERVAL=5
ELAPSED_TIME=0
while true; do
HTTP_STATUS=$(curl -Ik http://localhost:7880/health/caddy | head -n1 | awk '{print $2}')
if [ $HTTP_STATUS == 200 ]; then
if [ "$HTTP_STATUS" = "200" ]; then
break
fi
ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL))
if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then
exit 1
fi
sleep 5
done
'''
@ -793,11 +810,35 @@ set -e
INSTALL_DIR="/opt/openvidu"
CLUSTER_CONFIG_DIR="${INSTALL_DIR}/config/cluster"
az login --identity
# Retry login + storage key fetch to allow the Contributor role assignment to propagate
MAX_WAIT=100
WAIT_INTERVAL=1
ELAPSED_TIME=0
set +e
while true; do
az login --identity
# Config azure blob storage
AZURE_ACCOUNT_NAME="${storageAccountName}"
AZURE_ACCOUNT_KEY=$(az storage account keys list --account-name ${storageAccountName} --query '[0].value' -o tsv)
# Config azure blob storage
AZURE_ACCOUNT_NAME="${storageAccountName}"
AZURE_ACCOUNT_KEY=$(az storage account keys list --account-name ${storageAccountName} --query '[0].value' -o tsv)
# If the key was fetched successfully, exit the loop
if [ $? -eq 0 ]; then
break
fi
# If not, wait and check again incrementing the time
ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL))
# If exceeded the maximum time, exit with error
if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then
exit 1
fi
# Esperar antes de la próxima comprobación
sleep $WAIT_INTERVAL
done
set -e
AZURE_CONTAINER_NAME="${storageAccountContainerName}"
sed -i "s|AZURE_ACCOUNT_NAME=.*|AZURE_ACCOUNT_NAME=$AZURE_ACCOUNT_NAME|" "${CLUSTER_CONFIG_DIR}/openvidu.env"

6
openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json
View File

File diff suppressed because one or more lines are too long

15
openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf
View File

@ -989,6 +989,18 @@ if [[ $MASTER_NODE_NUM -eq 1 ]] && [[ "$ALL_SECRETS_GENERATED" == "false" ]]; th
fi
DOMAIN="$(/usr/local/bin/store_secret.sh save DOMAIN_NAME "$DOMAIN")"
# Publish derived URLs early so phase-2 consumers don't race on them (idempotent; after_install re-writes identical values later)
OPENVIDU_URL="https://$${DOMAIN}/"
LIVEKIT_URL="wss://$${DOMAIN}/"
DASHBOARD_URL="https://$${DOMAIN}/dashboard/"
GRAFANA_URL="https://$${DOMAIN}/grafana/"
MINIO_URL="https://$${DOMAIN}/minio-console/"
echo -n "$OPENVIDU_URL" | gcloud secrets versions add OPENVIDU_URL --data-file=-
echo -n "$LIVEKIT_URL" | gcloud secrets versions add LIVEKIT_URL --data-file=-
echo -n "$DASHBOARD_URL" | gcloud secrets versions add DASHBOARD_URL --data-file=-
echo -n "$GRAFANA_URL" | gcloud secrets versions add GRAFANA_URL --data-file=-
echo -n "$MINIO_URL" | gcloud secrets versions add MINIO_URL --data-file=-
# Meet initial admin user and password
MEET_INITIAL_ADMIN_USER="$(/usr/local/bin/store_secret.sh save MEET_INITIAL_ADMIN_USER "admin")"
if [[ "${var.initialMeetAdminPassword}" != '' ]]; then
@ -1038,6 +1050,9 @@ while true; do
sleep 5
done
# Wait for master-1 to finish generating all shared secrets before reading them
while ! gcloud secrets versions access latest --secret=ALL_SECRETS_GENERATED 2>/dev/null | grep -q "true"; do echo "Waiting for master-1 to finish generating secrets..."; sleep 5; done
# Fetch all values from Secret Manager
MASTER_NODE_1_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_1_PRIVATE_IP)
MASTER_NODE_2_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_2_PRIVATE_IP)