mirror of https://github.com/OpenVidu/openvidu.git
Merge branch 'elastic-hardening': elastic deployment hardening (AWS/Azure/GCP)
Also brings the Azure HA user-assigned identity optimization and the GCP elastic MinIO port closure. DOC-CHANGES-*.md helper files are intentionally left out of master.master
Piwccle
commit
3ff7e14baf
|
|
@ -1427,7 +1427,8 @@ Resources:
|
|||
Type: 'AWS::CloudFormation::WaitCondition'
|
||||
CreationPolicy:
|
||||
ResourceSignal:
|
||||
Timeout: PT10M
|
||||
# Elastic signals AFTER the full install completes, so it needs a wider window than HA
|
||||
Timeout: PT20M
|
||||
Count: '1'
|
||||
|
||||
OpenViduMasterNode:
|
||||
|
|
@ -1473,35 +1474,28 @@ Resources:
|
|||
else
|
||||
DOMAIN=${DomainName}
|
||||
fi
|
||||
OPENVIDU_PRO_LICENSE="$(/usr/local/bin/store_secret.sh save OPENVIDU_PRO_LICENSE "${OpenViduLicense}")"
|
||||
OPENVIDU_RTC_ENGINE="$(/usr/local/bin/store_secret.sh save OPENVIDU_RTC_ENGINE "${RTCEngine}")"
|
||||
# Store version so media nodes can use it to install the same version
|
||||
/usr/local/bin/store_secret.sh save OPENVIDU_VERSION "${!OPENVIDU_VERSION}"
|
||||
|
||||
# Get own private IP
|
||||
PRIVATE_IP=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/local-ipv4)
|
||||
|
||||
# Unfortunately, EC2 instance assigned role is not immediately available after the instance is launched.
|
||||
# Therefore, we need to retry this operation until the aws-cli command is successful.
|
||||
MAX_RETRIES=10
|
||||
RETRY_COUNT=0
|
||||
while : ; do
|
||||
# Get current shared secret and random seed
|
||||
# Instance role already proven by cfn-init above, so a single read needs no IAM-availability retry
|
||||
SHARED_SECRET=$(aws secretsmanager get-secret-value \
|
||||
--region ${AWS::Region} \
|
||||
--secret-id openvidu-elastic-${AWS::Region}-${AWS::StackName} \
|
||||
--query SecretString --output text || echo 'none')
|
||||
|
||||
if [[ "$SHARED_SECRET" != "none" ]]; then
|
||||
break
|
||||
fi
|
||||
|
||||
RETRY_COUNT=$((RETRY_COUNT+1))
|
||||
if [[ $RETRY_COUNT -ge $MAX_RETRIES ]]; then
|
||||
if [[ "$SHARED_SECRET" == "none" ]]; then
|
||||
echo "Error: Shared secret not found"
|
||||
exit 1
|
||||
fi
|
||||
sleep 6
|
||||
done
|
||||
ALL_SECRETS_GENERATED=$(echo "$SHARED_SECRET" | jq -r '.ALL_SECRETS_GENERATED')
|
||||
|
||||
# Generate secrets only once: a re-run must never regenerate passwords already in use by the cluster
|
||||
if [[ "$ALL_SECRETS_GENERATED" == "false" ]]; then
|
||||
# Persist the domain so after_install.sh reads it from the secret instead of the installer
|
||||
/usr/local/bin/store_secret.sh save DOMAIN_NAME "$DOMAIN"
|
||||
OPENVIDU_PRO_LICENSE="$(/usr/local/bin/store_secret.sh save OPENVIDU_PRO_LICENSE "${OpenViduLicense}")"
|
||||
OPENVIDU_RTC_ENGINE="$(/usr/local/bin/store_secret.sh save OPENVIDU_RTC_ENGINE "${RTCEngine}")"
|
||||
# Store version so media nodes can use it to install the same version
|
||||
/usr/local/bin/store_secret.sh save OPENVIDU_VERSION "${!OPENVIDU_VERSION}"
|
||||
|
||||
# Meet initial admin user and password
|
||||
MEET_INITIAL_ADMIN_USER="$(/usr/local/bin/store_secret.sh save MEET_INITIAL_ADMIN_USER "admin")"
|
||||
|
|
@ -1517,8 +1511,6 @@ Resources:
|
|||
fi
|
||||
|
||||
# Store usernames and generate random passwords
|
||||
OPENVIDU_PRO_LICENSE="$(/usr/local/bin/store_secret.sh save OPENVIDU_PRO_LICENSE "${OpenViduLicense}")"
|
||||
OPENVIDU_RTC_ENGINE="$(/usr/local/bin/store_secret.sh save OPENVIDU_RTC_ENGINE "${RTCEngine}")"
|
||||
REDIS_PASSWORD="$(/usr/local/bin/store_secret.sh generate REDIS_PASSWORD)"
|
||||
MONGO_ADMIN_USERNAME="$(/usr/local/bin/store_secret.sh save MONGO_ADMIN_USERNAME "mongoadmin")"
|
||||
MONGO_ADMIN_PASSWORD="$(/usr/local/bin/store_secret.sh generate MONGO_ADMIN_PASSWORD)"
|
||||
|
|
@ -1533,9 +1525,59 @@ Resources:
|
|||
LIVEKIT_API_SECRET="$(/usr/local/bin/store_secret.sh generate LIVEKIT_API_SECRET)"
|
||||
ENABLED_MODULES="$(/usr/local/bin/store_secret.sh save ENABLED_MODULES "observability,v2compatibility,openviduMeet")"
|
||||
ALL_SECRETS_GENERATED="$(/usr/local/bin/store_secret.sh save ALL_SECRETS_GENERATED "true")"
|
||||
fi
|
||||
|
||||
# Base command
|
||||
INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/elastic/$OPENVIDU_VERSION/install_ov_master_node.sh)"
|
||||
# Source every installer value from the secret so a re-run (guard skipped) still has them
|
||||
# GetSecretValue is eventually consistent: retry (up to 300s at 5s) until the read returns the generated values
|
||||
SECRET_READ_ATTEMPTS=0
|
||||
while true; do
|
||||
SHARED_SECRET=$(aws secretsmanager get-secret-value \
|
||||
--region ${AWS::Region} \
|
||||
--secret-id openvidu-elastic-${AWS::Region}-${AWS::StackName} \
|
||||
--query SecretString --output text)
|
||||
if echo "$SHARED_SECRET" | jq -e '(.ALL_SECRETS_GENERATED == "true") and ([.DOMAIN_NAME, .OPENVIDU_VERSION, .OPENVIDU_PRO_LICENSE, .REDIS_PASSWORD, .MONGO_ADMIN_PASSWORD, .MONGO_REPLICA_SET_KEY, .MINIO_SECRET_KEY, .DASHBOARD_ADMIN_PASSWORD, .GRAFANA_ADMIN_PASSWORD, .LIVEKIT_API_KEY, .LIVEKIT_API_SECRET] | all(. != "none"))' > /dev/null; then
|
||||
break
|
||||
fi
|
||||
SECRET_READ_ATTEMPTS=$((SECRET_READ_ATTEMPTS + 1))
|
||||
if [[ $SECRET_READ_ATTEMPTS -ge 60 ]]; then
|
||||
echo "Error: shared secret still incomplete after 5 minutes of stale reads"
|
||||
exit 1
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
|
||||
DOMAIN=$(echo "$SHARED_SECRET" | jq -r '.DOMAIN_NAME')
|
||||
OPENVIDU_VERSION=$(echo "$SHARED_SECRET" | jq -r '.OPENVIDU_VERSION')
|
||||
OPENVIDU_PRO_LICENSE=$(echo "$SHARED_SECRET" | jq -r '.OPENVIDU_PRO_LICENSE')
|
||||
OPENVIDU_RTC_ENGINE=$(echo "$SHARED_SECRET" | jq -r '.OPENVIDU_RTC_ENGINE')
|
||||
MEET_INITIAL_ADMIN_USER=$(echo "$SHARED_SECRET" | jq -r '.MEET_INITIAL_ADMIN_USER')
|
||||
MEET_INITIAL_ADMIN_PASSWORD=$(echo "$SHARED_SECRET" | jq -r '.MEET_INITIAL_ADMIN_PASSWORD')
|
||||
MEET_INITIAL_API_KEY=""
|
||||
if [[ "${InitialMeetApiKey}" != '' ]]; then
|
||||
MEET_INITIAL_API_KEY=$(echo "$SHARED_SECRET" | jq -r '.MEET_INITIAL_API_KEY')
|
||||
fi
|
||||
REDIS_PASSWORD=$(echo "$SHARED_SECRET" | jq -r '.REDIS_PASSWORD')
|
||||
MONGO_ADMIN_USERNAME=$(echo "$SHARED_SECRET" | jq -r '.MONGO_ADMIN_USERNAME')
|
||||
MONGO_ADMIN_PASSWORD=$(echo "$SHARED_SECRET" | jq -r '.MONGO_ADMIN_PASSWORD')
|
||||
MONGO_REPLICA_SET_KEY=$(echo "$SHARED_SECRET" | jq -r '.MONGO_REPLICA_SET_KEY')
|
||||
MINIO_ACCESS_KEY=$(echo "$SHARED_SECRET" | jq -r '.MINIO_ACCESS_KEY')
|
||||
MINIO_SECRET_KEY=$(echo "$SHARED_SECRET" | jq -r '.MINIO_SECRET_KEY')
|
||||
DASHBOARD_ADMIN_USERNAME=$(echo "$SHARED_SECRET" | jq -r '.DASHBOARD_ADMIN_USERNAME')
|
||||
DASHBOARD_ADMIN_PASSWORD=$(echo "$SHARED_SECRET" | jq -r '.DASHBOARD_ADMIN_PASSWORD')
|
||||
GRAFANA_ADMIN_USERNAME=$(echo "$SHARED_SECRET" | jq -r '.GRAFANA_ADMIN_USERNAME')
|
||||
GRAFANA_ADMIN_PASSWORD=$(echo "$SHARED_SECRET" | jq -r '.GRAFANA_ADMIN_PASSWORD')
|
||||
LIVEKIT_API_KEY=$(echo "$SHARED_SECRET" | jq -r '.LIVEKIT_API_KEY')
|
||||
LIVEKIT_API_SECRET=$(echo "$SHARED_SECRET" | jq -r '.LIVEKIT_API_SECRET')
|
||||
ENABLED_MODULES=$(echo "$SHARED_SECRET" | jq -r '.ENABLED_MODULES')
|
||||
|
||||
# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure
|
||||
INSTALLER_SCRIPT="/tmp/install_ov_master_node.sh"
|
||||
curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/elastic/$OPENVIDU_VERSION/install_ov_master_node.sh"
|
||||
if [ ! -s "$INSTALLER_SCRIPT" ]; then
|
||||
echo "Downloaded OpenVidu master node installer is empty or missing" >&2
|
||||
exit 1
|
||||
fi
|
||||
INSTALL_COMMAND="sh $INSTALLER_SCRIPT"
|
||||
|
||||
# Common arguments
|
||||
COMMON_ARGS=(
|
||||
|
|
@ -1858,20 +1900,6 @@ Resources:
|
|||
mode: "000755"
|
||||
owner: "root"
|
||||
group: "root"
|
||||
'/usr/local/bin/check_app_ready.sh':
|
||||
content: |
|
||||
#!/bin/bash
|
||||
set -e
|
||||
while true; do
|
||||
HTTP_STATUS=$(curl -Ik http://localhost:7880/health/caddy | head -n1 | awk '{print $2}')
|
||||
if [ $HTTP_STATUS == 200 ]; then
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
mode: "000755"
|
||||
owner: "root"
|
||||
group: "root"
|
||||
'/usr/local/bin/restart.sh':
|
||||
content: |
|
||||
#!/bin/bash
|
||||
|
|
@ -1950,9 +1978,30 @@ Resources:
|
|||
# Launch on reboot
|
||||
echo "@reboot /usr/local/bin/restart.sh &> /var/log/openvidu-restart.log" | crontab
|
||||
|
||||
# Wait for the app
|
||||
sleep 20
|
||||
/usr/local/bin/check_app_ready.sh
|
||||
# Local readiness gate: wait up to 300s for Caddy health, restart once if it does not converge
|
||||
OPENVIDU_READY=false
|
||||
for i in $(seq 1 60); do
|
||||
if curl -fsS http://127.0.0.1:7880/health/caddy >/dev/null 2>&1; then
|
||||
OPENVIDU_READY=true
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
if [ "$OPENVIDU_READY" != "true" ]; then
|
||||
echo "[OpenVidu] not healthy after 300s, restarting once"
|
||||
systemctl restart openvidu || true
|
||||
for i in $(seq 1 60); do
|
||||
if curl -fsS http://127.0.0.1:7880/health/caddy >/dev/null 2>&1; then
|
||||
OPENVIDU_READY=true
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
fi
|
||||
|
||||
# Disable errexit so cfn-signal always runs and reports readiness via $? (fast-fail instead of a PT20M timeout)
|
||||
set +e
|
||||
[ "$OPENVIDU_READY" = "true" ]
|
||||
|
||||
# sending the finish call
|
||||
cfn-signal -e $? --stack ${AWS::StackId} --resource WaitCondition --region ${AWS::Region}
|
||||
|
|
@ -2022,31 +2071,48 @@ Resources:
|
|||
sleep 6
|
||||
done
|
||||
|
||||
# Get current shared secret
|
||||
DOMAIN=$(echo $SHARED_SECRET | jq -r .DOMAIN_NAME)
|
||||
OPENVIDU_PRO_LICENSE=$(echo $SHARED_SECRET | jq -r .OPENVIDU_PRO_LICENSE)
|
||||
REDIS_PASSWORD=$(echo $SHARED_SECRET | jq -r .REDIS_PASSWORD)
|
||||
|
||||
# Get OpenVidu Media Nodes version to deploy
|
||||
OPENVIDU_VERSION=$(echo "$SHARED_SECRET" | jq -r '.OPENVIDU_VERSION')
|
||||
|
||||
if [[ "$OPENVIDU_VERSION" == "none" ]]; then
|
||||
echo "OpenVidu version not found"
|
||||
# Wait until the master node has generated all shared secrets.
|
||||
# Bounded (up to 1800s at 5s), validating content on each snapshot: stale eventually-consistent reads can return pre-generation values
|
||||
SECRETS_WAIT_ATTEMPTS=0
|
||||
SECRETS_WAIT_MAX=360
|
||||
while true; do
|
||||
if echo "$SHARED_SECRET" | jq -e '(.ALL_SECRETS_GENERATED == "true") and (.OPENVIDU_VERSION != "none") and (.REDIS_PASSWORD != "none")' > /dev/null; then
|
||||
break
|
||||
fi
|
||||
SECRETS_WAIT_ATTEMPTS=$((SECRETS_WAIT_ATTEMPTS + 1))
|
||||
if [[ $SECRETS_WAIT_ATTEMPTS -ge $SECRETS_WAIT_MAX ]]; then
|
||||
echo "Error: timed out after 30 minutes waiting for the master node to generate the shared secrets"
|
||||
/usr/local/bin/set_as_unhealthy.sh
|
||||
exit 1
|
||||
fi
|
||||
sleep 5
|
||||
SHARED_SECRET=$(aws secretsmanager get-secret-value \
|
||||
--region ${AWS::Region} \
|
||||
--secret-id openvidu-elastic-${AWS::Region}-${AWS::StackName} \
|
||||
--query SecretString --output text || echo 'none')
|
||||
done
|
||||
|
||||
ALL_SECRETS_GENERATED=$(echo $SHARED_SECRET | jq -r .ALL_SECRETS_GENERATED)
|
||||
if [[ "$ALL_SECRETS_GENERATED" == "false" ]]; then
|
||||
echo "Master node not ready"
|
||||
/usr/local/bin/set_as_unhealthy.sh
|
||||
# Read values only after the gate confirms the master node has published them
|
||||
DOMAIN=$(echo "$SHARED_SECRET" | jq -r '.DOMAIN_NAME')
|
||||
OPENVIDU_PRO_LICENSE=$(echo "$SHARED_SECRET" | jq -r '.OPENVIDU_PRO_LICENSE')
|
||||
REDIS_PASSWORD=$(echo "$SHARED_SECRET" | jq -r '.REDIS_PASSWORD')
|
||||
OPENVIDU_VERSION=$(echo "$SHARED_SECRET" | jq -r '.OPENVIDU_VERSION')
|
||||
if [[ "$OPENVIDU_VERSION" == "none" ]]; then
|
||||
echo "OpenVidu version not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get Master Node private IP
|
||||
MASTER_NODE_IP=${OpenViduMasterNode.PrivateIp}
|
||||
|
||||
# Base command
|
||||
INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/elastic/$OPENVIDU_VERSION/install_ov_media_node.sh)"
|
||||
# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure
|
||||
INSTALLER_SCRIPT="/tmp/install_ov_media_node.sh"
|
||||
curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/elastic/$OPENVIDU_VERSION/install_ov_media_node.sh"
|
||||
if [ ! -s "$INSTALLER_SCRIPT" ]; then
|
||||
echo "Downloaded OpenVidu media node installer is empty or missing" >&2
|
||||
exit 1
|
||||
fi
|
||||
INSTALL_COMMAND="sh $INSTALLER_SCRIPT"
|
||||
|
||||
# Common arguments
|
||||
COMMON_ARGS=(
|
||||
|
|
@ -2190,6 +2256,22 @@ Resources:
|
|||
# Install OpenVidu
|
||||
/usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu"; /usr/local/bin/set_as_unhealthy.sh; exit 1; }
|
||||
|
||||
# Wait for the master node to be healthy before starting (bounded: up to 1800s at 5s)
|
||||
MASTER_NODE_IP=${OpenViduMasterNode.PrivateIp}
|
||||
MASTER_HEALTHY=false
|
||||
for i in $(seq 1 360); do
|
||||
if curl -sf "http://$MASTER_NODE_IP:7880/health/caddy" >/dev/null 2>&1; then
|
||||
MASTER_HEALTHY=true
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
if [ "$MASTER_HEALTHY" != "true" ]; then
|
||||
echo "[OpenVidu] master node did not become healthy after 30 minutes"
|
||||
/usr/local/bin/set_as_unhealthy.sh
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Start OpenVidu
|
||||
systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; /usr/local/bin/set_as_unhealthy.sh; exit 1; }
|
||||
|
||||
|
|
@ -2202,6 +2284,8 @@ Resources:
|
|||
|
||||
OpenViduMediaNodeASG:
|
||||
DependsOn:
|
||||
- OpenViduMediaNodeInstanceProfile
|
||||
- OpenViduMasterNodeInstanceProfile
|
||||
- StopMediaNodeCloudWatchEventRule
|
||||
Type: AWS::AutoScaling::AutoScalingGroup
|
||||
Properties:
|
||||
|
|
|
|||
|
|
@ -212,7 +212,8 @@ var stringInterpolationParamsMaster = {
|
|||
}
|
||||
|
||||
var installScriptTemplateMaster = '''
|
||||
#!/bin/bash -x
|
||||
#!/bin/bash
|
||||
set -e
|
||||
OPENVIDU_VERSION=main
|
||||
DOMAIN=
|
||||
|
||||
|
|
@ -241,7 +242,7 @@ else
|
|||
fi
|
||||
|
||||
# Wait for the keyvault availability
|
||||
MAX_WAIT=100
|
||||
MAX_WAIT=300
|
||||
WAIT_INTERVAL=1
|
||||
ELAPSED_TIME=0
|
||||
while true; do
|
||||
|
|
@ -306,8 +307,16 @@ OPENVIDU_VERSION="$(/usr/local/bin/store_secret.sh save OPENVIDU-VERSION "${OPEN
|
|||
ENABLED_MODULES="$(/usr/local/bin/store_secret.sh save ENABLED-MODULES "observability,openviduMeet,v2compatibility")"
|
||||
ALL_SECRETS_GENERATED="$(/usr/local/bin/store_secret.sh save ALL-SECRETS-GENERATED "true")"
|
||||
|
||||
# Download to a file first: process substitution would silently run an empty script on a transient curl failure
|
||||
INSTALLER_SCRIPT="/tmp/install_ov_master_node.sh"
|
||||
curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/elastic/$OPENVIDU_VERSION/install_ov_master_node.sh"
|
||||
if [ ! -s "$INSTALLER_SCRIPT" ]; then
|
||||
echo "[OpenVidu] failed to download the master node installer script"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Base command
|
||||
INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/elastic/$OPENVIDU_VERSION/install_ov_master_node.sh)"
|
||||
INSTALL_COMMAND="sh $INSTALLER_SCRIPT"
|
||||
|
||||
# Common arguments
|
||||
COMMON_ARGS=(
|
||||
|
|
@ -630,11 +639,18 @@ az network public-ip show \
|
|||
var check_app_readyScriptMaster = '''
|
||||
#!/bin/bash
|
||||
set -e
|
||||
MAX_WAIT=1200
|
||||
WAIT_INTERVAL=5
|
||||
ELAPSED_TIME=0
|
||||
while true; do
|
||||
HTTP_STATUS=$(curl -Ik http://localhost:7880/health/caddy | head -n1 | awk '{print $2}')
|
||||
if [ $HTTP_STATUS == 200 ]; then
|
||||
if [ "$HTTP_STATUS" = "200" ]; then
|
||||
break
|
||||
fi
|
||||
ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL))
|
||||
if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then
|
||||
exit 1
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
'''
|
||||
|
|
@ -660,11 +676,35 @@ set -e
|
|||
INSTALL_DIR="/opt/openvidu"
|
||||
CLUSTER_CONFIG_DIR="${INSTALL_DIR}/config/cluster"
|
||||
|
||||
# Retry login + storage key fetch to allow the Contributor role assignment to propagate
|
||||
MAX_WAIT=300
|
||||
WAIT_INTERVAL=1
|
||||
ELAPSED_TIME=0
|
||||
set +e
|
||||
while true; do
|
||||
az login --identity
|
||||
|
||||
# Config azure blob storage
|
||||
AZURE_ACCOUNT_NAME="${storageAccountName}"
|
||||
AZURE_ACCOUNT_KEY=$(az storage account keys list --account-name ${storageAccountName} --query '[0].value' -o tsv)
|
||||
|
||||
# If the key was fetched successfully, exit the loop
|
||||
if [ $? -eq 0 ]; then
|
||||
break
|
||||
fi
|
||||
|
||||
# If not, wait and check again incrementing the time
|
||||
ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL))
|
||||
|
||||
# If exceeded the maximum time, exit with error
|
||||
if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Wait before the next check
|
||||
sleep $WAIT_INTERVAL
|
||||
done
|
||||
set -e
|
||||
AZURE_CONTAINER_NAME="${storageAccountContainerName}"
|
||||
|
||||
sed -i "s|AZURE_ACCOUNT_NAME=.*|AZURE_ACCOUNT_NAME=$AZURE_ACCOUNT_NAME|" "${CLUSTER_CONFIG_DIR}/openvidu.env"
|
||||
|
|
@ -747,7 +787,7 @@ var userDataParamsMasterNode = {
|
|||
}
|
||||
|
||||
var userDataTemplateMasterNode = '''
|
||||
#!/bin/bash -x
|
||||
#!/bin/bash
|
||||
set -eu -o pipefail
|
||||
|
||||
# Introduce the scripts in the instance
|
||||
|
|
@ -802,8 +842,6 @@ az login --identity --allow-no-subscriptions
|
|||
|
||||
echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout
|
||||
|
||||
apt-get update && apt-get install -y
|
||||
|
||||
export HOME="/root"
|
||||
|
||||
# Install OpenVidu
|
||||
|
|
@ -821,15 +859,14 @@ systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1;
|
|||
# Launch on reboot
|
||||
echo "@reboot /usr/local/bin/restart.sh >> /var/log/openvidu-restart.log" 2>&1 | crontab
|
||||
|
||||
set +e
|
||||
az storage blob upload --account-name ${storageAccountName} --container-name automation-locks --name lock.txt --file /dev/null --auth-mode key
|
||||
set -e
|
||||
# check_app_ready.sh internally caps its wait at 1200s
|
||||
/usr/local/bin/check_app_ready.sh || { echo "[OpenVidu] master node did not become healthy"; exit 1; }
|
||||
|
||||
az keyvault secret set --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --value "true"
|
||||
|
||||
# Wait for the app
|
||||
sleep 150
|
||||
/usr/local/bin/check_app_ready.sh
|
||||
set +e
|
||||
az storage blob upload --account-name ${storageAccountName} --container-name automation-locks --name lock.txt --file /dev/null --auth-mode key
|
||||
set -e
|
||||
'''
|
||||
|
||||
var userDataMasterNode = reduce(
|
||||
|
|
@ -880,7 +917,7 @@ var stringInterpolationParamsMedia = {
|
|||
}
|
||||
|
||||
var installScriptTemplateMedia = '''
|
||||
#!/bin/bash -x
|
||||
#!/bin/bash
|
||||
set -e
|
||||
DOMAIN=
|
||||
|
||||
|
|
@ -896,26 +933,21 @@ apt-get update && apt-get install -y \
|
|||
# Get own private IP
|
||||
PRIVATE_IP=$(curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text")
|
||||
|
||||
WAIT_INTERVAL=1
|
||||
MAX_WAIT=200
|
||||
ELAPSED_TIME=0
|
||||
# Gate 1: wait for master secrets before installing
|
||||
WAIT_INTERVAL=5
|
||||
MAX_RETRIES=360
|
||||
RETRIES=0
|
||||
set +e
|
||||
while true; do
|
||||
# get secret value
|
||||
FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv)
|
||||
|
||||
# Check if the secret has been generated
|
||||
if [ "$FINISH_MASTER_NODE" == "true" ]; then
|
||||
ALL_SECRETS_GENERATED=$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)
|
||||
if [ "$ALL_SECRETS_GENERATED" == "true" ]; then
|
||||
break
|
||||
fi
|
||||
|
||||
ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL))
|
||||
|
||||
# Check if the maximum waiting time has been reached
|
||||
if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then
|
||||
RETRIES=$((RETRIES + 1))
|
||||
if [ $RETRIES -ge $MAX_RETRIES ]; then
|
||||
echo "[OpenVidu] timed out after 30 min waiting for ALL-SECRETS-GENERATED"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sleep $WAIT_INTERVAL
|
||||
done
|
||||
set -e
|
||||
|
|
@ -929,8 +961,16 @@ OPENVIDU_VERSION="$(az keyvault secret show --vault-name ${keyVaultName} --name
|
|||
# Get Master Node private IP
|
||||
MASTER_NODE_IP=${privateIPMasterNode}
|
||||
|
||||
# Download to a file first: process substitution would silently run an empty script on a transient curl failure
|
||||
INSTALLER_SCRIPT="/tmp/install_ov_media_node.sh"
|
||||
curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/elastic/$OPENVIDU_VERSION/install_ov_media_node.sh"
|
||||
if [ ! -s "$INSTALLER_SCRIPT" ]; then
|
||||
echo "[OpenVidu] failed to download the media node installer script"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Base command
|
||||
INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/elastic/$OPENVIDU_VERSION/install_ov_media_node.sh)"
|
||||
INSTALL_COMMAND="sh $INSTALLER_SCRIPT"
|
||||
|
||||
# Common arguments
|
||||
COMMON_ARGS=(
|
||||
|
|
@ -1018,7 +1058,7 @@ az vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_
|
|||
'''
|
||||
|
||||
var userDataMediaNodeTemplate = '''
|
||||
#!/bin/bash -x
|
||||
#!/bin/bash
|
||||
set -eu -o pipefail
|
||||
|
||||
# Introduce the scripts in the instance
|
||||
|
|
@ -1036,8 +1076,7 @@ chmod +x /usr/local/bin/delete_media_node.sh
|
|||
|
||||
echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout
|
||||
|
||||
apt-get update && apt-get install -y
|
||||
apt-get install -y jq
|
||||
apt-get update && apt-get install -y jq
|
||||
|
||||
# Install azure cli
|
||||
AZURE_CLI_VERSION=2.87.0
|
||||
|
|
@ -1059,7 +1098,27 @@ az vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME -
|
|||
export HOME="/root"
|
||||
|
||||
# Install OpenVidu
|
||||
/usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu"; /usr/local/bin/delete_media_node.sh; }
|
||||
/usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu"; /usr/local/bin/delete_media_node.sh; exit 1; }
|
||||
|
||||
# Gate 2: wait for master readiness before starting
|
||||
WAIT_INTERVAL=5
|
||||
MAX_RETRIES=360
|
||||
RETRIES=0
|
||||
set +e
|
||||
while true; do
|
||||
FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv 2>/dev/null)
|
||||
if [ "$FINISH_MASTER_NODE" == "true" ]; then
|
||||
break
|
||||
fi
|
||||
RETRIES=$((RETRIES + 1))
|
||||
if [ $RETRIES -ge $MAX_RETRIES ]; then
|
||||
echo "[OpenVidu] timed out after 30 min waiting for FINISH-MASTER-NODE"
|
||||
/usr/local/bin/delete_media_node.sh
|
||||
exit 1
|
||||
fi
|
||||
sleep $WAIT_INTERVAL
|
||||
done
|
||||
set -e
|
||||
|
||||
# Start OpenVidu
|
||||
systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; /usr/local/bin/delete_media_node.sh; }
|
||||
|
|
@ -1095,6 +1154,7 @@ var userDataParamsMedia = {
|
|||
base64delete: base64delete_mediaNode_ScriptMedia
|
||||
resourceGroupName: resourceGroup().name
|
||||
vmScaleSetName: '${stackName}-mediaNodeScaleSet'
|
||||
keyVaultName: keyVaultName
|
||||
}
|
||||
|
||||
var userDataMediaNode = reduce(
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -27,6 +27,8 @@ resource "google_secret_manager_secret" "openvidu_shared_info" {
|
|||
replication {
|
||||
auto {}
|
||||
}
|
||||
|
||||
depends_on = [google_project_service.secretmanager_api]
|
||||
}
|
||||
|
||||
# GCS bucket
|
||||
|
|
@ -36,12 +38,16 @@ resource "google_storage_bucket" "bucket" {
|
|||
location = var.region
|
||||
force_destroy = true
|
||||
uniform_bucket_level_access = true
|
||||
|
||||
depends_on = [google_project_service.storage_api]
|
||||
}
|
||||
|
||||
# Service account for the instance
|
||||
resource "google_service_account" "service_account" {
|
||||
account_id = lower("${substr(var.stackName, 0, 12)}-sa")
|
||||
display_name = "OpenVidu instance service account"
|
||||
|
||||
depends_on = [google_project_service.iam_api]
|
||||
}
|
||||
|
||||
# IAM bindings for the service account so the instance can access Secret Manager and GCS
|
||||
|
|
@ -49,6 +55,8 @@ resource "google_project_iam_member" "iam_project_role" {
|
|||
project = var.projectId
|
||||
role = "roles/owner"
|
||||
member = "serviceAccount:${google_service_account.service_account.email}"
|
||||
|
||||
depends_on = [google_project_service.cloudresourcemanager_api]
|
||||
}
|
||||
|
||||
resource "google_compute_firewall" "firewall_master" {
|
||||
|
|
@ -57,11 +65,13 @@ resource "google_compute_firewall" "firewall_master" {
|
|||
|
||||
allow {
|
||||
protocol = "tcp"
|
||||
ports = ["22", "80", "443", "1935", "9000"]
|
||||
ports = ["22", "80", "443", "1935"]
|
||||
}
|
||||
|
||||
source_ranges = ["0.0.0.0/0"]
|
||||
target_tags = [lower("${var.stackName}-master-node")]
|
||||
|
||||
depends_on = [google_project_service.compute_api]
|
||||
}
|
||||
|
||||
resource "google_compute_firewall" "firewall_media" {
|
||||
|
|
@ -79,6 +89,8 @@ resource "google_compute_firewall" "firewall_media" {
|
|||
|
||||
source_ranges = ["0.0.0.0/0"]
|
||||
target_tags = [lower("${var.stackName}-media-node")]
|
||||
|
||||
depends_on = [google_project_service.compute_api]
|
||||
}
|
||||
|
||||
resource "google_compute_firewall" "firewall_media_to_master" {
|
||||
|
|
@ -96,6 +108,8 @@ resource "google_compute_firewall" "firewall_media_to_master" {
|
|||
target_tags = [
|
||||
lower("${var.stackName}-master-node"),
|
||||
]
|
||||
|
||||
depends_on = [google_project_service.compute_api]
|
||||
}
|
||||
|
||||
resource "google_compute_firewall" "firewall_master_to_media" {
|
||||
|
|
@ -113,6 +127,8 @@ resource "google_compute_firewall" "firewall_master_to_media" {
|
|||
target_tags = [
|
||||
lower("${var.stackName}-media-node")
|
||||
]
|
||||
|
||||
depends_on = [google_project_service.compute_api]
|
||||
}
|
||||
|
||||
# Create Public Ip address (if not provided)
|
||||
|
|
@ -120,6 +136,8 @@ resource "google_compute_address" "public_ip_address" {
|
|||
count = var.publicIpAddress == "" ? 1 : 0
|
||||
name = lower("${var.stackName}-public-ip")
|
||||
region = var.region
|
||||
|
||||
depends_on = [google_project_service.compute_api]
|
||||
}
|
||||
|
||||
locals {
|
||||
|
|
@ -181,6 +199,9 @@ resource "google_compute_instance" "openvidu_master_node" {
|
|||
stack = var.stackName
|
||||
node-type = "master"
|
||||
}
|
||||
|
||||
# Explicit: transitive coverage via public_ip_address is absent when publicIpAddress is provided
|
||||
depends_on = [google_project_service.compute_api]
|
||||
}
|
||||
|
||||
locals {
|
||||
|
|
@ -234,6 +255,24 @@ resource "google_compute_instance_template" "media_node_template" {
|
|||
}
|
||||
}
|
||||
|
||||
# Health check for media node auto-healing
|
||||
resource "google_compute_region_health_check" "media_node_health_check" {
|
||||
name = lower("${var.stackName}-media-node-health-check")
|
||||
region = var.region
|
||||
|
||||
# TCP-only, generous thresholds: auto-heal recreation kills live WebRTC sessions
|
||||
tcp_health_check {
|
||||
port = 7880
|
||||
}
|
||||
|
||||
check_interval_sec = 30
|
||||
timeout_sec = 10
|
||||
healthy_threshold = 2
|
||||
unhealthy_threshold = 5
|
||||
|
||||
depends_on = [google_project_service.compute_api]
|
||||
}
|
||||
|
||||
# Managed Instance Group for Media Nodes
|
||||
resource "google_compute_region_instance_group_manager" "media_node_group" {
|
||||
name = lower("${var.stackName}-media-node-group")
|
||||
|
|
@ -250,6 +289,12 @@ resource "google_compute_region_instance_group_manager" "media_node_group" {
|
|||
port = 7880
|
||||
}
|
||||
|
||||
# initial_delay_sec generous so media nodes finish installing before health checks can recreate them
|
||||
auto_healing_policies {
|
||||
health_check = google_compute_region_health_check.media_node_health_check.id
|
||||
initial_delay_sec = 600
|
||||
}
|
||||
|
||||
depends_on = [google_compute_instance.openvidu_master_node]
|
||||
}
|
||||
|
||||
|
|
@ -548,6 +593,8 @@ resource "google_storage_bucket_object" "function_source" {
|
|||
name = "function-source.zip"
|
||||
bucket = local.isEmpty ? google_storage_bucket.bucket[0].name : var.bucketName
|
||||
source = data.archive_file.function_source.output_path
|
||||
|
||||
depends_on = [google_project_service.storage_api]
|
||||
}
|
||||
|
||||
resource "google_cloudfunctions2_function" "scalein_function" {
|
||||
|
|
@ -577,6 +624,12 @@ resource "google_cloudfunctions2_function" "scalein_function" {
|
|||
}
|
||||
service_account_email = google_service_account.service_account.email
|
||||
}
|
||||
|
||||
depends_on = [
|
||||
google_project_service.cloudfunctions_api,
|
||||
google_project_service.cloudbuild_api,
|
||||
google_project_service.run_api
|
||||
]
|
||||
}
|
||||
|
||||
# Cloud Scheduler to trigger the function every 5 minutes
|
||||
|
|
@ -606,6 +659,8 @@ resource "google_cloud_scheduler_job" "scale_scheduler" {
|
|||
service_account_email = google_service_account.service_account.email
|
||||
}
|
||||
}
|
||||
|
||||
depends_on = [google_project_service.cloudscheduler_api]
|
||||
}
|
||||
|
||||
# ------------------------- local values -------------------------
|
||||
|
|
@ -640,8 +695,8 @@ gcloud auth activate-service-account --key-file=/dev/null 2>/dev/null || true
|
|||
METADATA_URL="http://metadata.google.internal/computeMetadata/v1"
|
||||
get_meta() { curl -s -H "Metadata-Flavor: Google" "$${METADATA_URL}/$1"; }
|
||||
|
||||
# Create counter file for tracking script executions
|
||||
echo 1 > /usr/local/bin/openvidu_install_counter.txt
|
||||
# Disable command tracing so secrets are not printed to the serial console
|
||||
set +x
|
||||
|
||||
# Configure domain
|
||||
if [[ "${var.domainName}" == "" ]]; then
|
||||
|
|
@ -690,8 +745,14 @@ OPENVIDU_VERSION="$(/usr/local/bin/store_secret.sh save OPENVIDU_VERSION "$OPENV
|
|||
|
||||
ALL_SECRETS_GENERATED="$(/usr/local/bin/store_secret.sh save ALL_SECRETS_GENERATED "true")"
|
||||
|
||||
# Build install command and args
|
||||
INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/elastic/$OPENVIDU_VERSION/install_ov_master_node.sh)"
|
||||
# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure
|
||||
INSTALLER_SCRIPT="/tmp/install_ov_master_node.sh"
|
||||
curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/elastic/$OPENVIDU_VERSION/install_ov_master_node.sh"
|
||||
if [ ! -s "$INSTALLER_SCRIPT" ]; then
|
||||
echo "Downloaded OpenVidu master node installer is empty or missing" >&2
|
||||
exit 1
|
||||
fi
|
||||
INSTALL_COMMAND="sh $INSTALLER_SCRIPT"
|
||||
|
||||
# Common arguments
|
||||
COMMON_ARGS=(
|
||||
|
|
@ -778,6 +839,9 @@ SERVICE_ACCOUNT_EMAIL=$(get_meta "instance/service-accounts/default/email")
|
|||
# Create key for service account
|
||||
gcloud iam service-accounts keys create credentials.json --iam-account=$SERVICE_ACCOUNT_EMAIL
|
||||
|
||||
# Disable command tracing so credentials are not printed to the serial console
|
||||
set +x
|
||||
|
||||
# Create HMAC key and parse output
|
||||
HMAC_OUTPUT=$(gcloud storage hmac create $SERVICE_ACCOUNT_EMAIL --format="json")
|
||||
EXTERNAL_S3_ACCESS_KEY=$(echo "$HMAC_OUTPUT" | jq -r '.metadata.accessId')
|
||||
|
|
@ -842,6 +906,9 @@ INSTALL_DIR="/opt/openvidu"
|
|||
CLUSTER_CONFIG_DIR="$${INSTALL_DIR}/config/cluster"
|
||||
MASTER_NODE_CONFIG_DIR="$${INSTALL_DIR}/config/node"
|
||||
|
||||
# Disable command tracing so secrets are not printed to the serial console
|
||||
set +x
|
||||
|
||||
# Replace DOMAIN_NAME
|
||||
export DOMAIN=$(gcloud secrets versions access latest --secret=DOMAIN_NAME)
|
||||
if [[ -n "$DOMAIN" ]]; then
|
||||
|
|
@ -970,7 +1037,7 @@ echo -n "$ENABLED_MODULES" | gcloud secrets versions add ENABLED_MODULES --data-
|
|||
EOF
|
||||
|
||||
get_value_from_config_script = <<-EOF
|
||||
#!/bin/bash -x
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Function to get the value of a given key from the environment file
|
||||
|
|
@ -1039,11 +1106,17 @@ EOF
|
|||
|
||||
check_app_ready_script = <<-EOF
|
||||
#!/bin/bash
|
||||
i=0
|
||||
while true; do
|
||||
HTTP_STATUS=$(curl -Ik http://localhost:7880/health/caddy | head -n1 | awk '{print $2}')
|
||||
if [ $HTTP_STATUS == 200 ]; then
|
||||
HTTP_STATUS=$(curl -Ik http://localhost:7880/health/caddy 2>/dev/null | head -n1 | awk '{print $2}')
|
||||
if [ "$HTTP_STATUS" == "200" ]; then
|
||||
break
|
||||
fi
|
||||
i=$((i + 1))
|
||||
if [ "$i" -ge 240 ]; then
|
||||
echo "Timed out after 20 minutes waiting for OpenVidu to become ready" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
EOF
|
||||
|
|
@ -1128,8 +1201,6 @@ CONFIG_S3_EOF
|
|||
|
||||
echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout
|
||||
|
||||
apt-get update && apt-get install -y
|
||||
|
||||
GCLOUD_VERSION=573.0.0
|
||||
# Install google cli
|
||||
if ! command -v gcloud >/dev/null 2>&1; then
|
||||
|
|
@ -1157,9 +1228,6 @@ CONFIG_S3_EOF
|
|||
# Update shared secret
|
||||
/usr/local/bin/after_install.sh || { echo "[OpenVidu] error updating shared secret"; exit 1; }
|
||||
|
||||
# restart.sh
|
||||
echo "@reboot /usr/local/bin/restart.sh >> /var/log/openvidu-restart.log" 2>&1 | crontab
|
||||
|
||||
# Mark installation as complete
|
||||
echo "installation_complete" > /usr/local/bin/openvidu_install_counter.txt
|
||||
fi
|
||||
|
|
@ -1196,12 +1264,21 @@ MASTER_NODE_PRIVATE_IP=$(get_meta "instance/attributes/masterNodePrivateIP")
|
|||
STACK_NAME=$(get_meta "instance/attributes/stackName")
|
||||
PRIVATE_IP=$(get_meta "instance/network-interfaces/0/ip")
|
||||
|
||||
# Wait for master node to be ready by checking secrets
|
||||
while ! gcloud secrets versions access latest --secret=ALL_SECRETS_GENERATED 2>/dev/null; do
|
||||
# Wait for master node to be ready by checking secrets.
|
||||
i=0
|
||||
while ! gcloud secrets versions access latest --secret=ALL_SECRETS_GENERATED 2>/dev/null | grep -q "true"; do
|
||||
i=$((i + 1))
|
||||
if [ "$i" -ge 180 ]; then
|
||||
echo "Timed out after 30 minutes waiting for master node to initialize secrets" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Waiting for master node to initialize secrets..."
|
||||
sleep 10
|
||||
done
|
||||
|
||||
# Disable command tracing so secrets are not printed to the serial console
|
||||
set +x
|
||||
|
||||
# Get all necessary values from secrets
|
||||
DOMAIN=$(gcloud secrets versions access latest --secret=DOMAIN_NAME)
|
||||
OPENVIDU_PRO_LICENSE=$(gcloud secrets versions access latest --secret=OPENVIDU_PRO_LICENSE)
|
||||
|
|
@ -1215,8 +1292,14 @@ if [[ "$OPENVIDU_VERSION" == "none" ]]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
# Build install command for media node
|
||||
INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/elastic/$OPENVIDU_VERSION/install_ov_media_node.sh)"
|
||||
# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure
|
||||
INSTALLER_SCRIPT="/tmp/install_ov_media_node.sh"
|
||||
curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/elastic/$OPENVIDU_VERSION/install_ov_media_node.sh"
|
||||
if [ ! -s "$INSTALLER_SCRIPT" ]; then
|
||||
echo "Downloaded OpenVidu media node installer is empty or missing" >&2
|
||||
exit 1
|
||||
fi
|
||||
INSTALL_COMMAND="sh $INSTALLER_SCRIPT"
|
||||
|
||||
# Media node arguments
|
||||
COMMON_ARGS=(
|
||||
|
|
@ -1320,6 +1403,11 @@ EOF
|
|||
#!/bin/bash -x
|
||||
set -eu -o pipefail
|
||||
|
||||
# Check if installation already completed
|
||||
if [ -f /usr/local/bin/openvidu_install_counter.txt ]; then
|
||||
# Launch on reboot
|
||||
systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; }
|
||||
else
|
||||
# install.sh (media node)
|
||||
cat > /usr/local/bin/install.sh << 'INSTALL_EOF'
|
||||
${local.install_script_media}
|
||||
|
|
@ -1334,8 +1422,6 @@ chmod +x /usr/local/bin/graceful_shutdown.sh
|
|||
|
||||
echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout
|
||||
|
||||
apt-get update && apt-get install -y
|
||||
|
||||
GCLOUD_VERSION=573.0.0
|
||||
# Install google cli
|
||||
if ! command -v gcloud >/dev/null 2>&1; then
|
||||
|
|
@ -1367,5 +1453,6 @@ CHECK_ABANDONED_EOF
|
|||
chmod +x /usr/local/bin/check_abandoned.sh
|
||||
|
||||
echo "*/1 * * * * /usr/local/bin/check_abandoned.sh > /var/log/openvidu-abandoned-check.log 2>&1" | crontab -
|
||||
fi
|
||||
EOF
|
||||
}
|
||||
|
|
|
|||
|
|
@ -158,6 +158,19 @@ var tenantId = subscription().tenantId
|
|||
|
||||
var deploymentUser = az.deployer().objectId
|
||||
|
||||
/*------------------------------------------- MANAGED IDENTITIES -------------------------------------------*/
|
||||
|
||||
// Split in two identities to keep the permission asymmetry: masters write secrets, media nodes only read them
|
||||
resource masterIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = {
|
||||
name: '${stackName}-master-identity'
|
||||
location: location
|
||||
}
|
||||
|
||||
resource mediaIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = {
|
||||
name: '${stackName}-media-identity'
|
||||
location: location
|
||||
}
|
||||
|
||||
/*------------------------------------------- KEY VAULT -------------------------------------------*/
|
||||
|
||||
resource openviduSharedInfo 'Microsoft.KeyVault/vaults@2023-07-01' = {
|
||||
|
|
@ -170,36 +183,16 @@ resource openviduSharedInfo 'Microsoft.KeyVault/vaults@2023-07-01' = {
|
|||
tenantId: tenantId
|
||||
enableSoftDelete: false
|
||||
accessPolicies: [
|
||||
// Pre-created identities: the vault no longer waits for the VMs to exist
|
||||
{
|
||||
objectId: openviduMasterNode1.identity.principalId
|
||||
objectId: masterIdentity.properties.principalId
|
||||
tenantId: tenantId
|
||||
permissions: {
|
||||
secrets: ['get', 'set', 'list']
|
||||
}
|
||||
}
|
||||
{
|
||||
objectId: openviduMasterNode2.identity.principalId
|
||||
tenantId: tenantId
|
||||
permissions: {
|
||||
secrets: ['get', 'set', 'list']
|
||||
}
|
||||
}
|
||||
{
|
||||
objectId: openviduMasterNode3.identity.principalId
|
||||
tenantId: tenantId
|
||||
permissions: {
|
||||
secrets: ['get', 'set', 'list']
|
||||
}
|
||||
}
|
||||
{
|
||||
objectId: openviduMasterNode4.identity.principalId
|
||||
tenantId: tenantId
|
||||
permissions: {
|
||||
secrets: ['get', 'set', 'list']
|
||||
}
|
||||
}
|
||||
{
|
||||
objectId: openviduScaleSetMediaNode.identity.principalId
|
||||
objectId: mediaIdentity.properties.principalId
|
||||
tenantId: tenantId
|
||||
permissions: {
|
||||
secrets: ['get']
|
||||
|
|
@ -237,6 +230,7 @@ var stringInterpolationParamsMaster1 = {
|
|||
initialMeetAdminPassword: initialMeetAdminPassword
|
||||
initialMeetApiKey: initialMeetApiKey
|
||||
keyVaultName: keyVaultName
|
||||
masterIdentityClientId: masterIdentity.properties.clientId
|
||||
masterNodeNum: '1'
|
||||
additionalInstallFlags: additionalInstallFlags
|
||||
}
|
||||
|
|
@ -252,6 +246,7 @@ var stringInterpolationParamsMaster2 = {
|
|||
initialMeetAdminPassword: initialMeetAdminPassword
|
||||
initialMeetApiKey: initialMeetApiKey
|
||||
keyVaultName: keyVaultName
|
||||
masterIdentityClientId: masterIdentity.properties.clientId
|
||||
masterNodeNum: '2'
|
||||
additionalInstallFlags: additionalInstallFlags
|
||||
}
|
||||
|
|
@ -267,6 +262,7 @@ var stringInterpolationParamsMaster3 = {
|
|||
initialMeetAdminPassword: initialMeetAdminPassword
|
||||
initialMeetApiKey: initialMeetApiKey
|
||||
keyVaultName: keyVaultName
|
||||
masterIdentityClientId: masterIdentity.properties.clientId
|
||||
masterNodeNum: '3'
|
||||
additionalInstallFlags: additionalInstallFlags
|
||||
}
|
||||
|
|
@ -282,6 +278,7 @@ var stringInterpolationParamsMaster4 = {
|
|||
initialMeetAdminPassword: initialMeetAdminPassword
|
||||
initialMeetApiKey: initialMeetApiKey
|
||||
keyVaultName: keyVaultName
|
||||
masterIdentityClientId: masterIdentity.properties.clientId
|
||||
masterNodeNum: '4'
|
||||
additionalInstallFlags: additionalInstallFlags
|
||||
}
|
||||
|
|
@ -551,7 +548,7 @@ var after_installScriptTemplateMaster = '''
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
az login --identity --allow-no-subscriptions > /dev/null
|
||||
az login --identity --client-id ${masterIdentityClientId} --allow-no-subscriptions > /dev/null
|
||||
|
||||
# Generate URLs
|
||||
DOMAIN=$(az keyvault secret show --vault-name ${keyVaultName} --name DOMAIN-NAME --query value -o tsv)
|
||||
|
|
@ -579,7 +576,7 @@ var update_config_from_secretScriptTemplateMaster = '''
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
az login --identity --allow-no-subscriptions > /dev/null
|
||||
az login --identity --client-id ${masterIdentityClientId} --allow-no-subscriptions > /dev/null
|
||||
|
||||
# Installation directory
|
||||
INSTALL_DIR="/opt/openvidu"
|
||||
|
|
@ -658,7 +655,7 @@ var update_secret_from_configScriptTemplateMaster = '''
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
az login --identity --allow-no-subscriptions > /dev/null
|
||||
az login --identity --client-id ${masterIdentityClientId} --allow-no-subscriptions > /dev/null
|
||||
|
||||
# Installation directory
|
||||
INSTALL_DIR="/opt/openvidu"
|
||||
|
|
@ -751,7 +748,7 @@ var store_secretScriptTemplateMaster = '''
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
az login --identity --allow-no-subscriptions > /dev/null
|
||||
az login --identity --client-id ${masterIdentityClientId} --allow-no-subscriptions > /dev/null
|
||||
|
||||
# Modes: save, generate
|
||||
# save mode: save the secret in the secret manager
|
||||
|
|
@ -789,7 +786,7 @@ fi
|
|||
|
||||
var get_public_ip = '''
|
||||
#!/bin/bash
|
||||
az login --identity --allow-no-subscriptions > /dev/null
|
||||
az login --identity --client-id ${masterIdentityClientId} --allow-no-subscriptions > /dev/null
|
||||
|
||||
az network public-ip show \
|
||||
--id ${publicIPId} \
|
||||
|
|
@ -842,7 +839,7 @@ WAIT_INTERVAL=1
|
|||
ELAPSED_TIME=0
|
||||
set +e
|
||||
while true; do
|
||||
az login --identity
|
||||
az login --identity --client-id ${masterIdentityClientId}
|
||||
|
||||
# Config azure blob storage
|
||||
AZURE_ACCOUNT_NAME="${storageAccountName}"
|
||||
|
|
@ -927,6 +924,7 @@ var store_secretScriptMaster = reduce(
|
|||
).value
|
||||
|
||||
var blobStorageParams = {
|
||||
masterIdentityClientId: masterIdentity.properties.clientId
|
||||
storageAccountName: isEmptyStorageAccountName ? storageAccount.name : existingStorageAccount.name
|
||||
storageAccountKey: listKeys(storageAccount.id, '2021-04-01').keys[0].value
|
||||
storageAccountContainerName: isEmptyAppDataContainerName ? 'openvidu-appdata' : '${appDataContainerName}'
|
||||
|
|
@ -963,6 +961,7 @@ var userDataParamsMasterNode1 = {
|
|||
base64check_app_ready: base64check_app_readyMaster
|
||||
base64restart: base64restartMaster
|
||||
keyVaultName: keyVaultName
|
||||
masterIdentityClientId: masterIdentity.properties.clientId
|
||||
masterNodeNum: '1'
|
||||
base64config_blobStorage: base64config_blobStorage
|
||||
}
|
||||
|
|
@ -978,6 +977,7 @@ var userDataParamsMasterNode2 = {
|
|||
base64check_app_ready: base64check_app_readyMaster
|
||||
base64restart: base64restartMaster
|
||||
keyVaultName: keyVaultName
|
||||
masterIdentityClientId: masterIdentity.properties.clientId
|
||||
masterNodeNum: '2'
|
||||
base64config_blobStorage: base64config_blobStorage
|
||||
}
|
||||
|
|
@ -993,6 +993,7 @@ var userDataParamsMasterNode3 = {
|
|||
base64check_app_ready: base64check_app_readyMaster
|
||||
base64restart: base64restartMaster
|
||||
keyVaultName: keyVaultName
|
||||
masterIdentityClientId: masterIdentity.properties.clientId
|
||||
masterNodeNum: '3'
|
||||
base64config_blobStorage: base64config_blobStorage
|
||||
}
|
||||
|
|
@ -1008,6 +1009,7 @@ var userDataParamsMasterNode4 = {
|
|||
base64check_app_ready: base64check_app_readyMaster
|
||||
base64restart: base64restartMaster
|
||||
keyVaultName: keyVaultName
|
||||
masterIdentityClientId: masterIdentity.properties.clientId
|
||||
masterNodeNum: '4'
|
||||
storageAccountName: isEmptyStorageAccountName ? storageAccount.name : existingStorageAccount.name
|
||||
base64config_blobStorage: base64config_blobStorage
|
||||
|
|
@ -1058,14 +1060,14 @@ echo ${base64config_blobStorage} | base64 -d > /usr/local/bin/config_blobStorage
|
|||
chmod +x /usr/local/bin/config_blobStorage.sh
|
||||
|
||||
# Install azure cli
|
||||
AZURE_CLI_VERSION=2.87.0
|
||||
AZURE_CLI_VERSION=2.88.0
|
||||
apt-get install -y apt-transport-https ca-certificates gnupg lsb-release
|
||||
curl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg
|
||||
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/azure-cli.list
|
||||
apt-get update
|
||||
apt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)
|
||||
|
||||
az login --identity --allow-no-subscriptions
|
||||
az login --identity --client-id ${masterIdentityClientId} --allow-no-subscriptions
|
||||
|
||||
echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout
|
||||
|
||||
|
|
@ -1127,7 +1129,12 @@ var userDataMasterNode4 = reduce(
|
|||
resource openviduMasterNode1 'Microsoft.Compute/virtualMachines@2023-09-01' = {
|
||||
name: '${stackName}-VM-MasterNode1'
|
||||
location: location
|
||||
identity: { type: 'SystemAssigned' }
|
||||
identity: {
|
||||
type: 'UserAssigned'
|
||||
userAssignedIdentities: {
|
||||
'${masterIdentity.id}': {}
|
||||
}
|
||||
}
|
||||
properties: {
|
||||
hardwareProfile: {
|
||||
vmSize: masterNodeInstanceType
|
||||
|
|
@ -1161,7 +1168,12 @@ resource openviduMasterNode1 'Microsoft.Compute/virtualMachines@2023-09-01' = {
|
|||
resource openviduMasterNode2 'Microsoft.Compute/virtualMachines@2023-09-01' = {
|
||||
name: '${stackName}-VM-MasterNode2'
|
||||
location: location
|
||||
identity: { type: 'SystemAssigned' }
|
||||
identity: {
|
||||
type: 'UserAssigned'
|
||||
userAssignedIdentities: {
|
||||
'${masterIdentity.id}': {}
|
||||
}
|
||||
}
|
||||
properties: {
|
||||
hardwareProfile: {
|
||||
vmSize: masterNodeInstanceType
|
||||
|
|
@ -1195,7 +1207,12 @@ resource openviduMasterNode2 'Microsoft.Compute/virtualMachines@2023-09-01' = {
|
|||
resource openviduMasterNode3 'Microsoft.Compute/virtualMachines@2023-09-01' = {
|
||||
name: '${stackName}-VM-MasterNode3'
|
||||
location: location
|
||||
identity: { type: 'SystemAssigned' }
|
||||
identity: {
|
||||
type: 'UserAssigned'
|
||||
userAssignedIdentities: {
|
||||
'${masterIdentity.id}': {}
|
||||
}
|
||||
}
|
||||
properties: {
|
||||
hardwareProfile: {
|
||||
vmSize: masterNodeInstanceType
|
||||
|
|
@ -1229,7 +1246,12 @@ resource openviduMasterNode3 'Microsoft.Compute/virtualMachines@2023-09-01' = {
|
|||
resource openviduMasterNode4 'Microsoft.Compute/virtualMachines@2023-09-01' = {
|
||||
name: '${stackName}-VM-MasterNode4'
|
||||
location: location
|
||||
identity: { type: 'SystemAssigned' }
|
||||
identity: {
|
||||
type: 'UserAssigned'
|
||||
userAssignedIdentities: {
|
||||
'${masterIdentity.id}': {}
|
||||
}
|
||||
}
|
||||
properties: {
|
||||
hardwareProfile: {
|
||||
vmSize: masterNodeInstanceType
|
||||
|
|
@ -1362,6 +1384,7 @@ exec bash -c "$FINAL_COMMAND"
|
|||
'''
|
||||
|
||||
var stopMediaNodeParams = {
|
||||
mediaIdentityClientId: mediaIdentity.properties.clientId
|
||||
subscriptionId: subscription().subscriptionId
|
||||
resourceGroupName: resourceGroup().name
|
||||
vmScaleSetName: '${stackName}-mediaNodeScaleSet'
|
||||
|
|
@ -1397,7 +1420,7 @@ if [ -x "$(command -v docker)" ]; then
|
|||
done
|
||||
fi
|
||||
|
||||
az login --identity
|
||||
az login --identity --client-id ${mediaIdentityClientId}
|
||||
|
||||
RESOURCE_GROUP_NAME=${resourceGroupName}
|
||||
VM_SCALE_SET_NAME=${vmScaleSetName}
|
||||
|
|
@ -1416,7 +1439,7 @@ var delete_mediaNode_ScriptMediaTemplate = '''
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
az login --identity
|
||||
az login --identity --client-id ${mediaIdentityClientId}
|
||||
|
||||
RESOURCE_GROUP_NAME=${resourceGroupName}
|
||||
VM_SCALE_SET_NAME=${vmScaleSetName}
|
||||
|
|
@ -1449,14 +1472,14 @@ echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout
|
|||
apt-get update && apt-get install -y jq
|
||||
|
||||
# Install azure cli
|
||||
AZURE_CLI_VERSION=2.87.0
|
||||
AZURE_CLI_VERSION=2.88.0
|
||||
apt-get install -y apt-transport-https ca-certificates gnupg lsb-release
|
||||
curl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg
|
||||
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/azure-cli.list
|
||||
apt-get update
|
||||
apt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)
|
||||
|
||||
az login --identity
|
||||
az login --identity --client-id ${mediaIdentityClientId}
|
||||
|
||||
# Protect from scale in actions
|
||||
RESOURCE_GROUP_NAME=${resourceGroupName}
|
||||
|
|
@ -1520,6 +1543,7 @@ var userDataParamsMedia = {
|
|||
base64install: base64installMedia
|
||||
base64stop: base64stopMediaNode
|
||||
base64delete_mediaNode: base64delete_mediaNode_ScriptMedia
|
||||
mediaIdentityClientId: mediaIdentity.properties.clientId
|
||||
resourceGroupName: resourceGroup().name
|
||||
vmScaleSetName: '${stackName}-mediaNodeScaleSet'
|
||||
keyVaultName: keyVaultName
|
||||
|
|
@ -1541,7 +1565,12 @@ resource openviduScaleSetMediaNode 'Microsoft.Compute/virtualMachineScaleSets@20
|
|||
InstanceDeleteTime: datetime
|
||||
storageAccount: isEmptyStorageAccountName ? storageAccount.name : existingStorageAccount.name
|
||||
}
|
||||
identity: { type: 'SystemAssigned' }
|
||||
identity: {
|
||||
type: 'UserAssigned'
|
||||
userAssignedIdentities: {
|
||||
'${mediaIdentity.id}': {}
|
||||
}
|
||||
}
|
||||
sku: {
|
||||
name: mediaNodeInstanceType
|
||||
tier: 'Standard'
|
||||
|
|
@ -1670,67 +1699,29 @@ resource openviduAutoScaleSettingsMediaNode 'Microsoft.Insights/autoscaleSetting
|
|||
|
||||
/*------------------------------------------- SCALE IN ------------------------------------------*/
|
||||
|
||||
resource roleAssignmentMasterNode1 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
|
||||
name: guid('roleAssignmentForMasterNode${openviduMasterNode1.name}')
|
||||
// One assignment for the shared master identity replaces the four per-VM ones
|
||||
resource roleAssignmentMasterNodes 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
|
||||
name: guid('roleAssignmentForMasterNodes', masterIdentity.id)
|
||||
scope: resourceGroup()
|
||||
properties: {
|
||||
roleDefinitionId: subscriptionResourceId(
|
||||
'Microsoft.Authorization/roleDefinitions',
|
||||
'b24988ac-6180-42a0-ab88-20f7382dd24c'
|
||||
)
|
||||
principalId: openviduMasterNode1.identity.principalId
|
||||
principalType: 'ServicePrincipal'
|
||||
}
|
||||
}
|
||||
|
||||
resource roleAssignmentMasterNode2 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
|
||||
name: guid('roleAssignmentForMasterNode${openviduMasterNode2.name}')
|
||||
scope: resourceGroup()
|
||||
properties: {
|
||||
roleDefinitionId: subscriptionResourceId(
|
||||
'Microsoft.Authorization/roleDefinitions',
|
||||
'b24988ac-6180-42a0-ab88-20f7382dd24c'
|
||||
)
|
||||
principalId: openviduMasterNode2.identity.principalId
|
||||
principalType: 'ServicePrincipal'
|
||||
}
|
||||
}
|
||||
|
||||
resource roleAssignmentMasterNode3 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
|
||||
name: guid('roleAssignmentForMasterNode${openviduMasterNode3.name}')
|
||||
scope: resourceGroup()
|
||||
properties: {
|
||||
roleDefinitionId: subscriptionResourceId(
|
||||
'Microsoft.Authorization/roleDefinitions',
|
||||
'b24988ac-6180-42a0-ab88-20f7382dd24c'
|
||||
)
|
||||
principalId: openviduMasterNode3.identity.principalId
|
||||
principalType: 'ServicePrincipal'
|
||||
}
|
||||
}
|
||||
|
||||
resource roleAssignmentMasterNode4 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
|
||||
name: guid('roleAssignmentForMasterNode${openviduMasterNode4.name}')
|
||||
scope: resourceGroup()
|
||||
properties: {
|
||||
roleDefinitionId: subscriptionResourceId(
|
||||
'Microsoft.Authorization/roleDefinitions',
|
||||
'b24988ac-6180-42a0-ab88-20f7382dd24c'
|
||||
)
|
||||
principalId: openviduMasterNode4.identity.principalId
|
||||
principalId: masterIdentity.properties.principalId
|
||||
principalType: 'ServicePrincipal'
|
||||
}
|
||||
}
|
||||
|
||||
resource roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
|
||||
name: guid('roleAssignmentForScaleSet${openviduScaleSetMediaNode.name}')
|
||||
name: guid('roleAssignmentForScaleSet', mediaIdentity.id)
|
||||
scope: resourceGroup()
|
||||
properties: {
|
||||
roleDefinitionId: subscriptionResourceId(
|
||||
'Microsoft.Authorization/roleDefinitions',
|
||||
'b24988ac-6180-42a0-ab88-20f7382dd24c'
|
||||
)
|
||||
principalId: openviduScaleSetMediaNode.identity.principalId
|
||||
principalId: mediaIdentity.properties.principalId
|
||||
principalType: 'ServicePrincipal'
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue