diff --git a/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml b/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml index 5f7a91afc..cce197135 100644 --- a/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml +++ b/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml @@ -1242,14 +1242,39 @@ Resources: "GRAFANA_ADMIN_USERNAME": "none", "GRAFANA_ADMIN_PASSWORD": "none", "ENABLED_MODULES": "none", - "MASTER_NODE_1_PRIVATE_IP": "none", - "MASTER_NODE_2_PRIVATE_IP": "none", - "MASTER_NODE_3_PRIVATE_IP": "none", - "MASTER_NODE_4_PRIVATE_IP": "none", "OPENVIDU_VERSION": "none", "ALL_SECRETS_GENERATED": "false" } + # One SSM parameter per master: atomic per-key writes + MasterNode1PrivateIpParameter: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub /openvidu/${AWS::StackName}/master-node-1-private-ip + Type: String + Value: "none" + + MasterNode2PrivateIpParameter: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub /openvidu/${AWS::StackName}/master-node-2-private-ip + Type: String + Value: "none" + + MasterNode3PrivateIpParameter: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub /openvidu/${AWS::StackName}/master-node-3-private-ip + Type: String + Value: "none" + + MasterNode4PrivateIpParameter: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub /openvidu/${AWS::StackName}/master-node-4-private-ip + Type: String + Value: "none" + S3AppDataBucketResource: Type: 'AWS::S3::Bucket' Properties: @@ -1281,8 +1306,7 @@ Resources: Condition: CreateClusterDataBucket # ------------------------- - # Preprocess subnets to allocate Volumes and ENIs across Availability Zones - # For OpenVidu Master Nodes + # Normalize the master node subnet list to exactly 4 entries, one per OpenVidu Master Node # ------------------------- SubnetProcessorFunction: Type: AWS::Lambda::Function @@ -1293,13 +1317,11 @@ Resources: Code: ZipFile: | import cfnresponse - import boto3 def lambda_handler(event, context): try: # Process event data subnets = event['ResourceProperties']['Subnets'] - ec2 = boto3.client('ec2') # Ensure we have at least four subnets by cycling through the available subnets subnets = (subnets * 4)[:4] # Repeat the list to have at least 4 elements and then take the first 4 @@ -1355,10 +1377,6 @@ Resources: - logs:CreateLogStream - logs:PutLogEvents Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/SubnetProcessor-${AWS::Region}-${AWS::StackName}:*' - - Effect: Allow - Action: - - ec2:DescribeSubnets - Resource: '*' OpenViduMasterNodeRole: Type: AWS::IAM::Role @@ -1390,6 +1408,11 @@ Resources: - secretsmanager:GetSecretValue - secretsmanager:UpdateSecret Resource: !Ref OpenViduSharedInfo + - Effect: Allow + Action: + - ssm:GetParameter + - ssm:PutParameter + Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/openvidu/${AWS::StackName}/* - Fn::If: - CreateRecordingsBucket - Effect: Allow @@ -1474,6 +1497,10 @@ Resources: Action: - secretsmanager:GetSecretValue Resource: !Ref OpenViduSharedInfo + - Effect: Allow + Action: + - ssm:GetParameter + Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/openvidu/${AWS::StackName}/* - Effect: Allow Action: - autoscaling:SetInstanceHealth @@ -1526,8 +1553,6 @@ Resources: OpenViduMediaInstanceProfile: Type: AWS::IAM::InstanceProfile - DependsOn: - - MasterNodesWaitCondition4 Properties: InstanceProfileName: !Sub OpenViduMediaInstanceProfile-${AWS::Region}-${AWS::StackName} Roles: @@ -1566,42 +1591,15 @@ Resources: rm -rf awscliv2.zip aws fi - # Singal to notify instance is waiting + # Signal to notify instance is waiting SIGNAL_NAME="$1" + # Exported earlier in this instance's UserData + MASTER_NODE_NUM="${!MASTER_NODE_NUM}" + # Token for IMDSv2 TOKEN="$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")" - # Subnets IDs - SUBNETS=( - "${SubnetProcessor.Subnet1}" - "${SubnetProcessor.Subnet2}" - "${SubnetProcessor.Subnet3}" - "${SubnetProcessor.Subnet4}" - ) - - MAC_ADDRESS="$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254//latest/meta-data/mac)" - SUBNET_ID="$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s "http://169.254.169.254/latest/meta-data/network/interfaces/macs/$MAC_ADDRESS/subnet-id")" - - # Check master node number - MASTER_NODE_NUM=1 - for subnet in "${!SUBNETS[@]}"; do - if [[ "$subnet" == "$SUBNET_ID" ]]; then - - SHARED_SECRET=$(aws secretsmanager get-secret-value \ - --region ${AWS::Region} \ - --secret-id openvidu-ha-${AWS::Region}-${AWS::StackName} \ - --query SecretString --output text || echo 'none') - - # Check if current master node is reacheable with ping - ACUTAL_MASTER_NODE_IP=$(echo "$SHARED_SECRET" | jq -r ".MASTER_NODE_${!MASTER_NODE_NUM}_PRIVATE_IP") - if [[ "$ACUTAL_MASTER_NODE_IP" == "none" ]]; then - break - fi - fi - MASTER_NODE_NUM=$((MASTER_NODE_NUM + 1)) - done - # Get own private IP PRIVATE_IP="$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/local-ipv4)" @@ -1610,8 +1608,12 @@ Resources: exit 1 fi - # Store current private IP - /usr/local/bin/store_secret.sh save MASTER_NODE_${!MASTER_NODE_NUM}_PRIVATE_IP "${!PRIVATE_IP}" + aws ssm put-parameter \ + --region ${AWS::Region} \ + --name "/openvidu/${AWS::StackName}/master-node-${!MASTER_NODE_NUM}-private-ip" \ + --value "${!PRIVATE_IP}" \ + --type String \ + --overwrite > /dev/null SHARED_SECRET=$(aws secretsmanager get-secret-value \ --region ${AWS::Region} \ @@ -1690,39 +1692,52 @@ Resources: # sending the signal call cfn-signal -e $? --stack ${AWS::StackId} --resource "$SIGNAL_NAME" --region ${AWS::Region} - # Wait for all master nodes to store their private IPs + IP_WAIT_ATTEMPTS=0 + IP_WAIT_MAX=360 while true; do - SHARED_SECRET=$(aws secretsmanager get-secret-value \ - --region ${AWS::Region} \ - --secret-id openvidu-ha-${AWS::Region}-${AWS::StackName} \ - --query SecretString --output text || echo 'none') + MASTER_NODE_1_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-1-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + MASTER_NODE_2_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-2-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + MASTER_NODE_3_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-3-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + MASTER_NODE_4_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-4-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') - MASTER_NODE_1_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_1_PRIVATE_IP') - MASTER_NODE_2_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_2_PRIVATE_IP') - MASTER_NODE_3_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_3_PRIVATE_IP') - MASTER_NODE_4_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_4_PRIVATE_IP') - - # Check if all master nodes have stored their private IPs - if [[ "$MASTER_NODE_1_PRIVATE_IP" != "none" ]] && - [[ "$MASTER_NODE_2_PRIVATE_IP" != "none" ]] && - [[ "$MASTER_NODE_3_PRIVATE_IP" != "none" ]] && - [[ "$MASTER_NODE_4_PRIVATE_IP" != "none" ]]; then + # Check if all master nodes have published their private IPs + if [[ "$MASTER_NODE_1_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_1_PRIVATE_IP" ]] && + [[ "$MASTER_NODE_2_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_2_PRIVATE_IP" ]] && + [[ "$MASTER_NODE_3_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_3_PRIVATE_IP" ]] && + [[ "$MASTER_NODE_4_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_4_PRIVATE_IP" ]]; then break fi + + IP_WAIT_ATTEMPTS=$((IP_WAIT_ATTEMPTS + 1)) + if [[ $IP_WAIT_ATTEMPTS -ge $IP_WAIT_MAX ]]; then + echo "Error: timed out after 30 minutes waiting for all master nodes to publish their private IPs to SSM" + exit 1 + fi sleep 5 done - SHARED_SECRET=$(aws secretsmanager get-secret-value \ - --region ${AWS::Region} \ - --secret-id openvidu-ha-${AWS::Region}-${AWS::StackName} \ - --query SecretString --output text) - - MASTER_NODE_1_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_1_PRIVATE_IP') - MASTER_NODE_2_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_2_PRIVATE_IP') - MASTER_NODE_3_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_3_PRIVATE_IP') - MASTER_NODE_4_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_4_PRIVATE_IP') + # Comma-separated 1-4: must match installer's --master-node-private-ip-list format MASTER_NODE_PRIVATE_IP_LIST="$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP" + # Re-read: other (non-IP) values still come from the shared secret. + # GetSecretValue is eventually consistent: retry until a read returns the generated values + SECRET_READ_ATTEMPTS=0 + while true; do + SHARED_SECRET=$(aws secretsmanager get-secret-value \ + --region ${AWS::Region} \ + --secret-id openvidu-ha-${AWS::Region}-${AWS::StackName} \ + --query SecretString --output text) + if echo "$SHARED_SECRET" | jq -e '(.ALL_SECRETS_GENERATED == "true") and ([.DOMAIN_NAME, .OPENVIDU_VERSION, .REDIS_PASSWORD, .MONGO_ADMIN_PASSWORD, .MONGO_REPLICA_SET_KEY, .MINIO_SECRET_KEY, .DASHBOARD_ADMIN_PASSWORD, .GRAFANA_ADMIN_PASSWORD, .LIVEKIT_API_KEY, .LIVEKIT_API_SECRET] | all(. != "none"))' > /dev/null; then + break + fi + SECRET_READ_ATTEMPTS=$((SECRET_READ_ATTEMPTS + 1)) + if [[ $SECRET_READ_ATTEMPTS -ge 60 ]]; then + echo "Error: shared secret still incomplete after 5 minutes of stale reads" + exit 1 + fi + sleep 5 + done + DOMAIN=$(echo "$SHARED_SECRET" | jq -r '.DOMAIN_NAME') OPENVIDU_PRO_LICENSE=$(echo "$SHARED_SECRET" | jq -r '.OPENVIDU_PRO_LICENSE') OPENVIDU_RTC_ENGINE=$(echo "$SHARED_SECRET" | jq -r '.OPENVIDU_RTC_ENGINE') @@ -1738,18 +1753,22 @@ Resources: GRAFANA_ADMIN_PASSWORD=$(echo "$SHARED_SECRET" | jq -r '.GRAFANA_ADMIN_PASSWORD') MEET_INITIAL_ADMIN_USER=$(echo "$SHARED_SECRET" | jq -r '.MEET_INITIAL_ADMIN_USER') MEET_INITIAL_ADMIN_PASSWORD=$(echo "$SHARED_SECRET" | jq -r '.MEET_INITIAL_ADMIN_PASSWORD') + MEET_INITIAL_API_KEY="" if [[ "${InitialMeetApiKey}" != '' ]]; then MEET_INITIAL_API_KEY=$(echo "$SHARED_SECRET" | jq -r '.MEET_INITIAL_API_KEY') - else - MEET_INITIAL_API_KEY="" fi - MEET_INITIAL_API_KEY=$(echo "$SHARED_SECRET" | jq -r '.MEET_INITIAL_API_KEY') LIVEKIT_API_KEY=$(echo "$SHARED_SECRET" | jq -r '.LIVEKIT_API_KEY') LIVEKIT_API_SECRET=$(echo "$SHARED_SECRET" | jq -r '.LIVEKIT_API_SECRET') ENABLED_MODULES=$(echo "$SHARED_SECRET" | jq -r '.ENABLED_MODULES') - # Base command - INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh)" + # Download first: sh <(curl ...) would silently run an empty script on a transient curl failure + INSTALLER_SCRIPT="/tmp/install_ov_master_node.sh" + curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh" + if [ ! -s "$INSTALLER_SCRIPT" ]; then + echo "Downloaded OpenVidu master node installer is empty or missing" >&2 + exit 1 + fi + INSTALL_COMMAND="sh $INSTALLER_SCRIPT" # Common arguments COMMON_ARGS=( @@ -1837,6 +1856,10 @@ Resources: content: !Sub | #!/bin/bash set -e + # Only master 1 writes the shared secret, to avoid a lost-update race + if [[ "${!MASTER_NODE_NUM:-}" != "1" ]]; then + exit 0 + fi # Get current shared secret SHARED_SECRET=$(aws secretsmanager get-secret-value \ --region ${AWS::Region} \ @@ -2103,6 +2126,8 @@ Resources: OpenViduMasterNode1: Type: AWS::EC2::Instance + # Depends on its own SSM parameter so its put-parameter can't race the parameter's creation + DependsOn: MasterNode1PrivateIpParameter Properties: LaunchTemplate: LaunchTemplateId: !Ref OpenViduMasterLaunchTemplate @@ -2133,6 +2158,8 @@ Resources: cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource OpenViduMasterLaunchTemplate + export MASTER_NODE_NUM=1 + # Install OpenVidu /usr/local/bin/install.sh "MasterNodesWaitCondition1" || { echo "[OpenVidu] error installing OpenVidu"; exit 1; } @@ -2179,7 +2206,7 @@ Resources: OpenViduMasterNode2: Type: AWS::EC2::Instance - DependsOn: MasterNodesWaitCondition1 + DependsOn: MasterNode2PrivateIpParameter Properties: LaunchTemplate: LaunchTemplateId: !Ref OpenViduMasterLaunchTemplate @@ -2210,6 +2237,8 @@ Resources: cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource OpenViduMasterLaunchTemplate + export MASTER_NODE_NUM=2 + # Install OpenVidu /usr/local/bin/install.sh "MasterNodesWaitCondition2" || { echo "[OpenVidu] error installing OpenVidu"; exit 1; } @@ -2256,7 +2285,7 @@ Resources: OpenViduMasterNode3: Type: AWS::EC2::Instance - DependsOn: MasterNodesWaitCondition2 + DependsOn: MasterNode3PrivateIpParameter Properties: LaunchTemplate: LaunchTemplateId: !Ref OpenViduMasterLaunchTemplate @@ -2287,6 +2316,8 @@ Resources: cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource OpenViduMasterLaunchTemplate + export MASTER_NODE_NUM=3 + # Install OpenVidu /usr/local/bin/install.sh "MasterNodesWaitCondition3" || { echo "[OpenVidu] error installing OpenVidu"; exit 1; } @@ -2333,7 +2364,7 @@ Resources: OpenViduMasterNode4: Type: AWS::EC2::Instance - DependsOn: MasterNodesWaitCondition3 + DependsOn: MasterNode4PrivateIpParameter Properties: LaunchTemplate: LaunchTemplateId: !Ref OpenViduMasterLaunchTemplate @@ -2364,6 +2395,8 @@ Resources: cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource OpenViduMasterLaunchTemplate + export MASTER_NODE_NUM=4 + # Install OpenVidu /usr/local/bin/install.sh "MasterNodesWaitCondition4" || { echo "[OpenVidu] error installing OpenVidu"; exit 1; } @@ -2453,6 +2486,25 @@ Resources: exit 1 fi + SECRETS_WAIT_ATTEMPTS=0 + SECRETS_WAIT_MAX=360 + while true; do + # Validate content, not just the flag: stale eventually-consistent reads can return pre-generation values + if echo "$SHARED_SECRET" | jq -e '(.ALL_SECRETS_GENERATED == "true") and (.OPENVIDU_VERSION != "none") and (.REDIS_PASSWORD != "none")' > /dev/null; then + break + fi + SECRETS_WAIT_ATTEMPTS=$((SECRETS_WAIT_ATTEMPTS + 1)) + if [[ $SECRETS_WAIT_ATTEMPTS -ge $SECRETS_WAIT_MAX ]]; then + echo "Error: timed out after 30 minutes waiting for shared secrets to be generated" + exit 1 + fi + sleep 5 + SHARED_SECRET="$(aws secretsmanager get-secret-value \ + --region ${AWS::Region} \ + --secret-id openvidu-ha-${AWS::Region}-${AWS::StackName} \ + --query SecretString --output text || echo 'none')" + done + # Get OpenVidu Media Nodes version to deploy OPENVIDU_VERSION=$(echo "$SHARED_SECRET" | jq -r '.OPENVIDU_VERSION') if [[ "$OPENVIDU_VERSION" == "none" ]]; then @@ -2460,20 +2512,39 @@ Resources: exit 1 fi - ALL_SECRETS_GENERATED=$(echo "$SHARED_SECRET" | jq -r '.ALL_SECRETS_GENERATED') - if [[ "$ALL_SECRETS_GENERATED" == "none" ]]; then - echo "Error: Secrets not generated" - exit 1 - fi - MASTER_NODE_1_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_1_PRIVATE_IP') - MASTER_NODE_2_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_2_PRIVATE_IP') - MASTER_NODE_3_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_3_PRIVATE_IP') - MASTER_NODE_4_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_4_PRIVATE_IP') + IP_WAIT_ATTEMPTS=0 + IP_WAIT_MAX=360 + while true; do + MASTER_NODE_1_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-1-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + MASTER_NODE_2_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-2-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + MASTER_NODE_3_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-3-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + MASTER_NODE_4_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-4-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + + if [[ "$MASTER_NODE_1_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_1_PRIVATE_IP" ]] && + [[ "$MASTER_NODE_2_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_2_PRIVATE_IP" ]] && + [[ "$MASTER_NODE_3_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_3_PRIVATE_IP" ]] && + [[ "$MASTER_NODE_4_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_4_PRIVATE_IP" ]]; then + break + fi + + IP_WAIT_ATTEMPTS=$((IP_WAIT_ATTEMPTS + 1)) + if [[ $IP_WAIT_ATTEMPTS -ge $IP_WAIT_MAX ]]; then + echo "Error: timed out after 30 minutes waiting for all master nodes to publish their private IPs to SSM" + exit 1 + fi + sleep 5 + done MASTER_NODE_PRIVATE_IP_LIST="$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP" REDIS_PASSWORD=$(echo "$SHARED_SECRET" | jq -r '.REDIS_PASSWORD') - # Base command - INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh)" + # Download first: sh <(curl ...) would silently run an empty script on a transient curl failure + INSTALLER_SCRIPT="/tmp/install_ov_media_node.sh" + curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh" + if [ ! -s "$INSTALLER_SCRIPT" ]; then + echo "Downloaded OpenVidu media node installer is empty or missing" >&2 + exit 1 + fi + INSTALL_COMMAND="sh $INSTALLER_SCRIPT" # Common arguments COMMON_ARGS=( @@ -2609,13 +2680,36 @@ Resources: python3 -m pip install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-${!CFN_BOOTSTRAP_VERSION}.tar.gz fi - cfn-init --region ${AWS::Region} --stack ${AWS::StackId} --resource OpenViduMediaNodeLaunchTemplate + cfn-init --region ${AWS::Region} --stack ${AWS::StackName} --resource OpenViduMediaNodeLaunchTemplate export HOME="/root" # Install OpenVidu /usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu"; /usr/local/bin/set_as_unhealthy.sh; exit 1; } + MASTER_IP_1=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-1-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + MASTER_IP_2=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-2-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + MASTER_IP_3=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-3-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + MASTER_IP_4=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-4-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none') + MASTER_HEALTHY=false + for i in $(seq 1 360); do + for MASTER_IP in "$MASTER_IP_1" "$MASTER_IP_2" "$MASTER_IP_3" "$MASTER_IP_4"; do + if [ "$MASTER_IP" != "none" ] && [ -n "$MASTER_IP" ] && curl -sf "http://$MASTER_IP:7880/health/caddy" >/dev/null 2>&1; then + MASTER_HEALTHY=true + break + fi + done + if [ "$MASTER_HEALTHY" = "true" ]; then + break + fi + sleep 5 + done + if [ "$MASTER_HEALTHY" != "true" ]; then + echo "[OpenVidu] no master node became healthy after 30 minutes" + /usr/local/bin/set_as_unhealthy.sh + exit 1 + fi + # Start OpenVidu systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; /usr/local/bin/set_as_unhealthy.sh; exit 1; } @@ -3191,8 +3285,6 @@ Resources: LoadBalancer: Type: AWS::ElasticLoadBalancingV2::LoadBalancer - DependsOn: - - MasterNodesWaitCondition4 Properties: Name: Fn::Join: diff --git a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep index ebe27892b..0d771c174 100644 --- a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep +++ b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep @@ -304,7 +304,7 @@ apt-get update && apt-get install -y \ # Wait for the keyvault availability -MAX_WAIT=100 +MAX_WAIT=300 WAIT_INTERVAL=1 ELAPSED_TIME=0 set +e @@ -397,6 +397,9 @@ if [[ $MASTER_NODE_NUM -eq 1 ]] && [[ "$ALL_SECRETS_GENERATED" == "" || "$ALL_SE ALL_SECRETS_GENERATED="$(/usr/local/bin/store_secret.sh save ALL-SECRETS-GENERATED "true")" fi +IP_WAIT_MAX_RETRIES=360 +IP_WAIT_INTERVAL=5 +IP_WAIT_RETRIES=0 while true; do MASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv) || true MASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv) || true @@ -409,11 +412,26 @@ while true; do [[ "$MASTER_NODE_4_PRIVATE_IP" != "" ]]; then break fi - sleep 5 + IP_WAIT_RETRIES=$((IP_WAIT_RETRIES + 1)) + if [ $IP_WAIT_RETRIES -ge $IP_WAIT_MAX_RETRIES ]; then + echo "[OpenVidu] timed out after 30 min waiting for the 4 master nodes to publish their private IPs" + exit 1 + fi + sleep $IP_WAIT_INTERVAL done # Wait until master-node-1 has generated all shared secrets before fetching them -while [[ "$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)" != "true" ]]; do sleep 5; done +SECRETS_WAIT_MAX_RETRIES=360 +SECRETS_WAIT_INTERVAL=5 +SECRETS_WAIT_RETRIES=0 +while [[ "$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)" != "true" ]]; do + SECRETS_WAIT_RETRIES=$((SECRETS_WAIT_RETRIES + 1)) + if [ $SECRETS_WAIT_RETRIES -ge $SECRETS_WAIT_MAX_RETRIES ]; then + echo "[OpenVidu] timed out after 30 min waiting for ALL-SECRETS-GENERATED to become true" + exit 1 + fi + sleep $SECRETS_WAIT_INTERVAL +done # Fetch the values in the keyvault @@ -448,8 +466,16 @@ fi ENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv) +# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure +INSTALLER_SCRIPT="/tmp/install_ov_master_node.sh" +curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh" +if [ ! -s "$INSTALLER_SCRIPT" ]; then + echo "[OpenVidu] failed to download the master node installer script" + exit 1 +fi + # Base command -INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh)" +INSTALL_COMMAND="sh $INSTALLER_SCRIPT" # Common arguments COMMON_ARGS=( @@ -811,7 +837,7 @@ INSTALL_DIR="/opt/openvidu" CLUSTER_CONFIG_DIR="${INSTALL_DIR}/config/cluster" # Retry login + storage key fetch to allow the Contributor role assignment to propagate -MAX_WAIT=100 +MAX_WAIT=300 WAIT_INTERVAL=1 ELAPSED_TIME=0 set +e @@ -1043,8 +1069,6 @@ az login --identity --allow-no-subscriptions echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout -apt-get update && apt-get install -y - export HOME="/root" # Install OpenVidu @@ -1062,20 +1086,18 @@ systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; # Launch on reboot echo "@reboot /usr/local/bin/restart.sh >> /var/log/openvidu-restart.log" 2>&1 | crontab +# check_app_ready.sh internally caps its wait at 1200s +/usr/local/bin/check_app_ready.sh || { echo "[OpenVidu] master node did not become healthy"; exit 1; } + +az keyvault secret set --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --value "true" + MASTER_NODE_NUM=${masterNodeNum} if [[ $MASTER_NODE_NUM -eq 4 ]]; then # Creating scale in lock set +e az storage blob upload --account-name ${storageAccountName} --container-name automation-locks --name lock.txt --file /dev/null --auth-mode key set -e - - #Finish all the nodes - az keyvault secret set --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --value "true" fi - -# Wait for the app -sleep 150 -/usr/local/bin/check_app_ready.sh ''' var userDataMasterNode1 = reduce( @@ -1168,7 +1190,6 @@ resource openviduMasterNode2 'Microsoft.Compute/virtualMachines@2023-09-01' = { } userData: base64(userDataMasterNode2) } - dependsOn: [openviduMasterNode1] } resource openviduMasterNode3 'Microsoft.Compute/virtualMachines@2023-09-01' = { @@ -1203,7 +1224,6 @@ resource openviduMasterNode3 'Microsoft.Compute/virtualMachines@2023-09-01' = { } userData: base64(userDataMasterNode3) } - dependsOn: [openviduMasterNode2] } resource openviduMasterNode4 'Microsoft.Compute/virtualMachines@2023-09-01' = { @@ -1238,7 +1258,6 @@ resource openviduMasterNode4 'Microsoft.Compute/virtualMachines@2023-09-01' = { } userData: base64(userDataMasterNode4) } - dependsOn: [openviduMasterNode3] } /*------------------------------------------- MEDIA NODES -------------------------------------------*/ @@ -1272,23 +1291,29 @@ apt-get update && apt-get install -y \ # Get own private IP PRIVATE_IP=$(curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text") -WAIT_INTERVAL=1 -MAX_WAIT=10000 -ELAPSED_TIME=0 +# Gate 1: wait for master secrets and IPs before installing +WAIT_INTERVAL=5 +MAX_RETRIES=360 +RETRIES=0 set +e while true; do - # get secret value - FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv) + ALL_SECRETS_GENERATED=$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null) + MASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv 2>/dev/null) + MASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv 2>/dev/null) + MASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv 2>/dev/null) + MASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv 2>/dev/null) - # Check if all master nodes finished - if [ "$FINISH_MASTER_NODE" == "true" ]; then + if [ "$ALL_SECRETS_GENERATED" == "true" ] && + [ "$MASTER_NODE_1_PRIVATE_IP" != "" ] && + [ "$MASTER_NODE_2_PRIVATE_IP" != "" ] && + [ "$MASTER_NODE_3_PRIVATE_IP" != "" ] && + [ "$MASTER_NODE_4_PRIVATE_IP" != "" ]; then break fi - ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL)) - - # Check if the maximum waiting time has been reached - if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then + RETRIES=$((RETRIES + 1)) + if [ $RETRIES -ge $MAX_RETRIES ]; then + echo "[OpenVidu] timed out after 30 min waiting for master nodes to generate secrets and publish their IPs" exit 1 fi @@ -1305,8 +1330,16 @@ REDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name RED ENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv) OPENVIDU_VERSION=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-VERSION --query value -o tsv) +# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure +INSTALLER_SCRIPT="/tmp/install_ov_media_node.sh" +curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh" +if [ ! -s "$INSTALLER_SCRIPT" ]; then + echo "[OpenVidu] failed to download the media node installer script" + exit 1 +fi + # Base command -INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh)" +INSTALL_COMMAND="sh $INSTALLER_SCRIPT" # Common arguments COMMON_ARGS=( @@ -1413,8 +1446,7 @@ chmod +x /usr/local/bin/delete_media_node.sh echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout -apt-get update && apt-get install -y -apt-get install -y jq +apt-get update && apt-get install -y jq # Install azure cli AZURE_CLI_VERSION=2.87.0 @@ -1438,6 +1470,26 @@ export HOME="/root" # Install OpenVidu /usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu"; /usr/local/bin/delete_media_node.sh; } +# Gate 2: wait for master readiness before starting +WAIT_INTERVAL=5 +MAX_RETRIES=360 +RETRIES=0 +set +e +while true; do + FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv 2>/dev/null) + if [ "$FINISH_MASTER_NODE" == "true" ]; then + break + fi + RETRIES=$((RETRIES + 1)) + if [ $RETRIES -ge $MAX_RETRIES ]; then + echo "[OpenVidu] timed out after 30 min waiting for FINISH-MASTER-NODE" + /usr/local/bin/delete_media_node.sh + exit 1 + fi + sleep $WAIT_INTERVAL +done +set -e + # Start OpenVidu systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; /usr/local/bin/delete_media_node.sh; } ''' @@ -1470,6 +1522,7 @@ var userDataParamsMedia = { base64delete_mediaNode: base64delete_mediaNode_ScriptMedia resourceGroupName: resourceGroup().name vmScaleSetName: '${stackName}-mediaNodeScaleSet' + keyVaultName: keyVaultName } var userDataMediaNode = reduce( diff --git a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json index 0f97b212c..0a8750e2f 100644 --- a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json +++ b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json @@ -4,8 +4,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.44.1.10279", - "templateHash": "8155530769596354082" + "version": "0.45.15.27210", + "templateHash": "10036394790552381773" } }, "parameters": { @@ -289,7 +289,7 @@ "masterNodeNum": "4", "additionalInstallFlags": "[parameters('additionalInstallFlags')]" }, - "installScriptTemplateMaster": "#!/bin/bash -x\nset -e\nOPENVIDU_VERSION=main\nDOMAIN=\n\n# Assume azure cli is installed\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y \\\n curl \\\n unzip \\\n jq \\\n wget\n\n\n# Wait for the keyvault availability\nMAX_WAIT=100\nWAIT_INTERVAL=1\nELAPSED_TIME=0\nset +e\nwhile true; do\n # Check keyvault availability\n az keyvault secret list --vault-name ${keyVaultName} \n\n # If it is available, exit the loop\n if [ $? -eq 0 ]; then\n break\n fi\n\n # If not, wait and check again incrementing the time\n ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL))\n\n # If exceeded the maximum time, exit with error\n if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then\n exit 1\n fi\n\n # Esperar antes de la próxima comprobación\n sleep $WAIT_INTERVAL\ndone\nset -e\n\nMASTER_NODE_NUM=${masterNodeNum}\n\n# Get own private IP\nPRIVATE_IP=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text\")\n\n# Store current private IP\nPRIVATE_IP=\"$(/usr/local/bin/store_secret.sh save MASTER-NODE-${masterNodeNum}-PRIVATE-IP $PRIVATE_IP)\"\n\n\nif [[ $MASTER_NODE_NUM -eq 1 ]] && [[ \"$ALL_SECRETS_GENERATED\" == \"\" || \"$ALL_SECRETS_GENERATED\" == \"false\" ]]; then\n\n # Configure Domain name\n if [[ \"${domainName}\" == '' ]]; then\n # Get public IP using the get_public_ip.sh script\n PUBLIC_IP=$(/usr/local/bin/get_public_ip.sh 2>/dev/null)\n if [[ $? -ne 0 || -z \"${PUBLIC_IP}\" ]]; then\n echo \"Could not determine public IP.\"\n exit 1\n fi\n\n DOMAIN=\"$PUBLIC_IP\"\n else\n DOMAIN=${domainName}\n fi\n DOMAIN=\"$(/usr/local/bin/store_secret.sh save DOMAIN-NAME \"${DOMAIN}\")\"\n\n # Publish service URLs early (idempotent; after_install re-writes byte-identical values)\n OPENVIDU_URL=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-URL \"https://${DOMAIN}/\")\"\n LIVEKIT_URL=\"$(/usr/local/bin/store_secret.sh save LIVEKIT-URL \"wss://${DOMAIN}/\")\"\n DASHBOARD_URL=\"$(/usr/local/bin/store_secret.sh save DASHBOARD-URL \"https://${DOMAIN}/dashboard/\")\"\n GRAFANA_URL=\"$(/usr/local/bin/store_secret.sh save GRAFANA-URL \"https://${DOMAIN}/grafana/\")\"\n MINIO_URL=\"$(/usr/local/bin/store_secret.sh save MINIO-URL \"https://${DOMAIN}/minio-console/\")\"\n\n # Meet initial admin user and password\n MEET_INITIAL_ADMIN_USER=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-ADMIN-USER \"admin\")\"\n if [[ \"${initialMeetAdminPassword}\" != '' ]]; then\n MEET_INITIAL_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-ADMIN-PASSWORD \"${initialMeetAdminPassword}\")\"\n else\n MEET_INITIAL_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate MEET-INITIAL-ADMIN-PASSWORD)\"\n fi\n if [[ \"${initialMeetApiKey}\" != '' ]]; then\n MEET_INITIAL_API_KEY=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-API-KEY \"${initialMeetApiKey}\")\"\n else\n MEET_INITIAL_API_KEY=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-API-KEY \"\")\"\n fi\n\n # Store usernames and generate random passwords\n OPENVIDU_PRO_LICENSE=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-PRO-LICENSE \"${openviduLicense}\")\"\n OPENVIDU_RTC_ENGINE=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-RTC-ENGINE \"${rtcEngine}\")\"\n REDIS_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate REDIS-PASSWORD)\"\n MONGO_ADMIN_USERNAME=\"$(/usr/local/bin/store_secret.sh save MONGO-ADMIN-USERNAME \"mongoadmin\")\"\n MONGO_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate MONGO-ADMIN-PASSWORD)\"\n MONGO_REPLICA_SET_KEY=\"$(/usr/local/bin/store_secret.sh generate MONGO-REPLICA-SET-KEY)\"\n MINIO_ACCESS_KEY=\"$(/usr/local/bin/store_secret.sh save MINIO-ACCESS-KEY \"minioadmin\")\"\n MINIO_SECRET_KEY=\"$(/usr/local/bin/store_secret.sh generate MINIO-SECRET-KEY)\"\n DASHBOARD_ADMIN_USERNAME=\"$(/usr/local/bin/store_secret.sh save DASHBOARD-ADMIN-USERNAME \"dashboardadmin\")\"\n DASHBOARD_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate DASHBOARD-ADMIN-PASSWORD)\"\n GRAFANA_ADMIN_USERNAME=\"$(/usr/local/bin/store_secret.sh save GRAFANA-ADMIN-USERNAME \"grafanaadmin\")\"\n GRAFANA_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate GRAFANA-ADMIN-PASSWORD)\"\n MEET_INITIAL_ADMIN_USER=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-ADMIN-USER \"admin\")\"\n LIVEKIT_API_KEY=\"$(/usr/local/bin/store_secret.sh generate LIVEKIT-API-KEY \"API\" 12)\"\n LIVEKIT_API_SECRET=\"$(/usr/local/bin/store_secret.sh generate LIVEKIT-API-SECRET)\"\n OPENVIDU_VERSION=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-VERSION \"${OPENVIDU_VERSION}\")\"\n ENABLED_MODULES=\"$(/usr/local/bin/store_secret.sh save ENABLED-MODULES \"observability,openviduMeet,v2compatibility\")\"\n ALL_SECRETS_GENERATED=\"$(/usr/local/bin/store_secret.sh save ALL-SECRETS-GENERATED \"true\")\"\nfi\n\nwhile true; do\n MASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv) || true\n MASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv) || true\n MASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv) || true\n MASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv) || true\n # Check if all master nodes have stored their private IPs\n if [[ \"$MASTER_NODE_1_PRIVATE_IP\" != \"\" ]] &&\n [[ \"$MASTER_NODE_2_PRIVATE_IP\" != \"\" ]] &&\n [[ \"$MASTER_NODE_3_PRIVATE_IP\" != \"\" ]] &&\n [[ \"$MASTER_NODE_4_PRIVATE_IP\" != \"\" ]]; then\n break\n fi\n sleep 5\ndone\n\n# Wait until master-node-1 has generated all shared secrets before fetching them\nwhile [[ \"$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)\" != \"true\" ]]; do sleep 5; done\n\n\n# Fetch the values in the keyvault\nMASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_PRIVATE_IP_LIST=\"$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP\"\n\nDOMAIN=$(az keyvault secret show --vault-name ${keyVaultName} --name DOMAIN-NAME --query value -o tsv)\nOPENVIDU_RTC_ENGINE=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-RTC-ENGINE --query value -o tsv)\nOPENVIDU_PRO_LICENSE=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-PRO-LICENSE --query value -o tsv)\nREDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name REDIS-PASSWORD --query value -o tsv)\nMONGO_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-ADMIN-USERNAME --query value -o tsv)\nMONGO_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-ADMIN-PASSWORD --query value -o tsv)\nMONGO_REPLICA_SET_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-REPLICA-SET-KEY --query value -o tsv)\nDASHBOARD_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-USERNAME --query value -o tsv)\nDASHBOARD_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-PASSWORD --query value -o tsv)\nMINIO_ACCESS_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MINIO-ACCESS-KEY --query value -o tsv)\nMINIO_SECRET_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MINIO-SECRET-KEY --query value -o tsv)\nGRAFANA_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name GRAFANA-ADMIN-USERNAME --query value -o tsv)\nGRAFANA_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name GRAFANA-ADMIN-PASSWORD --query value -o tsv)\nLIVEKIT_API_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name LIVEKIT-API-KEY --query value -o tsv)\nLIVEKIT_API_SECRET=$(az keyvault secret show --vault-name ${keyVaultName} --name LIVEKIT-API-SECRET --query value -o tsv)\nMEET_INITIAL_ADMIN_USER=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-USER --query value -o tsv)\nMEET_INITIAL_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-PASSWORD --query value -o tsv)\nif [[ \"${initialMeetApiKey}\" != '' ]]; then\n MEET_INITIAL_API_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-API-KEY --query value -o tsv)\nelse\n MEET_INITIAL_API_KEY=\"\"\nfi\nENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv)\n\n\n# Base command\nINSTALL_COMMAND=\"sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh)\"\n\n# Common arguments\nCOMMON_ARGS=(\n \"--no-tty\"\n \"--install\"\n \"--environment=azure\"\n \"--deployment-type='ha'\"\n \"--node-role='master-node'\"\n \"--external-load-balancer\"\n \"--internal-tls-termination\"\n \"--master-node-private-ip-list='$MASTER_NODE_PRIVATE_IP_LIST'\"\n \"--openvidu-pro-license='$OPENVIDU_PRO_LICENSE'\"\n \"--domain-name='$DOMAIN'\"\n \"--enabled-modules='$ENABLED_MODULES'\"\n \"--rtc-engine=$OPENVIDU_RTC_ENGINE\"\n \"--redis-password=$REDIS_PASSWORD\"\n \"--mongo-admin-user=$MONGO_ADMIN_USERNAME\"\n \"--mongo-admin-password=$MONGO_ADMIN_PASSWORD\"\n \"--mongo-replica-set-key=$MONGO_REPLICA_SET_KEY\"\n \"--minio-access-key=$MINIO_ACCESS_KEY\"\n \"--minio-secret-key=$MINIO_SECRET_KEY\"\n \"--dashboard-admin-user=$DASHBOARD_ADMIN_USERNAME\"\n \"--dashboard-admin-password=$DASHBOARD_ADMIN_PASSWORD\"\n \"--grafana-admin-user=$GRAFANA_ADMIN_USERNAME\"\n \"--grafana-admin-password=$GRAFANA_ADMIN_PASSWORD\"\n \"--meet-initial-admin-password=$MEET_INITIAL_ADMIN_PASSWORD\"\n \"--meet-initial-api-key=$MEET_INITIAL_API_KEY\"\n \"--livekit-api-key=$LIVEKIT_API_KEY\"\n \"--livekit-api-secret=$LIVEKIT_API_SECRET\"\n)\n\n# Include additional installer flags provided by the user\nif [[ \"${additionalInstallFlags}\" != \"\" ]]; then\n IFS=',' read -ra EXTRA_FLAGS <<< \"${additionalInstallFlags}\"\n for extra_flag in \"${EXTRA_FLAGS[@]}\"; do\n # Trim whitespace around each flag\n extra_flag=\"$(echo -e \"${extra_flag}\" | sed -e 's/^[ \\t]*//' -e 's/[ \\t]*$//')\"\n if [[ \"$extra_flag\" != \"\" ]]; then\n COMMON_ARGS+=(\"$extra_flag\")\n fi\n done\nfi\n\n# Certificate arguments\nif [[ \"${certificateType}\" == \"selfsigned\" ]]; then\n CERT_ARGS=(\n \"--certificate-type=selfsigned\"\n )\nelif [[ \"${certificateType}\" == \"letsencrypt\" ]]; then\n CERT_ARGS=(\n \"--certificate-type=letsencrypt\"\n )\nelse\n # Use base64 encoded certificates directly\n OWN_CERT_CRT=${ownPublicCertificate}\n OWN_CERT_KEY=${ownPrivateCertificate}\n\n CERT_ARGS=(\n \"--certificate-type=owncert\"\n \"--owncert-public-key=$OWN_CERT_CRT\"\n \"--owncert-private-key=$OWN_CERT_KEY\"\n )\nfi\n\n# Construct the final command\nFINAL_COMMAND=\"$INSTALL_COMMAND $(printf \"%s \" \"${COMMON_ARGS[@]}\") $(printf \"%s \" \"${CERT_ARGS[@]}\")\"\n\n# Install OpenVidu\nexec bash -c \"$FINAL_COMMAND\"\n", + "installScriptTemplateMaster": "#!/bin/bash -x\nset -e\nOPENVIDU_VERSION=main\nDOMAIN=\n\n# Assume azure cli is installed\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y \\\n curl \\\n unzip \\\n jq \\\n wget\n\n\n# Wait for the keyvault availability\nMAX_WAIT=300\nWAIT_INTERVAL=1\nELAPSED_TIME=0\nset +e\nwhile true; do\n # Check keyvault availability\n az keyvault secret list --vault-name ${keyVaultName} \n\n # If it is available, exit the loop\n if [ $? -eq 0 ]; then\n break\n fi\n\n # If not, wait and check again incrementing the time\n ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL))\n\n # If exceeded the maximum time, exit with error\n if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then\n exit 1\n fi\n\n # Esperar antes de la próxima comprobación\n sleep $WAIT_INTERVAL\ndone\nset -e\n\nMASTER_NODE_NUM=${masterNodeNum}\n\n# Get own private IP\nPRIVATE_IP=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text\")\n\n# Store current private IP\nPRIVATE_IP=\"$(/usr/local/bin/store_secret.sh save MASTER-NODE-${masterNodeNum}-PRIVATE-IP $PRIVATE_IP)\"\n\n\nif [[ $MASTER_NODE_NUM -eq 1 ]] && [[ \"$ALL_SECRETS_GENERATED\" == \"\" || \"$ALL_SECRETS_GENERATED\" == \"false\" ]]; then\n\n # Configure Domain name\n if [[ \"${domainName}\" == '' ]]; then\n # Get public IP using the get_public_ip.sh script\n PUBLIC_IP=$(/usr/local/bin/get_public_ip.sh 2>/dev/null)\n if [[ $? -ne 0 || -z \"${PUBLIC_IP}\" ]]; then\n echo \"Could not determine public IP.\"\n exit 1\n fi\n\n DOMAIN=\"$PUBLIC_IP\"\n else\n DOMAIN=${domainName}\n fi\n DOMAIN=\"$(/usr/local/bin/store_secret.sh save DOMAIN-NAME \"${DOMAIN}\")\"\n\n # Publish service URLs early (idempotent; after_install re-writes byte-identical values)\n OPENVIDU_URL=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-URL \"https://${DOMAIN}/\")\"\n LIVEKIT_URL=\"$(/usr/local/bin/store_secret.sh save LIVEKIT-URL \"wss://${DOMAIN}/\")\"\n DASHBOARD_URL=\"$(/usr/local/bin/store_secret.sh save DASHBOARD-URL \"https://${DOMAIN}/dashboard/\")\"\n GRAFANA_URL=\"$(/usr/local/bin/store_secret.sh save GRAFANA-URL \"https://${DOMAIN}/grafana/\")\"\n MINIO_URL=\"$(/usr/local/bin/store_secret.sh save MINIO-URL \"https://${DOMAIN}/minio-console/\")\"\n\n # Meet initial admin user and password\n MEET_INITIAL_ADMIN_USER=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-ADMIN-USER \"admin\")\"\n if [[ \"${initialMeetAdminPassword}\" != '' ]]; then\n MEET_INITIAL_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-ADMIN-PASSWORD \"${initialMeetAdminPassword}\")\"\n else\n MEET_INITIAL_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate MEET-INITIAL-ADMIN-PASSWORD)\"\n fi\n if [[ \"${initialMeetApiKey}\" != '' ]]; then\n MEET_INITIAL_API_KEY=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-API-KEY \"${initialMeetApiKey}\")\"\n else\n MEET_INITIAL_API_KEY=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-API-KEY \"\")\"\n fi\n\n # Store usernames and generate random passwords\n OPENVIDU_PRO_LICENSE=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-PRO-LICENSE \"${openviduLicense}\")\"\n OPENVIDU_RTC_ENGINE=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-RTC-ENGINE \"${rtcEngine}\")\"\n REDIS_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate REDIS-PASSWORD)\"\n MONGO_ADMIN_USERNAME=\"$(/usr/local/bin/store_secret.sh save MONGO-ADMIN-USERNAME \"mongoadmin\")\"\n MONGO_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate MONGO-ADMIN-PASSWORD)\"\n MONGO_REPLICA_SET_KEY=\"$(/usr/local/bin/store_secret.sh generate MONGO-REPLICA-SET-KEY)\"\n MINIO_ACCESS_KEY=\"$(/usr/local/bin/store_secret.sh save MINIO-ACCESS-KEY \"minioadmin\")\"\n MINIO_SECRET_KEY=\"$(/usr/local/bin/store_secret.sh generate MINIO-SECRET-KEY)\"\n DASHBOARD_ADMIN_USERNAME=\"$(/usr/local/bin/store_secret.sh save DASHBOARD-ADMIN-USERNAME \"dashboardadmin\")\"\n DASHBOARD_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate DASHBOARD-ADMIN-PASSWORD)\"\n GRAFANA_ADMIN_USERNAME=\"$(/usr/local/bin/store_secret.sh save GRAFANA-ADMIN-USERNAME \"grafanaadmin\")\"\n GRAFANA_ADMIN_PASSWORD=\"$(/usr/local/bin/store_secret.sh generate GRAFANA-ADMIN-PASSWORD)\"\n MEET_INITIAL_ADMIN_USER=\"$(/usr/local/bin/store_secret.sh save MEET-INITIAL-ADMIN-USER \"admin\")\"\n LIVEKIT_API_KEY=\"$(/usr/local/bin/store_secret.sh generate LIVEKIT-API-KEY \"API\" 12)\"\n LIVEKIT_API_SECRET=\"$(/usr/local/bin/store_secret.sh generate LIVEKIT-API-SECRET)\"\n OPENVIDU_VERSION=\"$(/usr/local/bin/store_secret.sh save OPENVIDU-VERSION \"${OPENVIDU_VERSION}\")\"\n ENABLED_MODULES=\"$(/usr/local/bin/store_secret.sh save ENABLED-MODULES \"observability,openviduMeet,v2compatibility\")\"\n ALL_SECRETS_GENERATED=\"$(/usr/local/bin/store_secret.sh save ALL-SECRETS-GENERATED \"true\")\"\nfi\n\nIP_WAIT_MAX_RETRIES=360\nIP_WAIT_INTERVAL=5\nIP_WAIT_RETRIES=0\nwhile true; do\n MASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv) || true\n MASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv) || true\n MASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv) || true\n MASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv) || true\n # Check if all master nodes have stored their private IPs\n if [[ \"$MASTER_NODE_1_PRIVATE_IP\" != \"\" ]] &&\n [[ \"$MASTER_NODE_2_PRIVATE_IP\" != \"\" ]] &&\n [[ \"$MASTER_NODE_3_PRIVATE_IP\" != \"\" ]] &&\n [[ \"$MASTER_NODE_4_PRIVATE_IP\" != \"\" ]]; then\n break\n fi\n IP_WAIT_RETRIES=$((IP_WAIT_RETRIES + 1))\n if [ $IP_WAIT_RETRIES -ge $IP_WAIT_MAX_RETRIES ]; then\n echo \"[OpenVidu] timed out after 30 min waiting for the 4 master nodes to publish their private IPs\"\n exit 1\n fi\n sleep $IP_WAIT_INTERVAL\ndone\n\n# Wait until master-node-1 has generated all shared secrets before fetching them\nSECRETS_WAIT_MAX_RETRIES=360\nSECRETS_WAIT_INTERVAL=5\nSECRETS_WAIT_RETRIES=0\nwhile [[ \"$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)\" != \"true\" ]]; do\n SECRETS_WAIT_RETRIES=$((SECRETS_WAIT_RETRIES + 1))\n if [ $SECRETS_WAIT_RETRIES -ge $SECRETS_WAIT_MAX_RETRIES ]; then\n echo \"[OpenVidu] timed out after 30 min waiting for ALL-SECRETS-GENERATED to become true\"\n exit 1\n fi\n sleep $SECRETS_WAIT_INTERVAL\ndone\n\n\n# Fetch the values in the keyvault\nMASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_PRIVATE_IP_LIST=\"$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP\"\n\nDOMAIN=$(az keyvault secret show --vault-name ${keyVaultName} --name DOMAIN-NAME --query value -o tsv)\nOPENVIDU_RTC_ENGINE=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-RTC-ENGINE --query value -o tsv)\nOPENVIDU_PRO_LICENSE=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-PRO-LICENSE --query value -o tsv)\nREDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name REDIS-PASSWORD --query value -o tsv)\nMONGO_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-ADMIN-USERNAME --query value -o tsv)\nMONGO_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-ADMIN-PASSWORD --query value -o tsv)\nMONGO_REPLICA_SET_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-REPLICA-SET-KEY --query value -o tsv)\nDASHBOARD_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-USERNAME --query value -o tsv)\nDASHBOARD_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-PASSWORD --query value -o tsv)\nMINIO_ACCESS_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MINIO-ACCESS-KEY --query value -o tsv)\nMINIO_SECRET_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MINIO-SECRET-KEY --query value -o tsv)\nGRAFANA_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name GRAFANA-ADMIN-USERNAME --query value -o tsv)\nGRAFANA_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name GRAFANA-ADMIN-PASSWORD --query value -o tsv)\nLIVEKIT_API_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name LIVEKIT-API-KEY --query value -o tsv)\nLIVEKIT_API_SECRET=$(az keyvault secret show --vault-name ${keyVaultName} --name LIVEKIT-API-SECRET --query value -o tsv)\nMEET_INITIAL_ADMIN_USER=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-USER --query value -o tsv)\nMEET_INITIAL_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-PASSWORD --query value -o tsv)\nif [[ \"${initialMeetApiKey}\" != '' ]]; then\n MEET_INITIAL_API_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-API-KEY --query value -o tsv)\nelse\n MEET_INITIAL_API_KEY=\"\"\nfi\nENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv)\n\n\n# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure\nINSTALLER_SCRIPT=\"/tmp/install_ov_master_node.sh\"\ncurl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o \"$INSTALLER_SCRIPT\" \"http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh\"\nif [ ! -s \"$INSTALLER_SCRIPT\" ]; then\n echo \"[OpenVidu] failed to download the master node installer script\"\n exit 1\nfi\n\n# Base command\nINSTALL_COMMAND=\"sh $INSTALLER_SCRIPT\"\n\n# Common arguments\nCOMMON_ARGS=(\n \"--no-tty\"\n \"--install\"\n \"--environment=azure\"\n \"--deployment-type='ha'\"\n \"--node-role='master-node'\"\n \"--external-load-balancer\"\n \"--internal-tls-termination\"\n \"--master-node-private-ip-list='$MASTER_NODE_PRIVATE_IP_LIST'\"\n \"--openvidu-pro-license='$OPENVIDU_PRO_LICENSE'\"\n \"--domain-name='$DOMAIN'\"\n \"--enabled-modules='$ENABLED_MODULES'\"\n \"--rtc-engine=$OPENVIDU_RTC_ENGINE\"\n \"--redis-password=$REDIS_PASSWORD\"\n \"--mongo-admin-user=$MONGO_ADMIN_USERNAME\"\n \"--mongo-admin-password=$MONGO_ADMIN_PASSWORD\"\n \"--mongo-replica-set-key=$MONGO_REPLICA_SET_KEY\"\n \"--minio-access-key=$MINIO_ACCESS_KEY\"\n \"--minio-secret-key=$MINIO_SECRET_KEY\"\n \"--dashboard-admin-user=$DASHBOARD_ADMIN_USERNAME\"\n \"--dashboard-admin-password=$DASHBOARD_ADMIN_PASSWORD\"\n \"--grafana-admin-user=$GRAFANA_ADMIN_USERNAME\"\n \"--grafana-admin-password=$GRAFANA_ADMIN_PASSWORD\"\n \"--meet-initial-admin-password=$MEET_INITIAL_ADMIN_PASSWORD\"\n \"--meet-initial-api-key=$MEET_INITIAL_API_KEY\"\n \"--livekit-api-key=$LIVEKIT_API_KEY\"\n \"--livekit-api-secret=$LIVEKIT_API_SECRET\"\n)\n\n# Include additional installer flags provided by the user\nif [[ \"${additionalInstallFlags}\" != \"\" ]]; then\n IFS=',' read -ra EXTRA_FLAGS <<< \"${additionalInstallFlags}\"\n for extra_flag in \"${EXTRA_FLAGS[@]}\"; do\n # Trim whitespace around each flag\n extra_flag=\"$(echo -e \"${extra_flag}\" | sed -e 's/^[ \\t]*//' -e 's/[ \\t]*$//')\"\n if [[ \"$extra_flag\" != \"\" ]]; then\n COMMON_ARGS+=(\"$extra_flag\")\n fi\n done\nfi\n\n# Certificate arguments\nif [[ \"${certificateType}\" == \"selfsigned\" ]]; then\n CERT_ARGS=(\n \"--certificate-type=selfsigned\"\n )\nelif [[ \"${certificateType}\" == \"letsencrypt\" ]]; then\n CERT_ARGS=(\n \"--certificate-type=letsencrypt\"\n )\nelse\n # Use base64 encoded certificates directly\n OWN_CERT_CRT=${ownPublicCertificate}\n OWN_CERT_KEY=${ownPrivateCertificate}\n\n CERT_ARGS=(\n \"--certificate-type=owncert\"\n \"--owncert-public-key=$OWN_CERT_CRT\"\n \"--owncert-private-key=$OWN_CERT_KEY\"\n )\nfi\n\n# Construct the final command\nFINAL_COMMAND=\"$INSTALL_COMMAND $(printf \"%s \" \"${COMMON_ARGS[@]}\") $(printf \"%s \" \"${CERT_ARGS[@]}\")\"\n\n# Install OpenVidu\nexec bash -c \"$FINAL_COMMAND\"\n", "after_installScriptTemplateMaster": "#!/bin/bash\nset -e\n\naz login --identity --allow-no-subscriptions > /dev/null\n\n# Generate URLs\nDOMAIN=$(az keyvault secret show --vault-name ${keyVaultName} --name DOMAIN-NAME --query value -o tsv)\nOPENVIDU_URL=\"https://${DOMAIN}/\"\nLIVEKIT_URL=\"wss://${DOMAIN}/\"\nDASHBOARD_URL=\"https://${DOMAIN}/dashboard/\"\nGRAFANA_URL=\"https://${DOMAIN}/grafana/\"\nMINIO_URL=\"https://${DOMAIN}/minio-console/\"\n\n# Update shared secret\naz keyvault secret set --vault-name ${keyVaultName} --name DOMAIN-NAME --value $DOMAIN\naz keyvault secret set --vault-name ${keyVaultName} --name OPENVIDU-URL --value $OPENVIDU_URL\naz keyvault secret set --vault-name ${keyVaultName} --name LIVEKIT-URL --value $LIVEKIT_URL\naz keyvault secret set --vault-name ${keyVaultName} --name DASHBOARD-URL --value $DASHBOARD_URL\naz keyvault secret set --vault-name ${keyVaultName} --name GRAFANA-URL --value $GRAFANA_URL\naz keyvault secret set --vault-name ${keyVaultName} --name MINIO-URL --value $MINIO_URL\n\naz keyvault secret show --vault-name ${keyVaultName} --name MINIO-URL\n\nif [[ $? -ne 0 ]]; then\n echo \"Error updating keyvault\"\nfi\n", "update_config_from_secretScriptTemplateMaster": "#!/bin/bash\nset -e\n\naz login --identity --allow-no-subscriptions > /dev/null\n\n# Installation directory\nINSTALL_DIR=\"/opt/openvidu\"\nCLUSTER_CONFIG_DIR=\"${INSTALL_DIR}/config/cluster\"\nMASTER_NODE_CONFIG_DIR=\"${INSTALL_DIR}/config/node\"\n\n# Replace DOMAIN_NAME\nexport DOMAIN=$(az keyvault secret show --vault-name ${keyVaultName} --name DOMAIN-NAME --query value -o tsv)\nif [[ -n \"$DOMAIN\" ]]; then\n sed -i \"s/DOMAIN_NAME=.*/DOMAIN_NAME=$DOMAIN/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nelse\n exit 1\nfi\n\n# Get the rest of the values\nexport REDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name REDIS-PASSWORD --query value -o tsv)\nexport OPENVIDU_RTC_ENGINE=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-RTC-ENGINE --query value -o tsv)\nexport OPENVIDU_PRO_LICENSE=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-PRO-LICENSE --query value -o tsv)\nexport MONGO_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-ADMIN-USERNAME --query value -o tsv)\nexport MONGO_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-ADMIN-PASSWORD --query value -o tsv)\nexport MONGO_REPLICA_SET_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MONGO-REPLICA-SET-KEY --query value -o tsv)\nexport DASHBOARD_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-USERNAME --query value -o tsv)\nexport DASHBOARD_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-PASSWORD --query value -o tsv)\nexport MINIO_ACCESS_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MINIO-ACCESS-KEY --query value -o tsv)\nexport MINIO_SECRET_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MINIO-SECRET-KEY --query value -o tsv)\nexport GRAFANA_ADMIN_USERNAME=$(az keyvault secret show --vault-name ${keyVaultName} --name GRAFANA-ADMIN-USERNAME --query value -o tsv)\nexport GRAFANA_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name GRAFANA-ADMIN-PASSWORD --query value -o tsv)\nexport LIVEKIT_API_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name LIVEKIT-API-KEY --query value -o tsv)\nexport LIVEKIT_API_SECRET=$(az keyvault secret show --vault-name ${keyVaultName} --name LIVEKIT-API-SECRET --query value -o tsv)\nexport MEET_INITIAL_ADMIN_USER=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-USER --query value -o tsv)\nexport MEET_INITIAL_ADMIN_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-PASSWORD --query value -o tsv)\nif [[ \"${initialMeetApiKey}\" != '' ]]; then\n export MEET_INITIAL_API_KEY=$(az keyvault secret show --vault-name ${keyVaultName} --name MEET-INITIAL-API-KEY --query value -o tsv)\nfi\nexport ENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv)\n\n# Replace rest of the values\nsed -i \"s/REDIS_PASSWORD=.*/REDIS_PASSWORD=$REDIS_PASSWORD/\" \"${MASTER_NODE_CONFIG_DIR}/master_node.env\"\nsed -i \"s/OPENVIDU_RTC_ENGINE=.*/OPENVIDU_RTC_ENGINE=$OPENVIDU_RTC_ENGINE/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/OPENVIDU_PRO_LICENSE=.*/OPENVIDU_PRO_LICENSE=$OPENVIDU_PRO_LICENSE/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/MONGO_ADMIN_USERNAME=.*/MONGO_ADMIN_USERNAME=$MONGO_ADMIN_USERNAME/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/MONGO_ADMIN_PASSWORD=.*/MONGO_ADMIN_PASSWORD=$MONGO_ADMIN_PASSWORD/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/MONGO_REPLICA_SET_KEY=.*/MONGO_REPLICA_SET_KEY=$MONGO_REPLICA_SET_KEY/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/DASHBOARD_ADMIN_USERNAME=.*/DASHBOARD_ADMIN_USERNAME=$DASHBOARD_ADMIN_USERNAME/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/DASHBOARD_ADMIN_PASSWORD=.*/DASHBOARD_ADMIN_PASSWORD=$DASHBOARD_ADMIN_PASSWORD/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/MINIO_ACCESS_KEY=.*/MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/MINIO_SECRET_KEY=.*/MINIO_SECRET_KEY=$MINIO_SECRET_KEY/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/GRAFANA_ADMIN_USERNAME=.*/GRAFANA_ADMIN_USERNAME=$GRAFANA_ADMIN_USERNAME/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/GRAFANA_ADMIN_PASSWORD=.*/GRAFANA_ADMIN_PASSWORD=$GRAFANA_ADMIN_PASSWORD/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/LIVEKIT_API_KEY=.*/LIVEKIT_API_KEY=$LIVEKIT_API_KEY/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/LIVEKIT_API_SECRET=.*/LIVEKIT_API_SECRET=$LIVEKIT_API_SECRET/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s/MEET_INITIAL_ADMIN_USER=.*/MEET_INITIAL_ADMIN_USER=$MEET_INITIAL_ADMIN_USER/\" \"${CLUSTER_CONFIG_DIR}/master_node/meet.env\"\nsed -i \"s/MEET_INITIAL_ADMIN_PASSWORD=.*/MEET_INITIAL_ADMIN_PASSWORD=$MEET_INITIAL_ADMIN_PASSWORD/\" \"${CLUSTER_CONFIG_DIR}/master_node/meet.env\"\nif [[ \"${initialMeetApiKey}\" != '' ]]; then\n sed -i \"s/MEET_INITIAL_API_KEY=.*/MEET_INITIAL_API_KEY=$MEET_INITIAL_API_KEY/\" \"${CLUSTER_CONFIG_DIR}/master_node/meet.env\"\nfi\nsed -i \"s/ENABLED_MODULES=.*/ENABLED_MODULES=$ENABLED_MODULES/\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\n\n# Update URLs in secret\nOPENVIDU_URL=\"https://${DOMAIN}/\"\nLIVEKIT_URL=\"wss://${DOMAIN}/\"\nDASHBOARD_URL=\"https://${DOMAIN}/dashboard/\"\nGRAFANA_URL=\"https://${DOMAIN}/grafana/\"\nMINIO_URL=\"https://${DOMAIN}/minio-console/\"\n\n# Update shared secret\naz keyvault secret set --vault-name ${keyVaultName} --name DOMAIN-NAME --value $DOMAIN\naz keyvault secret set --vault-name ${keyVaultName} --name OPENVIDU-URL --value $OPENVIDU_URL\naz keyvault secret set --vault-name ${keyVaultName} --name LIVEKIT-URL --value $LIVEKIT_URL\naz keyvault secret set --vault-name ${keyVaultName} --name DASHBOARD-URL --value $DASHBOARD_URL\naz keyvault secret set --vault-name ${keyVaultName} --name GRAFANA-URL --value $GRAFANA_URL\naz keyvault secret set --vault-name ${keyVaultName} --name MINIO-URL --value $MINIO_URL\n", "update_secret_from_configScriptTemplateMaster": "#!/bin/bash\nset -e\n\naz login --identity --allow-no-subscriptions > /dev/null\n\n# Installation directory\nINSTALL_DIR=\"/opt/openvidu\"\nCLUSTER_CONFIG_DIR=\"${INSTALL_DIR}/config/cluster\"\nMASTER_NODE_CONFIG_DIR=\"${INSTALL_DIR}/config/node\"\n\n# Get current values of the config\nREDIS_PASSWORD=\"$(/usr/local/bin/get_value_from_config.sh REDIS_PASSWORD \"${MASTER_NODE_CONFIG_DIR}/master_node.env\")\"\nDOMAIN_NAME=\"$(/usr/local/bin/get_value_from_config.sh DOMAIN_NAME \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nOPENVIDU_RTC_ENGINE=\"$(/usr/local/bin/get_value_from_config.sh OPENVIDU_RTC_ENGINE \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nOPENVIDU_PRO_LICENSE=\"$(/usr/local/bin/get_value_from_config.sh OPENVIDU_PRO_LICENSE \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nMONGO_ADMIN_USERNAME=\"$(/usr/local/bin/get_value_from_config.sh MONGO_ADMIN_USERNAME \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nMONGO_ADMIN_PASSWORD=\"$(/usr/local/bin/get_value_from_config.sh MONGO_ADMIN_PASSWORD \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nMONGO_REPLICA_SET_KEY=\"$(/usr/local/bin/get_value_from_config.sh MONGO_REPLICA_SET_KEY \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nMINIO_ACCESS_KEY=\"$(/usr/local/bin/get_value_from_config.sh MINIO_ACCESS_KEY \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nMINIO_SECRET_KEY=\"$(/usr/local/bin/get_value_from_config.sh MINIO_SECRET_KEY \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nDASHBOARD_ADMIN_USERNAME=\"$(/usr/local/bin/get_value_from_config.sh DASHBOARD_ADMIN_USERNAME \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nDASHBOARD_ADMIN_PASSWORD=\"$(/usr/local/bin/get_value_from_config.sh DASHBOARD_ADMIN_PASSWORD \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nGRAFANA_ADMIN_USERNAME=\"$(/usr/local/bin/get_value_from_config.sh GRAFANA_ADMIN_USERNAME \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nGRAFANA_ADMIN_PASSWORD=\"$(/usr/local/bin/get_value_from_config.sh GRAFANA_ADMIN_PASSWORD \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nLIVEKIT_API_KEY=\"$(/usr/local/bin/get_value_from_config.sh LIVEKIT_API_KEY \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nLIVEKIT_API_SECRET=\"$(/usr/local/bin/get_value_from_config.sh LIVEKIT_API_SECRET \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\nMEET_INITIAL_ADMIN_USER=\"$(/usr/local/bin/get_value_from_config.sh MEET_INITIAL_ADMIN_USER \"${CLUSTER_CONFIG_DIR}/master_node/meet.env\")\"\nMEET_INITIAL_ADMIN_PASSWORD=\"$(/usr/local/bin/get_value_from_config.sh MEET_INITIAL_ADMIN_PASSWORD \"${CLUSTER_CONFIG_DIR}/master_node/meet.env\")\"\nif [[ \"${initialMeetApiKey}\" != '' ]]; then\n MEET_INITIAL_API_KEY=\"$(/usr/local/bin/get_value_from_config.sh MEET_INITIAL_API_KEY \"${CLUSTER_CONFIG_DIR}/master_node/meet.env\")\"\nfi\nENABLED_MODULES=\"$(/usr/local/bin/get_value_from_config.sh ENABLED_MODULES \"${CLUSTER_CONFIG_DIR}/openvidu.env\")\"\n\n# Update shared secret\naz keyvault secret set --vault-name ${keyVaultName} --name REDIS-PASSWORD --value $REDIS_PASSWORD\naz keyvault secret set --vault-name ${keyVaultName} --name DOMAIN-NAME --value $DOMAIN_NAME\naz keyvault secret set --vault-name ${keyVaultName} --name OPENVIDU-RTC-ENGINE --value $OPENVIDU_RTC_ENGINE\naz keyvault secret set --vault-name ${keyVaultName} --name OPENVIDU-PRO-LICENSE --value $OPENVIDU_PRO_LICENSE\naz keyvault secret set --vault-name ${keyVaultName} --name MONGO-ADMIN-USERNAME --value $MONGO_ADMIN_USERNAME\naz keyvault secret set --vault-name ${keyVaultName} --name MONGO-ADMIN-PASSWORD --value $MONGO_ADMIN_PASSWORD\naz keyvault secret set --vault-name ${keyVaultName} --name MONGO-REPLICA-SET-KEY --value $MONGO_REPLICA_SET_KEY\naz keyvault secret set --vault-name ${keyVaultName} --name MINIO-ACCESS-KEY --value $MINIO_ACCESS_KEY\naz keyvault secret set --vault-name ${keyVaultName} --name MINIO-SECRET-KEY --value $MINIO_SECRET_KEY\naz keyvault secret set --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-USERNAME --value $DASHBOARD_ADMIN_USERNAME\naz keyvault secret set --vault-name ${keyVaultName} --name DASHBOARD-ADMIN-PASSWORD --value $DASHBOARD_ADMIN_PASSWORD\naz keyvault secret set --vault-name ${keyVaultName} --name GRAFANA-ADMIN-USERNAME --value $GRAFANA_ADMIN_USERNAME\naz keyvault secret set --vault-name ${keyVaultName} --name GRAFANA-ADMIN-PASSWORD --value $GRAFANA_ADMIN_PASSWORD\naz keyvault secret set --vault-name ${keyVaultName} --name LIVEKIT-API-KEY --value $LIVEKIT_API_KEY\naz keyvault secret set --vault-name ${keyVaultName} --name LIVEKIT-API-SECRET --value $LIVEKIT_API_SECRET\naz keyvault secret set --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-USER --value $MEET_INITIAL_ADMIN_USER\naz keyvault secret set --vault-name ${keyVaultName} --name MEET-INITIAL-ADMIN-PASSWORD --value $MEET_INITIAL_ADMIN_PASSWORD\nif [[ \"${initialMeetApiKey}\" != '' ]]; then\n az keyvault secret set --vault-name ${keyVaultName} --name MEET-INITIAL-API-KEY --value $MEET_INITIAL_API_KEY\nfi\naz keyvault secret set --vault-name ${keyVaultName} --name ENABLED-MODULES --value $ENABLED_MODULES\n", @@ -298,7 +298,7 @@ "get_public_ip": "#!/bin/bash\naz login --identity --allow-no-subscriptions > /dev/null\n\naz network public-ip show \\\n --id ${publicIPId} \\\n --query \"ipAddress\" -o tsv\n", "check_app_readyScriptMaster": "#!/bin/bash\nset -e\nMAX_WAIT=1200\nWAIT_INTERVAL=5\nELAPSED_TIME=0\nwhile true; do\n HTTP_STATUS=$(curl -Ik http://localhost:7880/health/caddy | head -n1 | awk '{print $2}')\n if [ \"$HTTP_STATUS\" = \"200\" ]; then\n break\n fi\n ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL))\n if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then\n exit 1\n fi\n sleep 5\ndone\n", "restartScriptMaster": "#!/bin/bash\nset -e\n# Stop all services\nsystemctl stop openvidu\n\n# Update config from secret\n/usr/local/bin/update_config_from_secret.sh\n\n# Start all services\nsystemctl start openvidu\n", - "config_blobStorageTemplate": "#!/bin/bash\nset -e\n\n# Install dir and config dir\nINSTALL_DIR=\"/opt/openvidu\"\nCLUSTER_CONFIG_DIR=\"${INSTALL_DIR}/config/cluster\"\n\n# Retry login + storage key fetch to allow the Contributor role assignment to propagate\nMAX_WAIT=100\nWAIT_INTERVAL=1\nELAPSED_TIME=0\nset +e\nwhile true; do\n az login --identity\n\n # Config azure blob storage\n AZURE_ACCOUNT_NAME=\"${storageAccountName}\"\n AZURE_ACCOUNT_KEY=$(az storage account keys list --account-name ${storageAccountName} --query '[0].value' -o tsv)\n\n # If the key was fetched successfully, exit the loop\n if [ $? -eq 0 ]; then\n break\n fi\n\n # If not, wait and check again incrementing the time\n ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL))\n\n # If exceeded the maximum time, exit with error\n if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then\n exit 1\n fi\n\n # Esperar antes de la próxima comprobación\n sleep $WAIT_INTERVAL\ndone\nset -e\nAZURE_CONTAINER_NAME=\"${storageAccountContainerName}\"\n\nsed -i \"s|AZURE_ACCOUNT_NAME=.*|AZURE_ACCOUNT_NAME=$AZURE_ACCOUNT_NAME|\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s|AZURE_ACCOUNT_KEY=.*|AZURE_ACCOUNT_KEY=$AZURE_ACCOUNT_KEY|\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s|AZURE_CONTAINER_NAME=.*|AZURE_CONTAINER_NAME=$AZURE_CONTAINER_NAME|\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\n", + "config_blobStorageTemplate": "#!/bin/bash\nset -e\n\n# Install dir and config dir\nINSTALL_DIR=\"/opt/openvidu\"\nCLUSTER_CONFIG_DIR=\"${INSTALL_DIR}/config/cluster\"\n\n# Retry login + storage key fetch to allow the Contributor role assignment to propagate\nMAX_WAIT=300\nWAIT_INTERVAL=1\nELAPSED_TIME=0\nset +e\nwhile true; do\n az login --identity\n\n # Config azure blob storage\n AZURE_ACCOUNT_NAME=\"${storageAccountName}\"\n AZURE_ACCOUNT_KEY=$(az storage account keys list --account-name ${storageAccountName} --query '[0].value' -o tsv)\n\n # If the key was fetched successfully, exit the loop\n if [ $? -eq 0 ]; then\n break\n fi\n\n # If not, wait and check again incrementing the time\n ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL))\n\n # If exceeded the maximum time, exit with error\n if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then\n exit 1\n fi\n\n # Esperar antes de la próxima comprobación\n sleep $WAIT_INTERVAL\ndone\nset -e\nAZURE_CONTAINER_NAME=\"${storageAccountContainerName}\"\n\nsed -i \"s|AZURE_ACCOUNT_NAME=.*|AZURE_ACCOUNT_NAME=$AZURE_ACCOUNT_NAME|\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s|AZURE_ACCOUNT_KEY=.*|AZURE_ACCOUNT_KEY=$AZURE_ACCOUNT_KEY|\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\nsed -i \"s|AZURE_CONTAINER_NAME=.*|AZURE_CONTAINER_NAME=$AZURE_CONTAINER_NAME|\" \"${CLUSTER_CONFIG_DIR}/openvidu.env\"\n", "installScriptMaster1": "[reduce(items(variables('stringInterpolationParamsMaster1')), createObject('value', variables('installScriptTemplateMaster')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value]", "installScriptMaster2": "[reduce(items(variables('stringInterpolationParamsMaster2')), createObject('value', variables('installScriptTemplateMaster')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value]", "installScriptMaster3": "[reduce(items(variables('stringInterpolationParamsMaster3')), createObject('value', variables('installScriptTemplateMaster')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value]", @@ -320,8 +320,8 @@ "base64store_secretMaster": "[base64(variables('store_secretScriptMaster'))]", "base64check_app_readyMaster": "[base64(variables('check_app_readyScriptMaster'))]", "base64restartMaster": "[base64(variables('restartScriptMaster'))]", - "userDataTemplateMasterNode": "#!/bin/bash -x\nset -eu -o pipefail\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# after_install.sh\necho ${base64after_install} | base64 -d > /usr/local/bin/after_install.sh\nchmod +x /usr/local/bin/after_install.sh\n\n# get_public_ip.sh\necho ${base64get_public_ip} | base64 -d > /usr/local/bin/get_public_ip.sh\nchmod +x /usr/local/bin/get_public_ip.sh\n\n# update_config_from_secret.sh\necho ${base64update_config_from_secret} | base64 -d > /usr/local/bin/update_config_from_secret.sh\nchmod +x /usr/local/bin/update_config_from_secret.sh\n\n# update_secret_from_config.sh\necho ${base64update_secret_from_config} | base64 -d > /usr/local/bin/update_secret_from_config.sh\nchmod +x /usr/local/bin/update_secret_from_config.sh\n\n# get_value_from_config.sh\necho ${base64get_value_from_config} | base64 -d > /usr/local/bin/get_value_from_config.sh\nchmod +x /usr/local/bin/get_value_from_config.sh\n\n# store_secret.sh\necho ${base64store_secret} | base64 -d > /usr/local/bin/store_secret.sh\nchmod +x /usr/local/bin/store_secret.sh\n\n# check_app_ready.sh\necho ${base64check_app_ready} | base64 -d > /usr/local/bin/check_app_ready.sh\nchmod +x /usr/local/bin/check_app_ready.sh\n\n# restart.sh\necho ${base64restart} | base64 -d > /usr/local/bin/restart.sh\nchmod +x /usr/local/bin/restart.sh\n\necho ${base64config_blobStorage} | base64 -d > /usr/local/bin/config_blobStorage.sh\nchmod +x /usr/local/bin/config_blobStorage.sh\n\n# Install azure cli\nAZURE_CLI_VERSION=2.87.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity --allow-no-subscriptions\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y\n\nexport HOME=\"/root\"\n\n# Install OpenVidu\n/usr/local/bin/install.sh || { echo \"[OpenVidu] error installing OpenVidu\"; exit 1; }\n\n#Config blob storage\n/usr/local/bin/config_blobStorage.sh || { echo \"[OpenVidu] error configuring Blob Storage\"; exit 1; }\n\n# Start OpenVidu\nsystemctl start openvidu || { echo \"[OpenVidu] error starting OpenVidu\"; exit 1; }\n\n# Update shared secret\n/usr/local/bin/after_install.sh || { echo \"[OpenVidu] error updating shared secret\"; exit 1; }\n\n# Launch on reboot\necho \"@reboot /usr/local/bin/restart.sh >> /var/log/openvidu-restart.log\" 2>&1 | crontab\n\nMASTER_NODE_NUM=${masterNodeNum}\nif [[ $MASTER_NODE_NUM -eq 4 ]]; then\n # Creating scale in lock\n set +e\n az storage blob upload --account-name ${storageAccountName} --container-name automation-locks --name lock.txt --file /dev/null --auth-mode key\n set -e\n\n #Finish all the nodes\n az keyvault secret set --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --value \"true\"\nfi\n\n# Wait for the app\nsleep 150\n/usr/local/bin/check_app_ready.sh\n", - "installScriptTemplateMedia": "#!/bin/bash -x\nDOMAIN=\n\n# Install dependencies\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y \\\n curl \\\n unzip \\\n jq \\\n wget\n\n# Get own private IP\nPRIVATE_IP=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text\")\n\nWAIT_INTERVAL=1\nMAX_WAIT=10000\nELAPSED_TIME=0\nset +e\nwhile true; do\n # get secret value\n FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv)\n\n # Check if all master nodes finished\n if [ \"$FINISH_MASTER_NODE\" == \"true\" ]; then\n break\n fi\n\n ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL))\n\n # Check if the maximum waiting time has been reached\n if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then\n exit 1\n fi\n\n sleep $WAIT_INTERVAL\ndone\nset -e\n\nMASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_PRIVATE_IP_LIST=\"$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP\"\nREDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name REDIS-PASSWORD --query value -o tsv)\nENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv)\nOPENVIDU_VERSION=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-VERSION --query value -o tsv)\n\n# Base command\nINSTALL_COMMAND=\"sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh)\"\n\n# Common arguments\nCOMMON_ARGS=(\n\"--no-tty\"\n\"--install\"\n\"--environment=azure\"\n\"--deployment-type='ha'\"\n\"--node-role='media-node'\"\n\"--master-node-private-ip-list=$MASTER_NODE_PRIVATE_IP_LIST\"\n\"--private-ip=$PRIVATE_IP\"\n\"--enabled-modules='$ENABLED_MODULES'\"\n\"--redis-password=$REDIS_PASSWORD\"\n)\n\n# Construct the final command with all arguments\nFINAL_COMMAND=\"$INSTALL_COMMAND $(printf \"%s \" \"${COMMON_ARGS[@]}\")\"\n\n# Install OpenVidu\nexec bash -c \"$FINAL_COMMAND\"\n", + "userDataTemplateMasterNode": "#!/bin/bash -x\nset -eu -o pipefail\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# after_install.sh\necho ${base64after_install} | base64 -d > /usr/local/bin/after_install.sh\nchmod +x /usr/local/bin/after_install.sh\n\n# get_public_ip.sh\necho ${base64get_public_ip} | base64 -d > /usr/local/bin/get_public_ip.sh\nchmod +x /usr/local/bin/get_public_ip.sh\n\n# update_config_from_secret.sh\necho ${base64update_config_from_secret} | base64 -d > /usr/local/bin/update_config_from_secret.sh\nchmod +x /usr/local/bin/update_config_from_secret.sh\n\n# update_secret_from_config.sh\necho ${base64update_secret_from_config} | base64 -d > /usr/local/bin/update_secret_from_config.sh\nchmod +x /usr/local/bin/update_secret_from_config.sh\n\n# get_value_from_config.sh\necho ${base64get_value_from_config} | base64 -d > /usr/local/bin/get_value_from_config.sh\nchmod +x /usr/local/bin/get_value_from_config.sh\n\n# store_secret.sh\necho ${base64store_secret} | base64 -d > /usr/local/bin/store_secret.sh\nchmod +x /usr/local/bin/store_secret.sh\n\n# check_app_ready.sh\necho ${base64check_app_ready} | base64 -d > /usr/local/bin/check_app_ready.sh\nchmod +x /usr/local/bin/check_app_ready.sh\n\n# restart.sh\necho ${base64restart} | base64 -d > /usr/local/bin/restart.sh\nchmod +x /usr/local/bin/restart.sh\n\necho ${base64config_blobStorage} | base64 -d > /usr/local/bin/config_blobStorage.sh\nchmod +x /usr/local/bin/config_blobStorage.sh\n\n# Install azure cli\nAZURE_CLI_VERSION=2.87.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity --allow-no-subscriptions\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\nexport HOME=\"/root\"\n\n# Install OpenVidu\n/usr/local/bin/install.sh || { echo \"[OpenVidu] error installing OpenVidu\"; exit 1; }\n\n#Config blob storage\n/usr/local/bin/config_blobStorage.sh || { echo \"[OpenVidu] error configuring Blob Storage\"; exit 1; }\n\n# Start OpenVidu\nsystemctl start openvidu || { echo \"[OpenVidu] error starting OpenVidu\"; exit 1; }\n\n# Update shared secret\n/usr/local/bin/after_install.sh || { echo \"[OpenVidu] error updating shared secret\"; exit 1; }\n\n# Launch on reboot\necho \"@reboot /usr/local/bin/restart.sh >> /var/log/openvidu-restart.log\" 2>&1 | crontab\n\n# check_app_ready.sh internally caps its wait at 1200s\n/usr/local/bin/check_app_ready.sh || { echo \"[OpenVidu] master node did not become healthy\"; exit 1; }\n\naz keyvault secret set --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --value \"true\"\n\nMASTER_NODE_NUM=${masterNodeNum}\nif [[ $MASTER_NODE_NUM -eq 4 ]]; then\n # Creating scale in lock\n set +e\n az storage blob upload --account-name ${storageAccountName} --container-name automation-locks --name lock.txt --file /dev/null --auth-mode key\n set -e\nfi\n", + "installScriptTemplateMedia": "#!/bin/bash -x\nDOMAIN=\n\n# Install dependencies\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y \\\n curl \\\n unzip \\\n jq \\\n wget\n\n# Get own private IP\nPRIVATE_IP=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text\")\n\n# Gate 1: wait for master secrets and IPs before installing\nWAIT_INTERVAL=5\nMAX_RETRIES=360\nRETRIES=0\nset +e\nwhile true; do\n ALL_SECRETS_GENERATED=$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)\n MASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv 2>/dev/null)\n MASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv 2>/dev/null)\n MASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv 2>/dev/null)\n MASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv 2>/dev/null)\n\n if [ \"$ALL_SECRETS_GENERATED\" == \"true\" ] &&\n [ \"$MASTER_NODE_1_PRIVATE_IP\" != \"\" ] &&\n [ \"$MASTER_NODE_2_PRIVATE_IP\" != \"\" ] &&\n [ \"$MASTER_NODE_3_PRIVATE_IP\" != \"\" ] &&\n [ \"$MASTER_NODE_4_PRIVATE_IP\" != \"\" ]; then\n break\n fi\n\n RETRIES=$((RETRIES + 1))\n if [ $RETRIES -ge $MAX_RETRIES ]; then\n echo \"[OpenVidu] timed out after 30 min waiting for master nodes to generate secrets and publish their IPs\"\n exit 1\n fi\n\n sleep $WAIT_INTERVAL\ndone\nset -e\n\nMASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv)\nMASTER_NODE_PRIVATE_IP_LIST=\"$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP\"\nREDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name REDIS-PASSWORD --query value -o tsv)\nENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv)\nOPENVIDU_VERSION=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-VERSION --query value -o tsv)\n\n# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure\nINSTALLER_SCRIPT=\"/tmp/install_ov_media_node.sh\"\ncurl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o \"$INSTALLER_SCRIPT\" \"http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh\"\nif [ ! -s \"$INSTALLER_SCRIPT\" ]; then\n echo \"[OpenVidu] failed to download the media node installer script\"\n exit 1\nfi\n\n# Base command\nINSTALL_COMMAND=\"sh $INSTALLER_SCRIPT\"\n\n# Common arguments\nCOMMON_ARGS=(\n\"--no-tty\"\n\"--install\"\n\"--environment=azure\"\n\"--deployment-type='ha'\"\n\"--node-role='media-node'\"\n\"--master-node-private-ip-list=$MASTER_NODE_PRIVATE_IP_LIST\"\n\"--private-ip=$PRIVATE_IP\"\n\"--enabled-modules='$ENABLED_MODULES'\"\n\"--redis-password=$REDIS_PASSWORD\"\n)\n\n# Construct the final command with all arguments\nFINAL_COMMAND=\"$INSTALL_COMMAND $(printf \"%s \" \"${COMMON_ARGS[@]}\")\"\n\n# Install OpenVidu\nexec bash -c \"$FINAL_COMMAND\"\n", "stopMediaNodeParams": { "subscriptionId": "[subscription().subscriptionId]", "resourceGroupName": "[resourceGroup().name]", @@ -330,7 +330,7 @@ }, "stop_media_nodesScriptMediaTemplate": "#!/bin/bash\nset -e\n\nif ! (set -o noclobber ; echo > /tmp/global.lock) ; then\n exit 1 # the global.lock already exists\nfi\n\n# Execute if docker is installed\nif [ -x \"$(command -v docker)\" ]; then\n\n echo \"Stopping media node services and waiting for termination...\"\n docker container kill --signal=SIGQUIT openvidu || true\n docker container kill --signal=SIGQUIT ingress || true\n docker container kill --signal=SIGQUIT egress || true\n for agent_container in $(docker ps --filter \"label=openvidu-agent=true\" --format '{{.Names}}'); do\n docker container kill --signal=SIGQUIT \"$agent_container\"\n done\n\n # Wait for running containers to not be openvidu, ingress or egress\n while [ $(docker ps --filter \"label=openvidu-agent=true\" -q | wc -l) -gt 0 ] || \\\n [ $(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null) == \"true\" ] || \\\n [ $(docker inspect -f '{{.State.Running}}' ingress 2>/dev/null) == \"true\" ] || \\\n [ $(docker inspect -f '{{.State.Running}}' egress 2>/dev/null) == \"true\" ]; do\n echo \"Waiting for containers to stop...\"\n sleep 5\n done\nfi\n\naz login --identity\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nSUBSCRIPTION_ID=${subscriptionId}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\nRESOURCE_ID=/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP_NAME/providers/Microsoft.Compute/virtualMachineScaleSets/$VM_SCALE_SET_NAME\n\nTIMESTAMP=$(date -u +\"%Y-%m-%dT%H:%M:%SZ\")\naz tag update --resource-id $RESOURCE_ID --operation replace --tags \"STATUS\"=\"HEALTHY\" \"InstanceDeleteTime\"=\"$TIMESTAMP\" \"storageAccount\"=\"${storageAccountName}\"\n\naz vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID\n", "delete_mediaNode_ScriptMediaTemplate": "#!/bin/bash\nset -e\n\naz login --identity\n\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\n\n\naz vmss delete-instances --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-ids $INSTANCE_ID\n", - "userDataMediaNodeTemplate": "#!/bin/bash -x\nset -eu -o pipefail\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# stop_media_nodes.sh\necho ${base64stop} | base64 -d > /usr/local/bin/stop_media_node.sh\nchmod +x /usr/local/bin/stop_media_node.sh\n\n# delete_media_node.sh\necho ${base64delete_mediaNode} | base64 -d > /usr/local/bin/delete_media_node.sh\nchmod +x /usr/local/bin/delete_media_node.sh\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y \napt-get install -y jq\n\n# Install azure cli\nAZURE_CLI_VERSION=2.87.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity\n\n# Protect from scale in actions\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\naz vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-id $INSTANCE_ID --protect-from-scale-in true\n\nexport HOME=\"/root\"\n\n# Install OpenVidu\n/usr/local/bin/install.sh || { echo \"[OpenVidu] error installing OpenVidu\"; /usr/local/bin/delete_media_node.sh; }\n\n# Start OpenVidu\nsystemctl start openvidu || { echo \"[OpenVidu] error starting OpenVidu\"; /usr/local/bin/delete_media_node.sh; }\n", + "userDataMediaNodeTemplate": "#!/bin/bash -x\nset -eu -o pipefail\n\n# Introduce the scripts in the instance\n# install.sh\necho ${base64install} | base64 -d > /usr/local/bin/install.sh\nchmod +x /usr/local/bin/install.sh\n\n# stop_media_nodes.sh\necho ${base64stop} | base64 -d > /usr/local/bin/stop_media_node.sh\nchmod +x /usr/local/bin/stop_media_node.sh\n\n# delete_media_node.sh\necho ${base64delete_mediaNode} | base64 -d > /usr/local/bin/delete_media_node.sh\nchmod +x /usr/local/bin/delete_media_node.sh\n\necho \"DPkg::Lock::Timeout \\\"-1\\\";\" > /etc/apt/apt.conf.d/99timeout\n\napt-get update && apt-get install -y jq\n\n# Install azure cli\nAZURE_CLI_VERSION=2.87.0\napt-get install -y apt-transport-https ca-certificates gnupg lsb-release\ncurl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg\necho \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main\" | tee /etc/apt/sources.list.d/azure-cli.list\napt-get update\napt-get install -y azure-cli=${AZURE_CLI_VERSION}-1~$(lsb_release -cs)\n\naz login --identity\n\n# Protect from scale in actions\nRESOURCE_GROUP_NAME=${resourceGroupName}\nVM_SCALE_SET_NAME=${vmScaleSetName}\nBEFORE_INSTANCE_ID=$(curl -H Metadata:true --noproxy \"*\" \"http://169.254.169.254/metadata/instance?api-version=2021-02-01\" | jq -r '.compute.resourceId')\nINSTANCE_ID=$(echo $BEFORE_INSTANCE_ID | awk -F'/' '{print $NF}')\naz vmss update --resource-group $RESOURCE_GROUP_NAME --name $VM_SCALE_SET_NAME --instance-id $INSTANCE_ID --protect-from-scale-in true\n\nexport HOME=\"/root\"\n\n# Install OpenVidu\n/usr/local/bin/install.sh || { echo \"[OpenVidu] error installing OpenVidu\"; /usr/local/bin/delete_media_node.sh; }\n\n# Gate 2: wait for master readiness before starting\nWAIT_INTERVAL=5\nMAX_RETRIES=360\nRETRIES=0\nset +e\nwhile true; do\n FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv 2>/dev/null)\n if [ \"$FINISH_MASTER_NODE\" == \"true\" ]; then\n break\n fi\n RETRIES=$((RETRIES + 1))\n if [ $RETRIES -ge $MAX_RETRIES ]; then\n echo \"[OpenVidu] timed out after 30 min waiting for FINISH-MASTER-NODE\"\n /usr/local/bin/delete_media_node.sh\n exit 1\n fi\n sleep $WAIT_INTERVAL\ndone\nset -e\n\n# Start OpenVidu\nsystemctl start openvidu || { echo \"[OpenVidu] error starting OpenVidu\"; /usr/local/bin/delete_media_node.sh; }\n", "stop_media_nodesScriptMedia": "[reduce(items(variables('stopMediaNodeParams')), createObject('value', variables('stop_media_nodesScriptMediaTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value]", "delete_mediaNode_ScriptMedia": "[reduce(items(variables('stopMediaNodeParams')), createObject('value', variables('delete_mediaNode_ScriptMediaTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value]", "base64stopMediaNode": "[base64(variables('stop_media_nodesScriptMedia'))]", @@ -531,7 +531,6 @@ }, "dependsOn": [ "[resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface2', parameters('stackName')))]", - "[resourceId('Microsoft.Compute/virtualMachines', format('{0}-VM-MasterNode1', parameters('stackName')))]", "[resourceId('Microsoft.Network/publicIPAddresses', parameters('publicIpAddressObject').name)]", "[resourceId('Microsoft.Storage/storageAccounts', uniqueString(resourceGroup().id))]" ] @@ -574,7 +573,6 @@ }, "dependsOn": [ "[resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface3', parameters('stackName')))]", - "[resourceId('Microsoft.Compute/virtualMachines', format('{0}-VM-MasterNode2', parameters('stackName')))]", "[resourceId('Microsoft.Network/publicIPAddresses', parameters('publicIpAddressObject').name)]", "[resourceId('Microsoft.Storage/storageAccounts', uniqueString(resourceGroup().id))]" ] @@ -617,7 +615,6 @@ }, "dependsOn": [ "[resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface4', parameters('stackName')))]", - "[resourceId('Microsoft.Compute/virtualMachines', format('{0}-VM-MasterNode3', parameters('stackName')))]", "[resourceId('Microsoft.Network/publicIPAddresses', parameters('publicIpAddressObject').name)]", "[resourceId('Microsoft.Storage/storageAccounts', uniqueString(resourceGroup().id))]" ] @@ -696,7 +693,7 @@ } ] }, - "userData": "[base64(reduce(items(createObject('base64install', base64(reduce(items(createObject('privateIPMasterNode1', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface1', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'privateIPMasterNode2', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface2', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'privateIPMasterNode3', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface3', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'privateIPMasterNode4', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface4', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'keyVaultName', variables('keyVaultName'))), createObject('value', variables('installScriptTemplateMedia')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value), 'base64stop', variables('base64stopMediaNode'), 'base64delete_mediaNode', variables('base64delete_mediaNode_ScriptMedia'), 'resourceGroupName', resourceGroup().name, 'vmScaleSetName', format('{0}-mediaNodeScaleSet', parameters('stackName')))), createObject('value', variables('userDataMediaNodeTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value)]" + "userData": "[base64(reduce(items(createObject('base64install', base64(reduce(items(createObject('privateIPMasterNode1', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface1', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'privateIPMasterNode2', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface2', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'privateIPMasterNode3', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface3', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'privateIPMasterNode4', reference(resourceId('Microsoft.Network/networkInterfaces', format('{0}-masterNodeNetInterface4', parameters('stackName'))), '2023-11-01').ipConfigurations[0].properties.privateIPAddress, 'keyVaultName', variables('keyVaultName'))), createObject('value', variables('installScriptTemplateMedia')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value), 'base64stop', variables('base64stopMediaNode'), 'base64delete_mediaNode', variables('base64delete_mediaNode_ScriptMedia'), 'resourceGroupName', resourceGroup().name, 'vmScaleSetName', format('{0}-mediaNodeScaleSet', parameters('stackName')), 'keyVaultName', variables('keyVaultName'))), createObject('value', variables('userDataMediaNodeTemplate')), lambda('curr', 'next', createObject('value', replace(lambdaVariables('curr').value, format('${{{0}}}', lambdaVariables('next').key), lambdaVariables('next').value)))).value)]" } }, "dependsOn": [ diff --git a/openvidu-deployment/pro/ha/azure/createUiDefinition.json b/openvidu-deployment/pro/ha/azure/createUiDefinition.json index 3e7d2db87..6e12791db 100644 --- a/openvidu-deployment/pro/ha/azure/createUiDefinition.json +++ b/openvidu-deployment/pro/ha/azure/createUiDefinition.json @@ -553,7 +553,7 @@ "datetime": "[steps('parameters SCALING').datetime]", "automationAccountName": "[steps('parameters SCALING').automationAccountName]", "storageAccountName": "[steps('parameters STORAGE').storageAccountName]", - "containerName": "[steps('parameters STORAGE').containerName]", + "appDataContainerName": "[steps('parameters STORAGE').containerName]", "additionalInstallFlags": "[steps('FLAGS').additionalInstallFlags]" } } diff --git a/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf b/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf index 781fc7472..027e89a71 100644 --- a/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf +++ b/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf @@ -20,14 +20,15 @@ resource "google_secret_manager_secret" "openvidu_shared_info" { "OPENVIDU_PRO_LICENSE", "OPENVIDU_RTC_ENGINE", "REDIS_PASSWORD", "MONGO_ADMIN_USERNAME", "MONGO_ADMIN_PASSWORD", "MONGO_REPLICA_SET_KEY", "MINIO_ACCESS_KEY", "MINIO_SECRET_KEY", "DASHBOARD_ADMIN_USERNAME", "DASHBOARD_ADMIN_PASSWORD", "GRAFANA_ADMIN_USERNAME", - "GRAFANA_ADMIN_PASSWORD", "ENABLED_MODULES", "OPENVIDU_VERSION", "ALL_SECRETS_GENERATED", - "MASTER_NODE_1_PRIVATE_IP", "MASTER_NODE_2_PRIVATE_IP", "MASTER_NODE_3_PRIVATE_IP", "MASTER_NODE_4_PRIVATE_IP" + "GRAFANA_ADMIN_PASSWORD", "ENABLED_MODULES", "OPENVIDU_VERSION", "ALL_SECRETS_GENERATED" ]) secret_id = each.key replication { auto {} } + + depends_on = [google_project_service.secretmanager_api] } # GCS buckets for HA deployment @@ -37,6 +38,8 @@ resource "google_storage_bucket" "appdata_bucket" { location = var.region force_destroy = true uniform_bucket_level_access = true + + depends_on = [google_project_service.storage_api] } resource "google_storage_bucket" "clusterdata_bucket" { @@ -45,12 +48,16 @@ resource "google_storage_bucket" "clusterdata_bucket" { location = var.region force_destroy = true uniform_bucket_level_access = true + + depends_on = [google_project_service.storage_api] } # Service account for the instances resource "google_service_account" "service_account" { account_id = lower("${substr(var.stackName, 0, 12)}-sa") display_name = "OpenVidu instance service account" + + depends_on = [google_project_service.iam_api] } # IAM bindings for the service account @@ -58,6 +65,8 @@ resource "google_project_iam_member" "iam_project_role" { project = var.projectId role = "roles/owner" member = "serviceAccount:${google_service_account.service_account.email}" + + depends_on = [google_project_service.cloudresourcemanager_api] } # External SSH access to Master Nodes @@ -72,6 +81,8 @@ resource "google_compute_firewall" "external_master_ssh" { source_ranges = ["0.0.0.0/0"] target_tags = [lower("${var.stackName}-master-node")] + + depends_on = [google_project_service.compute_api] } # External access to Media Nodes (SSH and media traffic) @@ -90,6 +101,8 @@ resource "google_compute_firewall" "external_media_access" { source_ranges = ["0.0.0.0/0"] target_tags = [lower("${var.stackName}-media-node")] + + depends_on = [google_project_service.compute_api] } # Load Balancer health checks and HTTP traffic to Master Nodes @@ -104,6 +117,8 @@ resource "google_compute_firewall" "lb_to_master_http" { source_ranges = ["0.0.0.0/0"] target_tags = [lower("${var.stackName}-master-node")] + + depends_on = [google_project_service.compute_api] } # Master node internal services communication @@ -129,6 +144,8 @@ resource "google_compute_firewall" "master_to_master_internal" { source_tags = [lower("${var.stackName}-master-node")] target_tags = [lower("${var.stackName}-master-node")] + + depends_on = [google_project_service.compute_api] } # Media Nodes to Master Nodes communication @@ -153,6 +170,8 @@ resource "google_compute_firewall" "media_to_master_services" { source_tags = [lower("${var.stackName}-media-node")] target_tags = [lower("${var.stackName}-master-node")] + + depends_on = [google_project_service.compute_api] } # Master Nodes to Media Nodes communication @@ -171,6 +190,8 @@ resource "google_compute_firewall" "master_to_media_services" { source_tags = [lower("${var.stackName}-master-node")] target_tags = [lower("${var.stackName}-media-node")] + + depends_on = [google_project_service.compute_api] } @@ -179,6 +200,8 @@ resource "google_compute_address" "nlb_ip" { count = var.publicIpAddress == "" ? 1 : 0 name = lower("${var.stackName}-nlb-ip") region = var.region + + depends_on = [google_project_service.compute_api] } # Data source for existing IP address when publicIpAddress is provided @@ -206,6 +229,8 @@ resource "google_compute_region_health_check" "tcp_health_check" { timeout_sec = 5 healthy_threshold = 3 unhealthy_threshold = 4 + + depends_on = [google_project_service.compute_api] } # Regional backend service for the TCP NLB @@ -289,6 +314,29 @@ locals { is_c4a_instance = startswith(var.masterNodesInstanceType, "c4a-") } +# Must match the subnetwork the master instances implicitly use (network = "default", no explicit subnetwork) +data "google_compute_subnetwork" "default" { + name = "default" + region = var.region + + depends_on = [google_project_service.compute_api] +} + +resource "google_compute_address" "master_internal_ip" { + count = 4 + name = lower("${var.stackName}-master-node-${count.index + 1}-internal-ip") + address_type = "INTERNAL" + region = var.region + subnetwork = data.google_compute_subnetwork.default.id + + depends_on = [google_project_service.compute_api] +} + +locals { + # Comma-separated: must match the installer's --master-node-private-ip-list format + master_node_private_ip_list = join(",", google_compute_address.master_internal_ip[*].address) +} + # Master Node 1 resource "google_compute_instance" "openvidu_master_node_1" { name = lower("${var.stackName}-master-node-1") @@ -305,13 +353,15 @@ resource "google_compute_instance" "openvidu_master_node_1" { } network_interface { - network = "default" + network = "default" + network_ip = google_compute_address.master_internal_ip[0].address access_config {} } metadata = { stackName = var.stackName masterNodeNum = "1" + masterNodePrivateIPList = local.master_node_private_ip_list domainName = var.domainName certificateType = var.certificateType ownPublicCertificate = var.ownPublicCertificate @@ -355,13 +405,15 @@ resource "google_compute_instance" "openvidu_master_node_2" { } network_interface { - network = "default" + network = "default" + network_ip = google_compute_address.master_internal_ip[1].address access_config {} } metadata = { stackName = var.stackName masterNodeNum = "2" + masterNodePrivateIPList = local.master_node_private_ip_list domainName = var.domainName certificateType = var.certificateType ownPublicCertificate = var.ownPublicCertificate @@ -387,8 +439,6 @@ resource "google_compute_instance" "openvidu_master_node_2" { node-type = "master" node-num = "2" } - - depends_on = [google_compute_instance.openvidu_master_node_1] } # Master Node 3 @@ -407,13 +457,15 @@ resource "google_compute_instance" "openvidu_master_node_3" { } network_interface { - network = "default" + network = "default" + network_ip = google_compute_address.master_internal_ip[2].address access_config {} } metadata = { stackName = var.stackName masterNodeNum = "3" + masterNodePrivateIPList = local.master_node_private_ip_list domainName = var.domainName certificateType = var.certificateType ownPublicCertificate = var.ownPublicCertificate @@ -439,8 +491,6 @@ resource "google_compute_instance" "openvidu_master_node_3" { node-type = "master" node-num = "3" } - - depends_on = [google_compute_instance.openvidu_master_node_2] } # Master Node 4 @@ -459,13 +509,15 @@ resource "google_compute_instance" "openvidu_master_node_4" { } network_interface { - network = "default" + network = "default" + network_ip = google_compute_address.master_internal_ip[3].address access_config {} } metadata = { stackName = var.stackName masterNodeNum = "4" + masterNodePrivateIPList = local.master_node_private_ip_list domainName = var.domainName certificateType = var.certificateType ownPublicCertificate = var.ownPublicCertificate @@ -491,8 +543,6 @@ resource "google_compute_instance" "openvidu_master_node_4" { node-type = "master" node-num = "4" } - - depends_on = [google_compute_instance.openvidu_master_node_3] } # ------------------------- scale in resources ------------------------- @@ -800,6 +850,12 @@ resource "google_cloudfunctions2_function" "scalein_function" { } service_account_email = google_service_account.service_account.email } + + depends_on = [ + google_project_service.cloudfunctions_api, + google_project_service.cloudbuild_api, + google_project_service.run_api + ] } # Cloud Scheduler to trigger the function every 5 minutes @@ -829,6 +885,8 @@ resource "google_cloud_scheduler_job" "scale_scheduler" { service_account_email = google_service_account.service_account.email } } + + depends_on = [google_project_service.cloudscheduler_api] } locals { @@ -858,7 +916,7 @@ resource "google_compute_instance_template" "media_node_template" { metadata = { stackName = var.stackName - masterNodePrivateIPList = "${google_compute_instance.openvidu_master_node_1.network_interface[0].network_ip},${google_compute_instance.openvidu_master_node_2.network_interface[0].network_ip},${google_compute_instance.openvidu_master_node_3.network_interface[0].network_ip},${google_compute_instance.openvidu_master_node_4.network_interface[0].network_ip}" + masterNodePrivateIPList = local.master_node_private_ip_list bucketAppDataName = local.isEmptyAppData ? google_storage_bucket.appdata_bucket[0].name : var.GCSAppDataBucketName bucketClusterDataName = local.isEmptyClusterData ? google_storage_bucket.clusterdata_bucket[0].name : var.GCSClusterDataBucketName region = var.region @@ -880,13 +938,6 @@ resource "google_compute_instance_template" "media_node_template" { lifecycle { create_before_destroy = true } - - depends_on = [ - google_compute_instance.openvidu_master_node_1, - google_compute_instance.openvidu_master_node_2, - google_compute_instance.openvidu_master_node_3, - google_compute_instance.openvidu_master_node_4 - ] } # Managed Instance Group for Media Nodes @@ -904,13 +955,6 @@ resource "google_compute_region_instance_group_manager" "media_node_group" { name = "http" port = 7880 } - - depends_on = [ - google_compute_instance.openvidu_master_node_1, - google_compute_instance.openvidu_master_node_2, - google_compute_instance.openvidu_master_node_3, - google_compute_instance.openvidu_master_node_4 - ] } # Autoscaler for Media Nodes @@ -969,11 +1013,7 @@ get_meta() { curl -s -H "Metadata-Flavor: Google" "$${METADATA_URL}/$1"; } # Get master node number from metadata MASTER_NODE_NUM=$(get_meta "instance/attributes/masterNodeNum") -# Get own private IP -PRIVATE_IP=$(get_meta "instance/network-interfaces/0/ip") - -# Store current private IP -PRIVATE_IP="$(/usr/local/bin/store_secret.sh save MASTER_NODE_$${MASTER_NODE_NUM}_PRIVATE_IP $PRIVATE_IP)" +MASTER_NODE_PRIVATE_IP_LIST=$(get_meta "instance/attributes/masterNodePrivateIPList") # Check if secrets have been generated ALL_SECRETS_GENERATED=$(gcloud secrets versions access latest --secret=ALL_SECRETS_GENERATED 2>/dev/null || echo "false") @@ -1033,33 +1073,19 @@ if [[ $MASTER_NODE_NUM -eq 1 ]] && [[ "$ALL_SECRETS_GENERATED" == "false" ]]; th ALL_SECRETS_GENERATED="$(/usr/local/bin/store_secret.sh save ALL_SECRETS_GENERATED "true")" fi -# Wait for all master nodes to store their private IPs -while true; do - MASTER_NODE_1_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_1_PRIVATE_IP 2>/dev/null || echo "") - MASTER_NODE_2_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_2_PRIVATE_IP 2>/dev/null || echo "") - MASTER_NODE_3_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_3_PRIVATE_IP 2>/dev/null || echo "") - MASTER_NODE_4_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_4_PRIVATE_IP 2>/dev/null || echo "") - - # Check if all master nodes have stored their private IPs - if [[ "$MASTER_NODE_1_PRIVATE_IP" != "" ]] && - [[ "$MASTER_NODE_2_PRIVATE_IP" != "" ]] && - [[ "$MASTER_NODE_3_PRIVATE_IP" != "" ]] && - [[ "$MASTER_NODE_4_PRIVATE_IP" != "" ]]; then - break +# Wait for master-1 to finish generating all shared secrets before reading them. +i=0 +while ! gcloud secrets versions access latest --secret=ALL_SECRETS_GENERATED 2>/dev/null | grep -q "true"; do + i=$((i + 1)) + if [ "$i" -ge 360 ]; then + echo "Timed out after 30 minutes waiting for master-1 to finish generating secrets" >&2 + exit 1 fi + echo "Waiting for master-1 to finish generating secrets..." sleep 5 done -# Wait for master-1 to finish generating all shared secrets before reading them -while ! gcloud secrets versions access latest --secret=ALL_SECRETS_GENERATED 2>/dev/null | grep -q "true"; do echo "Waiting for master-1 to finish generating secrets..."; sleep 5; done - # Fetch all values from Secret Manager -MASTER_NODE_1_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_1_PRIVATE_IP) -MASTER_NODE_2_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_2_PRIVATE_IP) -MASTER_NODE_3_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_3_PRIVATE_IP) -MASTER_NODE_4_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_4_PRIVATE_IP) -MASTER_NODE_PRIVATE_IP_LIST="$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP" - DOMAIN=$(gcloud secrets versions access latest --secret=DOMAIN_NAME) OPENVIDU_PRO_LICENSE=$(gcloud secrets versions access latest --secret=OPENVIDU_PRO_LICENSE) OPENVIDU_RTC_ENGINE=$(gcloud secrets versions access latest --secret=OPENVIDU_RTC_ENGINE) @@ -1082,8 +1108,14 @@ LIVEKIT_API_KEY=$(gcloud secrets versions access latest --secret=LIVEKIT_API_KEY LIVEKIT_API_SECRET=$(gcloud secrets versions access latest --secret=LIVEKIT_API_SECRET) ENABLED_MODULES=$(gcloud secrets versions access latest --secret=ENABLED_MODULES) -# Build install command -INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh)" +# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure +INSTALLER_SCRIPT="/tmp/install_ov_master_node.sh" +curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh" +if [ ! -s "$INSTALLER_SCRIPT" ]; then + echo "Downloaded OpenVidu master node installer is empty or missing" >&2 + exit 1 +fi +INSTALL_COMMAND="sh $INSTALLER_SCRIPT" # Common arguments COMMON_ARGS=( @@ -1434,11 +1466,17 @@ EOF check_app_ready_script = <<-EOF #!/bin/bash +i=0 while true; do HTTP_STATUS=$(curl -Ik http://localhost:7880/health/caddy 2>/dev/null | head -n1 | awk '{print $2}') if [ "$HTTP_STATUS" == "200" ]; then break fi + i=$((i + 1)) + if [ "$i" -ge 240 ]; then + echo "Timed out after 20 minutes waiting for OpenVidu to become ready" >&2 + exit 1 + fi sleep 5 done EOF @@ -1513,8 +1551,6 @@ CONFIG_S3_EOF echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout - apt-get update && apt-get install -y - GCLOUD_VERSION=573.0.0 # Install google cli if ! command -v gcloud >/dev/null 2>&1; then @@ -1578,8 +1614,14 @@ MASTER_NODE_PRIVATE_IP_LIST=$(get_meta "instance/attributes/masterNodePrivateIPL STACK_NAME=$(get_meta "instance/attributes/stackName") PRIVATE_IP=$(get_meta "instance/network-interfaces/0/ip") -# Wait for master nodes to be ready by checking secrets +# Wait for master nodes to be ready by checking secrets. +i=0 while ! gcloud secrets versions access latest --secret=ALL_SECRETS_GENERATED 2>/dev/null | grep -q "true"; do + i=$((i + 1)) + if [ "$i" -ge 180 ]; then + echo "Timed out after 30 minutes waiting for master nodes to initialize secrets" >&2 + exit 1 + fi echo "Waiting for master nodes to initialize secrets..." sleep 10 done @@ -1597,8 +1639,14 @@ if [[ "$OPENVIDU_VERSION" == "none" ]]; then exit 1 fi -# Build install command for media node -INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh)" +# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure +INSTALLER_SCRIPT="/tmp/install_ov_media_node.sh" +curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh" +if [ ! -s "$INSTALLER_SCRIPT" ]; then + echo "Downloaded OpenVidu media node installer is empty or missing" >&2 + exit 1 +fi +INSTALL_COMMAND="sh $INSTALLER_SCRIPT" # Media node arguments COMMON_ARGS=( @@ -1698,51 +1746,55 @@ EOF #!/bin/bash -x set -eu -o pipefail -# Create scripts -cat > /usr/local/bin/install.sh << 'INSTALL_EOF' +# Check if installation already completed +if [ -f /usr/local/bin/openvidu_install_counter.txt ]; then + # Launch on reboot + systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; } +else + # Create scripts + cat > /usr/local/bin/install.sh << 'INSTALL_EOF' ${local.install_script_media} INSTALL_EOF -chmod +x /usr/local/bin/install.sh + chmod +x /usr/local/bin/install.sh -cat > /usr/local/bin/graceful_shutdown.sh << 'GRACEFUL_SHUTDOWN_EOF' + cat > /usr/local/bin/graceful_shutdown.sh << 'GRACEFUL_SHUTDOWN_EOF' ${local.graceful_shutdown_script} GRACEFUL_SHUTDOWN_EOF -chmod +x /usr/local/bin/graceful_shutdown.sh + chmod +x /usr/local/bin/graceful_shutdown.sh -echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout + echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout -apt-get update && apt-get install -y + GCLOUD_VERSION=573.0.0 + # Install google cli + if ! command -v gcloud >/dev/null 2>&1; then + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + apt-get update && apt-get install -y google-cloud-cli=$${GCLOUD_VERSION}-0 + fi -GCLOUD_VERSION=573.0.0 -# Install google cli -if ! command -v gcloud >/dev/null 2>&1; then - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg - echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list - apt-get update && apt-get install -y google-cloud-cli=$${GCLOUD_VERSION}-0 -fi + # Authenticate with gcloud using instance service account + gcloud auth activate-service-account --key-file=/dev/null 2>/dev/null || true + gcloud config set account $(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email" -H "Metadata-Flavor: Google") + gcloud config set project $(curl -s "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google") -# Authenticate with gcloud using instance service account -gcloud auth activate-service-account --key-file=/dev/null 2>/dev/null || true -gcloud config set account $(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email" -H "Metadata-Flavor: Google") -gcloud config set project $(curl -s "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google") + export HOME="/root" -export HOME="/root" + # Install OpenVidu Media Node + /usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu Media Node"; exit 1; } -# Install OpenVidu Media Node -/usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu Media Node"; exit 1; } + # Mark installation as complete + echo "installation_complete" > /usr/local/bin/openvidu_install_counter.txt -# Mark installation as complete -echo "installation_complete" > /usr/local/bin/openvidu_install_counter.txt + # Start OpenVidu + systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; } -# Start OpenVidu -systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; } - -# Add cron job to check if instance is abandoned every minute -cat > /usr/local/bin/check_abandoned.sh << 'CHECK_ABANDONED_EOF' + # Add cron job to check if instance is abandoned every minute + cat > /usr/local/bin/check_abandoned.sh << 'CHECK_ABANDONED_EOF' ${local.crontab_job_media} CHECK_ABANDONED_EOF -chmod +x /usr/local/bin/check_abandoned.sh + chmod +x /usr/local/bin/check_abandoned.sh -echo "*/1 * * * * /usr/local/bin/check_abandoned.sh > /var/log/openvidu-abandoned-check.log 2>&1" | crontab - + echo "*/1 * * * * /usr/local/bin/check_abandoned.sh > /var/log/openvidu-abandoned-check.log 2>&1" | crontab - +fi EOF }