ambiser
/
openvidu
mirror of https://github.com/OpenVidu/openvidu.git
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'ha-optimizations': HA deployment time optimizations (AWS/Azure/GCP) 

DOC-CHANGES helper files intentionally excluded from the merge (branch-only).
master
Piwccle 2026-07-28 18:22:25 +02:00
parent 5a3347c12a 31a8488f23
commit 73f8b57498
5 changed files with 421 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

268
openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml
View File

@ -1242,14 +1242,39 @@ Resources:
"GRAFANA_ADMIN_USERNAME": "none", "GRAFANA_ADMIN_USERNAME": "none",
"GRAFANA_ADMIN_PASSWORD": "none", "GRAFANA_ADMIN_PASSWORD": "none",
"ENABLED_MODULES": "none", "ENABLED_MODULES": "none",
"MASTER_NODE_1_PRIVATE_IP": "none",
"MASTER_NODE_2_PRIVATE_IP": "none",
"MASTER_NODE_3_PRIVATE_IP": "none",
"MASTER_NODE_4_PRIVATE_IP": "none",
"OPENVIDU_VERSION": "none", "OPENVIDU_VERSION": "none",
"ALL_SECRETS_GENERATED": "false" "ALL_SECRETS_GENERATED": "false"
} }
# One SSM parameter per master: atomic per-key writes
MasterNode1PrivateIpParameter:
Type: AWS::SSM::Parameter
Properties:
Name: !Sub /openvidu/${AWS::StackName}/master-node-1-private-ip
Type: String
Value: "none"
MasterNode2PrivateIpParameter:
Type: AWS::SSM::Parameter
Properties:
Name: !Sub /openvidu/${AWS::StackName}/master-node-2-private-ip
Type: String
Value: "none"
MasterNode3PrivateIpParameter:
Type: AWS::SSM::Parameter
Properties:
Name: !Sub /openvidu/${AWS::StackName}/master-node-3-private-ip
Type: String
Value: "none"
MasterNode4PrivateIpParameter:
Type: AWS::SSM::Parameter
Properties:
Name: !Sub /openvidu/${AWS::StackName}/master-node-4-private-ip
Type: String
Value: "none"
S3AppDataBucketResource: S3AppDataBucketResource:
Type: 'AWS::S3::Bucket' Type: 'AWS::S3::Bucket'
Properties: Properties:
@ -1281,8 +1306,7 @@ Resources:
Condition: CreateClusterDataBucket Condition: CreateClusterDataBucket
# ------------------------- # -------------------------
# Preprocess subnets to allocate Volumes and ENIs across Availability Zones # Normalize the master node subnet list to exactly 4 entries, one per OpenVidu Master Node
# For OpenVidu Master Nodes
# ------------------------- # -------------------------
SubnetProcessorFunction: SubnetProcessorFunction:
Type: AWS::Lambda::Function Type: AWS::Lambda::Function
@ -1293,13 +1317,11 @@ Resources:
Code: Code:
ZipFile: | ZipFile: |
import cfnresponse import cfnresponse
import boto3
def lambda_handler(event, context): def lambda_handler(event, context):
try: try:
# Process event data # Process event data
subnets = event['ResourceProperties']['Subnets'] subnets = event['ResourceProperties']['Subnets']
ec2 = boto3.client('ec2')
# Ensure we have at least four subnets by cycling through the available subnets # Ensure we have at least four subnets by cycling through the available subnets
subnets = (subnets * 4)[:4] # Repeat the list to have at least 4 elements and then take the first 4 subnets = (subnets * 4)[:4] # Repeat the list to have at least 4 elements and then take the first 4
@ -1355,10 +1377,6 @@ Resources:
- logs:CreateLogStream - logs:CreateLogStream
- logs:PutLogEvents - logs:PutLogEvents
Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/SubnetProcessor-${AWS::Region}-${AWS::StackName}:*' Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/SubnetProcessor-${AWS::Region}-${AWS::StackName}:*'
- Effect: Allow
Action:
- ec2:DescribeSubnets
Resource: '*'
OpenViduMasterNodeRole: OpenViduMasterNodeRole:
Type: AWS::IAM::Role Type: AWS::IAM::Role
@ -1390,6 +1408,11 @@ Resources:
- secretsmanager:GetSecretValue - secretsmanager:GetSecretValue
- secretsmanager:UpdateSecret - secretsmanager:UpdateSecret
Resource: !Ref OpenViduSharedInfo Resource: !Ref OpenViduSharedInfo
- Effect: Allow
Action:
- ssm:GetParameter
- ssm:PutParameter
Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/openvidu/${AWS::StackName}/*
- Fn::If: - Fn::If:
- CreateRecordingsBucket - CreateRecordingsBucket
- Effect: Allow - Effect: Allow
@ -1474,6 +1497,10 @@ Resources:
Action: Action:
- secretsmanager:GetSecretValue - secretsmanager:GetSecretValue
Resource: !Ref OpenViduSharedInfo Resource: !Ref OpenViduSharedInfo
- Effect: Allow
Action:
- ssm:GetParameter
Resource: !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/openvidu/${AWS::StackName}/*
- Effect: Allow - Effect: Allow
Action: Action:
- autoscaling:SetInstanceHealth - autoscaling:SetInstanceHealth
@ -1526,8 +1553,6 @@ Resources:
OpenViduMediaInstanceProfile: OpenViduMediaInstanceProfile:
Type: AWS::IAM::InstanceProfile Type: AWS::IAM::InstanceProfile
DependsOn:
- MasterNodesWaitCondition4
Properties: Properties:
InstanceProfileName: !Sub OpenViduMediaInstanceProfile-${AWS::Region}-${AWS::StackName} InstanceProfileName: !Sub OpenViduMediaInstanceProfile-${AWS::Region}-${AWS::StackName}
Roles: Roles:
@ -1566,42 +1591,15 @@ Resources:
rm -rf awscliv2.zip aws rm -rf awscliv2.zip aws
fi fi
# Singal to notify instance is waiting # Signal to notify instance is waiting
SIGNAL_NAME="$1" SIGNAL_NAME="$1"
# Exported earlier in this instance's UserData
MASTER_NODE_NUM="${!MASTER_NODE_NUM}"
# Token for IMDSv2 # Token for IMDSv2
TOKEN="$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")" TOKEN="$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")"
# Subnets IDs
SUBNETS=(
"${SubnetProcessor.Subnet1}"
"${SubnetProcessor.Subnet2}"
"${SubnetProcessor.Subnet3}"
"${SubnetProcessor.Subnet4}"
)
MAC_ADDRESS="$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254//latest/meta-data/mac)"
SUBNET_ID="$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s "http://169.254.169.254/latest/meta-data/network/interfaces/macs/$MAC_ADDRESS/subnet-id")"
# Check master node number
MASTER_NODE_NUM=1
for subnet in "${!SUBNETS[@]}"; do
if [[ "$subnet" == "$SUBNET_ID" ]]; then
SHARED_SECRET=$(aws secretsmanager get-secret-value \
--region ${AWS::Region} \
--secret-id openvidu-ha-${AWS::Region}-${AWS::StackName} \
--query SecretString --output text || echo 'none')
# Check if current master node is reacheable with ping
ACUTAL_MASTER_NODE_IP=$(echo "$SHARED_SECRET" | jq -r ".MASTER_NODE_${!MASTER_NODE_NUM}_PRIVATE_IP")
if [[ "$ACUTAL_MASTER_NODE_IP" == "none" ]]; then
break
fi
fi
MASTER_NODE_NUM=$((MASTER_NODE_NUM + 1))
done
# Get own private IP # Get own private IP
PRIVATE_IP="$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/local-ipv4)" PRIVATE_IP="$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/local-ipv4)"
@ -1610,8 +1608,12 @@ Resources:
exit 1 exit 1
fi fi
# Store current private IP aws ssm put-parameter \
/usr/local/bin/store_secret.sh save MASTER_NODE_${!MASTER_NODE_NUM}_PRIVATE_IP "${!PRIVATE_IP}" --region ${AWS::Region} \
--name "/openvidu/${AWS::StackName}/master-node-${!MASTER_NODE_NUM}-private-ip" \
--value "${!PRIVATE_IP}" \
--type String \
--overwrite > /dev/null
SHARED_SECRET=$(aws secretsmanager get-secret-value \ SHARED_SECRET=$(aws secretsmanager get-secret-value \
--region ${AWS::Region} \ --region ${AWS::Region} \
@ -1690,38 +1692,51 @@ Resources:
# sending the signal call # sending the signal call
cfn-signal -e $? --stack ${AWS::StackId} --resource "$SIGNAL_NAME" --region ${AWS::Region} cfn-signal -e $? --stack ${AWS::StackId} --resource "$SIGNAL_NAME" --region ${AWS::Region}
# Wait for all master nodes to store their private IPs IP_WAIT_ATTEMPTS=0
IP_WAIT_MAX=360
while true; do while true; do
SHARED_SECRET=$(aws secretsmanager get-secret-value \ MASTER_NODE_1_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-1-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none')
--region ${AWS::Region} \ MASTER_NODE_2_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-2-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none')
--secret-id openvidu-ha-${AWS::Region}-${AWS::StackName} \ MASTER_NODE_3_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-3-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none')
--query SecretString --output text || echo 'none') MASTER_NODE_4_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-4-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none')
MASTER_NODE_1_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_1_PRIVATE_IP') # Check if all master nodes have published their private IPs
MASTER_NODE_2_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_2_PRIVATE_IP') if [[ "$MASTER_NODE_1_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_1_PRIVATE_IP" ]] &&
MASTER_NODE_3_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_3_PRIVATE_IP') [[ "$MASTER_NODE_2_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_2_PRIVATE_IP" ]] &&
MASTER_NODE_4_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_4_PRIVATE_IP') [[ "$MASTER_NODE_3_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_3_PRIVATE_IP" ]] &&
[[ "$MASTER_NODE_4_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_4_PRIVATE_IP" ]]; then
# Check if all master nodes have stored their private IPs
if [[ "$MASTER_NODE_1_PRIVATE_IP" != "none" ]] &&
[[ "$MASTER_NODE_2_PRIVATE_IP" != "none" ]] &&
[[ "$MASTER_NODE_3_PRIVATE_IP" != "none" ]] &&
[[ "$MASTER_NODE_4_PRIVATE_IP" != "none" ]]; then
break break
fi fi
IP_WAIT_ATTEMPTS=$((IP_WAIT_ATTEMPTS + 1))
if [[ $IP_WAIT_ATTEMPTS -ge $IP_WAIT_MAX ]]; then
echo "Error: timed out after 30 minutes waiting for all master nodes to publish their private IPs to SSM"
exit 1
fi
sleep 5 sleep 5
done done
# Comma-separated 1-4: must match installer's --master-node-private-ip-list format
MASTER_NODE_PRIVATE_IP_LIST="$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP"
# Re-read: other (non-IP) values still come from the shared secret.
# GetSecretValue is eventually consistent: retry until a read returns the generated values
SECRET_READ_ATTEMPTS=0
while true; do
SHARED_SECRET=$(aws secretsmanager get-secret-value \ SHARED_SECRET=$(aws secretsmanager get-secret-value \
--region ${AWS::Region} \ --region ${AWS::Region} \
--secret-id openvidu-ha-${AWS::Region}-${AWS::StackName} \ --secret-id openvidu-ha-${AWS::Region}-${AWS::StackName} \
--query SecretString --output text) --query SecretString --output text)
if echo "$SHARED_SECRET" | jq -e '(.ALL_SECRETS_GENERATED == "true") and ([.DOMAIN_NAME, .OPENVIDU_VERSION, .REDIS_PASSWORD, .MONGO_ADMIN_PASSWORD, .MONGO_REPLICA_SET_KEY, .MINIO_SECRET_KEY, .DASHBOARD_ADMIN_PASSWORD, .GRAFANA_ADMIN_PASSWORD, .LIVEKIT_API_KEY, .LIVEKIT_API_SECRET] | all(. != "none"))' > /dev/null; then
MASTER_NODE_1_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_1_PRIVATE_IP') break
MASTER_NODE_2_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_2_PRIVATE_IP') fi
MASTER_NODE_3_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_3_PRIVATE_IP') SECRET_READ_ATTEMPTS=$((SECRET_READ_ATTEMPTS + 1))
MASTER_NODE_4_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_4_PRIVATE_IP') if [[ $SECRET_READ_ATTEMPTS -ge 60 ]]; then
MASTER_NODE_PRIVATE_IP_LIST="$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP" echo "Error: shared secret still incomplete after 5 minutes of stale reads"
exit 1
fi
sleep 5
done
DOMAIN=$(echo "$SHARED_SECRET" | jq -r '.DOMAIN_NAME') DOMAIN=$(echo "$SHARED_SECRET" | jq -r '.DOMAIN_NAME')
OPENVIDU_PRO_LICENSE=$(echo "$SHARED_SECRET" | jq -r '.OPENVIDU_PRO_LICENSE') OPENVIDU_PRO_LICENSE=$(echo "$SHARED_SECRET" | jq -r '.OPENVIDU_PRO_LICENSE')
@ -1738,18 +1753,22 @@ Resources:
GRAFANA_ADMIN_PASSWORD=$(echo "$SHARED_SECRET" | jq -r '.GRAFANA_ADMIN_PASSWORD') GRAFANA_ADMIN_PASSWORD=$(echo "$SHARED_SECRET" | jq -r '.GRAFANA_ADMIN_PASSWORD')
MEET_INITIAL_ADMIN_USER=$(echo "$SHARED_SECRET" | jq -r '.MEET_INITIAL_ADMIN_USER') MEET_INITIAL_ADMIN_USER=$(echo "$SHARED_SECRET" | jq -r '.MEET_INITIAL_ADMIN_USER')
MEET_INITIAL_ADMIN_PASSWORD=$(echo "$SHARED_SECRET" | jq -r '.MEET_INITIAL_ADMIN_PASSWORD') MEET_INITIAL_ADMIN_PASSWORD=$(echo "$SHARED_SECRET" | jq -r '.MEET_INITIAL_ADMIN_PASSWORD')
MEET_INITIAL_API_KEY=""
if [[ "${InitialMeetApiKey}" != '' ]]; then if [[ "${InitialMeetApiKey}" != '' ]]; then
MEET_INITIAL_API_KEY=$(echo "$SHARED_SECRET" | jq -r '.MEET_INITIAL_API_KEY') MEET_INITIAL_API_KEY=$(echo "$SHARED_SECRET" | jq -r '.MEET_INITIAL_API_KEY')
else
MEET_INITIAL_API_KEY=""
fi fi
MEET_INITIAL_API_KEY=$(echo "$SHARED_SECRET" | jq -r '.MEET_INITIAL_API_KEY')
LIVEKIT_API_KEY=$(echo "$SHARED_SECRET" | jq -r '.LIVEKIT_API_KEY') LIVEKIT_API_KEY=$(echo "$SHARED_SECRET" | jq -r '.LIVEKIT_API_KEY')
LIVEKIT_API_SECRET=$(echo "$SHARED_SECRET" | jq -r '.LIVEKIT_API_SECRET') LIVEKIT_API_SECRET=$(echo "$SHARED_SECRET" | jq -r '.LIVEKIT_API_SECRET')
ENABLED_MODULES=$(echo "$SHARED_SECRET" | jq -r '.ENABLED_MODULES') ENABLED_MODULES=$(echo "$SHARED_SECRET" | jq -r '.ENABLED_MODULES')
# Base command # Download first: sh <(curl ...) would silently run an empty script on a transient curl failure
INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh)" INSTALLER_SCRIPT="/tmp/install_ov_master_node.sh"
curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh"
if [ ! -s "$INSTALLER_SCRIPT" ]; then
echo "Downloaded OpenVidu master node installer is empty or missing" >&2
exit 1
fi
INSTALL_COMMAND="sh $INSTALLER_SCRIPT"
# Common arguments # Common arguments
COMMON_ARGS=( COMMON_ARGS=(
@ -1837,6 +1856,10 @@ Resources:
content: !Sub | content: !Sub |
#!/bin/bash #!/bin/bash
set -e set -e
# Only master 1 writes the shared secret, to avoid a lost-update race
if [[ "${!MASTER_NODE_NUM:-}" != "1" ]]; then
exit 0
fi
# Get current shared secret # Get current shared secret
SHARED_SECRET=$(aws secretsmanager get-secret-value \ SHARED_SECRET=$(aws secretsmanager get-secret-value \
--region ${AWS::Region} \ --region ${AWS::Region} \
@ -2103,6 +2126,8 @@ Resources:
OpenViduMasterNode1: OpenViduMasterNode1:
Type: AWS::EC2::Instance Type: AWS::EC2::Instance
# Depends on its own SSM parameter so its put-parameter can't race the parameter's creation
DependsOn: MasterNode1PrivateIpParameter
Properties: Properties:
LaunchTemplate: LaunchTemplate:
LaunchTemplateId: !Ref OpenViduMasterLaunchTemplate LaunchTemplateId: !Ref OpenViduMasterLaunchTemplate
@ -2133,6 +2158,8 @@ Resources:
cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource OpenViduMasterLaunchTemplate cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource OpenViduMasterLaunchTemplate
export MASTER_NODE_NUM=1
# Install OpenVidu # Install OpenVidu
/usr/local/bin/install.sh "MasterNodesWaitCondition1" || { echo "[OpenVidu] error installing OpenVidu"; exit 1; } /usr/local/bin/install.sh "MasterNodesWaitCondition1" || { echo "[OpenVidu] error installing OpenVidu"; exit 1; }
@ -2179,7 +2206,7 @@ Resources:
OpenViduMasterNode2: OpenViduMasterNode2:
Type: AWS::EC2::Instance Type: AWS::EC2::Instance
DependsOn: MasterNodesWaitCondition1 DependsOn: MasterNode2PrivateIpParameter
Properties: Properties:
LaunchTemplate: LaunchTemplate:
LaunchTemplateId: !Ref OpenViduMasterLaunchTemplate LaunchTemplateId: !Ref OpenViduMasterLaunchTemplate
@ -2210,6 +2237,8 @@ Resources:
cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource OpenViduMasterLaunchTemplate cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource OpenViduMasterLaunchTemplate
export MASTER_NODE_NUM=2
# Install OpenVidu # Install OpenVidu
/usr/local/bin/install.sh "MasterNodesWaitCondition2" || { echo "[OpenVidu] error installing OpenVidu"; exit 1; } /usr/local/bin/install.sh "MasterNodesWaitCondition2" || { echo "[OpenVidu] error installing OpenVidu"; exit 1; }
@ -2256,7 +2285,7 @@ Resources:
OpenViduMasterNode3: OpenViduMasterNode3:
Type: AWS::EC2::Instance Type: AWS::EC2::Instance
DependsOn: MasterNodesWaitCondition2 DependsOn: MasterNode3PrivateIpParameter
Properties: Properties:
LaunchTemplate: LaunchTemplate:
LaunchTemplateId: !Ref OpenViduMasterLaunchTemplate LaunchTemplateId: !Ref OpenViduMasterLaunchTemplate
@ -2287,6 +2316,8 @@ Resources:
cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource OpenViduMasterLaunchTemplate cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource OpenViduMasterLaunchTemplate
export MASTER_NODE_NUM=3
# Install OpenVidu # Install OpenVidu
/usr/local/bin/install.sh "MasterNodesWaitCondition3" || { echo "[OpenVidu] error installing OpenVidu"; exit 1; } /usr/local/bin/install.sh "MasterNodesWaitCondition3" || { echo "[OpenVidu] error installing OpenVidu"; exit 1; }
@ -2333,7 +2364,7 @@ Resources:
OpenViduMasterNode4: OpenViduMasterNode4:
Type: AWS::EC2::Instance Type: AWS::EC2::Instance
DependsOn: MasterNodesWaitCondition3 DependsOn: MasterNode4PrivateIpParameter
Properties: Properties:
LaunchTemplate: LaunchTemplate:
LaunchTemplateId: !Ref OpenViduMasterLaunchTemplate LaunchTemplateId: !Ref OpenViduMasterLaunchTemplate
@ -2364,6 +2395,8 @@ Resources:
cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource OpenViduMasterLaunchTemplate cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource OpenViduMasterLaunchTemplate
export MASTER_NODE_NUM=4
# Install OpenVidu # Install OpenVidu
/usr/local/bin/install.sh "MasterNodesWaitCondition4" || { echo "[OpenVidu] error installing OpenVidu"; exit 1; } /usr/local/bin/install.sh "MasterNodesWaitCondition4" || { echo "[OpenVidu] error installing OpenVidu"; exit 1; }
@ -2453,6 +2486,25 @@ Resources:
exit 1 exit 1
fi fi
SECRETS_WAIT_ATTEMPTS=0
SECRETS_WAIT_MAX=360
while true; do
# Validate content, not just the flag: stale eventually-consistent reads can return pre-generation values
if echo "$SHARED_SECRET" | jq -e '(.ALL_SECRETS_GENERATED == "true") and (.OPENVIDU_VERSION != "none") and (.REDIS_PASSWORD != "none")' > /dev/null; then
break
fi
SECRETS_WAIT_ATTEMPTS=$((SECRETS_WAIT_ATTEMPTS + 1))
if [[ $SECRETS_WAIT_ATTEMPTS -ge $SECRETS_WAIT_MAX ]]; then
echo "Error: timed out after 30 minutes waiting for shared secrets to be generated"
exit 1
fi
sleep 5
SHARED_SECRET="$(aws secretsmanager get-secret-value \
--region ${AWS::Region} \
--secret-id openvidu-ha-${AWS::Region}-${AWS::StackName} \
--query SecretString --output text || echo 'none')"
done
# Get OpenVidu Media Nodes version to deploy # Get OpenVidu Media Nodes version to deploy
OPENVIDU_VERSION=$(echo "$SHARED_SECRET" | jq -r '.OPENVIDU_VERSION') OPENVIDU_VERSION=$(echo "$SHARED_SECRET" | jq -r '.OPENVIDU_VERSION')
if [[ "$OPENVIDU_VERSION" == "none" ]]; then if [[ "$OPENVIDU_VERSION" == "none" ]]; then
@ -2460,20 +2512,39 @@ Resources:
exit 1 exit 1
fi fi
ALL_SECRETS_GENERATED=$(echo "$SHARED_SECRET" | jq -r '.ALL_SECRETS_GENERATED') IP_WAIT_ATTEMPTS=0
if [[ "$ALL_SECRETS_GENERATED" == "none" ]]; then IP_WAIT_MAX=360
echo "Error: Secrets not generated" while true; do
MASTER_NODE_1_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-1-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none')
MASTER_NODE_2_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-2-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none')
MASTER_NODE_3_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-3-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none')
MASTER_NODE_4_PRIVATE_IP=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-4-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none')
if [[ "$MASTER_NODE_1_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_1_PRIVATE_IP" ]] &&
[[ "$MASTER_NODE_2_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_2_PRIVATE_IP" ]] &&
[[ "$MASTER_NODE_3_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_3_PRIVATE_IP" ]] &&
[[ "$MASTER_NODE_4_PRIVATE_IP" != "none" ]] && [[ -n "$MASTER_NODE_4_PRIVATE_IP" ]]; then
break
fi
IP_WAIT_ATTEMPTS=$((IP_WAIT_ATTEMPTS + 1))
if [[ $IP_WAIT_ATTEMPTS -ge $IP_WAIT_MAX ]]; then
echo "Error: timed out after 30 minutes waiting for all master nodes to publish their private IPs to SSM"
exit 1 exit 1
fi fi
MASTER_NODE_1_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_1_PRIVATE_IP') sleep 5
MASTER_NODE_2_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_2_PRIVATE_IP') done
MASTER_NODE_3_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_3_PRIVATE_IP')
MASTER_NODE_4_PRIVATE_IP=$(echo "$SHARED_SECRET" | jq -r '.MASTER_NODE_4_PRIVATE_IP')
MASTER_NODE_PRIVATE_IP_LIST="$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP" MASTER_NODE_PRIVATE_IP_LIST="$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP"
REDIS_PASSWORD=$(echo "$SHARED_SECRET" | jq -r '.REDIS_PASSWORD') REDIS_PASSWORD=$(echo "$SHARED_SECRET" | jq -r '.REDIS_PASSWORD')
# Base command # Download first: sh <(curl ...) would silently run an empty script on a transient curl failure
INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh)" INSTALLER_SCRIPT="/tmp/install_ov_media_node.sh"
curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh"
if [ ! -s "$INSTALLER_SCRIPT" ]; then
echo "Downloaded OpenVidu media node installer is empty or missing" >&2
exit 1
fi
INSTALL_COMMAND="sh $INSTALLER_SCRIPT"
# Common arguments # Common arguments
COMMON_ARGS=( COMMON_ARGS=(
@ -2609,13 +2680,36 @@ Resources:
python3 -m pip install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-${!CFN_BOOTSTRAP_VERSION}.tar.gz python3 -m pip install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-${!CFN_BOOTSTRAP_VERSION}.tar.gz
fi fi
cfn-init --region ${AWS::Region} --stack ${AWS::StackId} --resource OpenViduMediaNodeLaunchTemplate cfn-init --region ${AWS::Region} --stack ${AWS::StackName} --resource OpenViduMediaNodeLaunchTemplate
export HOME="/root" export HOME="/root"
# Install OpenVidu # Install OpenVidu
/usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu"; /usr/local/bin/set_as_unhealthy.sh; exit 1; } /usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu"; /usr/local/bin/set_as_unhealthy.sh; exit 1; }
MASTER_IP_1=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-1-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none')
MASTER_IP_2=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-2-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none')
MASTER_IP_3=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-3-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none')
MASTER_IP_4=$(aws ssm get-parameter --region ${AWS::Region} --name "/openvidu/${AWS::StackName}/master-node-4-private-ip" --query Parameter.Value --output text 2>/dev/null || echo 'none')
MASTER_HEALTHY=false
for i in $(seq 1 360); do
for MASTER_IP in "$MASTER_IP_1" "$MASTER_IP_2" "$MASTER_IP_3" "$MASTER_IP_4"; do
if [ "$MASTER_IP" != "none" ] && [ -n "$MASTER_IP" ] && curl -sf "http://$MASTER_IP:7880/health/caddy" >/dev/null 2>&1; then
MASTER_HEALTHY=true
break
fi
done
if [ "$MASTER_HEALTHY" = "true" ]; then
break
fi
sleep 5
done
if [ "$MASTER_HEALTHY" != "true" ]; then
echo "[OpenVidu] no master node became healthy after 30 minutes"
/usr/local/bin/set_as_unhealthy.sh
exit 1
fi
# Start OpenVidu # Start OpenVidu
systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; /usr/local/bin/set_as_unhealthy.sh; exit 1; } systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; /usr/local/bin/set_as_unhealthy.sh; exit 1; }
@ -3191,8 +3285,6 @@ Resources:
LoadBalancer: LoadBalancer:
Type: AWS::ElasticLoadBalancingV2::LoadBalancer Type: AWS::ElasticLoadBalancingV2::LoadBalancer
DependsOn:
- MasterNodesWaitCondition4
Properties: Properties:
Name: Name:
Fn::Join: Fn::Join:

115
openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep
View File

@ -304,7 +304,7 @@ apt-get update && apt-get install -y \
# Wait for the keyvault availability # Wait for the keyvault availability
MAX_WAIT=100 MAX_WAIT=300
WAIT_INTERVAL=1 WAIT_INTERVAL=1
ELAPSED_TIME=0 ELAPSED_TIME=0
set +e set +e
@ -397,6 +397,9 @@ if [[ $MASTER_NODE_NUM -eq 1 ]] && [[ "$ALL_SECRETS_GENERATED" == "" || "$ALL_SE
ALL_SECRETS_GENERATED="$(/usr/local/bin/store_secret.sh save ALL-SECRETS-GENERATED "true")" ALL_SECRETS_GENERATED="$(/usr/local/bin/store_secret.sh save ALL-SECRETS-GENERATED "true")"
fi fi
IP_WAIT_MAX_RETRIES=360
IP_WAIT_INTERVAL=5
IP_WAIT_RETRIES=0
while true; do while true; do
MASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv) || true MASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv) || true
MASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv) || true MASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv) || true
@ -409,11 +412,26 @@ while true; do
[[ "$MASTER_NODE_4_PRIVATE_IP" != "" ]]; then [[ "$MASTER_NODE_4_PRIVATE_IP" != "" ]]; then
break break
fi fi
sleep 5 IP_WAIT_RETRIES=$((IP_WAIT_RETRIES + 1))
if [ $IP_WAIT_RETRIES -ge $IP_WAIT_MAX_RETRIES ]; then
echo "[OpenVidu] timed out after 30 min waiting for the 4 master nodes to publish their private IPs"
exit 1
fi
sleep $IP_WAIT_INTERVAL
done done
# Wait until master-node-1 has generated all shared secrets before fetching them # Wait until master-node-1 has generated all shared secrets before fetching them
while [[ "$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)" != "true" ]]; do sleep 5; done SECRETS_WAIT_MAX_RETRIES=360
SECRETS_WAIT_INTERVAL=5
SECRETS_WAIT_RETRIES=0
while [[ "$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)" != "true" ]]; do
SECRETS_WAIT_RETRIES=$((SECRETS_WAIT_RETRIES + 1))
if [ $SECRETS_WAIT_RETRIES -ge $SECRETS_WAIT_MAX_RETRIES ]; then
echo "[OpenVidu] timed out after 30 min waiting for ALL-SECRETS-GENERATED to become true"
exit 1
fi
sleep $SECRETS_WAIT_INTERVAL
done
# Fetch the values in the keyvault # Fetch the values in the keyvault
@ -448,8 +466,16 @@ fi
ENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv) ENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv)
# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure
INSTALLER_SCRIPT="/tmp/install_ov_master_node.sh"
curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh"
if [ ! -s "$INSTALLER_SCRIPT" ]; then
echo "[OpenVidu] failed to download the master node installer script"
exit 1
fi
# Base command # Base command
INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh)" INSTALL_COMMAND="sh $INSTALLER_SCRIPT"
# Common arguments # Common arguments
COMMON_ARGS=( COMMON_ARGS=(
@ -811,7 +837,7 @@ INSTALL_DIR="/opt/openvidu"
CLUSTER_CONFIG_DIR="${INSTALL_DIR}/config/cluster" CLUSTER_CONFIG_DIR="${INSTALL_DIR}/config/cluster"
# Retry login + storage key fetch to allow the Contributor role assignment to propagate # Retry login + storage key fetch to allow the Contributor role assignment to propagate
MAX_WAIT=100 MAX_WAIT=300
WAIT_INTERVAL=1 WAIT_INTERVAL=1
ELAPSED_TIME=0 ELAPSED_TIME=0
set +e set +e
@ -1043,8 +1069,6 @@ az login --identity --allow-no-subscriptions
echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout
apt-get update && apt-get install -y
export HOME="/root" export HOME="/root"
# Install OpenVidu # Install OpenVidu
@ -1062,20 +1086,18 @@ systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1;
# Launch on reboot # Launch on reboot
echo "@reboot /usr/local/bin/restart.sh >> /var/log/openvidu-restart.log" 2>&1 | crontab echo "@reboot /usr/local/bin/restart.sh >> /var/log/openvidu-restart.log" 2>&1 | crontab
# check_app_ready.sh internally caps its wait at 1200s
/usr/local/bin/check_app_ready.sh || { echo "[OpenVidu] master node did not become healthy"; exit 1; }
az keyvault secret set --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --value "true"
MASTER_NODE_NUM=${masterNodeNum} MASTER_NODE_NUM=${masterNodeNum}
if [[ $MASTER_NODE_NUM -eq 4 ]]; then if [[ $MASTER_NODE_NUM -eq 4 ]]; then
# Creating scale in lock # Creating scale in lock
set +e set +e
az storage blob upload --account-name ${storageAccountName} --container-name automation-locks --name lock.txt --file /dev/null --auth-mode key az storage blob upload --account-name ${storageAccountName} --container-name automation-locks --name lock.txt --file /dev/null --auth-mode key
set -e set -e
#Finish all the nodes
az keyvault secret set --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --value "true"
fi fi
# Wait for the app
sleep 150
/usr/local/bin/check_app_ready.sh
''' '''
var userDataMasterNode1 = reduce( var userDataMasterNode1 = reduce(
@ -1168,7 +1190,6 @@ resource openviduMasterNode2 'Microsoft.Compute/virtualMachines@2023-09-01' = {
} }
userData: base64(userDataMasterNode2) userData: base64(userDataMasterNode2)
} }
dependsOn: [openviduMasterNode1]
} }
resource openviduMasterNode3 'Microsoft.Compute/virtualMachines@2023-09-01' = { resource openviduMasterNode3 'Microsoft.Compute/virtualMachines@2023-09-01' = {
@ -1203,7 +1224,6 @@ resource openviduMasterNode3 'Microsoft.Compute/virtualMachines@2023-09-01' = {
} }
userData: base64(userDataMasterNode3) userData: base64(userDataMasterNode3)
} }
dependsOn: [openviduMasterNode2]
} }
resource openviduMasterNode4 'Microsoft.Compute/virtualMachines@2023-09-01' = { resource openviduMasterNode4 'Microsoft.Compute/virtualMachines@2023-09-01' = {
@ -1238,7 +1258,6 @@ resource openviduMasterNode4 'Microsoft.Compute/virtualMachines@2023-09-01' = {
} }
userData: base64(userDataMasterNode4) userData: base64(userDataMasterNode4)
} }
dependsOn: [openviduMasterNode3]
} }
/*------------------------------------------- MEDIA NODES -------------------------------------------*/ /*------------------------------------------- MEDIA NODES -------------------------------------------*/
@ -1272,23 +1291,29 @@ apt-get update && apt-get install -y \
# Get own private IP # Get own private IP
PRIVATE_IP=$(curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text") PRIVATE_IP=$(curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text")
WAIT_INTERVAL=1 # Gate 1: wait for master secrets and IPs before installing
MAX_WAIT=10000 WAIT_INTERVAL=5
ELAPSED_TIME=0 MAX_RETRIES=360
RETRIES=0
set +e set +e
while true; do while true; do
# get secret value ALL_SECRETS_GENERATED=$(az keyvault secret show --vault-name ${keyVaultName} --name ALL-SECRETS-GENERATED --query value -o tsv 2>/dev/null)
FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv) MASTER_NODE_1_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-1-PRIVATE-IP --query value -o tsv 2>/dev/null)
MASTER_NODE_2_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-2-PRIVATE-IP --query value -o tsv 2>/dev/null)
MASTER_NODE_3_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-3-PRIVATE-IP --query value -o tsv 2>/dev/null)
MASTER_NODE_4_PRIVATE_IP=$(az keyvault secret show --vault-name ${keyVaultName} --name MASTER-NODE-4-PRIVATE-IP --query value -o tsv 2>/dev/null)
# Check if all master nodes finished if [ "$ALL_SECRETS_GENERATED" == "true" ] &&
if [ "$FINISH_MASTER_NODE" == "true" ]; then [ "$MASTER_NODE_1_PRIVATE_IP" != "" ] &&
[ "$MASTER_NODE_2_PRIVATE_IP" != "" ] &&
[ "$MASTER_NODE_3_PRIVATE_IP" != "" ] &&
[ "$MASTER_NODE_4_PRIVATE_IP" != "" ]; then
break break
fi fi
ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL)) RETRIES=$((RETRIES + 1))
if [ $RETRIES -ge $MAX_RETRIES ]; then
# Check if the maximum waiting time has been reached echo "[OpenVidu] timed out after 30 min waiting for master nodes to generate secrets and publish their IPs"
if [ $ELAPSED_TIME -ge $MAX_WAIT ]; then
exit 1 exit 1
fi fi
@ -1305,8 +1330,16 @@ REDIS_PASSWORD=$(az keyvault secret show --vault-name ${keyVaultName} --name RED
ENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv) ENABLED_MODULES=$(az keyvault secret show --vault-name ${keyVaultName} --name ENABLED-MODULES --query value -o tsv)
OPENVIDU_VERSION=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-VERSION --query value -o tsv) OPENVIDU_VERSION=$(az keyvault secret show --vault-name ${keyVaultName} --name OPENVIDU-VERSION --query value -o tsv)
# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure
INSTALLER_SCRIPT="/tmp/install_ov_media_node.sh"
curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh"
if [ ! -s "$INSTALLER_SCRIPT" ]; then
echo "[OpenVidu] failed to download the media node installer script"
exit 1
fi
# Base command # Base command
INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh)" INSTALL_COMMAND="sh $INSTALLER_SCRIPT"
# Common arguments # Common arguments
COMMON_ARGS=( COMMON_ARGS=(
@ -1413,8 +1446,7 @@ chmod +x /usr/local/bin/delete_media_node.sh
echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout
apt-get update && apt-get install -y apt-get update && apt-get install -y jq
apt-get install -y jq
# Install azure cli # Install azure cli
AZURE_CLI_VERSION=2.87.0 AZURE_CLI_VERSION=2.87.0
@ -1438,6 +1470,26 @@ export HOME="/root"
# Install OpenVidu # Install OpenVidu
/usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu"; /usr/local/bin/delete_media_node.sh; } /usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu"; /usr/local/bin/delete_media_node.sh; }
# Gate 2: wait for master readiness before starting
WAIT_INTERVAL=5
MAX_RETRIES=360
RETRIES=0
set +e
while true; do
FINISH_MASTER_NODE=$(az keyvault secret show --vault-name ${keyVaultName} --name FINISH-MASTER-NODE --query value -o tsv 2>/dev/null)
if [ "$FINISH_MASTER_NODE" == "true" ]; then
break
fi
RETRIES=$((RETRIES + 1))
if [ $RETRIES -ge $MAX_RETRIES ]; then
echo "[OpenVidu] timed out after 30 min waiting for FINISH-MASTER-NODE"
/usr/local/bin/delete_media_node.sh
exit 1
fi
sleep $WAIT_INTERVAL
done
set -e
# Start OpenVidu # Start OpenVidu
systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; /usr/local/bin/delete_media_node.sh; } systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; /usr/local/bin/delete_media_node.sh; }
''' '''
@ -1470,6 +1522,7 @@ var userDataParamsMedia = {
base64delete_mediaNode: base64delete_mediaNode_ScriptMedia base64delete_mediaNode: base64delete_mediaNode_ScriptMedia
resourceGroupName: resourceGroup().name resourceGroupName: resourceGroup().name
vmScaleSetName: '${stackName}-mediaNodeScaleSet' vmScaleSetName: '${stackName}-mediaNodeScaleSet'
keyVaultName: keyVaultName
} }
var userDataMediaNode = reduce( var userDataMediaNode = reduce(

19
openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json
View File

File diff suppressed because one or more lines are too long

2
openvidu-deployment/pro/ha/azure/createUiDefinition.json
View File

@ -553,7 +553,7 @@
"datetime": "[steps('parameters SCALING').datetime]", "datetime": "[steps('parameters SCALING').datetime]",
"automationAccountName": "[steps('parameters SCALING').automationAccountName]", "automationAccountName": "[steps('parameters SCALING').automationAccountName]",
"storageAccountName": "[steps('parameters STORAGE').storageAccountName]", "storageAccountName": "[steps('parameters STORAGE').storageAccountName]",
"containerName": "[steps('parameters STORAGE').containerName]", "appDataContainerName": "[steps('parameters STORAGE').containerName]",
"additionalInstallFlags": "[steps('FLAGS').additionalInstallFlags]" "additionalInstallFlags": "[steps('FLAGS').additionalInstallFlags]"
} }
} }

220
openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf
View File

@ -20,14 +20,15 @@ resource "google_secret_manager_secret" "openvidu_shared_info" {
"OPENVIDU_PRO_LICENSE", "OPENVIDU_RTC_ENGINE", "REDIS_PASSWORD", "MONGO_ADMIN_USERNAME", "OPENVIDU_PRO_LICENSE", "OPENVIDU_RTC_ENGINE", "REDIS_PASSWORD", "MONGO_ADMIN_USERNAME",
"MONGO_ADMIN_PASSWORD", "MONGO_REPLICA_SET_KEY", "MINIO_ACCESS_KEY", "MINIO_SECRET_KEY", "MONGO_ADMIN_PASSWORD", "MONGO_REPLICA_SET_KEY", "MINIO_ACCESS_KEY", "MINIO_SECRET_KEY",
"DASHBOARD_ADMIN_USERNAME", "DASHBOARD_ADMIN_PASSWORD", "GRAFANA_ADMIN_USERNAME", "DASHBOARD_ADMIN_USERNAME", "DASHBOARD_ADMIN_PASSWORD", "GRAFANA_ADMIN_USERNAME",
"GRAFANA_ADMIN_PASSWORD", "ENABLED_MODULES", "OPENVIDU_VERSION", "ALL_SECRETS_GENERATED", "GRAFANA_ADMIN_PASSWORD", "ENABLED_MODULES", "OPENVIDU_VERSION", "ALL_SECRETS_GENERATED"
"MASTER_NODE_1_PRIVATE_IP", "MASTER_NODE_2_PRIVATE_IP", "MASTER_NODE_3_PRIVATE_IP", "MASTER_NODE_4_PRIVATE_IP"
]) ])
secret_id = each.key secret_id = each.key
replication { replication {
auto {} auto {}
} }
depends_on = [google_project_service.secretmanager_api]
} }
# GCS buckets for HA deployment # GCS buckets for HA deployment
@ -37,6 +38,8 @@ resource "google_storage_bucket" "appdata_bucket" {
location = var.region location = var.region
force_destroy = true force_destroy = true
uniform_bucket_level_access = true uniform_bucket_level_access = true
depends_on = [google_project_service.storage_api]
} }
resource "google_storage_bucket" "clusterdata_bucket" { resource "google_storage_bucket" "clusterdata_bucket" {
@ -45,12 +48,16 @@ resource "google_storage_bucket" "clusterdata_bucket" {
location = var.region location = var.region
force_destroy = true force_destroy = true
uniform_bucket_level_access = true uniform_bucket_level_access = true
depends_on = [google_project_service.storage_api]
} }
# Service account for the instances # Service account for the instances
resource "google_service_account" "service_account" { resource "google_service_account" "service_account" {
account_id = lower("${substr(var.stackName, 0, 12)}-sa") account_id = lower("${substr(var.stackName, 0, 12)}-sa")
display_name = "OpenVidu instance service account" display_name = "OpenVidu instance service account"
depends_on = [google_project_service.iam_api]
} }
# IAM bindings for the service account # IAM bindings for the service account
@ -58,6 +65,8 @@ resource "google_project_iam_member" "iam_project_role" {
project = var.projectId project = var.projectId
role = "roles/owner" role = "roles/owner"
member = "serviceAccount:${google_service_account.service_account.email}" member = "serviceAccount:${google_service_account.service_account.email}"
depends_on = [google_project_service.cloudresourcemanager_api]
} }
# External SSH access to Master Nodes # External SSH access to Master Nodes
@ -72,6 +81,8 @@ resource "google_compute_firewall" "external_master_ssh" {
source_ranges = ["0.0.0.0/0"] source_ranges = ["0.0.0.0/0"]
target_tags = [lower("${var.stackName}-master-node")] target_tags = [lower("${var.stackName}-master-node")]
depends_on = [google_project_service.compute_api]
} }
# External access to Media Nodes (SSH and media traffic) # External access to Media Nodes (SSH and media traffic)
@ -90,6 +101,8 @@ resource "google_compute_firewall" "external_media_access" {
source_ranges = ["0.0.0.0/0"] source_ranges = ["0.0.0.0/0"]
target_tags = [lower("${var.stackName}-media-node")] target_tags = [lower("${var.stackName}-media-node")]
depends_on = [google_project_service.compute_api]
} }
# Load Balancer health checks and HTTP traffic to Master Nodes # Load Balancer health checks and HTTP traffic to Master Nodes
@ -104,6 +117,8 @@ resource "google_compute_firewall" "lb_to_master_http" {
source_ranges = ["0.0.0.0/0"] source_ranges = ["0.0.0.0/0"]
target_tags = [lower("${var.stackName}-master-node")] target_tags = [lower("${var.stackName}-master-node")]
depends_on = [google_project_service.compute_api]
} }
# Master node internal services communication # Master node internal services communication
@ -129,6 +144,8 @@ resource "google_compute_firewall" "master_to_master_internal" {
source_tags = [lower("${var.stackName}-master-node")] source_tags = [lower("${var.stackName}-master-node")]
target_tags = [lower("${var.stackName}-master-node")] target_tags = [lower("${var.stackName}-master-node")]
depends_on = [google_project_service.compute_api]
} }
# Media Nodes to Master Nodes communication # Media Nodes to Master Nodes communication
@ -153,6 +170,8 @@ resource "google_compute_firewall" "media_to_master_services" {
source_tags = [lower("${var.stackName}-media-node")] source_tags = [lower("${var.stackName}-media-node")]
target_tags = [lower("${var.stackName}-master-node")] target_tags = [lower("${var.stackName}-master-node")]
depends_on = [google_project_service.compute_api]
} }
# Master Nodes to Media Nodes communication # Master Nodes to Media Nodes communication
@ -171,6 +190,8 @@ resource "google_compute_firewall" "master_to_media_services" {
source_tags = [lower("${var.stackName}-master-node")] source_tags = [lower("${var.stackName}-master-node")]
target_tags = [lower("${var.stackName}-media-node")] target_tags = [lower("${var.stackName}-media-node")]
depends_on = [google_project_service.compute_api]
} }
@ -179,6 +200,8 @@ resource "google_compute_address" "nlb_ip" {
count = var.publicIpAddress == "" ? 1 : 0 count = var.publicIpAddress == "" ? 1 : 0
name = lower("${var.stackName}-nlb-ip") name = lower("${var.stackName}-nlb-ip")
region = var.region region = var.region
depends_on = [google_project_service.compute_api]
} }
# Data source for existing IP address when publicIpAddress is provided # Data source for existing IP address when publicIpAddress is provided
@ -206,6 +229,8 @@ resource "google_compute_region_health_check" "tcp_health_check" {
timeout_sec = 5 timeout_sec = 5
healthy_threshold = 3 healthy_threshold = 3
unhealthy_threshold = 4 unhealthy_threshold = 4
depends_on = [google_project_service.compute_api]
} }
# Regional backend service for the TCP NLB # Regional backend service for the TCP NLB
@ -289,6 +314,29 @@ locals {
is_c4a_instance = startswith(var.masterNodesInstanceType, "c4a-") is_c4a_instance = startswith(var.masterNodesInstanceType, "c4a-")
} }
# Must match the subnetwork the master instances implicitly use (network = "default", no explicit subnetwork)
data "google_compute_subnetwork" "default" {
name = "default"
region = var.region
depends_on = [google_project_service.compute_api]
}
resource "google_compute_address" "master_internal_ip" {
count = 4
name = lower("${var.stackName}-master-node-${count.index + 1}-internal-ip")
address_type = "INTERNAL"
region = var.region
subnetwork = data.google_compute_subnetwork.default.id
depends_on = [google_project_service.compute_api]
}
locals {
# Comma-separated: must match the installer's --master-node-private-ip-list format
master_node_private_ip_list = join(",", google_compute_address.master_internal_ip[*].address)
}
# Master Node 1 # Master Node 1
resource "google_compute_instance" "openvidu_master_node_1" { resource "google_compute_instance" "openvidu_master_node_1" {
name = lower("${var.stackName}-master-node-1") name = lower("${var.stackName}-master-node-1")
@ -306,12 +354,14 @@ resource "google_compute_instance" "openvidu_master_node_1" {
network_interface { network_interface {
network = "default" network = "default"
network_ip = google_compute_address.master_internal_ip[0].address
access_config {} access_config {}
} }
metadata = { metadata = {
stackName = var.stackName stackName = var.stackName
masterNodeNum = "1" masterNodeNum = "1"
masterNodePrivateIPList = local.master_node_private_ip_list
domainName = var.domainName domainName = var.domainName
certificateType = var.certificateType certificateType = var.certificateType
ownPublicCertificate = var.ownPublicCertificate ownPublicCertificate = var.ownPublicCertificate
@ -356,12 +406,14 @@ resource "google_compute_instance" "openvidu_master_node_2" {
network_interface { network_interface {
network = "default" network = "default"
network_ip = google_compute_address.master_internal_ip[1].address
access_config {} access_config {}
} }
metadata = { metadata = {
stackName = var.stackName stackName = var.stackName
masterNodeNum = "2" masterNodeNum = "2"
masterNodePrivateIPList = local.master_node_private_ip_list
domainName = var.domainName domainName = var.domainName
certificateType = var.certificateType certificateType = var.certificateType
ownPublicCertificate = var.ownPublicCertificate ownPublicCertificate = var.ownPublicCertificate
@ -387,8 +439,6 @@ resource "google_compute_instance" "openvidu_master_node_2" {
node-type = "master" node-type = "master"
node-num = "2" node-num = "2"
} }
depends_on = [google_compute_instance.openvidu_master_node_1]
} }
# Master Node 3 # Master Node 3
@ -408,12 +458,14 @@ resource "google_compute_instance" "openvidu_master_node_3" {
network_interface { network_interface {
network = "default" network = "default"
network_ip = google_compute_address.master_internal_ip[2].address
access_config {} access_config {}
} }
metadata = { metadata = {
stackName = var.stackName stackName = var.stackName
masterNodeNum = "3" masterNodeNum = "3"
masterNodePrivateIPList = local.master_node_private_ip_list
domainName = var.domainName domainName = var.domainName
certificateType = var.certificateType certificateType = var.certificateType
ownPublicCertificate = var.ownPublicCertificate ownPublicCertificate = var.ownPublicCertificate
@ -439,8 +491,6 @@ resource "google_compute_instance" "openvidu_master_node_3" {
node-type = "master" node-type = "master"
node-num = "3" node-num = "3"
} }
depends_on = [google_compute_instance.openvidu_master_node_2]
} }
# Master Node 4 # Master Node 4
@ -460,12 +510,14 @@ resource "google_compute_instance" "openvidu_master_node_4" {
network_interface { network_interface {
network = "default" network = "default"
network_ip = google_compute_address.master_internal_ip[3].address
access_config {} access_config {}
} }
metadata = { metadata = {
stackName = var.stackName stackName = var.stackName
masterNodeNum = "4" masterNodeNum = "4"
masterNodePrivateIPList = local.master_node_private_ip_list
domainName = var.domainName domainName = var.domainName
certificateType = var.certificateType certificateType = var.certificateType
ownPublicCertificate = var.ownPublicCertificate ownPublicCertificate = var.ownPublicCertificate
@ -491,8 +543,6 @@ resource "google_compute_instance" "openvidu_master_node_4" {
node-type = "master" node-type = "master"
node-num = "4" node-num = "4"
} }
depends_on = [google_compute_instance.openvidu_master_node_3]
} }
# ------------------------- scale in resources ------------------------- # ------------------------- scale in resources -------------------------
@ -800,6 +850,12 @@ resource "google_cloudfunctions2_function" "scalein_function" {
} }
service_account_email = google_service_account.service_account.email service_account_email = google_service_account.service_account.email
} }
depends_on = [
google_project_service.cloudfunctions_api,
google_project_service.cloudbuild_api,
google_project_service.run_api
]
} }
# Cloud Scheduler to trigger the function every 5 minutes # Cloud Scheduler to trigger the function every 5 minutes
@ -829,6 +885,8 @@ resource "google_cloud_scheduler_job" "scale_scheduler" {
service_account_email = google_service_account.service_account.email service_account_email = google_service_account.service_account.email
} }
} }
depends_on = [google_project_service.cloudscheduler_api]
} }
locals { locals {
@ -858,7 +916,7 @@ resource "google_compute_instance_template" "media_node_template" {
metadata = { metadata = {
stackName = var.stackName stackName = var.stackName
masterNodePrivateIPList = "${google_compute_instance.openvidu_master_node_1.network_interface[0].network_ip},${google_compute_instance.openvidu_master_node_2.network_interface[0].network_ip},${google_compute_instance.openvidu_master_node_3.network_interface[0].network_ip},${google_compute_instance.openvidu_master_node_4.network_interface[0].network_ip}" masterNodePrivateIPList = local.master_node_private_ip_list
bucketAppDataName = local.isEmptyAppData ? google_storage_bucket.appdata_bucket[0].name : var.GCSAppDataBucketName bucketAppDataName = local.isEmptyAppData ? google_storage_bucket.appdata_bucket[0].name : var.GCSAppDataBucketName
bucketClusterDataName = local.isEmptyClusterData ? google_storage_bucket.clusterdata_bucket[0].name : var.GCSClusterDataBucketName bucketClusterDataName = local.isEmptyClusterData ? google_storage_bucket.clusterdata_bucket[0].name : var.GCSClusterDataBucketName
region = var.region region = var.region
@ -880,13 +938,6 @@ resource "google_compute_instance_template" "media_node_template" {
lifecycle { lifecycle {
create_before_destroy = true create_before_destroy = true
} }
depends_on = [
google_compute_instance.openvidu_master_node_1,
google_compute_instance.openvidu_master_node_2,
google_compute_instance.openvidu_master_node_3,
google_compute_instance.openvidu_master_node_4
]
} }
# Managed Instance Group for Media Nodes # Managed Instance Group for Media Nodes
@ -904,13 +955,6 @@ resource "google_compute_region_instance_group_manager" "media_node_group" {
name = "http" name = "http"
port = 7880 port = 7880
} }
depends_on = [
google_compute_instance.openvidu_master_node_1,
google_compute_instance.openvidu_master_node_2,
google_compute_instance.openvidu_master_node_3,
google_compute_instance.openvidu_master_node_4
]
} }
# Autoscaler for Media Nodes # Autoscaler for Media Nodes
@ -969,11 +1013,7 @@ get_meta() { curl -s -H "Metadata-Flavor: Google" "$${METADATA_URL}/$1"; }
# Get master node number from metadata # Get master node number from metadata
MASTER_NODE_NUM=$(get_meta "instance/attributes/masterNodeNum") MASTER_NODE_NUM=$(get_meta "instance/attributes/masterNodeNum")
# Get own private IP MASTER_NODE_PRIVATE_IP_LIST=$(get_meta "instance/attributes/masterNodePrivateIPList")
PRIVATE_IP=$(get_meta "instance/network-interfaces/0/ip")
# Store current private IP
PRIVATE_IP="$(/usr/local/bin/store_secret.sh save MASTER_NODE_$${MASTER_NODE_NUM}_PRIVATE_IP $PRIVATE_IP)"
# Check if secrets have been generated # Check if secrets have been generated
ALL_SECRETS_GENERATED=$(gcloud secrets versions access latest --secret=ALL_SECRETS_GENERATED 2>/dev/null || echo "false") ALL_SECRETS_GENERATED=$(gcloud secrets versions access latest --secret=ALL_SECRETS_GENERATED 2>/dev/null || echo "false")
@ -1033,33 +1073,19 @@ if [[ $MASTER_NODE_NUM -eq 1 ]] && [[ "$ALL_SECRETS_GENERATED" == "false" ]]; th
ALL_SECRETS_GENERATED="$(/usr/local/bin/store_secret.sh save ALL_SECRETS_GENERATED "true")" ALL_SECRETS_GENERATED="$(/usr/local/bin/store_secret.sh save ALL_SECRETS_GENERATED "true")"
fi fi
# Wait for all master nodes to store their private IPs # Wait for master-1 to finish generating all shared secrets before reading them.
while true; do i=0
MASTER_NODE_1_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_1_PRIVATE_IP 2>/dev/null || echo "") while ! gcloud secrets versions access latest --secret=ALL_SECRETS_GENERATED 2>/dev/null | grep -q "true"; do
MASTER_NODE_2_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_2_PRIVATE_IP 2>/dev/null || echo "") i=$((i + 1))
MASTER_NODE_3_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_3_PRIVATE_IP 2>/dev/null || echo "") if [ "$i" -ge 360 ]; then
MASTER_NODE_4_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_4_PRIVATE_IP 2>/dev/null || echo "") echo "Timed out after 30 minutes waiting for master-1 to finish generating secrets" >&2
exit 1
# Check if all master nodes have stored their private IPs
if [[ "$MASTER_NODE_1_PRIVATE_IP" != "" ]] &&
[[ "$MASTER_NODE_2_PRIVATE_IP" != "" ]] &&
[[ "$MASTER_NODE_3_PRIVATE_IP" != "" ]] &&
[[ "$MASTER_NODE_4_PRIVATE_IP" != "" ]]; then
break
fi fi
echo "Waiting for master-1 to finish generating secrets..."
sleep 5 sleep 5
done done
# Wait for master-1 to finish generating all shared secrets before reading them
while ! gcloud secrets versions access latest --secret=ALL_SECRETS_GENERATED 2>/dev/null | grep -q "true"; do echo "Waiting for master-1 to finish generating secrets..."; sleep 5; done
# Fetch all values from Secret Manager # Fetch all values from Secret Manager
MASTER_NODE_1_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_1_PRIVATE_IP)
MASTER_NODE_2_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_2_PRIVATE_IP)
MASTER_NODE_3_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_3_PRIVATE_IP)
MASTER_NODE_4_PRIVATE_IP=$(gcloud secrets versions access latest --secret=MASTER_NODE_4_PRIVATE_IP)
MASTER_NODE_PRIVATE_IP_LIST="$MASTER_NODE_1_PRIVATE_IP,$MASTER_NODE_2_PRIVATE_IP,$MASTER_NODE_3_PRIVATE_IP,$MASTER_NODE_4_PRIVATE_IP"
DOMAIN=$(gcloud secrets versions access latest --secret=DOMAIN_NAME) DOMAIN=$(gcloud secrets versions access latest --secret=DOMAIN_NAME)
OPENVIDU_PRO_LICENSE=$(gcloud secrets versions access latest --secret=OPENVIDU_PRO_LICENSE) OPENVIDU_PRO_LICENSE=$(gcloud secrets versions access latest --secret=OPENVIDU_PRO_LICENSE)
OPENVIDU_RTC_ENGINE=$(gcloud secrets versions access latest --secret=OPENVIDU_RTC_ENGINE) OPENVIDU_RTC_ENGINE=$(gcloud secrets versions access latest --secret=OPENVIDU_RTC_ENGINE)
@ -1082,8 +1108,14 @@ LIVEKIT_API_KEY=$(gcloud secrets versions access latest --secret=LIVEKIT_API_KEY
LIVEKIT_API_SECRET=$(gcloud secrets versions access latest --secret=LIVEKIT_API_SECRET) LIVEKIT_API_SECRET=$(gcloud secrets versions access latest --secret=LIVEKIT_API_SECRET)
ENABLED_MODULES=$(gcloud secrets versions access latest --secret=ENABLED_MODULES) ENABLED_MODULES=$(gcloud secrets versions access latest --secret=ENABLED_MODULES)
# Build install command # Download first: sh <(curl ...) would silently run an empty script on a transient curl failure
INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh)" INSTALLER_SCRIPT="/tmp/install_ov_master_node.sh"
curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_master_node.sh"
if [ ! -s "$INSTALLER_SCRIPT" ]; then
echo "Downloaded OpenVidu master node installer is empty or missing" >&2
exit 1
fi
INSTALL_COMMAND="sh $INSTALLER_SCRIPT"
# Common arguments # Common arguments
COMMON_ARGS=( COMMON_ARGS=(
@ -1434,11 +1466,17 @@ EOF
check_app_ready_script = <<-EOF check_app_ready_script = <<-EOF
#!/bin/bash #!/bin/bash
i=0
while true; do while true; do
HTTP_STATUS=$(curl -Ik http://localhost:7880/health/caddy 2>/dev/null | head -n1 | awk '{print $2}') HTTP_STATUS=$(curl -Ik http://localhost:7880/health/caddy 2>/dev/null | head -n1 | awk '{print $2}')
if [ "$HTTP_STATUS" == "200" ]; then if [ "$HTTP_STATUS" == "200" ]; then
break break
fi fi
i=$((i + 1))
if [ "$i" -ge 240 ]; then
echo "Timed out after 20 minutes waiting for OpenVidu to become ready" >&2
exit 1
fi
sleep 5 sleep 5
done done
EOF EOF
@ -1513,8 +1551,6 @@ CONFIG_S3_EOF
echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout
apt-get update && apt-get install -y
GCLOUD_VERSION=573.0.0 GCLOUD_VERSION=573.0.0
# Install google cli # Install google cli
if ! command -v gcloud >/dev/null 2>&1; then if ! command -v gcloud >/dev/null 2>&1; then
@ -1578,8 +1614,14 @@ MASTER_NODE_PRIVATE_IP_LIST=$(get_meta "instance/attributes/masterNodePrivateIPL
STACK_NAME=$(get_meta "instance/attributes/stackName") STACK_NAME=$(get_meta "instance/attributes/stackName")
PRIVATE_IP=$(get_meta "instance/network-interfaces/0/ip") PRIVATE_IP=$(get_meta "instance/network-interfaces/0/ip")
# Wait for master nodes to be ready by checking secrets # Wait for master nodes to be ready by checking secrets.
i=0
while ! gcloud secrets versions access latest --secret=ALL_SECRETS_GENERATED 2>/dev/null | grep -q "true"; do while ! gcloud secrets versions access latest --secret=ALL_SECRETS_GENERATED 2>/dev/null | grep -q "true"; do
i=$((i + 1))
if [ "$i" -ge 180 ]; then
echo "Timed out after 30 minutes waiting for master nodes to initialize secrets" >&2
exit 1
fi
echo "Waiting for master nodes to initialize secrets..." echo "Waiting for master nodes to initialize secrets..."
sleep 10 sleep 10
done done
@ -1597,8 +1639,14 @@ if [[ "$OPENVIDU_VERSION" == "none" ]]; then
exit 1 exit 1
fi fi
# Build install command for media node # Download first: sh <(curl ...) would silently run an empty script on a transient curl failure
INSTALL_COMMAND="sh <(curl -fsSL http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh)" INSTALLER_SCRIPT="/tmp/install_ov_media_node.sh"
curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "http://get.openvidu.io/pro/ha/$OPENVIDU_VERSION/install_ov_media_node.sh"
if [ ! -s "$INSTALLER_SCRIPT" ]; then
echo "Downloaded OpenVidu media node installer is empty or missing" >&2
exit 1
fi
INSTALL_COMMAND="sh $INSTALLER_SCRIPT"
# Media node arguments # Media node arguments
COMMON_ARGS=( COMMON_ARGS=(
@ -1698,51 +1746,55 @@ EOF
#!/bin/bash -x #!/bin/bash -x
set -eu -o pipefail set -eu -o pipefail
# Create scripts # Check if installation already completed
cat > /usr/local/bin/install.sh << 'INSTALL_EOF' if [ -f /usr/local/bin/openvidu_install_counter.txt ]; then
# Launch on reboot
systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; }
else
# Create scripts
cat > /usr/local/bin/install.sh << 'INSTALL_EOF'
${local.install_script_media} ${local.install_script_media}
INSTALL_EOF INSTALL_EOF
chmod +x /usr/local/bin/install.sh chmod +x /usr/local/bin/install.sh
cat > /usr/local/bin/graceful_shutdown.sh << 'GRACEFUL_SHUTDOWN_EOF' cat > /usr/local/bin/graceful_shutdown.sh << 'GRACEFUL_SHUTDOWN_EOF'
${local.graceful_shutdown_script} ${local.graceful_shutdown_script}
GRACEFUL_SHUTDOWN_EOF GRACEFUL_SHUTDOWN_EOF
chmod +x /usr/local/bin/graceful_shutdown.sh chmod +x /usr/local/bin/graceful_shutdown.sh
echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout
apt-get update && apt-get install -y GCLOUD_VERSION=573.0.0
# Install google cli
GCLOUD_VERSION=573.0.0 if ! command -v gcloud >/dev/null 2>&1; then
# Install google cli
if ! command -v gcloud >/dev/null 2>&1; then
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg
echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
apt-get update && apt-get install -y google-cloud-cli=$${GCLOUD_VERSION}-0 apt-get update && apt-get install -y google-cloud-cli=$${GCLOUD_VERSION}-0
fi fi
# Authenticate with gcloud using instance service account # Authenticate with gcloud using instance service account
gcloud auth activate-service-account --key-file=/dev/null 2>/dev/null || true gcloud auth activate-service-account --key-file=/dev/null 2>/dev/null || true
gcloud config set account $(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email" -H "Metadata-Flavor: Google") gcloud config set account $(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email" -H "Metadata-Flavor: Google")
gcloud config set project $(curl -s "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google") gcloud config set project $(curl -s "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google")
export HOME="/root" export HOME="/root"
# Install OpenVidu Media Node # Install OpenVidu Media Node
/usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu Media Node"; exit 1; } /usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu Media Node"; exit 1; }
# Mark installation as complete # Mark installation as complete
echo "installation_complete" > /usr/local/bin/openvidu_install_counter.txt echo "installation_complete" > /usr/local/bin/openvidu_install_counter.txt
# Start OpenVidu # Start OpenVidu
systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; } systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; }
# Add cron job to check if instance is abandoned every minute # Add cron job to check if instance is abandoned every minute
cat > /usr/local/bin/check_abandoned.sh << 'CHECK_ABANDONED_EOF' cat > /usr/local/bin/check_abandoned.sh << 'CHECK_ABANDONED_EOF'
${local.crontab_job_media} ${local.crontab_job_media}
CHECK_ABANDONED_EOF CHECK_ABANDONED_EOF
chmod +x /usr/local/bin/check_abandoned.sh chmod +x /usr/local/bin/check_abandoned.sh
echo "*/1 * * * * /usr/local/bin/check_abandoned.sh > /var/log/openvidu-abandoned-check.log 2>&1" | crontab - echo "*/1 * * * * /usr/local/bin/check_abandoned.sh > /var/log/openvidu-abandoned-check.log 2>&1" | crontab -
fi
EOF EOF
} }