ambiser
/
openvidu
mirror of https://github.com/OpenVidu/openvidu.git
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
openvidu/openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf

1895 lines
72 KiB
HCL
Raw Blame History

# Enable APIs the deployment needs
resource "google_project_service" "compute_api" { service = "compute.googleapis.com" }
resource "google_project_service" "secretmanager_api" { service = "secretmanager.googleapis.com" }
resource "google_project_service" "storage_api" { service = "storage.googleapis.com" }
resource "google_project_service" "iam_api" { service = "iam.googleapis.com" }
resource "google_project_service" "cloudresourcemanager_api" { service = "cloudresourcemanager.googleapis.com" }
resource "google_project_service" "cloudfunctions_api" { service = "cloudfunctions.googleapis.com" }
resource "google_project_service" "cloudscheduler_api" { service = "cloudscheduler.googleapis.com" }
resource "google_project_service" "cloudbuild_api" { service = "cloudbuild.googleapis.com" }
resource "google_project_service" "run_api" { service = "run.googleapis.com" }
resource "random_id" "bucket_suffix" { byte_length = 3 }
# Secret Manager secrets for OpenVidu deployment information
resource "google_secret_manager_secret" "openvidu_shared_info" {
for_each = toset([
"OPENVIDU_URL", "MEET_INITIAL_ADMIN_USER", "MEET_INITIAL_ADMIN_PASSWORD",
"MEET_INITIAL_API_KEY", "LIVEKIT_URL", "LIVEKIT_API_KEY", "LIVEKIT_API_SECRET",
"DASHBOARD_URL", "GRAFANA_URL", "MINIO_URL", "DOMAIN_NAME",
"OPENVIDU_PRO_LICENSE", "OPENVIDU_RTC_ENGINE", "REDIS_PASSWORD", "MONGO_ADMIN_USERNAME",
"MONGO_ADMIN_PASSWORD", "MONGO_REPLICA_SET_KEY", "MINIO_ACCESS_KEY", "MINIO_SECRET_KEY",
"DASHBOARD_ADMIN_USERNAME", "DASHBOARD_ADMIN_PASSWORD", "GRAFANA_ADMIN_USERNAME",
"GRAFANA_ADMIN_PASSWORD", "ENABLED_MODULES", "OPENVIDU_VERSION", "ALL_SECRETS_GENERATED"
])
secret_id = each.key
replication {
auto {}
}
depends_on = [google_project_service.secretmanager_api]
}
# GCS bucket
resource "google_storage_bucket" "bucket" {
count = local.isEmpty ? 1 : 0
name = "${var.projectId}-${var.stackName}-${random_id.bucket_suffix.hex}"
location = var.region
force_destroy = true
uniform_bucket_level_access = true
depends_on = [google_project_service.storage_api]
}
# Service account for the instance
resource "google_service_account" "service_account" {
account_id = lower("${substr(var.stackName, 0, 12)}-sa")
display_name = "OpenVidu instance service account"
depends_on = [google_project_service.iam_api]
}
# IAM bindings for the service account so the instance can access Secret Manager and GCS
resource "google_project_iam_member" "iam_project_role" {
project = var.projectId
role = "roles/owner"
member = "serviceAccount:${google_service_account.service_account.email}"
depends_on = [google_project_service.cloudresourcemanager_api]
}
resource "google_compute_firewall" "firewall_master" {
name = lower("${var.stackName}-master-firewall")
network = "default"
allow {
protocol = "tcp"
ports = ["22", "80", "443", "1935"]
}
source_ranges = ["0.0.0.0/0"]
target_tags = [lower("${var.stackName}-master-node")]
depends_on = [google_project_service.compute_api]
}
resource "google_compute_firewall" "firewall_media" {
name = lower("${var.stackName}-media-firewall")
network = "default"
allow {
protocol = "tcp"
ports = ["22", "7881", "7880", "50000-60000"]
}
allow {
protocol = "udp"
ports = ["443", "7885", "50000-60000"]
}
source_ranges = ["0.0.0.0/0"]
target_tags = [lower("${var.stackName}-media-node")]
depends_on = [google_project_service.compute_api]
}
resource "google_compute_firewall" "firewall_media_to_master" {
name = lower("${var.stackName}-firewall-media-to-master")
network = "default"
allow {
protocol = "tcp"
ports = ["7000", "9100", "20000", "3100", "9009", "4443", "9080", "6080"]
}
source_tags = [
lower("${var.stackName}-media-node")
]
target_tags = [
lower("${var.stackName}-master-node"),
]
depends_on = [google_project_service.compute_api]
}
resource "google_compute_firewall" "firewall_master_to_media" {
name = lower("${var.stackName}-firewall-master-to-media")
network = "default"
allow {
protocol = "tcp"
ports = ["1935", "5349", "7880", "8080"]
}
source_tags = [
lower("${var.stackName}-master-node"),
]
target_tags = [
lower("${var.stackName}-media-node")
]
depends_on = [google_project_service.compute_api]
}
# Create Public Ip address (if not provided)
resource "google_compute_address" "public_ip_address" {
count = var.publicIpAddress == "" ? 1 : 0
name = lower("${var.stackName}-public-ip")
region = var.region
depends_on = [google_project_service.compute_api]
}
locals {
is_arm_instance = startswith(var.masterNodeInstanceType, "c4a-") || startswith(var.masterNodeInstanceType, "t2a-") || startswith(var.masterNodeInstanceType, "n4a-") || startswith(var.masterNodeInstanceType, "a4x-")
yq_arch = local.is_arm_instance ? "arm64" : "amd64"
ubuntu_image = local.is_arm_instance ? "ubuntu-os-cloud/ubuntu-2404-lts-arm64" : "ubuntu-os-cloud/ubuntu-2404-lts-amd64"
is_c4a_instance = startswith(var.masterNodeInstanceType, "c4a-")
}
# Compute instance for OpenVidu
resource "google_compute_instance" "openvidu_master_node" {
name = lower("${var.stackName}-master-node")
machine_type = var.masterNodeInstanceType
zone = var.zone
tags = [lower("${var.stackName}-master-node")]
boot_disk {
initialize_params {
image = local.ubuntu_image
size = 200
type = local.is_c4a_instance ? "hyperdisk-balanced" : "pd-standard"
}
}
# Keep existing Master Nodes when a newer Ubuntu 24.04 image is published or when the
# default boot disk size changes, so a re-apply never recreates a running Master Node.
lifecycle {
ignore_changes = [
boot_disk[0].initialize_params[0].image,
boot_disk[0].initialize_params[0].size,
]
}
network_interface {
network = "default"
access_config {
nat_ip = var.publicIpAddress == "" ? google_compute_address.public_ip_address[0].address : var.publicIpAddress
}
}
metadata = {
# metadata values are accessible from the instance
publicIpAddress = var.publicIpAddress == "" ? google_compute_address.public_ip_address[0].address : var.publicIpAddress
region = var.region
stackName = var.stackName
certificateType = var.certificateType
domainName = var.domainName
ownPublicCertificate = var.ownPublicCertificate
ownPrivateCertificate = var.ownPrivateCertificate
openviduLicense = var.openviduLicense
rtcEngine = var.rtcEngine
initialMeetAdminPassword = var.initialMeetAdminPassword
initialMeetApiKey = var.initialMeetApiKey
additionalInstallFlags = var.additionalInstallFlags
bucketName = local.isEmpty ? google_storage_bucket.bucket[0].name : var.bucketName
}
service_account {
email = google_service_account.service_account.email
scopes = ["https://www.googleapis.com/auth/cloud-platform"]
}
metadata_startup_script = local.user_data_master
labels = {
stack = var.stackName
node-type = "master"
}
# Explicit: transitive coverage via public_ip_address is absent when publicIpAddress is provided
depends_on = [google_project_service.compute_api]
}
locals {
is_arm_instance_media = startswith(var.mediaNodeInstanceType, "c4a-") || startswith(var.mediaNodeInstanceType, "t2a-") || startswith(var.mediaNodeInstanceType, "n4a-") || startswith(var.mediaNodeInstanceType, "a4x-")
ubuntu_image_media = local.is_arm_instance_media ? "ubuntu-os-cloud/ubuntu-2404-lts-arm64" : "ubuntu-os-cloud/ubuntu-2404-lts-amd64"
is_c4a_instance_media = startswith(var.mediaNodeInstanceType, "c4a-")
}
# Media Node Instance Template
resource "google_compute_instance_template" "media_node_template" {
name = lower("${var.stackName}-media-node-template")
machine_type = var.mediaNodeInstanceType
tags = [lower("${var.stackName}-media-node")]
disk {
source_image = local.ubuntu_image_media
auto_delete = true
boot = true
disk_size_gb = 100
disk_type = local.is_c4a_instance_media ? "hyperdisk-balanced" : "pd-standard"
}
network_interface {
network = "default"
access_config {}
}
metadata = {
stackName = var.stackName
masterNodePrivateIP = google_compute_instance.openvidu_master_node.network_interface[0].network_ip
bucketName = local.isEmpty ? google_storage_bucket.bucket[0].name : var.bucketName
region = var.region
shutdown-script = local.graceful_shutdown_script
}
service_account {
email = google_service_account.service_account.email
scopes = ["https://www.googleapis.com/auth/cloud-platform"]
}
metadata_startup_script = local.user_data_media
labels = {
stack = var.stackName
node-type = "media"
}
lifecycle {
create_before_destroy = true
}
}
# Health check for media node auto-healing
resource "google_compute_region_health_check" "media_node_health_check" {
name = lower("${var.stackName}-media-node-health-check")
region = var.region
# TCP (not HTTP /): during a master/Redis outage LiveKit answers 406 on every media node at
# once, so an HTTP check would recreate the whole fleet. Generous thresholds because auto-heal
# recreation kills live WebRTC sessions; the on-node watchdog owns application-level health.
tcp_health_check {
port = 7880
}
check_interval_sec = 30
timeout_sec = 10
healthy_threshold = 2
unhealthy_threshold = 10
depends_on = [google_project_service.compute_api]
}
# Managed Instance Group for Media Nodes
resource "google_compute_region_instance_group_manager" "media_node_group" {
name = lower("${var.stackName}-media-node-group")
base_instance_name = lower("${var.stackName}-media-node")
region = var.region
target_size = var.fixedNumberOfMediaNodes > 0 ? var.fixedNumberOfMediaNodes : var.initialNumberOfMediaNodes
version {
instance_template = google_compute_instance_template.media_node_template.self_link_unique
}
named_port {
name = "http"
port = 7880
}
# VM/process backstop only. initial_delay_sec at the 3600 s max so autohealing never recreates a
# still-bootstrapping node; the marker-aware on-node watchdog owns bootstrap-failure detection.
auto_healing_policies {
health_check = google_compute_region_health_check.media_node_health_check.id
initial_delay_sec = 3600
}
# Terraform sets the size at creation only. Afterwards the autoscaler (scale out) and the
# scale-in function (scale in, and the fixedNumberOfMediaNodes reconcile) own it.
lifecycle {
ignore_changes = [target_size]
}
depends_on = [google_compute_instance.openvidu_master_node]
}
# Autoscaler for Media Nodes
# Not created with a fixed number of Media Nodes: the scale-in function keeps the MIG at that size.
resource "google_compute_region_autoscaler" "media_node_autoscaler" {
count = var.fixedNumberOfMediaNodes > 0 ? 0 : 1
name = lower("${var.stackName}-media-node-autoscaler")
region = var.region
target = google_compute_region_instance_group_manager.media_node_group.self_link
autoscaling_policy {
max_replicas = var.maxNumberOfMediaNodes
min_replicas = var.minNumberOfMediaNodes
cooldown_period = 260
cpu_utilization {
target = var.scaleTargetCPU / 100.0
}
mode = "ONLY_SCALE_OUT"
}
}
# ------------------------- scale in resources -------------------------
# Create the function source code
data "archive_file" "function_source" {
type = "zip"
output_path = "/tmp/function-source.zip"
source {
content = local.scalein_function_code
filename = "main.py"
}
source {
content = local.function_requirements
filename = "requirements.txt"
}
}
# Local values for the function source code
locals {
scalein_function_code = <<EOF
import functions_framework
from google.cloud import compute_v1
import json
import time
import os
import logging
import subprocess
import requests
import google.auth
from datetime import datetime
from typing import Dict, List, Optional, Tuple
from google.auth.transport.requests import Request
# Configure logging for Cloud Run
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)
@functions_framework.http
def scalein_function(request):
"""
Cloud Function to handle scale-in operations for media node instances.
Abandons oldest instances when autoscaler recommends fewer instances.
With a fixed number of Media Nodes (no autoscaler), keeps the MIG at that number:
abandons the extra instances or resizes the MIG up.
"""
print("Starting scale-in function") # Also use print for Cloud Run logs
logger.info("Starting scale-in function")
try:
# Parse and validate input
project_id, region, stack_name = _parse_and_validate_request(request)
print(f"Processing request for project: {project_id}, region: {region}, stack: {stack_name}")
# Initialize GCP clients
clients = _initialize_gcp_clients()
# Construct resource names
mig_name = f"{stack_name.lower()}-media-node-group"
autoscaler_name = f"{stack_name.lower()}-media-node-autoscaler"
# Fixed number of Media Nodes: there is no autoscaler, keep the MIG at that number
fixed_number = _get_fixed_number_of_media_nodes()
if fixed_number > 0:
return _reconcile_fixed_size(
clients, project_id, region, mig_name, fixed_number
)
print(f"Checking MIG: {mig_name} and autoscaler: {autoscaler_name}")
logger.info(f"Checking MIG: {mig_name} and autoscaler: {autoscaler_name}")
# Get current state and recommendations
current_size, recommended_size = _get_scaling_info(
clients, project_id, region, mig_name, autoscaler_name
)
print(f"Current size: {current_size}, Recommended size: {recommended_size}")
logger.info(f"Current size: {current_size}, Recommended size: {recommended_size}")
# Check if scale-in is needed
if recommended_size >= current_size:
message = "No scale-in operation needed"
print(message)
logger.info(message)
return {
"message": message,
"current_size": current_size,
"recommended_size": recommended_size
}
# Perform scale-in
instances_to_abandon = current_size - recommended_size
print(f"Scale-in needed: abandoning {instances_to_abandon} instances")
result = _perform_scalein(
clients, project_id, region, mig_name, instances_to_abandon
)
success_message = f"Successfully abandoned {result['abandoned_count']} instances"
print(success_message)
logger.info(success_message)
return {
"message": success_message,
"abandoned_instances": result['abandoned_instances'],
"operation_name": result['operation_name']
}
except ValueError as e:
error_msg = f"Validation error: {str(e)}"
print(error_msg)
logger.error(error_msg)
return {"error": str(e)}, 400
except Exception as e:
error_msg = f"Unexpected error in scaling function: {str(e)}"
print(error_msg)
logger.error(error_msg)
return {"error": f"Internal server error: {str(e)}"}, 500
def _parse_and_validate_request(request) -> Tuple[str, str, str]:
"""Parse and validate the incoming request."""
print("Parsing request...")
request_json = request.get_json()
if not request_json:
raise ValueError("Request body must be valid JSON")
project_id = request_json.get('project_id')
region = request_json.get('region')
stack_name = request_json.get('stack_name')
if not all([project_id, region, stack_name]):
raise ValueError("Missing required parameters: project_id, region, stack_name")
print(f"Request parsed successfully: {project_id}, {region}, {stack_name}")
return project_id, region, stack_name
def _get_fixed_number_of_media_nodes() -> int:
"""Get the fixed number of Media Nodes (0 means autoscaling)."""
fixed_number = int(os.environ.get('FIXED_NUMBER_OF_MEDIA_NODES', '0'))
if fixed_number < 0:
raise ValueError(f"FIXED_NUMBER_OF_MEDIA_NODES must be >= 0, got {fixed_number}")
return fixed_number
def _initialize_gcp_clients() -> Dict:
"""Initialize all required GCP clients."""
print("Initializing GCP clients...")
clients = {
'mig': compute_v1.RegionInstanceGroupManagersClient(),
'autoscaler': compute_v1.RegionAutoscalersClient()
}
print("GCP clients initialized successfully")
return clients
def _get_scaling_info(clients: Dict, project_id: str, region: str,
mig_name: str, autoscaler_name: str) -> Tuple[int, int]:
"""Get current MIG size and autoscaler recommendation."""
print(f"Getting scaling info for MIG: {mig_name}")
try:
# Get current MIG state
mig_request = compute_v1.GetRegionInstanceGroupManagerRequest(
project=project_id,
region=region,
instance_group_manager=mig_name
)
mig = clients['mig'].get(request=mig_request)
current_size = mig.target_size
# Get autoscaler recommendation
autoscaler_request = compute_v1.GetRegionAutoscalerRequest(
project=project_id,
region=region,
autoscaler=autoscaler_name
)
autoscaler = clients['autoscaler'].get(request=autoscaler_request)
recommended_size = autoscaler.recommended_size
print(f"Retrieved scaling info - Current: {current_size}, Recommended: {recommended_size}")
return current_size, recommended_size
except Exception as e:
error_msg = f"Failed to get scaling information: {str(e)}"
print(error_msg)
raise Exception(error_msg)
def _get_current_size(clients: Dict, project_id: str, region: str,
mig_name: str) -> int:
"""Get current MIG size (fixed number of Media Nodes, no autoscaler)."""
print(f"Getting current size for MIG: {mig_name}")
try:
mig_request = compute_v1.GetRegionInstanceGroupManagerRequest(
project=project_id,
region=region,
instance_group_manager=mig_name
)
mig = clients['mig'].get(request=mig_request)
current_size = mig.target_size
print(f"Retrieved current size: {current_size}")
return current_size
except Exception as e:
error_msg = f"Failed to get current size: {str(e)}"
print(error_msg)
raise Exception(error_msg)
def _reconcile_fixed_size(clients: Dict, project_id: str, region: str,
mig_name: str, fixed_number: int) -> Dict:
"""Keep the MIG at the fixed number of Media Nodes."""
current_size = _get_current_size(clients, project_id, region, mig_name)
print(f"Current size: {current_size}, Fixed number of Media Nodes: {fixed_number}")
logger.info(f"Current size: {current_size}, Fixed number of Media Nodes: {fixed_number}")
if current_size == fixed_number:
message = "No scale-in or scale-out operation needed"
print(message)
logger.info(message)
return {
"message": message,
"current_size": current_size,
"fixed_number": fixed_number
}
if current_size < fixed_number:
# Fewer Media Nodes than the fixed number: resize the MIG up
print(f"Scale-out needed: resizing MIG from {current_size} to {fixed_number} instances")
operation = _resize_mig(clients['mig'], project_id, region, mig_name, fixed_number)
success_message = f"Successfully resized MIG from {current_size} to {fixed_number} instances"
print(success_message)
logger.info(success_message)
return {
"message": success_message,
"operation_name": operation.name
}
# More Media Nodes than the fixed number: abandon the extra ones so they drain gracefully
instances_to_abandon = current_size - fixed_number
print(f"Scale-in needed: abandoning {instances_to_abandon} instances")
result = _perform_scalein(
clients, project_id, region, mig_name, instances_to_abandon
)
success_message = f"Successfully abandoned {result['abandoned_count']} instances"
print(success_message)
logger.info(success_message)
return {
"message": success_message,
"abandoned_instances": result['abandoned_instances'],
"operation_name": result['operation_name']
}
def _perform_scalein(clients: Dict, project_id: str, region: str,
mig_name: str, instances_count: int) -> Dict:
"""Perform scale-in by abandoning oldest instances."""
print(f"Starting scale-in for {instances_count} instances")
# Get instances to abandon
instances_to_abandon = _get_oldest_instances(
clients['mig'], project_id, region, mig_name, instances_count
)
if not instances_to_abandon:
print("No instances to abandon")
return {"abandoned_count": 0, "abandoned_instances": [], "operation_name": None}
print(f"Preparing to abandon {len(instances_to_abandon)} instances")
logger.info(f"Preparing to abandon {len(instances_to_abandon)} instances")
# Abandon instances
operation = _abandon_instances(clients['mig'], project_id, region, mig_name, instances_to_abandon)
print(f"Instances abandoned successfully, operation: {operation.name}")
return {
"abandoned_count": len(instances_to_abandon),
"abandoned_instances": instances_to_abandon,
"operation_name": operation.name
}
def _get_oldest_instances(mig_client, project_id: str, region: str,
mig_name: str, count: int) -> List[str]:
"""Get the oldest instances from the MIG."""
print(f"Getting {count} oldest instances from MIG")
try:
list_request = compute_v1.ListManagedInstancesRegionInstanceGroupManagersRequest(
project=project_id,
region=region,
instance_group_manager=mig_name
)
instances = list(mig_client.list_managed_instances(request=list_request))
print(f"Found {len(instances)} total instances in MIG")
# Sort by creation timestamp (oldest first) - reverse the sort order
instances.sort(key=lambda x: _parse_timestamp(
x.instance_status.time_created if x.instance_status and
hasattr(x.instance_status, 'time_created') else None
), reverse=True)
# Select instances to abandon
instances_to_abandon = []
for i in range(min(count, len(instances))):
instance = instances[i]
if instance.instance:
instances_to_abandon.append(instance.instance)
print(f"Selected {len(instances_to_abandon)} instances to abandon")
return instances_to_abandon
except Exception as e:
error_msg = f"Error getting oldest instances: {str(e)}"
print(error_msg)
logger.error(error_msg)
return []
def _parse_timestamp(timestamp_str: Optional[str]) -> datetime:
"""Parse timestamp string to datetime object."""
if not timestamp_str:
return datetime.min
try:
return datetime.fromisoformat(timestamp_str.replace('Z', '+00:00'))
except Exception:
return datetime.min
def _abandon_instances(mig_client, project_id: str, region: str,
mig_name: str, instance_urls: List[str]):
"""Abandon instances from the MIG."""
print(f"Abandoning {len(instance_urls)} instances from MIG")
abandon_request = compute_v1.AbandonInstancesRegionInstanceGroupManagerRequest(
project=project_id,
region=region,
instance_group_manager=mig_name,
region_instance_group_managers_abandon_instances_request_resource=compute_v1.RegionInstanceGroupManagersAbandonInstancesRequest(
instances=instance_urls
)
)
operation = mig_client.abandon_instances(request=abandon_request)
print(f"Abandon operation initiated: {operation.name}")
return operation
def _resize_mig(mig_client, project_id: str, region: str,
mig_name: str, size: int):
"""Resize the MIG to the given number of instances."""
print(f"Resizing MIG to {size} instances")
resize_request = compute_v1.ResizeRegionInstanceGroupManagerRequest(
project=project_id,
region=region,
instance_group_manager=mig_name,
size=size
)
operation = mig_client.resize(request=resize_request)
print(f"Resize operation initiated: {operation.name}")
return operation
EOF
function_requirements = <<EOF
functions-framework==3.*
google-cloud-compute==1.*
requests==2.*
google-auth==2.*
EOF
}
resource "google_storage_bucket_object" "function_source" {
# The code's hash in the name: a template update that changes the function also changes
# the object, so the function is rebuilt. With a constant name it kept running the old code.
name = "function-source-${data.archive_file.function_source.output_md5}.zip"
bucket = local.isEmpty ? google_storage_bucket.bucket[0].name : var.bucketName
source = data.archive_file.function_source.output_path
depends_on = [google_project_service.storage_api]
}
resource "google_cloudfunctions2_function" "scalein_function" {
name = lower("${var.stackName}-scalein-function")
location = var.region
description = "Function to handle scale in operations"
build_config {
runtime = "python311"
entry_point = "scalein_function"
source {
storage_source {
bucket = local.isEmpty ? google_storage_bucket.bucket[0].name : var.bucketName
object = google_storage_bucket_object.function_source.name
}
}
}
service_config {
max_instance_count = 1
available_memory = "512M"
timeout_seconds = 300
environment_variables = {
PROJECT_ID = var.projectId
REGION = var.region
STACK_NAME = var.stackName
FIXED_NUMBER_OF_MEDIA_NODES = tostring(var.fixedNumberOfMediaNodes)
}
service_account_email = google_service_account.service_account.email
}
depends_on = [
google_project_service.cloudfunctions_api,
google_project_service.cloudbuild_api,
google_project_service.run_api
]
}
# Cloud Scheduler to trigger the function every 5 minutes
resource "google_cloud_scheduler_job" "scale_scheduler" {
name = lower("${var.stackName}-scale-scheduler")
description = "Trigger scaling function every 5 minutes"
schedule = "*/5 * * * *"
time_zone = "UTC"
attempt_deadline = "300s"
http_target {
http_method = "POST"
uri = google_cloudfunctions2_function.scalein_function.service_config[0].uri
# Send JSON payload with inputs
body = base64encode(jsonencode({
project_id = var.projectId
region = var.region
stack_name = var.stackName
}))
headers = {
"Content-Type" = "application/json"
}
oidc_token {
service_account_email = google_service_account.service_account.email
}
}
depends_on = [google_project_service.cloudscheduler_api]
}
# ------------------------- local values -------------------------
locals {
isEmpty = var.bucketName == ""
install_script_master = <<-EOF
#!/bin/bash -x
set -e
OPENVIDU_VERSION=main
DOMAIN=
YQ_VERSION=v4.53.6
echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout
apt-get update && apt-get install -y \
curl \
unzip \
jq \
wget \
ca-certificates \
gnupg \
lsb-release \
openssl
wget https://github.com/mikefarah/yq/releases/download/$${YQ_VERSION}/yq_linux_${local.yq_arch}.tar.gz -O - |\
tar xz && mv yq_linux_${local.yq_arch} /usr/bin/yq
# Configure gcloud with instance service account
gcloud auth activate-service-account --key-file=/dev/null 2>/dev/null || true
METADATA_URL="http://metadata.google.internal/computeMetadata/v1"
get_meta() { curl -s -H "Metadata-Flavor: Google" "$${METADATA_URL}/$1"; }
# Disable command tracing so secrets are not printed to the serial console
set +x
# Configure domain
if [[ "${var.domainName}" == "" ]]; then
[ ! -d "/usr/share/openvidu" ] && mkdir -p /usr/share/openvidu
EXTERNAL_IP=$(get_meta "instance/network-interfaces/0/access-configs/0/external-ip")
DOMAIN="$EXTERNAL_IP"
else
DOMAIN="${var.domainName}"
fi
DOMAIN="$(/usr/local/bin/store_secret.sh save DOMAIN_NAME "$DOMAIN")"
# Meet initial admin user and password
MEET_INITIAL_ADMIN_USER="$(/usr/local/bin/store_secret.sh save MEET_INITIAL_ADMIN_USER "admin")"
if [[ "${var.initialMeetAdminPassword}" != '' ]]; then
MEET_INITIAL_ADMIN_PASSWORD="$(/usr/local/bin/store_secret.sh save MEET_INITIAL_ADMIN_PASSWORD "${var.initialMeetAdminPassword}")"
else
MEET_INITIAL_ADMIN_PASSWORD="$(/usr/local/bin/store_secret.sh generate MEET_INITIAL_ADMIN_PASSWORD)"
fi
if [[ "${var.initialMeetApiKey}" != '' ]]; then
MEET_INITIAL_API_KEY="$(/usr/local/bin/store_secret.sh save MEET_INITIAL_API_KEY "${var.initialMeetApiKey}")"
fi
#Get own private IP
PRIVATE_IP=$(get_meta "instance/network-interfaces/0/ip")
# Store usernames and generate random passwords
REDIS_PASSWORD="$(/usr/local/bin/store_secret.sh generate REDIS_PASSWORD)"
MONGO_ADMIN_USERNAME="$(/usr/local/bin/store_secret.sh save MONGO_ADMIN_USERNAME "mongoadmin")"
MONGO_ADMIN_PASSWORD="$(/usr/local/bin/store_secret.sh generate MONGO_ADMIN_PASSWORD)"
MONGO_REPLICA_SET_KEY="$(/usr/local/bin/store_secret.sh generate MONGO_REPLICA_SET_KEY)"
MINIO_ACCESS_KEY="$(/usr/local/bin/store_secret.sh save MINIO_ACCESS_KEY "minioadmin")"
MINIO_SECRET_KEY="$(/usr/local/bin/store_secret.sh generate MINIO_SECRET_KEY)"
DASHBOARD_ADMIN_USERNAME="$(/usr/local/bin/store_secret.sh save DASHBOARD_ADMIN_USERNAME "dashboardadmin")"
DASHBOARD_ADMIN_PASSWORD="$(/usr/local/bin/store_secret.sh generate DASHBOARD_ADMIN_PASSWORD)"
GRAFANA_ADMIN_USERNAME="$(/usr/local/bin/store_secret.sh save GRAFANA_ADMIN_USERNAME "grafanaadmin")"
GRAFANA_ADMIN_PASSWORD="$(/usr/local/bin/store_secret.sh generate GRAFANA_ADMIN_PASSWORD)"
ENABLED_MODULES="$(/usr/local/bin/store_secret.sh save ENABLED_MODULES "observability,openviduMeet,v2compatibility")"
LIVEKIT_API_KEY="$(/usr/local/bin/store_secret.sh generate LIVEKIT_API_KEY "API" 12)"
LIVEKIT_API_SECRET="$(/usr/local/bin/store_secret.sh generate LIVEKIT_API_SECRET)"
# Store OpenVidu Pro license and RTC engine and Openvidu version
OPENVIDU_PRO_LICENSE="$(/usr/local/bin/store_secret.sh save OPENVIDU_PRO_LICENSE "${var.openviduLicense}")"
OPENVIDU_RTC_ENGINE="$(/usr/local/bin/store_secret.sh save OPENVIDU_RTC_ENGINE "${var.rtcEngine}")"
OPENVIDU_VERSION="$(/usr/local/bin/store_secret.sh save OPENVIDU_VERSION "$OPENVIDU_VERSION")"
ALL_SECRETS_GENERATED="$(/usr/local/bin/store_secret.sh save ALL_SECRETS_GENERATED "true")"
# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure
INSTALLER_SCRIPT="/tmp/install_ov_master_node.sh"
curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "https://get.openvidu.io/pro/elastic/$OPENVIDU_VERSION/install_ov_master_node.sh"
if [ ! -s "$INSTALLER_SCRIPT" ]; then
echo "Downloaded OpenVidu master node installer is empty or missing" >&2
exit 1
fi
INSTALL_COMMAND="sh $INSTALLER_SCRIPT"
# Common arguments
COMMON_ARGS=(
"--no-tty"
"--install"
"--environment=gcp"
"--deployment-type=elastic"
"--node-role=master-node"
"--openvidu-pro-license=$OPENVIDU_PRO_LICENSE"
"--private-ip=$PRIVATE_IP"
"--domain-name=$DOMAIN"
"--enabled-modules='$ENABLED_MODULES'"
"--rtc-engine=$OPENVIDU_RTC_ENGINE"
"--redis-password=$REDIS_PASSWORD"
"--mongo-admin-user=$MONGO_ADMIN_USERNAME"
"--mongo-admin-password=$MONGO_ADMIN_PASSWORD"
"--mongo-replica-set-key=$MONGO_REPLICA_SET_KEY"
"--minio-access-key=$MINIO_ACCESS_KEY"
"--minio-secret-key=$MINIO_SECRET_KEY"
"--dashboard-admin-user=$DASHBOARD_ADMIN_USERNAME"
"--dashboard-admin-password=$DASHBOARD_ADMIN_PASSWORD"
"--grafana-admin-user=$GRAFANA_ADMIN_USERNAME"
"--grafana-admin-password=$GRAFANA_ADMIN_PASSWORD"
"--meet-initial-admin-password=$MEET_INITIAL_ADMIN_PASSWORD"
"--meet-initial-api-key=$MEET_INITIAL_API_KEY"
"--livekit-api-key=$LIVEKIT_API_KEY"
"--livekit-api-secret=$LIVEKIT_API_SECRET"
)
# Include additional installer flags provided by the user
if [[ "${var.additionalInstallFlags}" != "" ]]; then
IFS=',' read -ra EXTRA_FLAGS <<< "${var.additionalInstallFlags}"
for extra_flag in "$${EXTRA_FLAGS[@]}"; do
# Trim whitespace around each flag
extra_flag="$(echo -e "$${extra_flag}" | sed -e 's/^[ \t]*//' -e 's/[ \t]*$//')"
if [[ "$extra_flag" != "" ]]; then
COMMON_ARGS+=("$extra_flag")
fi
done
fi
# Certificate arguments
if [[ "${var.certificateType}" == "selfsigned" ]]; then
CERT_ARGS=(
"--certificate-type=selfsigned"
)
elif [[ "${var.certificateType}" == "letsencrypt" ]]; then
CERT_ARGS=(
"--certificate-type=letsencrypt"
)
else
# Use base64 encoded certificates directly
OWN_CERT_CRT=${var.ownPublicCertificate}
OWN_CERT_KEY=${var.ownPrivateCertificate}
CERT_ARGS=(
"--certificate-type=owncert"
"--owncert-public-key=$OWN_CERT_CRT"
"--owncert-private-key=$OWN_CERT_KEY"
)
fi
# Final command
FINAL_COMMAND="$INSTALL_COMMAND $(printf "%s " "$${COMMON_ARGS[@]}") $(printf "%s " "$${CERT_ARGS[@]}")"
# Execute installation
exec bash -c "$FINAL_COMMAND"
EOF
config_s3_script = <<-EOF
#!/bin/bash -x
set -e
# Configure gcloud with instance service account
gcloud auth activate-service-account --key-file=/dev/null 2>/dev/null || true
# Install dir and config dir
INSTALL_DIR="/opt/openvidu"
CLUSTER_CONFIG_DIR="$${INSTALL_DIR}/config/cluster"
METADATA_URL="http://metadata.google.internal/computeMetadata/v1"
get_meta() { curl -s -H "Metadata-Flavor: Google" "$${METADATA_URL}/$1"; }
SERVICE_ACCOUNT_EMAIL=$(get_meta "instance/service-accounts/default/email")
# Create key for service account
gcloud iam service-accounts keys create credentials.json --iam-account=$SERVICE_ACCOUNT_EMAIL
# Disable command tracing so credentials are not printed to the serial console
set +x
# Create HMAC key and parse output
HMAC_OUTPUT=$(gcloud storage hmac create $SERVICE_ACCOUNT_EMAIL --format="json")
EXTERNAL_S3_ACCESS_KEY=$(echo "$HMAC_OUTPUT" | jq -r '.metadata.accessId')
EXTERNAL_S3_SECRET_KEY=$(echo "$HMAC_OUTPUT" | jq -r '.secret')
# Config S3 bucket
EXTERNAL_S3_ENDPOINT="https://storage.googleapis.com"
EXTERNAL_S3_REGION="${var.region}"
EXTERNAL_S3_PATH_STYLE_ACCESS="true"
EXTERNAL_S3_BUCKET_APP_DATA=$(get_meta "instance/attributes/bucketName")
# Update egress.yaml to use hardcoded credentials instead of env variable
if [ -f "$${CLUSTER_CONFIG_DIR}/media_node/egress.yaml" ]; then
yq eval --inplace '.storage.gcp.credentials_json = (load("/credentials.json") | tostring) | .storage.gcp.credentials_json style="single"' $${CLUSTER_CONFIG_DIR}/media_node/egress.yaml
fi
sed -i "s|EXTERNAL_S3_ENDPOINT=.*|EXTERNAL_S3_ENDPOINT=$EXTERNAL_S3_ENDPOINT|" "$${CLUSTER_CONFIG_DIR}/openvidu.env"
sed -i "s|EXTERNAL_S3_REGION=.*|EXTERNAL_S3_REGION=$EXTERNAL_S3_REGION|" "$${CLUSTER_CONFIG_DIR}/openvidu.env"
sed -i "s|EXTERNAL_S3_PATH_STYLE_ACCESS=.*|EXTERNAL_S3_PATH_STYLE_ACCESS=$EXTERNAL_S3_PATH_STYLE_ACCESS|" "$${CLUSTER_CONFIG_DIR}/openvidu.env"
sed -i "s|EXTERNAL_S3_BUCKET_APP_DATA=.*|EXTERNAL_S3_BUCKET_APP_DATA=$EXTERNAL_S3_BUCKET_APP_DATA|" "$${CLUSTER_CONFIG_DIR}/openvidu.env"
sed -i "s|EXTERNAL_S3_ACCESS_KEY=.*|EXTERNAL_S3_ACCESS_KEY=$EXTERNAL_S3_ACCESS_KEY|" "$${CLUSTER_CONFIG_DIR}/openvidu.env"
sed -i "s|EXTERNAL_S3_SECRET_KEY=.*|EXTERNAL_S3_SECRET_KEY=$EXTERNAL_S3_SECRET_KEY|" "$${CLUSTER_CONFIG_DIR}/openvidu.env"
EOF
after_install_script = <<-EOF
#!/bin/bash
set -e
# Configure gcloud with instance service account
gcloud auth activate-service-account --key-file=/dev/null 2>/dev/null || true
# Generate URLs
DOMAIN=$(gcloud secrets versions access latest --secret=DOMAIN_NAME)
OPENVIDU_URL="https://$${DOMAIN}/"
LIVEKIT_URL="wss://$${DOMAIN}/"
DASHBOARD_URL="https://$${DOMAIN}/dashboard/"
GRAFANA_URL="https://$${DOMAIN}/grafana/"
MINIO_URL="https://$${DOMAIN}/minio-console/"
# Update shared secret
echo -n "$DOMAIN" | gcloud secrets versions add DOMAIN_NAME --data-file=-
echo -n "$OPENVIDU_URL" | gcloud secrets versions add OPENVIDU_URL --data-file=-
echo -n "$LIVEKIT_URL" | gcloud secrets versions add LIVEKIT_URL --data-file=-
echo -n "$DASHBOARD_URL" | gcloud secrets versions add DASHBOARD_URL --data-file=-
echo -n "$GRAFANA_URL" | gcloud secrets versions add GRAFANA_URL --data-file=-
echo -n "$MINIO_URL" | gcloud secrets versions add MINIO_URL --data-file=-
gcloud secrets versions access latest --secret=MINIO_URL
if [[ $? -ne 0 ]]; then
echo "Error updating secret_manager"
fi
EOF
update_config_from_secret_script = <<-EOF
#!/bin/bash -x
set -e
# Configure gcloud with instance service account
gcloud auth activate-service-account --key-file=/dev/null 2>/dev/null || true
# Installation directory
INSTALL_DIR="/opt/openvidu"
CLUSTER_CONFIG_DIR="$${INSTALL_DIR}/config/cluster"
MASTER_NODE_CONFIG_DIR="$${INSTALL_DIR}/config/node"
# Disable command tracing so secrets are not printed to the serial console
set +x
# Replace DOMAIN_NAME
export DOMAIN=$(gcloud secrets versions access latest --secret=DOMAIN_NAME)
if [[ -n "$DOMAIN" ]]; then
sed -i "s/DOMAIN_NAME=.*/DOMAIN_NAME=$DOMAIN/" "$${CLUSTER_CONFIG_DIR}/openvidu.env"
else
exit 1
fi
# Get the rest of the values
export REDIS_PASSWORD=$(gcloud secrets versions access latest --secret=REDIS_PASSWORD)
export OPENVIDU_RTC_ENGINE=$(gcloud secrets versions access latest --secret=OPENVIDU_RTC_ENGINE)
export OPENVIDU_PRO_LICENSE=$(gcloud secrets versions access latest --secret=OPENVIDU_PRO_LICENSE)
export MONGO_ADMIN_USERNAME=$(gcloud secrets versions access latest --secret=MONGO_ADMIN_USERNAME)
export MONGO_ADMIN_PASSWORD=$(gcloud secrets versions access latest --secret=MONGO_ADMIN_PASSWORD)
export MONGO_REPLICA_SET_KEY=$(gcloud secrets versions access latest --secret=MONGO_REPLICA_SET_KEY)
export DASHBOARD_ADMIN_USERNAME=$(gcloud secrets versions access latest --secret=DASHBOARD_ADMIN_USERNAME)
export DASHBOARD_ADMIN_PASSWORD=$(gcloud secrets versions access latest --secret=DASHBOARD_ADMIN_PASSWORD)
export MINIO_ACCESS_KEY=$(gcloud secrets versions access latest --secret=MINIO_ACCESS_KEY)
export MINIO_SECRET_KEY=$(gcloud secrets versions access latest --secret=MINIO_SECRET_KEY)
export GRAFANA_ADMIN_USERNAME=$(gcloud secrets versions access latest --secret=GRAFANA_ADMIN_USERNAME)
export GRAFANA_ADMIN_PASSWORD=$(gcloud secrets versions access latest --secret=GRAFANA_ADMIN_PASSWORD)
export LIVEKIT_API_KEY=$(gcloud secrets versions access latest --secret=LIVEKIT_API_KEY)
export LIVEKIT_API_SECRET=$(gcloud secrets versions access latest --secret=LIVEKIT_API_SECRET)
export MEET_INITIAL_ADMIN_USER=$(gcloud secrets versions access latest --secret=MEET_INITIAL_ADMIN_USER)
export MEET_INITIAL_ADMIN_PASSWORD=$(gcloud secrets versions access latest --secret=MEET_INITIAL_ADMIN_PASSWORD)
if [[ "${var.initialMeetApiKey}" != '' ]]; then
export MEET_INITIAL_API_KEY=$(gcloud secrets versions access latest --secret=MEET_INITIAL_API_KEY)
fi
export ENABLED_MODULES=$(gcloud secrets versions access latest --secret=ENABLED_MODULES)
# Replace rest of the values
sed -i "s/REDIS_PASSWORD=.*/REDIS_PASSWORD=$REDIS_PASSWORD/" "$${MASTER_NODE_CONFIG_DIR}/master_node.env"
sed -i "s/OPENVIDU_RTC_ENGINE=.*/OPENVIDU_RTC_ENGINE=$OPENVIDU_RTC_ENGINE/" "$${CLUSTER_CONFIG_DIR}/openvidu.env"
sed -i "s/OPENVIDU_PRO_LICENSE=.*/OPENVIDU_PRO_LICENSE=$OPENVIDU_PRO_LICENSE/" "$${CLUSTER_CONFIG_DIR}/openvidu.env"
sed -i "s/MONGO_ADMIN_USERNAME=.*/MONGO_ADMIN_USERNAME=$MONGO_ADMIN_USERNAME/" "$${CLUSTER_CONFIG_DIR}/openvidu.env"
sed -i "s/MONGO_ADMIN_PASSWORD=.*/MONGO_ADMIN_PASSWORD=$MONGO_ADMIN_PASSWORD/" "$${CLUSTER_CONFIG_DIR}/openvidu.env"
sed -i "s/MONGO_REPLICA_SET_KEY=.*/MONGO_REPLICA_SET_KEY=$MONGO_REPLICA_SET_KEY/" "$${CLUSTER_CONFIG_DIR}/openvidu.env"
sed -i "s/DASHBOARD_ADMIN_USERNAME=.*/DASHBOARD_ADMIN_USERNAME=$DASHBOARD_ADMIN_USERNAME/" "$${CLUSTER_CONFIG_DIR}/openvidu.env"
sed -i "s/DASHBOARD_ADMIN_PASSWORD=.*/DASHBOARD_ADMIN_PASSWORD=$DASHBOARD_ADMIN_PASSWORD/" "$${CLUSTER_CONFIG_DIR}/openvidu.env"
sed -i "s/MINIO_ACCESS_KEY=.*/MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY/" "$${CLUSTER_CONFIG_DIR}/openvidu.env"
sed -i "s/MINIO_SECRET_KEY=.*/MINIO_SECRET_KEY=$MINIO_SECRET_KEY/" "$${CLUSTER_CONFIG_DIR}/openvidu.env"
sed -i "s/GRAFANA_ADMIN_USERNAME=.*/GRAFANA_ADMIN_USERNAME=$GRAFANA_ADMIN_USERNAME/" "$${CLUSTER_CONFIG_DIR}/openvidu.env"
sed -i "s/GRAFANA_ADMIN_PASSWORD=.*/GRAFANA_ADMIN_PASSWORD=$GRAFANA_ADMIN_PASSWORD/" "$${CLUSTER_CONFIG_DIR}/openvidu.env"
sed -i "s/LIVEKIT_API_KEY=.*/LIVEKIT_API_KEY=$LIVEKIT_API_KEY/" "$${CLUSTER_CONFIG_DIR}/openvidu.env"
sed -i "s/LIVEKIT_API_SECRET=.*/LIVEKIT_API_SECRET=$LIVEKIT_API_SECRET/" "$${CLUSTER_CONFIG_DIR}/openvidu.env"
sed -i "s/MEET_INITIAL_ADMIN_USER=.*/MEET_INITIAL_ADMIN_USER=$MEET_INITIAL_ADMIN_USER/" "$${CLUSTER_CONFIG_DIR}/master_node/meet.env"
sed -i "s/MEET_INITIAL_ADMIN_PASSWORD=.*/MEET_INITIAL_ADMIN_PASSWORD=$MEET_INITIAL_ADMIN_PASSWORD/" "$${CLUSTER_CONFIG_DIR}/master_node/meet.env"
if [[ "${var.initialMeetApiKey}" != '' ]]; then
sed -i "s/MEET_INITIAL_API_KEY=.*/MEET_INITIAL_API_KEY=$MEET_INITIAL_API_KEY/" "$${CLUSTER_CONFIG_DIR}/master_node/meet.env"
fi
sed -i "s/ENABLED_MODULES=.*/ENABLED_MODULES=$ENABLED_MODULES/" "$${CLUSTER_CONFIG_DIR}/openvidu.env"
# Update URLs in secret
OPENVIDU_URL="https://$${DOMAIN}/"
LIVEKIT_URL="wss://$${DOMAIN}/"
DASHBOARD_URL="https://$${DOMAIN}/dashboard/"
GRAFANA_URL="https://$${DOMAIN}/grafana/"
MINIO_URL="https://$${DOMAIN}/minio-console/"
# Update shared secret
echo -n "$DOMAIN" | gcloud secrets versions add DOMAIN_NAME --data-file=-
echo -n "$OPENVIDU_URL" | gcloud secrets versions add OPENVIDU_URL --data-file=-
echo -n "$LIVEKIT_URL" | gcloud secrets versions add LIVEKIT_URL --data-file=-
echo -n "$DASHBOARD_URL" | gcloud secrets versions add DASHBOARD_URL --data-file=-
echo -n "$GRAFANA_URL" | gcloud secrets versions add GRAFANA_URL --data-file=-
echo -n "$MINIO_URL" | gcloud secrets versions add MINIO_URL --data-file=-
EOF
update_secret_from_config_script = <<-EOF
#!/bin/bash
set -e
# Configure gcloud with instance service account
gcloud auth activate-service-account --key-file=/dev/null 2>/dev/null || true
# Installation directory
INSTALL_DIR="/opt/openvidu"
CLUSTER_CONFIG_DIR="$${INSTALL_DIR}/config/cluster"
MASTER_NODE_CONFIG_DIR="$${INSTALL_DIR}/config/node"
# Get current values of the config
REDIS_PASSWORD="$(/usr/local/bin/get_value_from_config.sh REDIS_PASSWORD "$${MASTER_NODE_CONFIG_DIR}/master_node.env")"
DOMAIN_NAME="$(/usr/local/bin/get_value_from_config.sh DOMAIN_NAME "$${CLUSTER_CONFIG_DIR}/openvidu.env")"
OPENVIDU_RTC_ENGINE="$(/usr/local/bin/get_value_from_config.sh OPENVIDU_RTC_ENGINE "$${CLUSTER_CONFIG_DIR}/openvidu.env")"
OPENVIDU_PRO_LICENSE="$(/usr/local/bin/get_value_from_config.sh OPENVIDU_PRO_LICENSE "$${CLUSTER_CONFIG_DIR}/openvidu.env")"
MONGO_ADMIN_USERNAME="$(/usr/local/bin/get_value_from_config.sh MONGO_ADMIN_USERNAME "$${CLUSTER_CONFIG_DIR}/openvidu.env")"
MONGO_ADMIN_PASSWORD="$(/usr/local/bin/get_value_from_config.sh MONGO_ADMIN_PASSWORD "$${CLUSTER_CONFIG_DIR}/openvidu.env")"
MONGO_REPLICA_SET_KEY="$(/usr/local/bin/get_value_from_config.sh MONGO_REPLICA_SET_KEY "$${CLUSTER_CONFIG_DIR}/openvidu.env")"
MINIO_ACCESS_KEY="$(/usr/local/bin/get_value_from_config.sh MINIO_ACCESS_KEY "$${CLUSTER_CONFIG_DIR}/openvidu.env")"
MINIO_SECRET_KEY="$(/usr/local/bin/get_value_from_config.sh MINIO_SECRET_KEY "$${CLUSTER_CONFIG_DIR}/openvidu.env")"
DASHBOARD_ADMIN_USERNAME="$(/usr/local/bin/get_value_from_config.sh DASHBOARD_ADMIN_USERNAME "$${CLUSTER_CONFIG_DIR}/openvidu.env")"
DASHBOARD_ADMIN_PASSWORD="$(/usr/local/bin/get_value_from_config.sh DASHBOARD_ADMIN_PASSWORD "$${CLUSTER_CONFIG_DIR}/openvidu.env")"
GRAFANA_ADMIN_USERNAME="$(/usr/local/bin/get_value_from_config.sh GRAFANA_ADMIN_USERNAME "$${CLUSTER_CONFIG_DIR}/openvidu.env")"
GRAFANA_ADMIN_PASSWORD="$(/usr/local/bin/get_value_from_config.sh GRAFANA_ADMIN_PASSWORD "$${CLUSTER_CONFIG_DIR}/openvidu.env")"
LIVEKIT_API_KEY="$(/usr/local/bin/get_value_from_config.sh LIVEKIT_API_KEY "$${CLUSTER_CONFIG_DIR}/openvidu.env")"
LIVEKIT_API_SECRET="$(/usr/local/bin/get_value_from_config.sh LIVEKIT_API_SECRET "$${CLUSTER_CONFIG_DIR}/openvidu.env")"
MEET_INITIAL_ADMIN_USER="$(/usr/local/bin/get_value_from_config.sh MEET_INITIAL_ADMIN_USER "$${CLUSTER_CONFIG_DIR}/master_node/meet.env")"
MEET_INITIAL_ADMIN_PASSWORD="$(/usr/local/bin/get_value_from_config.sh MEET_INITIAL_ADMIN_PASSWORD "$${CLUSTER_CONFIG_DIR}/master_node/meet.env")"
if [[ "${var.initialMeetApiKey}" != '' ]]; then
MEET_INITIAL_API_KEY="$(/usr/local/bin/get_value_from_config.sh MEET_INITIAL_API_KEY "$${CLUSTER_CONFIG_DIR}/master_node/meet.env")"
fi
ENABLED_MODULES="$(/usr/local/bin/get_value_from_config.sh ENABLED_MODULES "$${CLUSTER_CONFIG_DIR}/openvidu.env")"
# Update shared secret
echo -n "$REDIS_PASSWORD" | gcloud secrets versions add REDIS_PASSWORD --data-file=-
echo -n "$DOMAIN_NAME" | gcloud secrets versions add DOMAIN_NAME --data-file=-
echo -n "$OPENVIDU_RTC_ENGINE" | gcloud secrets versions add OPENVIDU_RTC_ENGINE --data-file=-
echo -n "$OPENVIDU_PRO_LICENSE" | gcloud secrets versions add OPENVIDU_PRO_LICENSE --data-file=-
echo -n "$MONGO_ADMIN_USERNAME" | gcloud secrets versions add MONGO_ADMIN_USERNAME --data-file=-
echo -n "$MONGO_ADMIN_PASSWORD" | gcloud secrets versions add MONGO_ADMIN_PASSWORD --data-file=-
echo -n "$MONGO_REPLICA_SET_KEY" | gcloud secrets versions add MONGO_REPLICA_SET_KEY --data-file=-
echo -n "$MINIO_ACCESS_KEY" | gcloud secrets versions add MINIO_ACCESS_KEY --data-file=-
echo -n "$MINIO_SECRET_KEY" | gcloud secrets versions add MINIO_SECRET_KEY --data-file=-
echo -n "$DASHBOARD_ADMIN_USERNAME" | gcloud secrets versions add DASHBOARD_ADMIN_USERNAME --data-file=-
echo -n "$DASHBOARD_ADMIN_PASSWORD" | gcloud secrets versions add DASHBOARD_ADMIN_PASSWORD --data-file=-
echo -n "$GRAFANA_ADMIN_USERNAME" | gcloud secrets versions add GRAFANA_ADMIN_USERNAME --data-file=-
echo -n "$GRAFANA_ADMIN_PASSWORD" | gcloud secrets versions add GRAFANA_ADMIN_PASSWORD --data-file=-
echo -n "$LIVEKIT_API_KEY" | gcloud secrets versions add LIVEKIT_API_KEY --data-file=-
echo -n "$LIVEKIT_API_SECRET" | gcloud secrets versions add LIVEKIT_API_SECRET --data-file=-
echo -n "$MEET_INITIAL_ADMIN_USER" | gcloud secrets versions add MEET_INITIAL_ADMIN_USER --data-file=-
echo -n "$MEET_INITIAL_ADMIN_PASSWORD" | gcloud secrets versions add MEET_INITIAL_ADMIN_PASSWORD --data-file=-
if [[ "${var.initialMeetApiKey}" != '' ]]; then
echo -n "$MEET_INITIAL_API_KEY" | gcloud secrets versions add MEET_INITIAL_API_KEY --data-file=-
fi
echo -n "$ENABLED_MODULES" | gcloud secrets versions add ENABLED_MODULES --data-file=-
EOF
get_value_from_config_script = <<-EOF
#!/bin/bash
set -e
# Function to get the value of a given key from the environment file
get_value() {
local key="$1"
local file_path="$2"
# Use grep to find the line with the key, ignoring lines starting with #
# Use awk to split on '=' and print the second field, which is the value
local value=$(grep -E "^\s*$key\s*=" "$file_path" | awk -F= '{print $2}' | sed 's/#.*//; s/^\s*//; s/\s*$//')
# If the value is empty, return "none"
if [ -z "$value" ]; then
echo "none"
else
echo "$value"
fi
}
# Check if the correct number of arguments are supplied
if [ "$#" -ne 2 ]; then
echo "Usage: $0 <key> <file_path>"
exit 1
fi
# Get the key and file path from the arguments
key="$1"
file_path="$2"
# Get and print the value
get_value "$key" "$file_path"
EOF
store_secret_script = <<-EOF
#!/bin/bash
set -e
# Authenticate using instance service account
gcloud auth activate-service-account --key-file=/dev/null 2>/dev/null || true
# Modes: save, generate
# save mode: save the secret in the secret manager
# generate mode: generate a random password and save it in the secret manager
MODE="$1"
if [[ "$MODE" == "generate" ]]; then
SECRET_KEY_NAME="$2"
PREFIX="$${3:-}"
LENGTH="$${4:-44}"
RANDOM_PASSWORD="$(openssl rand -base64 64 | tr -d '+/=\n' | cut -c -$${LENGTH})"
RANDOM_PASSWORD="$${PREFIX}$${RANDOM_PASSWORD}"
echo -n "$RANDOM_PASSWORD" | gcloud secrets versions add $SECRET_KEY_NAME --data-file=-
if [[ $? -ne 0 ]]; then
echo "Error generating secret"
fi
echo "$RANDOM_PASSWORD"
elif [[ "$MODE" == "save" ]]; then
SECRET_KEY_NAME="$2"
SECRET_VALUE="$3"
echo -n "$SECRET_VALUE" | gcloud secrets versions add $SECRET_KEY_NAME --data-file=-
if [[ $? -ne 0 ]]; then
echo "Error generating secret"
fi
echo "$SECRET_VALUE"
else
exit 1
fi
EOF
check_app_ready_script = <<-EOF
#!/bin/bash
i=0
while true; do
HTTP_STATUS=$(curl -Ik http://localhost:7880/health/caddy 2>/dev/null | head -n1 | awk '{print $2}')
if [ "$HTTP_STATUS" == "200" ]; then
break
fi
i=$((i + 1))
if [ "$i" -ge 240 ]; then
echo "Timed out after 20 minutes waiting for OpenVidu to become ready" >&2
exit 1
fi
sleep 5
done
EOF
restart_script = <<-EOF
#!/bin/bash -x
set -e
# Stop all services
systemctl stop openvidu
# Update config from secret
/usr/local/bin/update_config_from_secret.sh
# Start all services
systemctl start openvidu
EOF
user_data_master = <<-EOF
#!/bin/bash -x
set -eu -o pipefail
# restart.sh
cat > /usr/local/bin/restart.sh << 'RESTART_EOF'
${local.restart_script}
RESTART_EOF
chmod +x /usr/local/bin/restart.sh
# Check if installation already completed
if [ -f /usr/local/bin/openvidu_install_counter.txt ]; then
# Launch on reboot
/usr/local/bin/restart.sh || { echo "[OpenVidu] error restarting OpenVidu"; exit 1; }
else
# install.sh
cat > /usr/local/bin/install.sh << 'INSTALL_EOF'
${local.install_script_master}
INSTALL_EOF
chmod +x /usr/local/bin/install.sh
# after_install.sh
cat > /usr/local/bin/after_install.sh << 'AFTER_INSTALL_EOF'
${local.after_install_script}
AFTER_INSTALL_EOF
chmod +x /usr/local/bin/after_install.sh
# update_config_from_secret.sh
cat > /usr/local/bin/update_config_from_secret.sh << 'UPDATE_CONFIG_EOF'
${local.update_config_from_secret_script}
UPDATE_CONFIG_EOF
chmod +x /usr/local/bin/update_config_from_secret.sh
# update_secret_from_config.sh
cat > /usr/local/bin/update_secret_from_config.sh << 'UPDATE_SECRET_EOF'
${local.update_secret_from_config_script}
UPDATE_SECRET_EOF
chmod +x /usr/local/bin/update_secret_from_config.sh
# get_value_from_config.sh
cat > /usr/local/bin/get_value_from_config.sh << 'GET_VALUE_EOF'
${local.get_value_from_config_script}
GET_VALUE_EOF
chmod +x /usr/local/bin/get_value_from_config.sh
# store_secret.sh
cat > /usr/local/bin/store_secret.sh << 'STORE_SECRET_EOF'
${local.store_secret_script}
STORE_SECRET_EOF
chmod +x /usr/local/bin/store_secret.sh
# check_app_ready.sh
cat > /usr/local/bin/check_app_ready.sh << 'CHECK_APP_EOF'
${local.check_app_ready_script}
CHECK_APP_EOF
chmod +x /usr/local/bin/check_app_ready.sh
# config_s3.sh
cat > /usr/local/bin/config_s3.sh << 'CONFIG_S3_EOF'
${local.config_s3_script}
CONFIG_S3_EOF
chmod +x /usr/local/bin/config_s3.sh
echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout
GCLOUD_VERSION=573.0.0
# Install google cli
if ! command -v gcloud >/dev/null 2>&1; then
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg
echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
apt-get update && apt-get install -y google-cloud-cli=$${GCLOUD_VERSION}-0
fi
# Authenticate with gcloud using instance service account
gcloud auth activate-service-account --key-file=/dev/null 2>/dev/null || true
gcloud config set account $(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email" -H "Metadata-Flavor: Google")
gcloud config set project $(curl -s "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google")
export HOME="/root"
# Install OpenVidu
/usr/local/bin/install.sh || { echo "[OpenVidu] error installing OpenVidu"; exit 1; }
# Config S3 bucket
/usr/local/bin/config_s3.sh || { echo "[OpenVidu] error configuring S3 bucket"; exit 1; }
# Start OpenVidu
systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; }
# Update shared secret
/usr/local/bin/after_install.sh || { echo "[OpenVidu] error updating shared secret"; exit 1; }
# Mark installation as complete
echo "installation_complete" > /usr/local/bin/openvidu_install_counter.txt
fi
# Wait for the app
/usr/local/bin/check_app_ready.sh
EOF
install_script_media = <<-EOF
#!/bin/bash -x
set -e
# Install dependencies
echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout
apt-get update && apt-get install -y \
curl \
unzip \
jq \
wget \
ca-certificates \
gnupg \
lsb-release \
openssl
# Configure gcloud with instance service account
gcloud auth activate-service-account --key-file=/dev/null 2>/dev/null || true
METADATA_URL="http://metadata.google.internal/computeMetadata/v1"
get_meta() { curl -s -H "Metadata-Flavor: Google" "$${METADATA_URL}/$1"; }
# Get metadata
MASTER_NODE_PRIVATE_IP=$(get_meta "instance/attributes/masterNodePrivateIP")
STACK_NAME=$(get_meta "instance/attributes/stackName")
PRIVATE_IP=$(get_meta "instance/network-interfaces/0/ip")
# Wait for master node to be ready by checking secrets.
i=0
while ! gcloud secrets versions access latest --secret=ALL_SECRETS_GENERATED 2>/dev/null | grep -q "true"; do
i=$((i + 1))
if [ "$i" -ge 180 ]; then
echo "Timed out after 30 minutes waiting for master node to initialize secrets" >&2
exit 1
fi
echo "Waiting for master node to initialize secrets..."
sleep 10
done
# Disable command tracing so secrets are not printed to the serial console
set +x
# Get all necessary values from secrets
DOMAIN=$(gcloud secrets versions access latest --secret=DOMAIN_NAME)
OPENVIDU_PRO_LICENSE=$(gcloud secrets versions access latest --secret=OPENVIDU_PRO_LICENSE)
REDIS_PASSWORD=$(gcloud secrets versions access latest --secret=REDIS_PASSWORD)
# Get OpenVidu version from secret
OPENVIDU_VERSION=$(gcloud secrets versions access latest --secret=OPENVIDU_VERSION)
if [[ "$OPENVIDU_VERSION" == "none" ]]; then
echo "OpenVidu version not found"
exit 1
fi
# Download first: sh <(curl ...) would silently run an empty script on a transient curl failure
INSTALLER_SCRIPT="/tmp/install_ov_media_node.sh"
curl -fsSL --retry 8 --retry-all-errors --retry-delay 5 -o "$INSTALLER_SCRIPT" "https://get.openvidu.io/pro/elastic/$OPENVIDU_VERSION/install_ov_media_node.sh"
if [ ! -s "$INSTALLER_SCRIPT" ]; then
echo "Downloaded OpenVidu media node installer is empty or missing" >&2
exit 1
fi
INSTALL_COMMAND="sh $INSTALLER_SCRIPT"
# Media node arguments
COMMON_ARGS=(
"--no-tty"
"--install"
"--environment=gcp"
"--deployment-type=elastic"
"--node-role=media-node"
"--master-node-private-ip=$MASTER_NODE_PRIVATE_IP"
"--private-ip=$PRIVATE_IP"
"--redis-password=$REDIS_PASSWORD"
)
# Construct the final command
FINAL_COMMAND="$INSTALL_COMMAND $(printf "%s " "$${COMMON_ARGS[@]}")"
# Execute installation
exec bash -c "$FINAL_COMMAND"
EOF
graceful_shutdown_script = <<-EOF
#!/bin/bash -x
set -e
# Prevent concurrent/repeated runs. The check-abandoned cron fires every minute and would
# otherwise launch this script again while a drain is already in progress, sending a second
# SIGQUIT to the media node. LiveKit interprets the second signal as a forced shutdown
# (force=true), which closes rooms and kicks out participants instead of waiting for them.
exec 9>/var/lock/openvidu_shutdown.lock
if ! flock -n 9; then
echo "Graceful shutdown already in progress, skipping..."
exit 0
fi
# Stop the health watchdog so it does not request a replacement while this node drains. The
# pause file is honored on the watchdog's next loop regardless of when the unit stops; it is
# cleared on reboot.
touch /run/openvidu-media-health.paused
systemctl stop openvidu-media-health 2>/dev/null || true
echo "Starting graceful shutdown of OpenVidu Media Node..."
INSTANCE_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google")
PROJECT_ID=$(curl -s "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google")
ZONE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/zone" -H "Metadata-Flavor: Google" | cut -d'/' -f4)
# Execute if docker is installed
if [ -x "$(command -v docker)" ]; then
echo "Stopping media node services and waiting for termination..."
docker container kill --signal=SIGQUIT openvidu 2>/dev/null || true
docker container kill --signal=SIGQUIT ingress 2>/dev/null || true
docker container kill --signal=SIGQUIT egress 2>/dev/null || true
for agent_container in $(docker ps --filter "label=openvidu-agent=true" --format '{{.Names}}' 2>/dev/null); do
docker container kill --signal=SIGQUIT "$agent_container" 2>/dev/null || true
done
# A node being replaced because it is unhealthy must not wait for sessions to drain.
if [ -f /run/openvidu-unhealthy ]; then
docker ps -q | xargs -r docker kill >/dev/null 2>&1 || true
fi
# Wait for running containers to not be openvidu, ingress, egress or an openvidu agent
while [ "$(docker ps --filter 'label=openvidu-agent=true' -q 2>/dev/null | wc -l)" -gt 0 ] || \
[ "$(docker inspect -f '{{.State.Running}}' openvidu 2>/dev/null)" = "true" ] || \
[ "$(docker inspect -f '{{.State.Running}}' ingress 2>/dev/null)" = "true" ] || \
[ "$(docker inspect -f '{{.State.Running}}' egress 2>/dev/null)" = "true" ]; do
echo "Waiting for containers to stop..."
sleep 10
done
fi
# Self-delete using gcloud API
# Delete this instance
gcloud compute instances delete "$INSTANCE_NAME" \
--zone="$ZONE" \
--project="$PROJECT_ID" \
--quiet \
--no-user-output-enabled || echo "Failed to self-delete, instance may already be terminating"
echo "Graceful shutdown completed."
EOF
crontab_job_media = <<-EOF
#!/bin/bash -x
set -e
# Get current instance details
INSTANCE_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google")
PROJECT_ID=$(curl -s "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google")
ZONE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/zone" -H "Metadata-Flavor: Google" | cut -d'/' -f4)
STACK_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/stackName" -H "Metadata-Flavor: Google")
# Check if instance is still managed by the MIG
MIG_NAME=$(echo "${var.stackName}" | tr '[:upper:]' '[:lower:]')-media-node-group
REGION=$(echo $ZONE | sed 's/-[a-z]$//')
# Check if this instance is still part of the MIG
INSTANCE_IN_MIG=$(gcloud compute instance-groups managed list-instances "$MIG_NAME" --region="$REGION" --project="$PROJECT_ID" --format="value(NAME)" | grep "$INSTANCE_NAME" || echo "")
if [ -n "$INSTANCE_IN_MIG" ]; then
echo "Instance $INSTANCE_NAME is still managed by MIG. Continuing normal operation..."
exit 0
else
echo "Instance $INSTANCE_NAME has been abandoned from MIG. Starting graceful shutdown..."
/usr/local/bin/graceful_shutdown.sh
fi
EOF
media_health_watchdog_script = <<-EOF
#!/bin/bash
# OpenVidu Media Node health watchdog.
#
# Asks the cloud to replace this Media Node with a new one, keeping the number of
# Media Nodes, through /usr/local/bin/openvidu-media-replace.sh, when:
# - its bootstrap failed (the user-data wrote the bootstrap-failed marker): after a
# cool-down, so a persistent failure does not churn nodes and can be inspected,
# - its bootstrap did not finish BOOTSTRAP_DEADLINE_SEC after boot,
# - LiveKit fails its health check (HTTP 200 on PROBE_URL): for HARD_UNHEALTHY_AFTER_SEC
# while it is down (connection refused or closed), or for SOFT_UNHEALTHY_AFTER_SEC
# while it answers an error or times out. Soft failures are not counted while the CPU
# is saturated (an overloaded LiveKit reports Not Ready), and no failure is counted in
# the WARMUP_SEC after openvidu.service (re)started.
# Nothing is replaced while no master answers a Redis PING (MASTER_ADDRS): a master outage
# fails the health check of every Media Node at once, and a new node could not bootstrap
# either; LiveKit failures are not counted until a master answers again. Once requested,
# the replacement is requested again every REPLACE_RETRY_SEC until the node goes away.
#
# Settings: /etc/openvidu/media-health.env (sourced every cycle; MASTER_ADDRS="ip:port ...")
# Pause: touch /etc/openvidu/media-health.disabled (drain scripts use /run/openvidu-media-health.paused)
# Logs: journalctl -u openvidu-media-health
#
# This file is embedded verbatim in CloudFormation !Sub blocks, Bicep strings and
# Terraform heredocs, so it must never contain a dollar sign or a percent sign followed
# by an opening brace, nor two opening braces in a row.
STATE_DIR=/var/lib/openvidu-media-health
INSTALLED_MARKER="$STATE_DIR/installed"
BOOTSTRAP_FAILED_MARKER="$STATE_DIR/bootstrap-failed"
REPLACE_SCRIPT=/usr/local/bin/openvidu-media-replace.sh
ENV_FILE=/etc/openvidu/media-health.env
PAUSE_FILE=/etc/openvidu/media-health.disabled
DRAIN_PAUSE_FILE=/run/openvidu-media-health.paused
log() { echo "[media-health] $*"; }
# Seconds since boot: monotonic, unaffected by clock steps
uptime_sec() { cut -d. -f1 /proc/uptime; }
# Busy CPU percentage since the previous call, in BUSY (0 on the first call)
cpu_busy() {
read -r _ c_user c_nice c_system c_idle c_iowait c_irq c_softirq c_steal _ < /proc/stat
total=$((c_user + c_nice + c_system + c_idle + c_iowait + c_irq + c_softirq + c_steal))
idle=$((c_idle + c_iowait))
BUSY=0
if [ -n "$PREV_TOTAL" ] && [ "$total" -gt "$PREV_TOTAL" ]; then
BUSY=$((100 * (total - PREV_TOTAL - idle + PREV_IDLE) / (total - PREV_TOTAL)))
fi
PREV_TOTAL=$total
PREV_IDLE=$idle
}
# openvidu.service is running and entered the active state less than WARMUP_SEC ago
# (a stopped or failed openvidu.service is not starting: no warm-up for it)
openvidu_starting() {
case "$(systemctl show -p ActiveState --value openvidu 2>/dev/null)" in
active|activating|reloading) ;;
*) return 1 ;;
esac
since=$(systemctl show -p ActiveEnterTimestampMonotonic --value openvidu 2>/dev/null)
case "$since" in ''|0|*[!0-9]*) return 1 ;; esac
[ $(($(uptime_sec) - since / 1000000)) -lt "$WARMUP_SEC" ]
}
# A master answers a Redis PING (+PONG, or -NOAUTH when a password is required). A bare
# TCP connect would also succeed against a wedged Redis.
master_serving() {
[ -n "$MASTER_ADDRS" ] || return 1
for addr in $MASTER_ADDRS; do
host=$(echo "$addr" | cut -d: -f1)
port=$(echo "$addr" | cut -d: -f2)
reply=$(timeout 5 bash -c 'exec 3<>"/dev/tcp/$0/$1" && printf "PING\r\n" >&3 && head -c 7 <&3' "$host" "$port" 2>/dev/null)
case "$reply" in
+PONG*|-NOAUTH*) return 0 ;;
esac
done
return 1
}
# Request the replacement of this node ($1 = reason). Gated on a master answering, except
# in the bootstrap phase before the master addresses are known.
request_replacement() {
now=$(uptime_sec)
if [ -n "$REQUESTED_AT" ] && [ $((now - REQUESTED_AT)) -lt "$REPLACE_RETRY_SEC" ]; then
return
fi
if [ -n "$MASTER_ADDRS" ] || [ -f "$INSTALLED_MARKER" ]; then
if ! master_serving; then
if [ -z "$HOLD_LOGGED_AT" ] || [ $((now - HOLD_LOGGED_AT)) -ge 600 ]; then
log "$1, but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not replacing this node"
HOLD_LOGGED_AT=$now
fi
return
fi
fi
log "$1: requesting the replacement of this node"
if "$REPLACE_SCRIPT" "$1"; then
REQUESTED_AT=$now
log "replacement requested"
else
log "the replacement request failed, retrying in $INTERVAL_SEC s"
fi
}
mkdir -p "$STATE_DIR"
REQUESTED_AT=""
HOLD_LOGGED_AT=""
FAILED_SEEN_AT=""
FAIL_SINCE=""
SATURATED=""
PREV_TOTAL=""
PREV_IDLE=""
log "started"
while true; do
PROBE_URL=http://127.0.0.1:7880/
INTERVAL_SEC=30
HARD_UNHEALTHY_AFTER_SEC=300
SOFT_UNHEALTHY_AFTER_SEC=600
CPU_SATURATED_PCT=90
WARMUP_SEC=900
BOOTSTRAP_DEADLINE_SEC=5400
BOOTSTRAP_FAILURE_COOLDOWN_SEC=600
REPLACE_RETRY_SEC=600
MASTER_ADDRS=""
[ -f "$ENV_FILE" ] && . "$ENV_FILE"
NOW=$(uptime_sec)
cpu_busy
if [ -f "$PAUSE_FILE" ] || [ -f "$DRAIN_PAUSE_FILE" ]; then
FAIL_SINCE=""
elif [ -f "$BOOTSTRAP_FAILED_MARKER" ]; then
if [ -z "$FAILED_SEEN_AT" ]; then
FAILED_SEEN_AT=$NOW
log "bootstrap failed ($(head -c 300 "$BOOTSTRAP_FAILED_MARKER")): replacing this node in $BOOTSTRAP_FAILURE_COOLDOWN_SEC s"
fi
if [ $((NOW - FAILED_SEEN_AT)) -ge "$BOOTSTRAP_FAILURE_COOLDOWN_SEC" ]; then
request_replacement "bootstrap failed: $(head -c 300 "$BOOTSTRAP_FAILED_MARKER")"
fi
elif [ ! -f "$INSTALLED_MARKER" ]; then
if [ "$NOW" -ge "$BOOTSTRAP_DEADLINE_SEC" ]; then
request_replacement "bootstrap not finished $BOOTSTRAP_DEADLINE_SEC s after boot"
fi
elif curl -sf -o /dev/null --connect-timeout 3 --max-time 15 "$PROBE_URL"; then
[ -n "$FAIL_SINCE" ] && log "LiveKit healthy again after $((NOW - FAIL_SINCE)) s"
FAIL_SINCE=""
SATURATED=""
else
RC=$?
case "$RC" in
7|52|56) KIND=hard ;;
*) KIND=soft ;;
esac
if openvidu_starting; then
FAIL_SINCE=""
elif ! master_serving; then
# A master outage fails the health check of every Media Node at once: not counted,
# so LiveKit gets the whole window to reconnect once a master answers again
FAIL_SINCE=""
if [ -z "$HOLD_LOGGED_AT" ] || [ $((NOW - HOLD_LOGGED_AT)) -ge 600 ]; then
log "LiveKit health check failed (curl exit $RC), but no master answers a Redis PING (MASTER_ADDRS='$MASTER_ADDRS'): not counted"
HOLD_LOGGED_AT=$NOW
fi
elif [ "$KIND" = soft ] && [ "$BUSY" -ge "$CPU_SATURATED_PCT" ]; then
[ -n "$SATURATED" ] || log "LiveKit not ready (curl exit $RC) with the CPU $BUSY% busy: not counted"
SATURATED=1
FAIL_SINCE=""
else
SATURATED=""
if [ -z "$FAIL_SINCE" ]; then
FAIL_SINCE=$NOW
log "LiveKit health check failed (curl exit $RC)"
fi
LIMIT=$SOFT_UNHEALTHY_AFTER_SEC
[ "$KIND" = hard ] && LIMIT=$HARD_UNHEALTHY_AFTER_SEC
if [ $((NOW - FAIL_SINCE)) -ge "$LIMIT" ]; then
request_replacement "LiveKit unhealthy for $((NOW - FAIL_SINCE)) s (curl exit $RC)"
fi
fi
fi
sleep "$INTERVAL_SEC"
done
EOF
media_replace_script = <<-EOF
#!/bin/bash
# Requests the replacement of THIS media node while keeping the media node count: it asks the
# regional MIG to recreate the instance (delete + create in place, same name, new instance id,
# fresh disk). Called by the health watchdog and by the bootstrap-failure hook; it never stops
# or signals the watchdog (its caller, in the same cgroup).
REASON="$1"
mkdir -p /var/lib/openvidu-media-health
echo "$(date -u +%FT%TZ) $REASON" >> /var/lib/openvidu-media-health/replace-requests.log
# Tell the drain script this node is being replaced so it skips waiting for sessions.
touch /run/openvidu-unhealthy
# Withdrawn if the request fails: the node then stays, and if it recovers
# a later scale-in must still wait for its sessions
trap '[ $? -eq 0 ] || rm -f /run/openvidu-unhealthy' EXIT
META="http://metadata.google.internal/computeMetadata/v1"
meta() { curl -s -H "Metadata-Flavor: Google" "$META/$1"; }
SELF_NAME=$(meta "instance/name")
PROJECT_ID=$(meta "project/project-id")
ZONE=$(meta "instance/zone" | cut -d/ -f4)
REGION=$(echo "$ZONE" | sed 's/-[a-z]$//')
STACK_NAME=$(meta "instance/attributes/stackName")
MIG_NAME=$(echo "$STACK_NAME" | tr '[:upper:]' '[:lower:]')-media-node-group
# Recreate keeps the MIG size. Retry transient API errors. A "not a member / already being
# deleted, abandoned or recreated" error means the node is already leaving the group (e.g.
# scaled in): nothing to do.
i=0
while [ "$i" -lt 5 ]; do
i=$((i + 1))
if OUT=$(gcloud compute instance-groups managed recreate-instances "$MIG_NAME" \
--instances="$SELF_NAME" --region="$REGION" --project="$PROJECT_ID" --quiet 2>&1); then
echo "[media-replace] recreate requested for $SELF_NAME"
exit 0
fi
case "$OUT" in
*"not a member"*|*"being deleted"*|*"being recreated"*|*abandon*|*ABANDON*)
echo "[media-replace] $SELF_NAME already leaving the group, nothing to do: $OUT"
exit 0
;;
esac
echo "[media-replace] recreate attempt $i for $SELF_NAME failed: $OUT"
sleep 12
done
echo "[media-replace] gave up requesting recreate of $SELF_NAME after $i attempts"
exit 1
EOF
media_health_unit = <<-EOF
[Unit]
Description=OpenVidu Media Node health watchdog
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
ExecStart=/usr/local/bin/openvidu-media-health.sh
Restart=always
RestartSec=30
[Install]
WantedBy=multi-user.target
EOF
user_data_media = <<-EOF
#!/bin/bash -x
set -eu -o pipefail
# Records a media-node bootstrap failure so the health watchdog replaces this node after a
# cool-down, then aborts the bootstrap.
ov_media_bootstrap_failed() {
echo "[OpenVidu] media node bootstrap failed: $1"
mkdir -p /var/lib/openvidu-media-health
echo "$1" > /var/lib/openvidu-media-health/bootstrap-failed
exit 1
}
# Check if installation already completed
if [ -f /usr/local/bin/openvidu_install_counter.txt ]; then
# Launch on reboot
systemctl start openvidu || { echo "[OpenVidu] error starting OpenVidu"; exit 1; }
else
# install.sh (media node)
cat > /usr/local/bin/install.sh << 'INSTALL_EOF'
${local.install_script_media}
INSTALL_EOF
chmod +x /usr/local/bin/install.sh
# graceful_shutdown.sh
cat > /usr/local/bin/graceful_shutdown.sh << 'GRACEFUL_SHUTDOWN_EOF'
${local.graceful_shutdown_script}
GRACEFUL_SHUTDOWN_EOF
chmod +x /usr/local/bin/graceful_shutdown.sh
# Media node health watchdog + its replacement script and unit
cat > /usr/local/bin/openvidu-media-health.sh << 'MEDIA_HEALTH_EOF'
${local.media_health_watchdog_script}
MEDIA_HEALTH_EOF
chmod +x /usr/local/bin/openvidu-media-health.sh
cat > /usr/local/bin/openvidu-media-replace.sh << 'MEDIA_REPLACE_EOF'
${local.media_replace_script}
MEDIA_REPLACE_EOF
chmod +x /usr/local/bin/openvidu-media-replace.sh
cat > /etc/systemd/system/openvidu-media-health.service << 'MEDIA_HEALTH_UNIT_EOF'
${local.media_health_unit}
MEDIA_HEALTH_UNIT_EOF
echo "DPkg::Lock::Timeout \"-1\";" > /etc/apt/apt.conf.d/99timeout
GCLOUD_VERSION=573.0.0
# Install google cli
if ! command -v gcloud >/dev/null 2>&1; then
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg
echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
apt-get update && apt-get install -y google-cloud-cli=$${GCLOUD_VERSION}-0
fi
# Authenticate with gcloud using instance service account
gcloud auth activate-service-account --key-file=/dev/null 2>/dev/null || true
gcloud config set account $(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email" -H "Metadata-Flavor: Google")
gcloud config set project $(curl -s "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google")
export HOME="/root"
# Start the media node health watchdog now: gcloud (which its replacement needs) is installed,
# and starting before install.sh lets the watchdog cover a hung bootstrap too. Write the master
# address(es) first (before the installed marker) so the anti-storm guard is always populated.
mkdir -p /etc/openvidu
MASTER_NODE_PRIVATE_IP=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/masterNodePrivateIP" -H "Metadata-Flavor: Google")
[ -n "$MASTER_NODE_PRIVATE_IP" ] || ov_media_bootstrap_failed "could not determine master node private IP"
echo "MASTER_ADDRS=\"$MASTER_NODE_PRIVATE_IP:7000\"" > /etc/openvidu/media-health.env
systemctl daemon-reload
systemctl enable --now openvidu-media-health
# Install OpenVidu Media Node
/usr/local/bin/install.sh || ov_media_bootstrap_failed "install.sh failed"
# Mark installation as complete
echo "installation_complete" > /usr/local/bin/openvidu_install_counter.txt
# Start OpenVidu
systemctl start openvidu || ov_media_bootstrap_failed "systemctl start openvidu failed"
# Bootstrap finished: hand runtime replacement to the watchdog (never on a failure path).
mkdir -p /var/lib/openvidu-media-health
touch /var/lib/openvidu-media-health/installed
# Add cron job to check if instance is abandoned every minute
cat > /usr/local/bin/check_abandoned.sh << 'CHECK_ABANDONED_EOF'
${local.crontab_job_media}
CHECK_ABANDONED_EOF
chmod +x /usr/local/bin/check_abandoned.sh
echo "*/1 * * * * /usr/local/bin/check_abandoned.sh > /var/log/openvidu-abandoned-check.log 2>&1" | crontab -
fi
EOF
}
# fixedNumberOfMediaNodes gave the autoscaler a count. A deployment created before keeps
# it under its old address, so a re-apply moves it instead of destroying and recreating it.
moved {
from = google_compute_region_autoscaler.media_node_autoscaler
to = google_compute_region_autoscaler.media_node_autoscaler[0]
}
Reference in New Issue View Git Blame Copy Permalink