diff --git a/openvidu-deployment/CHANGELOG.md b/openvidu-deployment/CHANGELOG.md index 5f3861532..862c12d23 100644 --- a/openvidu-deployment/CHANGELOG.md +++ b/openvidu-deployment/CHANGELOG.md @@ -35,6 +35,18 @@ in the [openvidu-oracle](https://github.com/OpenVidu/openvidu-oracle) and - `journalctl -u openvidu-media-health` on a Media Node shows what it is doing, and `sudo touch /etc/openvidu/media-health.disabled` pauses it (for example during manual maintenance); deleting the file resumes it. +- Elastic and High Availability deployments on AWS, Azure and Google Cloud can run a fixed number of Media + Nodes, as DigitalOcean and Oracle Cloud already could, with the same parameter: `FixedNumberOfMediaNodes` + on AWS, `fixedNumberOfMediaNodes` on the other clouds. With a value greater than 0 the deployment runs + exactly that many Media Nodes with no autoscaling, and the initial, minimum, maximum and target CPU + parameters are ignored. The default, 0, keeps autoscaling. An unhealthy Media Node is still replaced. + Changing the number, or switching between the two modes, is a redeploy of the template with the new value. + Raising the number adds Media Nodes. On AWS and Google Cloud, lowering it drains the Media Nodes it removes + gracefully, as in a scale-in: each one stops accepting new Rooms and is removed when its ongoing Rooms, + recordings and ingests end. Azure does not accept a number below the Media Nodes running, because they + protect themselves from scale-in: drain the extra ones first (manual scale, then + `/usr/local/bin/stop_media_node.sh` on each, which removes itself the same graceful way) and then redeploy. + The administration documentation of each cloud describes the steps. #### AWS @@ -90,5 +102,5 @@ in the [openvidu-oracle](https://github.com/OpenVidu/openvidu-oracle) and under a new name. The webhooks of earlier deployments stay unused in the Automation account until they expire. - A redeploy still cannot set the minimum or maximum number of Media Nodes below the Media Nodes + A redeploy still cannot set the fixed, minimum or maximum number of Media Nodes below the Media Nodes running (`UnsupportedCapacity`); the administration documentation describes how to remove them first. diff --git a/openvidu-deployment/pro/elastic/aws/cf-openvidu-elastic.yaml b/openvidu-deployment/pro/elastic/aws/cf-openvidu-elastic.yaml index 3347203d2..09df620da 100644 --- a/openvidu-deployment/pro/elastic/aws/cf-openvidu-elastic.yaml +++ b/openvidu-deployment/pro/elastic/aws/cf-openvidu-elastic.yaml @@ -1020,22 +1020,28 @@ Parameters: InitialNumberOfMediaNodes: Type: Number Default: 1 - Description: Number of initial media nodes to deploy + Description: Number of initial media nodes to deploy. Ignored when FixedNumberOfMediaNodes > 0 MinNumberOfMediaNodes: Type: Number Default: 1 - Description: Minimum number of media nodes to deploy + Description: Minimum number of media nodes to deploy. Ignored when FixedNumberOfMediaNodes > 0 MaxNumberOfMediaNodes: Type: Number Default: 5 - Description: Maximum number of media nodes to deploy + Description: Maximum number of media nodes to deploy. Ignored when FixedNumberOfMediaNodes > 0 ScaleTargetCPU: Type: Number Default: 50 - Description: Target CPU percentage to scale up or down + Description: Target CPU percentage to scale up or down. Ignored when FixedNumberOfMediaNodes > 0 + + FixedNumberOfMediaNodes: + Type: Number + Default: 0 + MinValue: 0 + Description: If > 0, deploys a fixed number of media nodes with no autoscaling (InitialNumberOfMediaNodes, MinNumberOfMediaNodes, MaxNumberOfMediaNodes and ScaleTargetCPU are ignored). If 0 (default), the deployment is elastic and autoscaling is enabled S3AppDataBucketName: Type: String @@ -1090,6 +1096,7 @@ Metadata: - Label: default: Media Nodes Autoscaling Group configuration Parameters: + - FixedNumberOfMediaNodes - InitialNumberOfMediaNodes - MinNumberOfMediaNodes - MaxNumberOfMediaNodes @@ -1112,6 +1119,8 @@ Conditions: PublicElasticIPPresent: !Not [ !Equals [!Ref PublicElasticIP, ""] ] PublicElasticIPAbsent: !Equals [!Ref PublicElasticIP, ""] CreateRecordingsBucket: !Equals [!Ref S3AppDataBucketName, ""] + FixedNumberOfMediaNodesEnabled: !Not [ !Equals [!Ref FixedNumberOfMediaNodes, 0] ] + AutoscalingEnabled: !Equals [!Ref FixedNumberOfMediaNodes, 0] IsMasterGraviton: !Or - !Equals [ !Select [ 0, !Split ['.', !Ref MasterNodeInstanceType ]], 't4g'] - !Equals [ !Select [ 0, !Split ['.', !Ref MasterNodeInstanceType ]], 'c6g'] @@ -2576,9 +2585,9 @@ Resources: LaunchTemplate: LaunchTemplateId: !Ref OpenViduMediaNodeLaunchTemplate Version: !GetAtt OpenViduMediaNodeLaunchTemplate.DefaultVersionNumber - MinSize: !Ref MinNumberOfMediaNodes - MaxSize: !Ref MaxNumberOfMediaNodes - DesiredCapacity: !Ref InitialNumberOfMediaNodes + MinSize: !If [FixedNumberOfMediaNodesEnabled, !Ref FixedNumberOfMediaNodes, !Ref MinNumberOfMediaNodes] + MaxSize: !If [FixedNumberOfMediaNodesEnabled, !Ref FixedNumberOfMediaNodes, !Ref MaxNumberOfMediaNodes] + DesiredCapacity: !If [FixedNumberOfMediaNodesEnabled, !Ref FixedNumberOfMediaNodes, !Ref InitialNumberOfMediaNodes] VPCZoneIdentifier: !Ref OpenViduMediaNodeSubnets Tags: - Key: Name @@ -2741,6 +2750,7 @@ Resources: OpenViduMediaNodeASGScalingPolicy: Type: AWS::AutoScaling::ScalingPolicy + Condition: AutoscalingEnabled Properties: AutoScalingGroupName: !Ref OpenViduMediaNodeASG PolicyType: TargetTrackingScaling diff --git a/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.bicep b/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.bicep index 111503eeb..2e1e9564f 100644 --- a/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.bicep +++ b/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.bicep @@ -59,18 +59,22 @@ param adminUsername string @secure() param adminSshKey object -@description('Number of initial media nodes to deploy') +@description('Number of initial media nodes to deploy. Ignored when fixedNumberOfMediaNodes > 0') param initialNumberOfMediaNodes int = 1 -@description('Minimum number of media nodes to deploy') +@description('Minimum number of media nodes to deploy. Ignored when fixedNumberOfMediaNodes > 0') param minNumberOfMediaNodes int = 1 -@description('Maximum number of media nodes to deploy') +@description('Maximum number of media nodes to deploy. Ignored when fixedNumberOfMediaNodes > 0') param maxNumberOfMediaNodes int = 5 -@description('Target CPU percentage to scale up or down') +@description('Target CPU percentage to scale up or down. Ignored when fixedNumberOfMediaNodes > 0') param scaleTargetCPU int = 50 +@description('If > 0, keeps this fixed number of media nodes: autoscaling is pinned to it (minimum = maximum = this number) and initialNumberOfMediaNodes, minNumberOfMediaNodes, maxNumberOfMediaNodes and scaleTargetCPU are ignored. When it is lowered, the extra media nodes are drained gracefully, one at a time, by the scale-in runbook. If 0 (default), the deployment is elastic and autoscaling is enabled') +@minValue(0) +param fixedNumberOfMediaNodes int = 0 + param additionalInstallFlags string = '' /*------------------------------------------- VARIABLES AND VALIDATIONS -------------------------------------------*/ @@ -1506,6 +1510,15 @@ resource openviduScaleSetMediaNode 'Microsoft.Compute/virtualMachineScaleSets@20 } } +// Always deployed and always enabled, so a redeploy switches modes both ways (an incremental +// ARM deployment never deletes a resource the template stops declaring). With a fixed number +// of Media Nodes autoscaling is pinned to it, not disabled: minimum = maximum = default, so +// the CPU rules are inert. Azure refuses a minimum or a maximum below the Media Nodes that are +// running, because they protect themselves from scale-in: a redeploy can raise the fixed +// number (or minNumberOfMediaNodes) but not lower it below the Media Nodes running. To lower +// it, switch the scale set to manual scale, drain the extra Media Nodes with +// /usr/local/bin/stop_media_node.sh (each one removes itself once its Rooms end), and then +// redeploy. resource openviduAutoScaleSettings 'Microsoft.Insights/autoscaleSettings@2022-10-01' = { name: '${stackName}-autoscaleSettings' location: resourceGroup().location @@ -1514,9 +1527,9 @@ resource openviduAutoScaleSettings 'Microsoft.Insights/autoscaleSettings@2022-10 { name: 'openvidu-medianode-autoscale' capacity: { - minimum: string(minNumberOfMediaNodes) - maximum: string(maxNumberOfMediaNodes) - default: string(initialNumberOfMediaNodes) + minimum: string(fixedNumberOfMediaNodes > 0 ? fixedNumberOfMediaNodes : minNumberOfMediaNodes) + maximum: string(fixedNumberOfMediaNodes > 0 ? fixedNumberOfMediaNodes : maxNumberOfMediaNodes) + default: string(fixedNumberOfMediaNodes > 0 ? fixedNumberOfMediaNodes : initialNumberOfMediaNodes) } rules: [ { diff --git a/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.json b/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.json index 70a3322c7..0e09181f4 100644 --- a/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.json +++ b/openvidu-deployment/pro/elastic/azure/cf-openvidu-elastic.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.46.1.21595", - "templateHash": "13646640563547871932" + "templateHash": "12314273468008882970" } }, "parameters": { @@ -117,28 +117,36 @@ "type": "int", "defaultValue": 1, "metadata": { - "description": "Number of initial media nodes to deploy" + "description": "Number of initial media nodes to deploy. Ignored when fixedNumberOfMediaNodes > 0" } }, "minNumberOfMediaNodes": { "type": "int", "defaultValue": 1, "metadata": { - "description": "Minimum number of media nodes to deploy" + "description": "Minimum number of media nodes to deploy. Ignored when fixedNumberOfMediaNodes > 0" } }, "maxNumberOfMediaNodes": { "type": "int", "defaultValue": 5, "metadata": { - "description": "Maximum number of media nodes to deploy" + "description": "Maximum number of media nodes to deploy. Ignored when fixedNumberOfMediaNodes > 0" } }, "scaleTargetCPU": { "type": "int", "defaultValue": 50, "metadata": { - "description": "Target CPU percentage to scale up or down" + "description": "Target CPU percentage to scale up or down. Ignored when fixedNumberOfMediaNodes > 0" + } + }, + "fixedNumberOfMediaNodes": { + "type": "int", + "defaultValue": 0, + "minValue": 0, + "metadata": { + "description": "If > 0, keeps this fixed number of media nodes: autoscaling is pinned to it (minimum = maximum = this number) and initialNumberOfMediaNodes, minNumberOfMediaNodes, maxNumberOfMediaNodes and scaleTargetCPU are ignored. When it is lowered, the extra media nodes are drained gracefully, one at a time, by the scale-in runbook. If 0 (default), the deployment is elastic and autoscaling is enabled" } }, "additionalInstallFlags": { @@ -502,9 +510,9 @@ { "name": "openvidu-medianode-autoscale", "capacity": { - "minimum": "[string(parameters('minNumberOfMediaNodes'))]", - "maximum": "[string(parameters('maxNumberOfMediaNodes'))]", - "default": "[string(parameters('initialNumberOfMediaNodes'))]" + "minimum": "[string(if(greater(parameters('fixedNumberOfMediaNodes'), 0), parameters('fixedNumberOfMediaNodes'), parameters('minNumberOfMediaNodes')))]", + "maximum": "[string(if(greater(parameters('fixedNumberOfMediaNodes'), 0), parameters('fixedNumberOfMediaNodes'), parameters('maxNumberOfMediaNodes')))]", + "default": "[string(if(greater(parameters('fixedNumberOfMediaNodes'), 0), parameters('fixedNumberOfMediaNodes'), parameters('initialNumberOfMediaNodes')))]" }, "rules": [ { diff --git a/openvidu-deployment/pro/elastic/azure/createUiDefinition.json b/openvidu-deployment/pro/elastic/azure/createUiDefinition.json index 6c23859da..3ff47219d 100644 --- a/openvidu-deployment/pro/elastic/azure/createUiDefinition.json +++ b/openvidu-deployment/pro/elastic/azure/createUiDefinition.json @@ -351,6 +351,22 @@ "name": "parameters SCALING", "label": "Media Nodes Sacle Set configuration", "elements": [ + { + "name": "fixedNumberOfMediaNodes", + "type": "Microsoft.Common.TextBox", + "label": "Fixed Number Of Media Nodes", + "subLabel": "If > 0, deploys a fixed number of media nodes with no autoscaling (the Initial/Min/Max/Scale Target CPU values are ignored). If 0 (default), the deployment is elastic and autoscaling is enabled", + "defaultValue": "0", + "toolTip": "", + "constraints": { + "required": false, + "regex": "^[0-9]+$", + "validationMessage": "Must be an integer greater than or equal to 0", + "validations": [] + }, + "infoMessages": [], + "visible": true + }, { "name": "initialNumberOfMediaNodes", "type": "Microsoft.Common.TextBox", @@ -530,6 +546,7 @@ "mediaNodeInstanceType": "[steps('parameters INSTANCE').mediaNodeInstanceType]", "adminUsername": "[steps('parameters INSTANCE').adminUsername]", "adminSshKey": "[steps('parameters INSTANCE').adminSshKey]", + "fixedNumberOfMediaNodes": "[steps('parameters SCALING').fixedNumberOfMediaNodes]", "initialNumberOfMediaNodes": "[steps('parameters SCALING').initialNumberOfMediaNodes]", "minNumberOfMediaNodes": "[steps('parameters SCALING').minNumberOfMediaNodes]", "maxNumberOfMediaNodes": "[steps('parameters SCALING').maxNumberOfMediaNodes]", diff --git a/openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf b/openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf index f426449ec..ca8fe421a 100644 --- a/openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf +++ b/openvidu-deployment/pro/elastic/gcp/tf-gpc-openvidu-elastic.tf @@ -280,7 +280,7 @@ resource "google_compute_region_instance_group_manager" "media_node_group" { name = lower("${var.stackName}-media-node-group") base_instance_name = lower("${var.stackName}-media-node") region = var.region - target_size = var.initialNumberOfMediaNodes + target_size = var.fixedNumberOfMediaNodes > 0 ? var.fixedNumberOfMediaNodes : var.initialNumberOfMediaNodes version { instance_template = google_compute_instance_template.media_node_template.self_link_unique @@ -298,11 +298,19 @@ resource "google_compute_region_instance_group_manager" "media_node_group" { initial_delay_sec = 3600 } + # Terraform sets the size at creation only. Afterwards the autoscaler (scale out) and the + # scale-in function (scale in, and the fixedNumberOfMediaNodes reconcile) own it. + lifecycle { + ignore_changes = [target_size] + } + depends_on = [google_compute_instance.openvidu_master_node] } # Autoscaler for Media Nodes +# Not created with a fixed number of Media Nodes: the scale-in function keeps the MIG at that size. resource "google_compute_region_autoscaler" "media_node_autoscaler" { + count = var.fixedNumberOfMediaNodes > 0 ? 0 : 1 name = lower("${var.stackName}-media-node-autoscaler") region = var.region target = google_compute_region_instance_group_manager.media_node_group.self_link @@ -364,6 +372,8 @@ def scalein_function(request): """ Cloud Function to handle scale-in operations for media node instances. Abandons oldest instances when autoscaler recommends fewer instances. + With a fixed number of Media Nodes (no autoscaler), keeps the MIG at that number: + abandons the extra instances or resizes the MIG up. """ print("Starting scale-in function") # Also use print for Cloud Run logs logger.info("Starting scale-in function") @@ -380,6 +390,13 @@ def scalein_function(request): mig_name = f"{stack_name.lower()}-media-node-group" autoscaler_name = f"{stack_name.lower()}-media-node-autoscaler" + # Fixed number of Media Nodes: there is no autoscaler, keep the MIG at that number + fixed_number = _get_fixed_number_of_media_nodes() + if fixed_number > 0: + return _reconcile_fixed_size( + clients, project_id, region, mig_name, fixed_number + ) + print(f"Checking MIG: {mig_name} and autoscaler: {autoscaler_name}") logger.info(f"Checking MIG: {mig_name} and autoscaler: {autoscaler_name}") @@ -450,6 +467,14 @@ def _parse_and_validate_request(request) -> Tuple[str, str, str]: return project_id, region, stack_name +def _get_fixed_number_of_media_nodes() -> int: + """Get the fixed number of Media Nodes (0 means autoscaling).""" + fixed_number = int(os.environ.get('FIXED_NUMBER_OF_MEDIA_NODES', '0')) + if fixed_number < 0: + raise ValueError(f"FIXED_NUMBER_OF_MEDIA_NODES must be >= 0, got {fixed_number}") + return fixed_number + + def _initialize_gcp_clients() -> Dict: """Initialize all required GCP clients.""" print("Initializing GCP clients...") @@ -493,6 +518,80 @@ def _get_scaling_info(clients: Dict, project_id: str, region: str, raise Exception(error_msg) +def _get_current_size(clients: Dict, project_id: str, region: str, + mig_name: str) -> int: + """Get current MIG size (fixed number of Media Nodes, no autoscaler).""" + print(f"Getting current size for MIG: {mig_name}") + try: + mig_request = compute_v1.GetRegionInstanceGroupManagerRequest( + project=project_id, + region=region, + instance_group_manager=mig_name + ) + mig = clients['mig'].get(request=mig_request) + current_size = mig.target_size + + print(f"Retrieved current size: {current_size}") + return current_size + + except Exception as e: + error_msg = f"Failed to get current size: {str(e)}" + print(error_msg) + raise Exception(error_msg) + + +def _reconcile_fixed_size(clients: Dict, project_id: str, region: str, + mig_name: str, fixed_number: int) -> Dict: + """Keep the MIG at the fixed number of Media Nodes.""" + current_size = _get_current_size(clients, project_id, region, mig_name) + + print(f"Current size: {current_size}, Fixed number of Media Nodes: {fixed_number}") + logger.info(f"Current size: {current_size}, Fixed number of Media Nodes: {fixed_number}") + + if current_size == fixed_number: + message = "No scale-in or scale-out operation needed" + print(message) + logger.info(message) + return { + "message": message, + "current_size": current_size, + "fixed_number": fixed_number + } + + if current_size < fixed_number: + # Fewer Media Nodes than the fixed number: resize the MIG up + print(f"Scale-out needed: resizing MIG from {current_size} to {fixed_number} instances") + + operation = _resize_mig(clients['mig'], project_id, region, mig_name, fixed_number) + + success_message = f"Successfully resized MIG from {current_size} to {fixed_number} instances" + print(success_message) + logger.info(success_message) + + return { + "message": success_message, + "operation_name": operation.name + } + + # More Media Nodes than the fixed number: abandon the extra ones so they drain gracefully + instances_to_abandon = current_size - fixed_number + print(f"Scale-in needed: abandoning {instances_to_abandon} instances") + + result = _perform_scalein( + clients, project_id, region, mig_name, instances_to_abandon + ) + + success_message = f"Successfully abandoned {result['abandoned_count']} instances" + print(success_message) + logger.info(success_message) + + return { + "message": success_message, + "abandoned_instances": result['abandoned_instances'], + "operation_name": result['operation_name'] + } + + def _perform_scalein(clients: Dict, project_id: str, region: str, mig_name: str, instances_count: int) -> Dict: """Perform scale-in by abandoning oldest instances.""" @@ -582,6 +681,21 @@ def _abandon_instances(mig_client, project_id: str, region: str, operation = mig_client.abandon_instances(request=abandon_request) print(f"Abandon operation initiated: {operation.name}") return operation + +def _resize_mig(mig_client, project_id: str, region: str, + mig_name: str, size: int): + """Resize the MIG to the given number of instances.""" + print(f"Resizing MIG to {size} instances") + resize_request = compute_v1.ResizeRegionInstanceGroupManagerRequest( + project=project_id, + region=region, + instance_group_manager=mig_name, + size=size + ) + + operation = mig_client.resize(request=resize_request) + print(f"Resize operation initiated: {operation.name}") + return operation EOF function_requirements = <= 0 && floor(var.fixedNumberOfMediaNodes) == var.fixedNumberOfMediaNodes + error_message = "fixedNumberOfMediaNodes must be a whole number >= 0." + } +} + variable "bucketName" { description = "Name of the GCS bucket to store data and recordings. If empty, a bucket will be created" type = string diff --git a/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml b/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml index cec852235..7dad8bddb 100644 --- a/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml +++ b/openvidu-deployment/pro/ha/aws/cf-openvidu-ha.yaml @@ -992,22 +992,28 @@ Parameters: InitialNumberOfMediaNodes: Type: Number Default: 1 - Description: Number of initial media nodes to deploy + Description: Number of initial media nodes to deploy. Ignored when FixedNumberOfMediaNodes > 0 MinNumberOfMediaNodes: Type: Number Default: 1 - Description: Minimum number of media nodes to deploy + Description: Minimum number of media nodes to deploy. Ignored when FixedNumberOfMediaNodes > 0 MaxNumberOfMediaNodes: Type: Number Default: 5 - Description: Maximum number of media nodes to deploy + Description: Maximum number of media nodes to deploy. Ignored when FixedNumberOfMediaNodes > 0 ScaleTargetCPU: Type: Number Default: 50 - Description: Target CPU percentage to scale up or down + Description: Target CPU percentage to scale up or down. Ignored when FixedNumberOfMediaNodes > 0 + + FixedNumberOfMediaNodes: + Type: Number + Default: 0 + MinValue: 0 + Description: If > 0, deploys a fixed number of media nodes with no autoscaling (InitialNumberOfMediaNodes, MinNumberOfMediaNodes, MaxNumberOfMediaNodes and ScaleTargetCPU are ignored). If 0 (default), the deployment is elastic and autoscaling is enabled S3AppDataBucketName: Type: String @@ -1094,6 +1100,7 @@ Metadata: - Label: default: Media Nodes Autoscaling Group configuration Parameters: + - FixedNumberOfMediaNodes - InitialNumberOfMediaNodes - MinNumberOfMediaNodes - MaxNumberOfMediaNodes @@ -1122,6 +1129,8 @@ Metadata: Conditions: CreateRecordingsBucket: !Equals [!Ref S3AppDataBucketName, ""] CreateClusterDataBucket: !Equals [!Ref S3ClusterDataBucketName, ""] + FixedNumberOfMediaNodesEnabled: !Not [ !Equals [!Ref FixedNumberOfMediaNodes, 0] ] + AutoscalingEnabled: !Equals [!Ref FixedNumberOfMediaNodes, 0] HasLoadBalancerSubnets: !Not [!Equals [!Join ["", !Ref LoadBalancerSubnets], ""]] TurnTLSIsEnabled: !And [!Not [!Equals [!Ref TurnDomainName, ""]], !Not [!Equals [!Ref TurnCertificateARN, ""]]] # Media Nodes behind the TURN Load Balancer in other subnets than the Load Balancers (private subnets) @@ -3329,9 +3338,9 @@ Resources: - !Ref OpenViduMediaNodeRTMPTG - !If [TurnTLSIsEnabled, !Ref OpenViduTurnMediaNodeTG, !Ref AWS::NoValue] - !If [TurnUDPIsEnabled, !Ref OpenViduTurnUDPMediaNodeTG, !Ref AWS::NoValue] - MinSize: !Ref MinNumberOfMediaNodes - MaxSize: !Ref MaxNumberOfMediaNodes - DesiredCapacity: !Ref InitialNumberOfMediaNodes + MinSize: !If [FixedNumberOfMediaNodesEnabled, !Ref FixedNumberOfMediaNodes, !Ref MinNumberOfMediaNodes] + MaxSize: !If [FixedNumberOfMediaNodesEnabled, !Ref FixedNumberOfMediaNodes, !Ref MaxNumberOfMediaNodes] + DesiredCapacity: !If [FixedNumberOfMediaNodesEnabled, !Ref FixedNumberOfMediaNodes, !Ref InitialNumberOfMediaNodes] VPCZoneIdentifier: !Ref OpenViduMediaNodeSubnets Tags: - Key: Name @@ -3494,6 +3503,7 @@ Resources: OpenViduMediaNodeASGScalingPolicy: Type: AWS::AutoScaling::ScalingPolicy + Condition: AutoscalingEnabled Properties: AutoScalingGroupName: !Ref OpenViduMediaNodeASG PolicyType: TargetTrackingScaling diff --git a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep index 73a4297db..233163d38 100644 --- a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep +++ b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.bicep @@ -62,18 +62,22 @@ param adminUsername string @secure() param adminSshKey object -@description('Number of initial media nodes to deploy') +@description('Number of initial media nodes to deploy. Ignored when fixedNumberOfMediaNodes > 0') param initialNumberOfMediaNodes int = 1 -@description('Minimum number of media nodes to deploy') +@description('Minimum number of media nodes to deploy. Ignored when fixedNumberOfMediaNodes > 0') param minNumberOfMediaNodes int = 1 -@description('Maximum number of media nodes to deploy') +@description('Maximum number of media nodes to deploy. Ignored when fixedNumberOfMediaNodes > 0') param maxNumberOfMediaNodes int = 5 -@description('Target CPU percentage to scale up or down') +@description('Target CPU percentage to scale up or down. Ignored when fixedNumberOfMediaNodes > 0') param scaleTargetCPU int = 50 +@description('If > 0, keeps this fixed number of media nodes: autoscaling is pinned to it (minimum = maximum = this number) and initialNumberOfMediaNodes, minNumberOfMediaNodes, maxNumberOfMediaNodes and scaleTargetCPU are ignored. When it is lowered, the extra media nodes are drained gracefully, one at a time, by the scale-in runbook. If 0 (default), the deployment is elastic and autoscaling is enabled') +@minValue(0) +param fixedNumberOfMediaNodes int = 0 + param additionalInstallFlags string = '' /*------------------------------------------- VARIABLES AND VALIDATIONS -------------------------------------------*/ @@ -1906,6 +1910,15 @@ resource openviduScaleSetMediaNode 'Microsoft.Compute/virtualMachineScaleSets@20 } } +// Always deployed and always enabled, so a redeploy switches modes both ways (an incremental +// ARM deployment never deletes a resource the template stops declaring). With a fixed number +// of Media Nodes autoscaling is pinned to it, not disabled: minimum = maximum = default, so +// the CPU rules are inert. Azure refuses a minimum or a maximum below the Media Nodes that are +// running, because they protect themselves from scale-in: a redeploy can raise the fixed +// number (or minNumberOfMediaNodes) but not lower it below the Media Nodes running. To lower +// it, switch the scale set to manual scale, drain the extra Media Nodes with +// /usr/local/bin/stop_media_node.sh (each one removes itself once its Rooms end), and then +// redeploy. resource openviduAutoScaleSettingsMediaNode 'Microsoft.Insights/autoscaleSettings@2022-10-01' = { name: '${stackName}-autoscaleSettings' location: resourceGroup().location @@ -1914,9 +1927,9 @@ resource openviduAutoScaleSettingsMediaNode 'Microsoft.Insights/autoscaleSetting { name: 'openvidu-medianode-autoscale' capacity: { - minimum: string(minNumberOfMediaNodes) - maximum: string(maxNumberOfMediaNodes) - default: string(initialNumberOfMediaNodes) + minimum: string(fixedNumberOfMediaNodes > 0 ? fixedNumberOfMediaNodes : minNumberOfMediaNodes) + maximum: string(fixedNumberOfMediaNodes > 0 ? fixedNumberOfMediaNodes : maxNumberOfMediaNodes) + default: string(fixedNumberOfMediaNodes > 0 ? fixedNumberOfMediaNodes : initialNumberOfMediaNodes) } rules: [ { diff --git a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json index 353f2d739..f6f1ac547 100644 --- a/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json +++ b/openvidu-deployment/pro/ha/azure/cf-openvidu-ha.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.46.1.21595", - "templateHash": "4663408180279198376" + "templateHash": "1734713128178491421" } }, "parameters": { @@ -124,28 +124,36 @@ "type": "int", "defaultValue": 1, "metadata": { - "description": "Number of initial media nodes to deploy" + "description": "Number of initial media nodes to deploy. Ignored when fixedNumberOfMediaNodes > 0" } }, "minNumberOfMediaNodes": { "type": "int", "defaultValue": 1, "metadata": { - "description": "Minimum number of media nodes to deploy" + "description": "Minimum number of media nodes to deploy. Ignored when fixedNumberOfMediaNodes > 0" } }, "maxNumberOfMediaNodes": { "type": "int", "defaultValue": 5, "metadata": { - "description": "Maximum number of media nodes to deploy" + "description": "Maximum number of media nodes to deploy. Ignored when fixedNumberOfMediaNodes > 0" } }, "scaleTargetCPU": { "type": "int", "defaultValue": 50, "metadata": { - "description": "Target CPU percentage to scale up or down" + "description": "Target CPU percentage to scale up or down. Ignored when fixedNumberOfMediaNodes > 0" + } + }, + "fixedNumberOfMediaNodes": { + "type": "int", + "defaultValue": 0, + "minValue": 0, + "metadata": { + "description": "If > 0, keeps this fixed number of media nodes: autoscaling is pinned to it (minimum = maximum = this number) and initialNumberOfMediaNodes, minNumberOfMediaNodes, maxNumberOfMediaNodes and scaleTargetCPU are ignored. When it is lowered, the extra media nodes are drained gracefully, one at a time, by the scale-in runbook. If 0 (default), the deployment is elastic and autoscaling is enabled" } }, "additionalInstallFlags": { @@ -639,9 +647,9 @@ { "name": "openvidu-medianode-autoscale", "capacity": { - "minimum": "[string(parameters('minNumberOfMediaNodes'))]", - "maximum": "[string(parameters('maxNumberOfMediaNodes'))]", - "default": "[string(parameters('initialNumberOfMediaNodes'))]" + "minimum": "[string(if(greater(parameters('fixedNumberOfMediaNodes'), 0), parameters('fixedNumberOfMediaNodes'), parameters('minNumberOfMediaNodes')))]", + "maximum": "[string(if(greater(parameters('fixedNumberOfMediaNodes'), 0), parameters('fixedNumberOfMediaNodes'), parameters('maxNumberOfMediaNodes')))]", + "default": "[string(if(greater(parameters('fixedNumberOfMediaNodes'), 0), parameters('fixedNumberOfMediaNodes'), parameters('initialNumberOfMediaNodes')))]" }, "rules": [ { diff --git a/openvidu-deployment/pro/ha/azure/createUiDefinition.json b/openvidu-deployment/pro/ha/azure/createUiDefinition.json index d53921931..728b940c2 100644 --- a/openvidu-deployment/pro/ha/azure/createUiDefinition.json +++ b/openvidu-deployment/pro/ha/azure/createUiDefinition.json @@ -366,6 +366,22 @@ "name": "parameters SCALING", "label": "Media Nodes Sacle Set configuration", "elements": [ + { + "name": "fixedNumberOfMediaNodes", + "type": "Microsoft.Common.TextBox", + "label": "Fixed Number Of Media Nodes", + "subLabel": "If > 0, deploys a fixed number of media nodes with no autoscaling (the Initial/Min/Max/Scale Target CPU values are ignored). If 0 (default), the deployment is elastic and autoscaling is enabled", + "defaultValue": "0", + "toolTip": "", + "constraints": { + "required": false, + "regex": "^[0-9]+$", + "validationMessage": "Must be an integer greater than or equal to 0", + "validations": [] + }, + "infoMessages": [], + "visible": true + }, { "name": "initialNumberOfMediaNodes", "type": "Microsoft.Common.TextBox", @@ -546,6 +562,7 @@ "mediaNodeInstanceType": "[steps('parameters INSTANCE').mediaNodeInstanceType]", "adminUsername": "[steps('parameters INSTANCE').adminUsername]", "adminSshKey": "[steps('parameters INSTANCE').adminSshKey]", + "fixedNumberOfMediaNodes": "[steps('parameters SCALING').fixedNumberOfMediaNodes]", "initialNumberOfMediaNodes": "[steps('parameters SCALING').initialNumberOfMediaNodes]", "minNumberOfMediaNodes": "[steps('parameters SCALING').minNumberOfMediaNodes]", "maxNumberOfMediaNodes": "[steps('parameters SCALING').maxNumberOfMediaNodes]", diff --git a/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf b/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf index 94255cb2b..c0ae7a2aa 100644 --- a/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf +++ b/openvidu-deployment/pro/ha/gcp/tf-gpc-openvidu-ha.tf @@ -606,6 +606,8 @@ def scalein_function(request): """ Cloud Function to handle scale-in operations for media node instances. Abandons oldest instances when autoscaler recommends fewer instances. + With a fixed number of Media Nodes (no autoscaler), keeps the MIG at that number: + abandons the extra instances or resizes the MIG up. """ print("Starting scale-in function") # Also use print for Cloud Run logs logger.info("Starting scale-in function") @@ -622,6 +624,13 @@ def scalein_function(request): mig_name = f"{stack_name.lower()}-media-node-group" autoscaler_name = f"{stack_name.lower()}-media-node-autoscaler" + # Fixed number of Media Nodes: there is no autoscaler, keep the MIG at that number + fixed_number = _get_fixed_number_of_media_nodes() + if fixed_number > 0: + return _reconcile_fixed_size( + clients, project_id, region, mig_name, fixed_number + ) + print(f"Checking MIG: {mig_name} and autoscaler: {autoscaler_name}") logger.info(f"Checking MIG: {mig_name} and autoscaler: {autoscaler_name}") @@ -692,6 +701,14 @@ def _parse_and_validate_request(request) -> Tuple[str, str, str]: return project_id, region, stack_name +def _get_fixed_number_of_media_nodes() -> int: + """Get the fixed number of Media Nodes (0 means autoscaling).""" + fixed_number = int(os.environ.get('FIXED_NUMBER_OF_MEDIA_NODES', '0')) + if fixed_number < 0: + raise ValueError(f"FIXED_NUMBER_OF_MEDIA_NODES must be >= 0, got {fixed_number}") + return fixed_number + + def _initialize_gcp_clients() -> Dict: """Initialize all required GCP clients.""" print("Initializing GCP clients...") @@ -735,6 +752,80 @@ def _get_scaling_info(clients: Dict, project_id: str, region: str, raise Exception(error_msg) +def _get_current_size(clients: Dict, project_id: str, region: str, + mig_name: str) -> int: + """Get current MIG size (fixed number of Media Nodes, no autoscaler).""" + print(f"Getting current size for MIG: {mig_name}") + try: + mig_request = compute_v1.GetRegionInstanceGroupManagerRequest( + project=project_id, + region=region, + instance_group_manager=mig_name + ) + mig = clients['mig'].get(request=mig_request) + current_size = mig.target_size + + print(f"Retrieved current size: {current_size}") + return current_size + + except Exception as e: + error_msg = f"Failed to get current size: {str(e)}" + print(error_msg) + raise Exception(error_msg) + + +def _reconcile_fixed_size(clients: Dict, project_id: str, region: str, + mig_name: str, fixed_number: int) -> Dict: + """Keep the MIG at the fixed number of Media Nodes.""" + current_size = _get_current_size(clients, project_id, region, mig_name) + + print(f"Current size: {current_size}, Fixed number of Media Nodes: {fixed_number}") + logger.info(f"Current size: {current_size}, Fixed number of Media Nodes: {fixed_number}") + + if current_size == fixed_number: + message = "No scale-in or scale-out operation needed" + print(message) + logger.info(message) + return { + "message": message, + "current_size": current_size, + "fixed_number": fixed_number + } + + if current_size < fixed_number: + # Fewer Media Nodes than the fixed number: resize the MIG up + print(f"Scale-out needed: resizing MIG from {current_size} to {fixed_number} instances") + + operation = _resize_mig(clients['mig'], project_id, region, mig_name, fixed_number) + + success_message = f"Successfully resized MIG from {current_size} to {fixed_number} instances" + print(success_message) + logger.info(success_message) + + return { + "message": success_message, + "operation_name": operation.name + } + + # More Media Nodes than the fixed number: abandon the extra ones so they drain gracefully + instances_to_abandon = current_size - fixed_number + print(f"Scale-in needed: abandoning {instances_to_abandon} instances") + + result = _perform_scalein( + clients, project_id, region, mig_name, instances_to_abandon + ) + + success_message = f"Successfully abandoned {result['abandoned_count']} instances" + print(success_message) + logger.info(success_message) + + return { + "message": success_message, + "abandoned_instances": result['abandoned_instances'], + "operation_name": result['operation_name'] + } + + def _perform_scalein(clients: Dict, project_id: str, region: str, mig_name: str, instances_count: int) -> Dict: """Perform scale-in by abandoning oldest instances.""" @@ -824,6 +915,21 @@ def _abandon_instances(mig_client, project_id: str, region: str, operation = mig_client.abandon_instances(request=abandon_request) print(f"Abandon operation initiated: {operation.name}") return operation + +def _resize_mig(mig_client, project_id: str, region: str, + mig_name: str, size: int): + """Resize the MIG to the given number of instances.""" + print(f"Resizing MIG to {size} instances") + resize_request = compute_v1.ResizeRegionInstanceGroupManagerRequest( + project=project_id, + region=region, + instance_group_manager=mig_name, + size=size + ) + + operation = mig_client.resize(request=resize_request) + print(f"Resize operation initiated: {operation.name}") + return operation EOF function_requirements = < 0 ? var.fixedNumberOfMediaNodes : var.initialNumberOfMediaNodes version { instance_template = google_compute_instance_template.media_node_template.self_link_unique @@ -999,10 +1108,18 @@ resource "google_compute_region_instance_group_manager" "media_node_group" { health_check = google_compute_region_health_check.media_node_health_check.id initial_delay_sec = 3600 } + + # Terraform sets the size at creation only. Afterwards the autoscaler (scale out) and the + # scale-in function (scale in, and the fixedNumberOfMediaNodes reconcile) own it. + lifecycle { + ignore_changes = [target_size] + } } # Autoscaler for Media Nodes +# Not created with a fixed number of Media Nodes: the scale-in function keeps the MIG at that size. resource "google_compute_region_autoscaler" "media_node_autoscaler" { + count = var.fixedNumberOfMediaNodes > 0 ? 0 : 1 name = lower("${var.stackName}-media-node-autoscaler") region = var.region target = google_compute_region_instance_group_manager.media_node_group.self_link @@ -2143,3 +2260,10 @@ CHECK_ABANDONED_EOF fi EOF } + +# fixedNumberOfMediaNodes gave the autoscaler a count. A deployment created before keeps +# it under its old address, so a re-apply moves it instead of destroying and recreating it. +moved { + from = google_compute_region_autoscaler.media_node_autoscaler + to = google_compute_region_autoscaler.media_node_autoscaler[0] +} diff --git a/openvidu-deployment/pro/ha/gcp/variables.tf b/openvidu-deployment/pro/ha/gcp/variables.tf index 994378c1f..b0283e883 100644 --- a/openvidu-deployment/pro/ha/gcp/variables.tf +++ b/openvidu-deployment/pro/ha/gcp/variables.tf @@ -108,29 +108,39 @@ variable "mediaNodeInstanceType" { } variable "initialNumberOfMediaNodes" { - description = "Number of initial media nodes to deploy" + description = "Number of initial media nodes to deploy. Ignored when fixedNumberOfMediaNodes > 0" type = number default = 1 } variable "minNumberOfMediaNodes" { - description = "Minimum number of media nodes to deploy" + description = "Minimum number of media nodes to deploy. Ignored when fixedNumberOfMediaNodes > 0" type = number default = 1 } variable "maxNumberOfMediaNodes" { - description = "Maximum number of media nodes to deploy" + description = "Maximum number of media nodes to deploy. Ignored when fixedNumberOfMediaNodes > 0" type = number default = 5 } variable "scaleTargetCPU" { - description = "Target CPU percentage to scale up or down" + description = "Target CPU percentage to scale up or down. Ignored when fixedNumberOfMediaNodes > 0" type = number default = 50 } +variable "fixedNumberOfMediaNodes" { + description = "If > 0, deploys a fixed number of media nodes with no autoscaling (initialNumberOfMediaNodes, minNumberOfMediaNodes, maxNumberOfMediaNodes and scaleTargetCPU are ignored). If 0 (default), the deployment is elastic and autoscaling is enabled" + type = number + default = 0 + validation { + condition = var.fixedNumberOfMediaNodes >= 0 && floor(var.fixedNumberOfMediaNodes) == var.fixedNumberOfMediaNodes + error_message = "fixedNumberOfMediaNodes must be a whole number >= 0." + } +} + variable "GCSAppDataBucketName" { description = "Name of the GCS bucket to store application data and recordings. If empty, a bucket will be created" type = string