From 665f3db149f12c1d477b52ea6a8d733a38f2cfc8 Mon Sep 17 00:00:00 2001 From: cruizba Date: Tue, 13 Apr 2021 15:58:29 +0200 Subject: [PATCH] deployment: Update aws-asg master files --- .../aws-asg-openvidu-server-pro/.env | 9 +++- .../beats/filebeat.yml | 16 ++++--- .../beats/metricbeat.yml | 3 +- .../cluster/aws/openvidu_launch_kms.sh | 20 ++++----- .../docker-compose.yml | 9 +++- .../install_openvidu_pro.sh | 45 +++++++++++++++++-- 6 files changed, 77 insertions(+), 25 deletions(-) diff --git a/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/.env b/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/.env index 00cef5de..50c85ea4 100644 --- a/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/.env +++ b/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/.env @@ -110,7 +110,7 @@ OPENVIDU_PRO_CLUSTER_ENVIRONMENT=on_premise # Whether to enable or disable autoscaling. With autoscaling the number of Media Nodes will # be automatically adjusted according to existing load # Values: true | false -# OPENVIDU_PRO_CLUSTER_AUTOSCALING=false +OPENVIDU_PRO_CLUSTER_AUTOSCALING=false # How often the autoscaling algorithm runs, in seconds # Type number >= 0 @@ -188,7 +188,7 @@ OPENVIDU_PRO_CLUSTER_LOAD_STRATEGY=streams # AWS region in which the S3 bucket is located (e.g. eu-west-1). If not provided, # the region will try to be discovered automatically, although this is not always possible. # This property is only taken into account if OPENVIDU_PRO_RECORDING_STORAGE=s3 -# OPENVIDU_PRO_AWS_REGION= +#OPENVIDU_PRO_AWS_REGION= # Whether to enable recording module or not OPENVIDU_RECORDING=false @@ -250,6 +250,10 @@ OPENVIDU_STREAMS_VIDEO_MIN_SEND_BANDWIDTH=300 # Default value is false # OPENVIDU_STREAMS_ALLOW_TRANSCODING=false +# Send openvidu-browser logs of clients to Elasticsearch +# Default value is 'disabled' +# OPENVIDU_BROWSER_LOGS=disabled + # true to enable OpenVidu Webhook service. false' otherwise # Values: true | false OPENVIDU_WEBHOOK=false @@ -337,6 +341,7 @@ ELASTICSEARCH_PASSWORD= #AWS_DEFAULT_REGION= #AWS_IMAGE_ID= #AWS_INSTANCE_TYPE= +#AWS_INSTANCE_ID= #AWS_KEY_NAME= #AWS_SUBNET_ID= #AWS_SECURITY_GROUP= diff --git a/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/beats/filebeat.yml b/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/beats/filebeat.yml index 022e52fa..1f0fb0a6 100644 --- a/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/beats/filebeat.yml +++ b/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/beats/filebeat.yml @@ -2,6 +2,11 @@ filebeat.inputs: - type: container paths: - '/var/lib/docker/containers/*/*.log' + fields: + cluster_id: ${OPENVIDU_PRO_CLUSTER_ID:${DOMAIN_OR_PUBLIC_IP:undefined}} + node_id: master_${AWS_INSTANCE_ID:${OPENVIDU_PRO_CLUSTER_ID:${DOMAIN_OR_PUBLIC_IP:undefined}}} + node_role: masternode + fields_under_root: true processors: - add_docker_metadata: @@ -25,10 +30,8 @@ processors: container.image.name: docker.elastic.co/beats/filebeat-oss - contains: container.image.name: docker.elastic.co/beats/metricbeat-oss - - add_fields: - fields: - cluster_id: ${OPENVIDU_PRO_CLUSTER_ID:${DOMAIN_OR_PUBLIC_IP}} - node_role: masternode + - contains: + container.image.name: openvidu/openvidu-server-pro output: elasticsearch: @@ -42,7 +45,10 @@ output: when.or: - contains: container.image.name: openvidu/openvidu-proxy - + - index: "filebeat-openvidu-recording-%{+yyyy.MM.dd}" + when.or: + - contains: + container.image.name: openvidu/openvidu-recording logging.json: true logging.metrics.enabled: false diff --git a/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/beats/metricbeat.yml b/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/beats/metricbeat.yml index ed62cf69..4dbcac0e 100644 --- a/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/beats/metricbeat.yml +++ b/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/beats/metricbeat.yml @@ -31,7 +31,8 @@ metricbeat.modules: period: ${OPENVIDU_PRO_STATS_MONITORING_INTERVAL}s cpu.metrics: [normalized_percentages] fields: - cluster_id: ${OPENVIDU_PRO_CLUSTER_ID:${DOMAIN_OR_PUBLIC_IP}} + cluster_id: ${OPENVIDU_PRO_CLUSTER_ID:${DOMAIN_OR_PUBLIC_IP:undefined}} + node_id: master_${AWS_INSTANCE_ID:${OPENVIDU_PRO_CLUSTER_ID:${DOMAIN_OR_PUBLIC_IP:undefined}}} node_role: masternode output: elasticsearch: diff --git a/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/cluster/aws/openvidu_launch_kms.sh b/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/cluster/aws/openvidu_launch_kms.sh index 48b92bf7..72f23e33 100644 --- a/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/cluster/aws/openvidu_launch_kms.sh +++ b/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/cluster/aws/openvidu_launch_kms.sh @@ -35,6 +35,14 @@ exit_on_error () { esac } +# Check custom parameters +if [[ -n "${CUSTOM_INSTANCE_TYPE}" ]]; then + AWS_INSTANCE_TYPE="${CUSTOM_INSTANCE_TYPE}" +fi +if [[ -n "${CUSTOM_VOLUME_SIZE}" ]]; then + AWS_VOLUME_SIZE="${CUSTOM_VOLUME_SIZE}" +fi + docker run --rm amazon/aws-cli:${AWS_CLI_DOCKER_TAG} ec2 run-instances \ --image-id ${AWS_IMAGE_ID} --count 1 \ --instance-type ${AWS_INSTANCE_TYPE} \ @@ -51,18 +59,6 @@ docker run --rm amazon/aws-cli:${AWS_CLI_DOCKER_TAG} ec2 wait instance-running - KMS_IP=$(cat ${OUTPUT} | jq --raw-output ' .Instances[] | .NetworkInterfaces[0] | .PrivateIpAddress') KMS_ID=$(cat ${OUTPUT} | jq --raw-output ' .Instances[] | .InstanceId') -# Wait media-node controller -attempt_counter=0 -max_attempts=10 - -until $(curl --output /dev/null --silent --head --fail -u OPENVIDUAPP:${OPENVIDU_SECRET} http://${KMS_IP}:3000/media-node/status); do - if [ ${attempt_counter} -eq ${max_attempts} ];then - exit 1 - fi - attempt_counter=$(($attempt_counter+1)) - sleep 5 -done - jq -n \ --arg id "${KMS_ID}" \ --arg ip "${KMS_IP}" \ diff --git a/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/docker-compose.yml b/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/docker-compose.yml index 2e9da321..a60ab41c 100644 --- a/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/docker-compose.yml +++ b/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/docker-compose.yml @@ -22,7 +22,7 @@ version: '3.1' services: openvidu-server: - image: openvidu/openvidu-server-pro:2.17.0 + image: openvidu/openvidu-server-pro:2.18.0-beta5 restart: on-failure network_mode: host entrypoint: ['/usr/local/bin/entrypoint.sh'] @@ -109,10 +109,15 @@ services: - .env environment: - OPENVIDU_PRO_ELASTICSEARCH_HOST=${OPENVIDU_PRO_ELASTICSEARCH_HOST:-http://127.0.0.1:9200} + - OPENVIDU_PRO_STATS_MONITORING_INTERVAL=${OPENVIDU_PRO_STATS_MONITORING_INTERVAL:-10} volumes: + - /var/run/docker.sock:/var/run/docker.sock - ./beats/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro + - /proc:/hostfs/proc:ro + - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro + - /:/hostfs:ro command: > - /bin/bash -c "metricbeat -e -strict.perms=false + /bin/bash -c "metricbeat -e -strict.perms=false -e -system.hostfs=/hostfs `if [ ! -z $ELASTICSEARCH_USERNAME ]; then echo '-E output.elasticsearch.username=$ELASTICSEARCH_USERNAME'; fi` `if [ ! -z $ELASTICSEARCH_PASSWORD ]; then echo '-E output.elasticsearch.password=$ELASTICSEARCH_PASSWORD'; fi`" logging: diff --git a/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/install_openvidu_pro.sh b/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/install_openvidu_pro.sh index ec895dfb..79b8a676 100755 --- a/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/install_openvidu_pro.sh +++ b/openvidu-server/deployments/pro/docker-compose/aws-asg-openvidu-server-pro/install_openvidu_pro.sh @@ -130,6 +130,17 @@ new_ov_installation() { exit 0 } +get_previous_env_variable() { + local ENV_VARIABLE_NAME=$1 + echo $(grep -E "${ENV_VARIABLE_NAME}=.*$" "${OPENVIDU_PREVIOUS_FOLDER}/.env" | cut -d'=' -f2) +} + +replace_variable_in_new_env_file() { + local ENV_VARIABLE_NAME=$1 + local ENV_VARIABLE_VALUE=$2 + [[ ! -z "${ENV_VARIABLE_VALUE}" ]] && sed -i "s|#${ENV_VARIABLE_NAME}=|${ENV_VARIABLE_NAME}=${ENV_VARIABLE_VALUE}|" "${OPENVIDU_PREVIOUS_FOLDER}/.env-${OPENVIDU_VERSION}" +} + upgrade_ov() { # Search local Openvidu installation printf '\n' @@ -336,14 +347,42 @@ upgrade_ov() { OLD_MODE=$(grep -E "Installation Mode:.*$" "${ROLL_BACK_FOLDER}/docker-compose.yml" | awk '{ print $4,$5 }') [ ! -z "${OLD_MODE}" ] && sed -i -r "s/Installation Mode:.+/Installation Mode: ${OLD_MODE}/" "${OPENVIDU_PREVIOUS_FOLDER}/docker-compose.yml" - # In Aws, update AMI ID - AWS_REGION=$(grep -E "AWS_DEFAULT_REGION=.*$" "${OPENVIDU_PREVIOUS_FOLDER}/.env" | cut -d'=' -f2) + # Update .env variables to new .env-version + AWS_REGION=$(get_previous_env_variable AWS_DEFAULT_REGION) if [[ ! -z ${AWS_REGION} ]]; then + + # Get new AMI ID NEW_AMI_ID=$(curl https://s3-eu-west-1.amazonaws.com/aws.openvidu.io/CF-OpenVidu-Pro-${OPENVIDU_VERSION//v}.yaml --silent | sed -n -e '/KMSAMIMAP:/,/Metadata:/ p' | grep -A 1 ${AWS_REGION} | grep AMI | tr -d " " | cut -d":" -f2) [[ -z ${NEW_AMI_ID} ]] && fatal_error "Error while getting new AWS_IMAGE_ID for Media Nodes" - sed -i "s/.*AWS_IMAGE_ID=.*/AWS_IMAGE_ID=${NEW_AMI_ID}/" "${OPENVIDU_PREVIOUS_FOLDER}/.env" || fatal_error "Error while updating new AWS_IMAGE_ID for Media Nodes" + + # Get previous values + PREV_AWS_DEFAULT_REGION=$(get_previous_env_variable AWS_DEFAULT_REGION) + PREV_AWS_INSTANCE_TYPE=$(get_previous_env_variable AWS_INSTANCE_TYPE) + PREV_AWS_INSTANCE_ID=$(get_previous_env_variable AWS_INSTANCE_ID) + PREV_AWS_KEY_NAME=$(get_previous_env_variable AWS_KEY_NAME) + PREV_AWS_SUBNET_ID=$(get_previous_env_variable AWS_SUBNET_ID) + PREV_AWS_SECURITY_GROUP=$(get_previous_env_variable AWS_SECURITY_GROUP) + PREV_AWS_STACK_ID=$(get_previous_env_variable AWS_STACK_ID) + PREV_AWS_STACK_NAME=$(get_previous_env_variable AWS_STACK_NAME) + PREV_AWS_CLI_DOCKER_TAG=$(get_previous_env_variable AWS_CLI_DOCKER_TAG) + PREV_AWS_VOLUME_SIZE=$(get_previous_env_variable AWS_VOLUME_SIZE) + + # Replace variables in new .env-version file + replace_variable_in_new_env_file "AWS_DEFAULT_REGION" "${PREV_AWS_DEFAULT_REGION}" + replace_variable_in_new_env_file "AWS_INSTANCE_TYPE" "${PREV_AWS_INSTANCE_TYPE}" + replace_variable_in_new_env_file "AWS_INSTANCE_ID" "${PREV_AWS_INSTANCE_ID}" + replace_variable_in_new_env_file "AWS_KEY_NAME" "${PREV_AWS_KEY_NAME}" + replace_variable_in_new_env_file "AWS_SUBNET_ID" "${PREV_AWS_SUBNET_ID}" + replace_variable_in_new_env_file "AWS_SECURITY_GROUP" "${PREV_AWS_SECURITY_GROUP}" + replace_variable_in_new_env_file "AWS_STACK_ID" "${PREV_AWS_STACK_ID}" + replace_variable_in_new_env_file "AWS_STACK_NAME" "${PREV_AWS_STACK_NAME}" + replace_variable_in_new_env_file "AWS_CLI_DOCKER_TAG" "${PREV_AWS_CLI_DOCKER_TAG}" + replace_variable_in_new_env_file "AWS_VOLUME_SIZE" "${PREV_AWS_VOLUME_SIZE}" + + # Replace new AMI + replace_variable_in_new_env_file "AWS_IMAGE_ID" "${NEW_AMI_ID}" fi