From 484f4c53dbae24e70a16c9e4498058f5a4da1011 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Fri, 20 Dec 2024 18:48:34 +0300 Subject: [PATCH] PMM-12566 script to migrate from PMM 2 to PMM 3. (#3332) * PMM-12566 Update env variables and support data container installations. * PMM-12566 Update env variables and support data container installations. * PMM-12566 Update env variables and support data container installations. * Apply suggestions from code review * PMM-12566 Start 3.0.0 beta by default. * PMM-12566 Fix check. --- get-pmm.sh | 171 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 154 insertions(+), 17 deletions(-) diff --git a/get-pmm.sh b/get-pmm.sh index 1b498de6e9..a0fea34ea8 100755 --- a/get-pmm.sh +++ b/get-pmm.sh @@ -15,12 +15,13 @@ trap cleanup SIGINT SIGTERM ERR EXIT # Set defaults. network_name=${NETWORK_NAME:-pmm-net} -tag=${PMM_TAG:-3} -repo=${PMM_REPO:-percona/pmm-server} +tag=${PMM_TAG:-3.0.0-beta} +repo=${PMM_REPO:-perconalab/pmm-server} port=${PMM_PORT:-443} container_name=${CONTAINER_NAME:-pmm-server} docker_socket_path=${DOCKER_SOCKET_PATH:-/var/run/docker.sock} watchtower_token=${WATCHTOWER_TOKEN:-} +volume_name=${VOLUME_NAME:-pmm-data} backup_data=0 interactive=0 root_is_needed=no @@ -64,6 +65,9 @@ Available options: -net, --network-name Name of the network to create (default: pmm-net) + +-vn, --volume-name + Name of the volume to create (default: pmm-data) EOF exit } @@ -148,6 +152,10 @@ parse_params() { network_name="${2-}" shift ;; + -vn | --volume-name) + volume_name="${2-}" + shift + ;; -?*) die "Unknown option: $1" ;; *) break ;; esac @@ -310,14 +318,141 @@ migrate_pmm_data() { run_docker "stop $container_name" } +# Full Mapping of PMM v2 to v3 Environment Variables +# Simulated mapping using an indexed array +ENV_MAPPING=( + "DATA_RETENTION=PMM_DATA_RETENTION" + "DISABLE_ALERTING=PMM_ENABLE_ALERTING" + "DISABLE_UPDATES=PMM_ENABLE_UPDATES" + "DISABLE_TELEMETRY=PMM_ENABLE_TELEMETRY" + "PERCONA_PLATFORM_API_TIMEOUT=PMM_DEV_PERCONA_PLATFORM_API_TIMEOUT" + "DISABLE_BACKUP_MANAGEMENT=PMM_ENABLE_BACKUP_MANAGEMENT" + "ENABLE_AZUREDISCOVER=PMM_ENABLE_AZURE_DISCOVER" + "ENABLE_RBAC=PMM_ENABLE_ACCESS_CONTROL" + "METRICS_RESOLUTION=PMM_METRICS_RESOLUTION" + "METRICS_RESOLUTION_HR=PMM_METRICS_RESOLUTION_HR" + "METRICS_RESOLUTION_LR=PMM_METRICS_RESOLUTION_LR" + "METRICS_RESOLUTION_MR=PMM_METRICS_RESOLUTION_MR" + "OAUTH_PMM_CLIENT_ID=PMM_DEV_OAUTH_CLIENT_ID" + "OAUTH_PMM_CLIENT_SECRET=PMM_DEV_OAUTH_CLIENT_SECRET" + "PERCONA_TEST_AUTH_HOST=PMM_DEV_PERCONA_PLATFORM_ADDRESS" + "PERCONA_TEST_CHECKS_FILE=PMM_DEV_ADVISOR_CHECKS_FILE" + "PERCONA_TEST_CHECKS_HOST=PMM_DEV_PERCONA_PLATFORM_ADDRESS" + "PERCONA_TEST_CHECKS_PUBLIC_KEY=PMM_DEV_PERCONA_PLATFORM_PUBLIC_KEY" + "PERCONA_TEST_PLATFORM_ADDRESS=PMM_DEV_PERCONA_PLATFORM_ADDRESS" + "PERCONA_TEST_PLATFORM_INSECURE=PMM_DEV_PERCONA_PLATFORM_INSECURE" + "PERCONA_TEST_PLATFORM_PUBLIC_KEY=PMM_DEV_PERCONA_PLATFORM_PUBLIC_KEY" + "PERCONA_TEST_SAAS_HOST=PMM_DEV_PERCONA_PLATFORM_ADDRESS" + "PERCONA_TEST_POSTGRES_ADDR=PMM_POSTGRES_ADDR" + "PERCONA_TEST_POSTGRES_DBNAME=PMM_POSTGRES_DBNAME" + "PERCONA_TEST_POSTGRES_SSL_CA_PATH=PMM_POSTGRES_SSL_CA_PATH" + "PERCONA_TEST_POSTGRES_SSL_CERT_PATH=PMM_POSTGRES_SSL_CERT_PATH" + "PERCONA_TEST_POSTGRES_SSL_KEY_PATH=PMM_POSTGRES_SSL_KEY_PATH" + "PERCONA_TEST_POSTGRES_SSL_MODE=PMM_POSTGRES_SSL_MODE" + "PERCONA_TEST_POSTGRES_USERNAME=PMM_POSTGRES_USERNAME" + "PERCONA_TEST_POSTGRES_DBPASSWORD=PMM_POSTGRES_DBPASSWORD" + "PMM_TEST_TELEMETRY_DISABLE_SEND=PMM_DEV_TELEMETRY_DISABLE_SEND" + "PERCONA_TEST_TELEMETRY_DISABLE_START_DELAY=PMM_DEV_TELEMETRY_DISABLE_START_DELAY" + "PERCONA_TEST_PMM_CLICKHOUSE_ADDR=PMM_CLICKHOUSE_ADDR" + "PERCONA_TEST_PMM_CLICKHOUSE_DATABASE=PMM_CLICKHOUSE_DATABASE" + "PERCONA_TEST_PMM_CLICKHOUSE_DATASOURCE=PMM_CLICKHOUSE_DATASOURCE" + "PERCONA_TEST_PMM_CLICKHOUSE_HOST=PMM_CLICKHOUSE_HOST" + "PERCONA_TEST_PMM_CLICKHOUSE_PORT=PMM_CLICKHOUSE_PORT" + "PERCONA_TEST_PMM_DISABLE_BUILTIN_CLICKHOUSE=PMM_DISABLE_BUILTIN_CLICKHOUSE" + "PERCONA_TEST_PMM_DISABLE_BUILTIN_POSTGRES=PMM_DISABLE_BUILTIN_POSTGRES" + "PERCONA_TEST_INTERFACE_TO_BIND=PMM_INTERFACE_TO_BIND" + "PERCONA_TEST_VERSION_SERVICE_URL=PMM_DEV_PERCONA_PLATFORM_ADDRESS" + "PMM_TEST_TELEMETRY_FILE=PMM_DEV_TELEMETRY_FILE" + "PERCONA_TEST_TELEMETRY_HOST=PMM_DEV_TELEMETRY_HOST" + "PERCONA_TEST_TELEMETRY_INTERVAL=PMM_DEV_TELEMETRY_INTERVAL" + "PERCONA_TEST_TELEMETRY_RETRY_BACKOFF=PMM_DEV_TELEMETRY_RETRY_BACKOFF" + "PERCONA_TEST_STARLARK_ALLOW_RECURSION=PMM_DEV_ADVISOR_STARLARK_ALLOW_RECURSION" +) + +ENV_TO_DROP=( +"PATH" +"LANG" +"LC_ALL" +"GF_PLUGIN_DIR" +"PS1" +"LESS_LOG_NOISE" +"PERCONA_TEST_CHECKS_INTERVAL" +"PERCONA_TEST_NICER_API" +"PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE" +"PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE" +) + +# Function to get the new key from the mapping +get_mapped_key() { + local key="$1" + for mapping in "${ENV_MAPPING[@]}"; do + local old_key="${mapping%%=*}" + local new_key="${mapping#*=}" + if [[ "$old_key" == "$key" ]]; then + echo "$new_key" + return + fi + done + echo "" +} + +needs_to_drop() { + local key="$1" + for drop_key in "${ENV_TO_DROP[@]}"; do + if [[ "$drop_key" == "$key" ]]; then + return 0 + fi + done + return 1 +} + +# Function to migrate environment variables +migrate_env_vars() { + local current_env + IFS=$'\n' current_env=$(run_docker "inspect --format '{{range .Config.Env}}{{println .}}{{end}}' $container_name") + local docker_env_flags=$1 + + for env in $current_env; do + local key="${env%%=*}" + local value="${env#*=}" + + if [[ -z "$value" ]]; then + continue + fi + + local new_key + new_key=$(get_mapped_key "$key") + + local needs_drop + needs_drop=$(needs_to_drop "$key") + + if [[ -n "$new_key" ]]; then + msg "Migrating env variable $key to $new_key" + # Handle DISABLE_* to ENABLE_* boolean reversal + if [[ "$key" =~ ^DISABLE_ ]]; then + value=$((1 - value)) # Reverse the boolean + fi + + docker_env_flags+="--env $new_key=\"$value\" " + elif [[ "$needs_drop" -eq 0 ]]; then + docker_env_flags+="--env $key=\"$value\" " + else + msg "Dropping env variable $key" + fi + done + + echo "$docker_env_flags" +} + + ####################################### # Backs up existing PMM Data Volume. ####################################### backup_pmm_data() { - pmm_volume_archive="pmm-data-$(date "+%F-%H%M%S")" + pmm_volume_archive="$volume_name-$(date "+%F-%H%M%S")" msg "Backing up existing PMM Data Volume to $pmm_volume_archive" run_docker "volume create $pmm_volume_archive 1> /dev/null" - run_docker "run --rm -v pmm-data:/from -v $pmm_volume_archive:/to alpine ash -c 'cd /from ; cp -av . /to'" + run_docker "run --rm -v $volume_name:/from -v $pmm_volume_archive:/to alpine ash -c 'cd /from ; cp -av . /to'" msg "Successfully backed up existing PMM Data Volume to $pmm_volume_archive" } @@ -326,16 +461,10 @@ backup_pmm_data() { # If a PMM Server instance is running - stop and back it up. ####################################### start_pmm() { - msg "Starting PMM Server..." + msg "Pulling $repo:$tag" run_docker "pull $repo:$tag 1> /dev/null" - if ! run_docker "inspect pmm-data 1> /dev/null 2> /dev/null"; then - if ! run_docker "volume create pmm-data 1> /dev/null"; then - die "${RED}ERROR: cannot create PMM Data Volume${NOFORMAT}" - fi - msg "Created PMM Data Volume: pmm-data" - fi - + docker_env_flags="-e PMM_WATCHTOWER_HOST=http://watchtower:8080 -e PMM_WATCHTOWER_TOKEN=$watchtower_token " if run_docker "inspect $container_name 1> /dev/null 2> /dev/null"; then pmm_archive="$container_name-$(date "+%F-%H%M%S")" msg "\tExisting PMM Server found, renaming to $pmm_archive\n" @@ -346,12 +475,23 @@ start_pmm() { # get container tag from inspect old_version=$(run_docker "inspect --format='{{.Config.Image}}' $container_name | cut -d':' -f2") # if tag starts with 2.x, we need to migrate data - if [[ "$old_version" == 2.* ]]; then + if [[ "$old_version" == "2" || "$old_version" == 2.* || "$old_version" == "dev-latest" ]]; then + docker_env_flags=$(migrate_env_vars "$docker_env_flags") migrate_pmm_data fi + volume_name=$(run_docker "inspect -f '{{ range .Mounts }}{{ if and (eq .Type \"volume\") (eq .Destination \"/srv\" )}}{{ .Name }}{{ \"\n\" }}{{ end }}{{ end }}' $container_name") run_docker "rename $container_name $pmm_archive\n" fi - run_pmm="run -d -p $port:8443 --volume pmm-data:/srv --name $container_name --network $network_name -e PMM_WATCHTOWER_HOST=http://watchtower:8080 -e PMM_WATCHTOWER_TOKEN=$watchtower_token --restart always $repo:$tag" + + if ! run_docker "volume inspect $volume_name 1> /dev/null 2> /dev/null"; then + if ! run_docker "volume create $volume_name 1> /dev/null"; then + die "${RED}ERROR: cannot create PMM Data Volume${NOFORMAT}" + fi + msg "Created PMM Data Volume: $volume_name" + fi + msg "Starting PMM Server..." + + run_pmm="run -d -p $port:8443 --volume $volume_name:/srv --name $container_name --network $network_name $docker_env_flags --restart always $repo:$tag" run_docker "$run_pmm 1> /dev/null" msg "Created PMM Server: $container_name" @@ -402,6 +542,3 @@ parse_params "$@" main die "Enjoy Percona Monitoring and Management!" 0 -# TODO: Update script from PMM 2 to PMM 3 - -}# TODO: reuse watchtower token from older pmm instance or watchtower \ No newline at end of file