Percona-Lab · saikumar-vs · Aug 5, 2024 · Aug 8, 2024 · Aug 12, 2024 · Aug 12, 2024
@@ -0,0 +1,16 @@
+FROM php:7.4-alpine
+
+RUN set -ex \
+	&& apk --no-cache add postgresql-libs postgresql-dev \
+	&& docker-php-ext-install pgsql pdo_pgsql \
+	&& docker-php-ext-install mysqli && docker-php-ext-enable mysqli \
+	&& apk del postgresql-dev
+
+RUN apk --update add --virtual build-dependencies build-base openssl-dev autoconf \
+  && pecl install mongodb \
+  && docker-php-ext-enable mongodb \
+  && apk del build-dependencies build-base openssl-dev autoconf \
+  && rm -rf /var/cache/apk/*
+
+RUN curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/bin/ --filename=composer
+RUN composer require mongodb/mongodb
@@ -0,0 +1,111 @@
+#!/bin/sh
+
+while [ $# -gt 0 ]; do
+   if [[ $1 == *"--"* ]]; then
+        param="${1/--/}"
+        declare $param="$2"
+   fi
+  shift
+done
+
+if [ -z "$mongodb_version" ]; then
+  export mongodb_version=6.0
+fi
+
+if [ -z "$mongdb_setup" ]; then
+  export mongdb_setup=pss
+fi
+
+if [ -z "$metrics_mode" ]; then
+  export metrics_mode=push
+fi
+
+# Mongo user credtials for the replicat set cluster
+export user="dba"
+export pwd="test1234"
+
+# Install the dependencies
+source ~/.bash_profile || true;
+apt-get update
+apt-get -y install wget curl jq git gnupg2 lsb-release
+apt-get -y install libreadline6-dev systemtap-sdt-dev zlib1g-dev libssl-dev libpam0g-dev python-dev bison make flex libipc-run-perl
+sleep 10
+
+#wget https://raw.githubusercontent.com/Percona-QA/percona-qa/master/mongo_startup.sh
+#chmod +x mongo_startup.sh
+export SERVICE_RANDOM_NUMBER=$(echo $((1 + $RANDOM % 9999)))
+
+### Detect latest tarball link for specified mongodb_version: 7.0 | 6.0 | 5.0 | 4.4 | 4.2 at the moment
+# psmdb_latest=$(wget -q --post-data "version=percona-server-mongodb-${mongodb_version}" https://www.percona.com/products-api.php -O - | grep  -oP "(?<=value\=\")[^\"]*" | sort -V | tail -1)
+psmdb_tarball=$(wget -q --post-data "version_files=percona-server-mongodb-${mongodb_version}&software_files=binary" https://www.percona.com/products-api.php -O - | jq -r '.[] | select(.link | contains("sha") | not) | .link' | grep glibc2\.17-minimal)
+
+echo "Downloading ${mongodb_version} ..."
+wget -O percona_server_mongodb.tar.gz ${psmdb_tarball}
+tar -xvf percona_server_mongodb.tar.gz
+
+export extracted_folder_name=$(ls | grep percona-server-mongodb)
+echo "Extracted folder name ${extracted_folder_name}"
+mv ${extracted_folder_name} psmdb_${mongodb_version}
+
+# TODO: refactor if to match range of versions 6.0+
+if [[ "$mongodb_version" == "6.0" || "$mongodb_version" == "7.0" ]]; then
+    ### PSMDB 6+ requires "percona-mongodb-mongosh" additionally
+    echo "Downloading mongosh ..."
+    mongosh_link=$(wget -q --post-data "version_files=percona-server-mongodb-${mongodb_version}&software_files=binary" https://www.percona.com/products-api.php -O - | jq -r '.[] | select(.link | contains("sha") | not) | .link' | grep mongosh)
+    wget -O mongosh.tar.gz ${mongosh_link}
+    tar -xvf mongosh.tar.gz
+    mv percona-mongodb-mongosh* mongosh
+    cp mongosh/bin/mongosh ./psmdb_${mongodb_version}/bin/mongo
+    rm mongosh.tar.gz
+fi
+rm percona_server_mongodb.tar.gz*
+
+# For mongodb dependency in Debian
+wget http://http.us.debian.org/debian/pool/main/o/openldap/libldap-2.4-2_2.4.47+dfsg-3+deb10u7_amd64.deb
+apt install -y ./libldap-2.4-2_2.4.47+dfsg-3+deb10u7_amd64.deb
+
+if [ "$mongodb_setup" == "pss" ]; then
+    mlaunch init --bind_ip 0.0.0.0 --binarypath "./psmdb_${mongodb_version}/bin" --replicaset --name rs1 --nodes 3
+    sleep 20
+    pmm-admin remove mongodb mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27017
+    sleep 2
+    pmm-admin remove mongodb mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27018
+    sleep 2
+    pmm-admin remove mongodb mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27019
+    sleep 20
+fi
+
+if [ "$mongodb_setup" == "psa" ]; then
+    mlaunch init --bind_ip 0.0.0.0 --binarypath "./psmdb_${mongodb_version}/bin" --replicaset --name rs1 --nodes 2 --arbiter
+    sleep 20
+    pmm-admin remove mongodb mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27017
+    sleep 2
+    pmm-admin remove mongodb mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27018
+    sleep 2
+    pmm-admin remove mongodb mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27019
+    sleep 20
+fi
+
+if [ "$mongodb_setup" == "sharded" ]; then
+    mlaunch init --bind_ip 0.0.0.0 --binarypath "./psmdb_${mongodb_version}/bin" --replicaset --sharded rs1 rs2 --config 3
+    pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --environment=mongos_shraded_node mongos_shraded_node_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27017
+    sleep 2
+    pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=config --environment=mongodb_config_node mongodb_config_1_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27024
+    sleep 2
+    pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=config --environment=mongodb_config_node mongodb_config_2_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27025
+    sleep 2
+    pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=config --environment=mongodb_config_node mongodb_config_3_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27026
+    sleep 2
+    pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27018
+    sleep 2
+    pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27019
+    sleep 2
+    pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27020
+    sleep 2
+    pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs2 --environment=mongodb_rs_node mongodb_rs2_1_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27021
+    sleep 2
+    pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs2 --environment=mongodb_rs_node mongodb_rs2_2_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27022
+    sleep 2
+    pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs2 --environment=mongodb_rs_node mongodb_rs2_3_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27023
+    sleep 20
+fi
@@ -0,0 +1,85 @@
+---
+
+- hosts: all
+  become: true
+  become_method: sudo
+  vars:
+    psmdb_version: "{{ lookup('vars', 'extra_psmdb_version', default=lookup('env','PSMDB_VERSION') | default('4.4', true) ) }}"
+    psmdb_tarball: "{{ lookup('vars', 'extra_psmdb_tarball', default=lookup('env','PSMDB_TARBALL') | default('', true) ) }}"
+    psmdb_setup: "{{ lookup('vars', 'extra_psmdb_setup', default=lookup('env','PSMDB_SETUP') | default('pss', true) ) }}"
+    psmdb_container: "{{ lookup('vars', 'extra_psmdb_container', default=lookup('env','PSMDB_CONTAINER') | default('psmdb', true) ) }}"
+    pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}"
+    client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}"
+    admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}"
+    pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}"
+
+  tasks:
+  - name: cleanup container for client and DB setup
+    shell: >
+      docker ps -a --filter "name={{ psmdb_container }}" | grep -q . && docker stop {{ psmdb_container }} && docker rm -fv {{ psmdb_container }}
+    ignore_errors: true
+    tags:
+      - cleanup
+
+  - name: Create pmm-qa network if not exist
+    shell: docker network create pmm-qa
+    ignore_errors: true
+
+  - name: Prepare Container for PSMDB
+    shell: >
+      docker run -d --name={{ psmdb_container }}
+      -p 27017:27017
+      phusion/baseimage:jammy-1.0.1
+
+  - name: Copy all required Artifacts to the docker psmdb_container
+    shell: "{{ item }}"
+    with_items:
+      - docker cp ./mlaunch_setup.sh {{ psmdb_container }}:/
+      - docker cp ./pmm3-client-setup.sh {{ psmdb_container }}:/
+      - docker exec {{ psmdb_container }} apt-get update
+      - docker exec {{ psmdb_container }} apt-get -y install wget curl git gnupg2 lsb-release jq python3 pip
+
+  - name: Install required software's to the docker psmdb_container
+    shell: "{{ item }}"
+    with_items:
+      - docker exec {{ psmdb_container }} python3 -m pip install --upgrade pip
+      - docker exec {{ psmdb_container }} pip3 install 'mtools[all]'
+
+  - name: Install pmm2-client on the psmdb_container
+    shell: "{{ item }}"
+    with_items:
+      - docker network connect pmm-qa {{ psmdb_container }}
+      - docker exec {{ psmdb_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no
+
+  - name: Setup psmdb for monitoring
+    shell: "{{ item }}"
+    with_items:
+      - docker exec {{ psmdb_container }} bash -x ./mlaunch_setup.sh --mongodb_version {{ psmdb_version }} --mongodb_setup {{ psmdb_setup }} > setup_psmdb_{{ psmdb_version }}_{{ psmdb_setup }}.log
+
+  - name: Setup Load Running Docker Container
+    shell: "{{ item }}"
+    with_items:
+      - rm -rf ~/psmdb_{{ psmdb_version }} || true; mkdir -p ~/psmdb_{{ psmdb_version }}
+      - wget -P ~/psmdb_{{ psmdb_version }}/ "https://raw.githubusercontent.com/Percona-Lab/qa-integration/pmm3-mongo-mlaunch/pmm_qa/Dockerfile"
+      - wget -P ~/psmdb_{{ psmdb_version }}/ "https://raw.githubusercontent.com/Percona-Lab/qa-integration/pmm3-mongo-mlaunch/pmm_qa/mongodb_query.php"
+      - docker build --tag php-db ~/psmdb_{{ psmdb_version }}/ > ~/docker-build_mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }}.log || true
+      - docker rm mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} || true
+      - docker run --rm --name mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db composer require mongodb/mongodb || true
+
+  - name: Run load on Replica Set Master(PSS)
+    shell: "{{ item }}"
+    with_items:
+      - docker run --name mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} -d -e MONGODB_HOST={{ psmdb_container }} -e MONGODB_PORT=27017 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php >> setup_psmdb_{{ psmdb_version }}_{{ psmdb_setup }}.log
+    when: psmdb_setup == "pss"
+
+  - name: Run load on Replica Set Master(PSA)
+    shell: "{{ item }}"
+    with_items:
+      - docker run --name mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} -d -e MONGODB_HOST={{ psmdb_container }} -e MONGODB_PORT=27017 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php >> setup_psmdb_{{ psmdb_version }}_{{ psmdb_setup }}.log
+    when: psmdb_setup == "psa"
+
+  - name: Run load on Sharded Clusters Master
+    shell: "{{ item }}"
+    with_items:
+      - docker run --name mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} -d -e MONGODB_HOST={{ psmdb_container }} -e MONGODB_PORT=27017 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php >> setup_psmdb_{{ psmdb_version }}_{{ psmdb_setup }}.log
+    when: psmdb_setup == "sharded"
@@ -0,0 +1,108 @@
+<?php
+require 'vendor/autoload.php'; // include Composer's autoloader
+/* This script is designed to generate test workload to hit certain amount of tables in certain amount of schemas
+   having certain amount of unique queries */
+$db=5;
+$collection=50;
+
+/* How many queries try to run per second */
+$target_qps=10;
+
+$mongodb_host="localhost";
+$mongodb_user="";
+$mongodb_password="";
+$mongodb_port=27017;
+
+if(getenv('TEST_DB'))
+  $db=getenv('TEST_DB');
+if(getenv('TEST_COLLECTION'))
+  $collection=getenv('TEST_COLLECTION');
+if(getenv('TEST_TARGET_QPS'))
+  $target_qps=getenv('TEST_TARGET_QPS');
+if(getenv('MONGODB_HOST'))
+  $mongodb_host=getenv('MONGODB_HOST');
+if(getenv('MONGODB_USER'))
+  $mongodb_user=getenv('MONGODB_USER') . ":";
+if(getenv('MONGODB_PASSWORD'))
+  $mongodb_password=getenv('MONGODB_PASSWORD') . "@";
+if(getenv('MONGODB_PORT'))
+  $mongodb_port=getenv('MONGODB_PORT');
+
+/* We do not want uniform distribution so skew things a bit */
+function skewed_rnd($min,$max)
+{
+        $rounds=5;
+        $r=0;
+        for($i=0;$i<$rounds;$i++)
+        $r+=rand($min,$max);
+
+        return round($r/$rounds);
+}
+
+$client = new MongoDB\Client("mongodb://$mongodb_user$mongodb_password$mongodb_host:$mongodb_port", ["retryWrites" => false]);
+
+function run_query($db,$collection)
+{
+        global $client;
+        $collectionName = "beers" . $collection;
+        $dbName = "demo" . $db;
+        $collectionObj = $client->$dbName->$collectionName;
+        //read
+        $cursor = $collectionObj->find();
+        //update
+        $collectionObj->updateMany(array("a"=>"a"),
+        array('$set'=>array("a"=>"a_u")));
+
+        //count
+        $collectionObj->count();
+
+        //distinct
+        $collectionObj->distinct("a");
+
+        //aggregate
+        $collectionObj->aggregate([array('$match' =>array("a"=>"a_u"))]);
+
+        //findAndModify
+        $collectionObj->findOneAndUpdate(array("a"=>"a_u"), array('$set'=>array("a"=>"a_m")));  
+
+        //delete
+        $collectionObj->deleteOne(array("a"=>"a_m"));
+        //create
+        $result = $collectionObj->insertOne( [ 'a' => 'a', 'b' => 'B', 'c' => $i ] );
+}
+
+echo("Running Queries...\n");
+
+//lets create all db's and data
+for($i = 1; $i <= $db; $i++)
+{
+        $dbName = "demo" . $i;
+        for ($j = 1; $j <= $collection; $j++)
+        {
+                $collectionName = "beers" . $j;
+                $collectionObj = $client->$dbName->$collectionName;
+                $result = $collectionObj->insertOne( [ 'a' => 'a', 'b' => 'B', 'c' => $j ] );
+                echo "Inserted with Object ID '{$result->getInsertedId()}'";
+        }
+}
+
+/* How long we want target to take */
+$target_round_time=1/$target_qps;
+
+while(1)
+{
+   $start=microtime(1);
+   $dbNumber=skewed_rnd(1,$db);
+   $collectionNumber=skewed_rnd(1,$collection);
+   run_query($dbNumber,$collectionNumber);
+   $end=microtime(1);
+   $round_time=$end-$start;
+#   echo("Round Took: $round_time\n");
+   if($round_time<$target_round_time) /* Went faster than needed */
+   {
+     $sleep=($target_round_time-$round_time)*1000000;
+#     echo("Sleeping $sleep microseconds\n");
+     usleep($sleep);
+   }
+}
+?>