Kakfa & Elastic Deployement Added

README.md

@@ -2,4 +2,36 @@
 Big Data Decisional Benchmark 
 
 
-![alt text](./doc/img/benchmark.png)
+![alt text](./doc/img/benchmark.png)
+
+## BDBench Installation 
+
+### Requirements 
+
+ -  Java 1.8
+
+
+### Steps 
+
+
+1. Install **Ansible** :
+
+http://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html
+
+2. Active the Remove Session in your Mac Configuration
+
+`Préférences Système > Partage > Coche "Session à distance"`
+
+3. create your ssh public key :
+
+`ssh-keygen`
+
+4. add your public Key to your Host 
+
+`ssh-copy-id localhost`
+
+5. test Ansible Installation 
+
+`ansible localhost -m ping`
+
+

benchmark-installation/ansible.cfg

@@ -0,0 +1,5 @@
+[defaults]
+
+inventory  =  ./inventory
+roles_path  =  ./roles
+

benchmark-installation/group_vars/all

@@ -0,0 +1,5 @@
+kafka_install_parent_dir: /Users/mehdi/kafka/
+elasticsearch_install_parent_dir: /Users/mehdi/elasticsearch/
+logstash_install_parent_dir: /Users/mehdi/logstash/
+kibana_install_parent_dir: /Users/mehdi/kibana/
+

benchmark-installation/inventory

@@ -0,0 +1,15 @@
+node1 ansible_ssh_host=localhost
+
+
+[elasticsearch-nodes]
+node1
+
+[logstash-nodes]
+node1
+
+[kibana-nodes]
+node1
+
+[kakfa-nodes]
+node1
+

benchmark-installation/main.yml

@@ -0,0 +1,15 @@
+- hosts: kakfa-nodes
+  roles:
+     - { role: kafka.install}
+
+- hosts: elasticsearch-nodes
+  roles:
+     - { role: elasticsearch.install}
+
+- hosts: logstash-nodes
+  roles:
+     - { role: logstash.install}
+
+- hosts: kibana-nodes
+  roles:
+     - { role: kibana.install}

benchmark-installation/roles/bdbench.install/README.md

@@ -0,0 +1,43 @@
+Command to generate this Ansible Role 
+=========
+
+ansible-galaxy init bdbench-install
+
+Role Name
+=========
+
+A brief description of the role goes here.
+
+Requirements
+------------
+
+Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
+
+Role Variables
+--------------
+
+A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
+
+Dependencies
+------------
+
+A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+    - hosts: servers
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+BSD
+
+Author Information
+------------------
+
+An optional section for the role authors to include contact information, or a website (HTML is not allowed).

benchmark-installation/roles/bdbench.install/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+# defaults file for bdbench-install

benchmark-installation/roles/bdbench.install/handlers/main.yml

@@ -0,0 +1,2 @@
+---
+# handlers file for bdbench-install

benchmark-installation/roles/bdbench.install/meta/main.yml

@@ -0,0 +1,57 @@
+galaxy_info:
+  author: your name
+  description: your description
+  company: your company (optional)
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+
+  # Some suggested licenses:
+  # - BSD (default)
+  # - MIT
+  # - GPLv2
+  # - GPLv3
+  # - Apache
+  # - CC-BY
+  license: license (GPLv2, CC-BY, etc)
+
+  min_ansible_version: 1.2
+
+  # If this a Container Enabled role, provide the minimum Ansible Container version.
+  # min_ansible_container_version:
+
+  # Optionally specify the branch Galaxy will use when accessing the GitHub
+  # repo for this role. During role install, if no tags are available,
+  # Galaxy will use this branch. During import Galaxy will access files on
+  # this branch. If Travis integration is configured, only notifications for this
+  # branch will be accepted. Otherwise, in all cases, the repo's default branch
+  # (usually master) will be used.
+  #github_branch:
+
+  #
+  # platforms is a list of platforms, and each platform has a name and a list of versions.
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.

benchmark-installation/roles/bdbench.install/tasks/main.yml

@@ -0,0 +1,6 @@
+---
+# tasks file for bdbench-install
+
+- name: Create the hosts file for all machines
+  debug:
+    msg: "System : {{ inventory_hostname }} "

benchmark-installation/roles/bdbench.install/tests/inventory

@@ -0,0 +1,2 @@
+localhost
+

benchmark-installation/roles/bdbench.install/tests/test.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  remote_user: root
+  roles:
+    - bdbench-install

benchmark-installation/roles/bdbench.install/vars/main.yml

@@ -0,0 +1,2 @@
+---
+# vars file for bdbench-install

benchmark-installation/roles/elasticsearch.install/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+elasticsearch_download_timeout_seconds: 600
+elasticsearch_install_parent_dir: /usr/local
+
+elasticsearch_mirror : https://artifacts.elastic.co/downloads/elasticsearch
+elasticsearch_ver : '6.2.4'
+

benchmark-installation/roles/elasticsearch.install/handlers/main.yml

@@ -0,0 +1,2 @@
+---
+# handlers file for kafka

benchmark-installation/roles/elasticsearch.install/tasks/main.yml

@@ -0,0 +1,32 @@
+---
+- name: check for elasticsearch installation..
+  stat:
+    path: '{{elasticsearch_install_dir}}'
+  changed_when: false
+  register: elasticsearch_binary
+- when: not elasticsearch_binary.stat.exists
+  block:
+    - name: download elasticsearch tgz...
+      get_url:
+        url: '{{elasticsearch_tgz_url}}'
+        dest: /tmp/{{elasticsearch_tgz}}
+        mode: '0644'
+        timeout: '{{elasticsearch_download_timeout_seconds}}'
+    - name: 'ensure {{elasticsearch_install_parent_dir}} exists'
+      file:
+        path: '{{elasticsearch_install_parent_dir}}'
+        state: directory
+        mode: '0755'
+    - name: unarchive...
+      shell: tar xvf /tmp/{{elasticsearch_tgz}} -C {{elasticsearch_install_parent_dir}}
+  always:
+    - name: delete archive...
+      file:
+        path: /tmp/{{elasticsearch_tgz}}
+        state: absent
+
+- name: link...
+  file:
+    src: '{{elasticsearch_install_dir}}'
+    dest: '{{elasticsearch_install_link_dir}}'
+    state: link

benchmark-installation/roles/elasticsearch.install/vars/main.yml

@@ -0,0 +1,8 @@
+---
+
+elasticsearch_name : 'elasticsearch-{{elasticsearch_ver}}'
+elasticsearch_tgz : '{{elasticsearch_name}}.tar.gz'
+elasticsearch_tgz_url : '{{elasticsearch_mirror}}/{{elasticsearch_tgz}}'
+
+elasticsearch_install_dir: '{{elasticsearch_install_parent_dir}}/{{elasticsearch_name}}'
+elasticsearch_install_link_dir: '{{elasticsearch_install_parent_dir}}/elasticsearch'

benchmark-installation/roles/kafka.install/defaults/main.yml

@@ -0,0 +1,21 @@
+---
+kafka_download_timeout_seconds: 600
+kafka_install_parent_dir: /usr/local
+
+kafka_mirror : http://apache.mirrors.hoobly.com/kafka
+kafka_scala_ver : '2.12'
+kafka_ver : '1.1.0'
+
+kafka_checksums:
+  '1.0.0':
+    '2.11': sha256:b5b535f8db770cda8513e391917d0f5a35ef24c537ef3d29dcd9aa287da529f5
+    '2.12': sha256:d5b1d00752252d9c129e9284f26f8280e9899dd374167f257e29d5346eb544b3
+  '1.1.0':
+    '2.11': >
+      sha512:5E613DBB 21C2AED7 97E18BB5 63838626 7C4F61EE 00D10760
+                      DBC6BD47 2AF8B6AA 9CDCA30C 1F93F5E3 1A3AD8C5 06374A73
+                      50357E2A C2B9A3DD F87CAF22 820D470E
+    '2.12': >
+      sha512:48D1DDC7 1F5A5B1B 25D111F7 92553BE6 9BE62293 640A3C6A
+                      F985203C 6EE88C6A A78E0132 7066BFAD 3FEAE6B0 B45D71C0
+                      CAC6EBD2 D08843D9 22691327 41A3791B

benchmark-installation/roles/kafka.install/handlers/main.yml

@@ -0,0 +1,2 @@
+---
+# handlers file for kafka

benchmark-installation/roles/kafka.install/tasks/main.yml

@@ -0,0 +1,33 @@
+---
+- name: check for kafka installation..
+  stat:
+    path: '{{kafka_install_dir}}'
+  changed_when: false
+  register: kafka_binary
+- when: not kafka_binary.stat.exists
+  block:
+    - name: download kafka tgz...
+      get_url:
+        url: '{{kafka_tgz_url}}'
+        dest: /tmp/{{kafka_tgz}}
+        mode: '0644'
+        timeout: '{{kafka_download_timeout_seconds}}'
+        checksum: '{{kafka_checksum}}'
+    - name: 'ensure {{kafka_install_parent_dir}} exists'
+      file:
+        path: '{{kafka_install_parent_dir}}'
+        state: directory
+        mode: '0755'
+    - name: unarchive...
+      shell: tar xvf /tmp/{{kafka_tgz}} -C {{kafka_install_parent_dir}}
+  always:
+    - name: delete archive...
+      file:
+        path: /tmp/{{kafka_tgz}}
+        state: absent
+
+- name: link...
+  file:
+    src: '{{kafka_install_dir}}'
+    dest: '{{kafka_install_link_dir}}'
+    state: link

benchmark-installation/roles/kafka.install/templates/kafka.sh.j2

@@ -0,0 +1 @@
+export PATH=$PATH:{{kafka_install_link_dir}}/bin

benchmark-installation/roles/kafka.install/vars/main.yml

@@ -0,0 +1,8 @@
+---
+kafka_name : 'kafka_{{kafka_scala_ver}}-{{kafka_ver}}'
+kafka_tgz : '{{kafka_name}}.tgz'
+kafka_tgz_url : '{{kafka_mirror}}/{{kafka_ver}}/{{kafka_tgz}}'
+kafka_checksum: '{{kafka_checksums[kafka_ver][kafka_scala_ver]}}'
+
+kafka_install_dir: '{{kafka_install_parent_dir}}/{{kafka_name}}'
+kafka_install_link_dir: '{{kafka_install_parent_dir}}/kafka'

benchmark-installation/roles/kibana.install/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+kibana_download_timeout_seconds: 600
+kibana_install_parent_dir: /usr/local
+
+kibana_mirror : https://artifacts.elastic.co/downloads/kibana
+kibana_ver : '6.2.4'
+

benchmark-installation/roles/kibana.install/handlers/main.yml

@@ -0,0 +1,2 @@
+---
+# handlers file for kafka

benchmark-installation/roles/kibana.install/tasks/main.yml

@@ -0,0 +1,32 @@
+---
+- name: check for kibana installation..
+  stat:
+    path: '{{kibana_install_dir}}'
+  changed_when: false
+  register: kibana_binary
+- when: not kibana_binary.stat.exists
+  block:
+    - name: download kibana tgz...
+      get_url:
+        url: '{{kibana_tgz_url}}'
+        dest: /tmp/{{kibana_tgz}}
+        mode: '0644'
+        timeout: '{{kibana_download_timeout_seconds}}'
+    - name: 'ensure {{kibana_install_parent_dir}} exists'
+      file:
+        path: '{{kibana_install_parent_dir}}'
+        state: directory
+        mode: '0755'
+    - name: unarchive...
+      shell: tar xvf /tmp/{{kibana_tgz}} -C {{kibana_install_parent_dir}}
+  always:
+    - name: delete archive...
+      file:
+        path: /tmp/{{kibana_tgz}}
+        state: absent
+
+- name: link...
+  file:
+    src: '{{kibana_install_dir}}'
+    dest: '{{kibana_install_link_dir}}'
+    state: link

benchmark-installation/roles/kibana.install/vars/main.yml

@@ -0,0 +1,8 @@
+---
+
+kibana_name : 'kibana-{{kibana_ver}}-darwin-x86_64'
+kibana_tgz : '{{kibana_name}}.tar.gz'
+kibana_tgz_url : '{{kibana_mirror}}/{{kibana_tgz}}'
+
+kibana_install_dir: '{{kibana_install_parent_dir}}/{{kibana_name}}'
+kibana_install_link_dir: '{{kibana_install_parent_dir}}/kibana'