start restructuring for better handling; #227

README.md

@@ -50,6 +50,7 @@ We test the collection on the following Linux distributions. Each one with Elast
 * Ubuntu 20.04 LTS
 * Ubuntu 22.04 LTS
 * Debian 11
+* RHEL 9 ( Elastic Stack 8)
 
 We know from personal experience, that the collections work in following combinations. Missing tests mostly come from incompatibilties between the distribution and our testing environment, not from problems with the collection itself.
 

roles/elasticsearch/defaults/main.yml

@@ -22,6 +22,11 @@ elasticsearch_conf_dir: "/etc/elasticsearch/"
 elasticsearch_config_jvm: "jvm.options.j2"
 elasticsearch_user: elasticsearch
 elasticsearch_group: elasticsearch
+elasticsearch_initial_setup: true
+
+# elasticsearh security and api
+elasticsearch_api_basic_auth_username: 'elastic' #
+elasticsearch_api_basic_auth_password: '' # after starting the cluster for the first time you will find that password in /usr/share/elasticsearch/initial_passwords
 
 # JVM custom parameters
 elasticsearch_java_home: ''

roles/elasticsearch/tasks/elasticsearch-configuration.yml

@@ -0,0 +1,42 @@
+---
+- name: Configure Elasticsearch
+  template:
+    src: elasticsearch.yml.j2
+    dest: /etc/elasticsearch/elasticsearch.yml
+    owner: root
+    group: root
+    mode: 0644
+    backup: "{{ elasticsearch_config_backup }}"
+  notify:
+    - Restart Elasticsearch
+  when: elasticsearch_manage_yaml | bool
+
+- name: Create Elasticsearch directory
+  file:
+    path: "{{ item.path }}"
+    state: directory
+    owner: elasticsearch
+    group: elasticsearch
+    mode: "2750"
+  when: item.create | bool
+  loop:
+    - {create: "{{elasticsearch_create_logpath}}", path: "{{ elasticsearch_logpath }}" }
+    - {create: "{{elasticsearch_create_datapath}}", path: "{{ elasticsearch_datapath }}" }
+
+- name: Copy jvm.options File
+  become: yes
+  template:
+    src: "{{ elasticsearch_config_jvm }}"
+    dest: "{{ elasticsearch_conf_dir }}/jvm.options"
+    owner: root
+    group: "{{ elasticsearch_group }}"
+    mode: "660"
+    force: yes
+  notify: Restart Elasticsearch
+
+- name: Start Elasticsearch
+  service:
+    name: elasticsearch
+    state: started
+    enabled: yes
+  failed_when: false

roles/elasticsearch/tasks/elasticsearch-keystore.yml

@@ -0,0 +1,184 @@
+---
+
+- name: Create keystore
+  command: /usr/share/elasticsearch/bin/elasticsearch-keystore create
+  args:
+    creates: /etc/elasticsearch/elasticsearch.keystore
+
+- name: Check for bootstrap password
+  command: /usr/share/elasticsearch/bin/elasticsearch-keystore list
+  changed_when: false
+  register: elasticsearch_keystore
+
+- name: Set bootstrap password # noqa: risky-shell-pipe
+  shell: >
+    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+    echo "{{ elasticsearch_bootstrap_pw }}" |
+    /usr/share/elasticsearch/bin/elasticsearch-keystore
+    add -x 'bootstrap.password'
+  when: "'bootstrap.password' not in elasticsearch_keystore.stdout_lines"
+  changed_when: false
+  no_log: true
+  notify:
+    - Restart Elasticsearch
+  ignore_errors: "{{ ansible_check_mode }}"
+
+- name: Get xpack.security.http.ssl.keystore.secure_password # noqa: risky-shell-pipe
+  shell: >
+    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+    /usr/share/elasticsearch/bin/elasticsearch-keystore
+    show 'xpack.security.http.ssl.keystore.secure_password'
+  when:
+    - "'xpack.security.http.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines"
+    - elasticsearch_http_security
+  register: elasticsearch_http_ssl_keystore_secure_password
+  ignore_errors: "{{ ansible_check_mode }}"
+  no_log: true
+  changed_when: false
+
+- name: Set xpack.security.http.ssl.keystore.secure_password # noqa: risky-shell-pipe
+  shell: >
+    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+    echo "{{ elasticsearch_tls_key_passphrase }}" |
+    /usr/share/elasticsearch/bin/elasticsearch-keystore
+    add -f -x 'xpack.security.http.ssl.keystore.secure_password'
+  changed_when: false
+  no_log: true
+  when:
+    - elasticsearch_http_ssl_keystore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_http_ssl_keystore_secure_password.stdout
+    - elasticsearch_http_security
+  notify:
+    - Restart Elasticsearch
+
+- name: Remove xpack.security.http.ssl.keystore.secure_password # noqa: risky-shell-pipe
+  shell: >
+    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+    /usr/share/elasticsearch/bin/elasticsearch-keystore
+    remove 'xpack.security.http.ssl.keystore.secure_password'
+  changed_when: false
+  no_log: true
+  when:
+    - "'xpack.security.http.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines"
+    - not elasticsearch_http_security
+  notify:
+    - Restart Elasticsearch
+
+- name: Get xpack.security.http.ssl.truststore.secure_password # noqa: risky-shell-pipe
+  shell: >
+    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+    /usr/share/elasticsearch/bin/elasticsearch-keystore
+    show 'xpack.security.http.ssl.truststore.secure_password'
+  when:
+    - "'xpack.security.http.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines"
+    - elasticsearch_http_security
+  register: elasticsearch_http_ssl_truststore_secure_password
+  ignore_errors: "{{ ansible_check_mode }}"
+  no_log: true
+  changed_when: false
+
+- name: Set xpack.security.http.ssl.truststore.secure_password # noqa: risky-shell-pipe
+  shell: >
+    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+    echo "{{ elasticsearch_tls_key_passphrase }}" |
+    /usr/share/elasticsearch/bin/elasticsearch-keystore
+    add -f -x 'xpack.security.http.ssl.truststore.secure_password'
+  changed_when: false
+  no_log: true
+  when:
+    - elasticsearch_http_ssl_truststore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_http_ssl_truststore_secure_password.stdout
+    - elasticsearch_http_security
+  notify:
+    - Restart Elasticsearch
+
+- name: Remove xpack.security.http.ssl.truststore.secure_password # noqa: risky-shell-pipe
+  shell: >
+    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+    /usr/share/elasticsearch/bin/elasticsearch-keystore
+    remove 'xpack.security.http.ssl.truststore.secure_password'
+  changed_when: false
+  no_log: true
+  when:
+    - "'xpack.security.http.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines"
+    - not elasticsearch_http_security
+  notify:
+    - Restart Elasticsearch
+
+- name: Get xpack.security.transport.ssl.keystore.secure_password # noqa: risky-shell-pipe
+  shell: >
+    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+    /usr/share/elasticsearch/bin/elasticsearch-keystore
+    show 'xpack.security.transport.ssl.keystore.secure_password'
+  when:
+    - "'xpack.security.transport.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines"
+    - elasticsearch_security
+  register: elasticsearch_transport_ssl_keystore_secure_password
+  ignore_errors: "{{ ansible_check_mode }}"
+  no_log: true
+  changed_when: false
+
+- name: Set xpack.security.transport.ssl.keystore.secure_password # noqa: risky-shell-pipe
+  shell: >
+    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+    echo "{{ elasticsearch_tls_key_passphrase }}" |
+    /usr/share/elasticsearch/bin/elasticsearch-keystore
+    add -f -x 'xpack.security.transport.ssl.keystore.secure_password'
+  changed_when: false
+  no_log: true
+  when:
+    - elasticsearch_transport_ssl_keystore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_transport_ssl_keystore_secure_password.stdout
+    - elasticsearch_security
+  notify:
+    - Restart Elasticsearch
+
+- name: Remove xpack.security.transport.ssl.keystore.secure_password # noqa: risky-shell-pipe
+  shell: >
+    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+    /usr/share/elasticsearch/bin/elasticsearch-keystore
+    remove 'xpack.security.transport.ssl.keystore.secure_password'
+  changed_when: false
+  no_log: true
+  when:
+    - "'xpack.security.transport.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines"
+    - not elasticsearch_security
+  notify:
+    - Restart Elasticsearch
+
+- name: Get xpack.security.transport.ssl.truststore.secure_password # noqa: risky-shell-pipe
+  shell: >
+    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+    /usr/share/elasticsearch/bin/elasticsearch-keystore
+    show 'xpack.security.transport.ssl.truststore.secure_password'
+  when:
+    - "'xpack.security.transport.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines"
+    - elasticsearch_security
+  register: elasticsearch_transport_ssl_truststore_secure_password
+  ignore_errors: "{{ ansible_check_mode }}"
+  no_log: true
+  changed_when: false
+
+- name: Set xpack.security.transport.ssl.truststore.secure_password # noqa: risky-shell-pipe
+  shell: >
+    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+    echo "{{ elasticsearch_tls_key_passphrase }}" |
+    /usr/share/elasticsearch/bin/elasticsearch-keystore
+    add -f -x 'xpack.security.transport.ssl.truststore.secure_password'
+  changed_when: false
+  no_log: true
+  when:
+    - elasticsearch_transport_ssl_truststore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_transport_ssl_truststore_secure_password.stdout
+    - elasticsearch_security
+  notify:
+    - Restart Elasticsearch
+
+- name: Remove xpack.security.transport.ssl.truststore.secure_password # noqa: risky-shell-pipe
+  shell: >
+    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+    /usr/share/elasticsearch/bin/elasticsearch-keystore
+    remove 'xpack.security.transport.ssl.truststore.secure_password'
+  changed_when: false
+  no_log: true
+  when:
+    - "'xpack.security.transport.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines"
+    - not elasticsearch_security
+  notify:
+    - Restart Elasticsearch

roles/elasticsearch/tasks/elasticsearch-security.yml

@@ -80,13 +80,13 @@
         state: absent
       when: elasticsearch_move_ca_directory.changed
 
-    - name: Check the existance of ca on Ansible controler
+    - name: Check the existance of ca on Ansible controller
       stat:
         path: /tmp/ca.crt
       register: elasticsearch_check_temporary_ca
       delegate_to: localhost
 
-    - name: Move temporary ca file on Ansible controler
+    - name: Move temporary ca file on Ansible controller
       copy:
         src: /tmp/ca.crt
         dest: "/tmp/ca.crt_{{ ansible_date_time.iso8601_micro }}"
@@ -95,7 +95,7 @@
       delegate_to: localhost
       register: elasticsearch_move_ca_file
 
-    - name: Remove temporary ca file on Ansible controler
+    - name: Remove temporary ca file on Ansible controller
       file:
         path: /tmp/ca.crt
         state: absent
@@ -521,7 +521,7 @@
     state: started
   failed_when: false
 
-- name: Wait for all instances to start
+- name: Wait for all instances to startelasticstack_initial_passwords
   include_tasks: wait_for_instance.yml
   loop: "{{ groups['elasticsearch'] }}"
   tags: notest
@@ -542,7 +542,7 @@
     - renew_ca
     - renew_es_cert
 
-- name: Check for passwords being set
+- name: Check for if path for initial passwords being set
   stat:
     path: "{{ elasticstack_initial_passwords }}"
   delegate_to: "{{ elasticstack_ca }}"

roles/elasticsearch/tasks/main.yml

@@ -127,35 +127,22 @@
   when:
     - ansible_os_family == "Debian"
 
-- name: Configure Elasticsearch
-  template:
-    src: elasticsearch.yml.j2
-    dest: /etc/elasticsearch/elasticsearch.yml
-    owner: root
-    group: root
-    mode: 0644
-    backup: "{{ elasticsearch_config_backup }}"
-  notify:
-    - Restart Elasticsearch
-  when: elasticsearch_manage_yaml | bool
-
-- name: Create Elasticsearch directory
-  file:
-    path: "{{ item.path }}"
-    state: directory
-    owner: elasticsearch
-    group: elasticsearch
-    mode: "2750"
-  when: item.create | bool
-  loop:
-    - {create: "{{elasticsearch_create_logpath}}", path: "{{ elasticsearch_logpath }}" }
-    - {create: "{{elasticsearch_create_datapath}}", path: "{{ elasticsearch_datapath }}" }
+- name: Import Tasks elasticsearch-configuration.yml
+  import_tasks: elasticsearch-configuration.yml
+  when: 
+    - elasticsearch_initial_setup | bool
+    - elasticsearch_manage_yaml | bool
+
+- name: Import Tasks elasticsearch-keystore.yml
+  import_tasks: elasticsearch-keystore.yml
+  when: elasticsearch_initial_setup | bool
 
 - name: Import Tasks elasticsearch-security.yml
   import_tasks: elasticsearch-security.yml
   when:
     - elasticsearch_security | bool
     - elasticstack_variant == "elastic"
+    - elasticsearch_initial_setup | bool
   tags:
     - certificates
     - renew_ca
@@ -170,24 +157,6 @@
     - Restart Elasticsearch
   when: elasticsearch_jna_workaround | bool
 
-- name: Copy jvm.options File
-  become: yes
-  template:
-    src: "{{ elasticsearch_config_jvm }}"
-    dest: "{{ elasticsearch_conf_dir }}/jvm.options"
-    owner: root
-    group: "{{ elasticsearch_group }}"
-    mode: "660"
-    force: yes
-  notify: Restart Elasticsearch
-
-- name: Start Elasticsearch
-  service:
-    name: elasticsearch
-    state: started
-    enabled: yes
-  failed_when: false
-
 - name: Handle cluster setup without security
   when: not elasticsearch_security | bool
   block:
@@ -222,18 +191,6 @@
 # See https://github.com/NETWAYS/ansible-collection-elasticstack/issues/137
 # for details why we have this task again here
 #
-- name: Configure Elasticsearch
-  template:
-    src: elasticsearch.yml.j2
-    dest: /etc/elasticsearch/elasticsearch.yml
-    owner: root
-    group: root
-    mode: 0644
-    backup: "{{ elasticsearch_config_backup }}"
-  notify:
-    - Restart Elasticsearch
-  when: elasticsearch_manage_yaml | bool
-
 - name: Show Info about heap
   debug:
     msg: "Using {{ elasticsearch_heap | int * 1024 }} of {{ ansible_memtotal_mb }} MB as heap for Elasticsearch"