Create configs for monitoring server (not ready yet, see TODO etc)

README.md

@@ -22,12 +22,11 @@ Please note: Replace <ENVIRONMENT> with either **production** or **staging** in
 
 Requirements: jmespath (deployer host), unzip (target host)
 
-### Grafana
+Adjust inventory and config variables in `environments/monitoring/group_vars/monitoring/main.yml` before prodceeding.
 
-- `ansible-galaxy install cloudalchemy.grafana`
-- `ansible-playbook -i environments/monitoring/inventory.yml playbooks/install_grafana.yml`
-
-### Prometheus
+### Prometheus + Grafana + Alert Manager
 
+- `ansible-galaxy install cloudalchemy.grafana`
 - `ansible-galaxy install cloudalchemy.prometheus`
-- `ansible-playbook -i environments/monitoring/inventory.yml playbooks/install_prometheus.yml`
+- `ansible-galaxy install cloudalchemy.alertmanager`
+- `ansible-playbook -i environments/monitoring/inventory.yml playbooks/install_monitoring.yml`

environments/monitoring/group_vars/monitoring/main.yml

@@ -0,0 +1,53 @@
+domains:
+  prometheus: prometheus.palava.tv
+  grafana: grafana.palava.tv
+  alertmanager: alerts.palava.tv
+
+### PROMETHEUS ###
+
+prometheus_version: latest
+prometheus_web_listen_address: "127.0.0.1:9090"
+# TODO use
+prometheus_web_external_url: "https://{{ domains.prometheus }}"
+prometheus_storage_retention: 30d
+prometheus_scrape_jobs:
+  - job_name: "signal-tower"
+    metrics_path: /metrics
+    basic_auth:
+      username: "TODO"
+      password: "TODOTODOTODO"
+    params:
+      module: [http_2xx]
+    static_configs:
+      - targets:
+          - machine.palava.tv
+
+### GRAFANA ###
+
+grafana_version: latest
+grafana_address: 127.0.0.1
+grafana_port: 7000
+grafana_url: "https://{{ domains.grafana }}"
+grafana_security:
+  admin_user: TODO
+  admin_password: "TODO"
+grafana_datasources:
+  - name: prometheus
+    type: prometheus
+    url: "http://{{ prometheus_web_listen_address }}"
+    basicAuth: false
+    basicAuthUser: "TODO"
+    basicAuthPassword: "TODO"
+
+### ALERTMANAGER ###
+
+alertmanager_version: 0.23.0
+alertmanager_web_listen_address: 127.0.0.1:9093
+alertmanager_web_external_url: "https://{{ domains.alertmanager }}"
+#alertmanager_receivers: TODO
+alertmanager_route:
+  group_by: ["alertname", "cluster", "service"]
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 3h
+  receiver: slack

environments/monitoring/inventory.yml

@@ -0,0 +1,3 @@
+monitoring:
+  hosts:
+    128.140.124.42: null

environments/production/group_vars/all/main.yml

@@ -1,31 +1,8 @@
 ---
-
 palava_signaltower_install_dir: /srv/signaltower-production
 palava_signaltower_log_dir: /var/log/signaltower-production
 palava_environment: production
 # Used to configure the TURN server (turn role) to enable TURN in the signaltower role
 # palava_signaltower_turn_secret: SOME_SECRET_KEY
 
 palava_signaltower_autostart: yes
-
-# Prometheus vars
-prometheus_version: 2.22.0
-prometheus_web_listen_address: '127.0.0.1:9090'
-prometheus_scrape_jobs:
-- job_name: 'signal-tower'
-  metrics_path: /metrics
-  params:
-    module: [http_2xx]
-  static_configs:
-    - targets:
-      - localhost:4233
-# Grafana vars
-grafana_security:
-  admin_user: admin
-  admin_password: "admin"
-grafana_datasources:
-  - name: prometheus
-    type: prometheus
-    access: proxy
-    url: 'http://{{ prometheus_web_listen_address }}'
-    basicAuth: false

environments/production/inventory.yml

@@ -1,5 +1,3 @@
----
-
 all:
   hosts:
     157.90.226.126: null

environments/staging/group_vars/all/main.yml

@@ -1,5 +1,4 @@
 ---
-
 palava_signaltower_install_dir: /srv/signaltower-staging
 palava_signaltower_log_dir: /var/log/signaltower-staging
 palava_environment: staging

environments/staging/inventory.yml

@@ -1,5 +1,3 @@
----
-
 all:
   hosts:
     157.90.226.126: null

playbooks/install_monitoring.yml

@@ -0,0 +1,11 @@
+- hosts: monitoring
+  strategy: debug
+  remote_user: root
+  become: yes
+  roles:
+    - include_role:
+        name: common
+        tasks_from: monitoring.yml
+    - cloudalchemy.prometheus
+    - cloudalchemy.grafana
+    - cloudalchemy.alertmanager

roles/common/tasks/monitoring.yml

@@ -0,0 +1,4 @@
+---
+- include: upgrade.yml
+- include: packages-monitoring.yml
+- include: ssh.yml

roles/common/tasks/packages-monitoring.yml

@@ -0,0 +1,18 @@
+- name: Install common packages (monitoring)
+  apt:
+    name:
+      - apt-transport-https
+      - apache2-utils
+      - fail2ban
+      - nginx
+      - unattended-upgrades
+      - vim
+    state: present
+
+- name: Make sure some packages are not installed
+  apt:
+    name:
+      - apache2
+      - cups
+      - telnet
+    state: absent