router rebuild/recovery fixes

.github/CODEOWNERS

@@ -1,2 +1,2 @@
 # https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners
-* @onedr0p
+* @solidDoWant

bootstrap/local/requirements.txt

@@ -1,4 +1,4 @@
-jmsepath
+jmespath
 docker
 lxml
 openshift

bootstrap/remote/inventory

@@ -6,7 +6,8 @@ working_dir_path="{{ repo_root_path }}/working"
 k8s_masters
 
 [k8s_masters]
-10.1.1.[1:4] ansible_user="bootstrap" k3s_control_node=true k3s_use_unsupported_config=true
+10.1.1.[1:2] ansible_user="bootstrap" k3s_control_node=true k3s_use_unsupported_config=true
+10.1.1.4 ansible_user="bootstrap" k3s_control_node=true k3s_use_unsupported_config=true
 
 [proxmox_hosts:children]
 opnsense_hosts

bootstrap/remote/playbooks/all_hosts/roles/configure_package_manager/tasks/debian.yaml

@@ -23,9 +23,14 @@
         state: present
         filename: pve-no-subscription
         update_cache: false
-    - name: Add Ceph Pacific repository
+    - name: Remove enterprise Ceph repository
       apt_repository:
-        repo: "deb http://download.proxmox.com/debian/ceph-pacific {{ ansible_distribution_release }} main"
+        repo: "deb https://enterprise.proxmox.com/debian/ceph-quincy {{ ansible_distribution_release }} enterprise"
+        state: absent
+        filename: pve-enterprise
+    - name: Add Ceph repository
+      apt_repository:
+        repo: "deb http://download.proxmox.com/debian/ceph-quincy {{ ansible_distribution_release }} main"
         state: present
         filename: ceph
         update_cache: false

bootstrap/remote/playbooks/coredns/roles/configure_coredns/tasks/main.yaml

@@ -3,7 +3,7 @@
   copy:
     src: "{{ local_kubeconfig_path }}"
     dest: /usr/local/etc/coredns/kubeconfig
-    mode: 0640
+    mode: "0600"
     owner: root
     group: wheel
   register: copied_kubeconfig
@@ -14,10 +14,10 @@
     dest: "/usr/local/etc/coredns/root_config.d/k8s_gateway_{{ secret_domain }}"
     owner: root
     group: wheel
-    mode: 0755
-  when: copied_kubeconfig.changed
+    mode: "0755"
+  register: created_coredns_config
 
 - name: Restart CoreDNS
   shell: |
     /usr/local/etc/rc.d/coredns restart
-  when: copied_kubeconfig.changed
+  when: created_coredns_config.changed

bootstrap/remote/playbooks/coredns/roles/generate_kubeconfig/templates/kubeconfig.yaml

@@ -5,7 +5,7 @@ clusters:
   - name: home
     cluster:
       certificate-authority-data: "{{ DNSCertificateAuthority }}"
-      server: "https://{{ kube_vip_ip }}:6443"
+      server: "https://{{ kube_vip_ip }}:443"
 contexts:
   - name: home
     context:

bootstrap/remote/playbooks/gateway_hosts/roles/coredns_download/defaults/main.yaml

@@ -0,0 +1,2 @@
+---
+coredns_working_dir_path: "{{ working_dir_path }}/coredns"

bootstrap/remote/playbooks/gateway_hosts/roles/coredns_download/tasks/main.yaml

@@ -0,0 +1,36 @@
+---
+- name: Check if the CoreDNS binary file exists
+  ansible.builtin.stat:
+    path: "{{ coredns_working_dir_path }}/coredns"
+  register: coredns_binary
+
+- name: Create the coredns directory
+  ansible.builtin.file:
+    path: "{{ coredns_working_dir_path }}"
+    state: directory
+    mode: "0755"
+  register: coredns_directory
+  when: not coredns_binary.stat.exists
+
+# TODO version this with Renovate
+- name: Get the latest k8s_gateway release data from GitHub
+  ansible.builtin.uri:
+    url: https://api.github.com/repos/ori-edge/k8s_gateway/releases/latest
+    method: GET
+    return_content: true
+    status_code: 200
+    body_format: json
+  register: github_k8s_gateway_page
+  when: not coredns_binary.stat.exists
+
+# TODO don't hardcode OS info
+- name: Download and extract the k8s_gateway version of CoreDNS from GitHub
+  ansible.builtin.unarchive:
+    src: "{{ github_k8s_gateway_page.json | json_query(query) | first }}"
+    dest: "{{ coredns_working_dir_path }}"
+    remote_src: true
+    include:
+      - coredns
+  vars:
+    query: assets[?ends_with(name, 'freebsd_amd64.tar.gz')].browser_download_url
+  when: not coredns_binary.stat.exists

bootstrap/remote/playbooks/gateway_hosts/roles/coredns_runner/files/coredns-rc.d

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # PROVIDE: coredns
-# REQUIRE: DAEMON NETWORKING
+# REQUIRE: DAEMON NETWORKING frr
 # KEYWORD: shutdown
 #
 # Add the following to /etc/rc.conf[.local] to enable this service

bootstrap/remote/playbooks/gateway_hosts/roles/coredns_runner/tasks/main.yaml

@@ -1,136 +1,141 @@
 ---
 - name: Create config directory
-  file:
+  ansible.builtin.file:
     path: /usr/local/etc/coredns
     state: directory
-    mode: 0755
+    mode: "0755"
 
 - name: Create zones directory
-  file:
+  ansible.builtin.file:
     path: /usr/local/etc/coredns/config.d
     state: directory
-    mode: 0755
+    mode: "0755"
 
 - name: Create root config directory
-  file:
+  ansible.builtin.file:
     path: /usr/local/etc/coredns/root_config.d
     state: directory
-    mode: 0755
+    mode: "0755"
 
 - name: Create zonefiles directory
-  file:
+  ansible.builtin.file:
     path: /usr/local/etc/coredns/zones
     state: directory
-    mode: 0755
+    mode: "0755"
 
 - name: Gather facts on all hosts for DNS record creation
-  setup: {}
+  ansible.builtin.setup: {}
   delegate_to: "{{ item }}"
   delegate_facts: true
   when: hostvars[item]['ansible_default_ipv4'] is not defined
   with_items: "{{ groups['all'] }}"
 
 - name: Create Corefile
-  template:
+  ansible.builtin.template:
     src: Corefile
     dest: /usr/local/etc/coredns/Corefile
-    mode: 0755
+    mode: "0755"
   vars:
     listening_addresses: "{{ ansible_interfaces | select('match', '^(lo\\d+|vtnet1\\S*)$') | join(' ') }}"
 
 - name: Create config files
-  template:
+  ansible.builtin.template:
     src: "{{ item }}"
     dest: /usr/local/etc/coredns/config.d
-    mode: 0755
+    mode: "0755"
   with_fileglob:
     - ../templates/config.d/*
 
 - name: Create zone files
-  template:
+  ansible.builtin.template:
     src: "{{ item }}"
     dest: /usr/local/etc/coredns/zones
-    mode: 0755
+    mode: "0755"
   with_fileglob:
     - ../templates/zones/*
 
 - name: Create CoreDNS rc.conf script
-  copy:
+  ansible.builtin.copy:
     src: coredns-rc.conf
     dest: /etc/rc.conf.d/coredns
-    mode: 0755
+    mode: "0755"
 
 - name: Create CoreDNS rc.d script
-  copy:
+  ansible.builtin.copy:
     src: coredns-rc.d
     dest: /usr/local/etc/rc.d/coredns
-    mode: 0755
+    mode: "0755"
 
 - name: Create CoreDNS action script
-  copy:
+  ansible.builtin.copy:
     src: actions_coredns.conf
     dest: /usr/local/opnsense/service/conf/actions.d/actions_coredns.conf
-    mode: 0755
+    mode: "0755"
 
 - name: Create CoreDNS start up script
-  copy:
+  ansible.builtin.copy:
     src: 99-coredns
     dest: /usr/local/etc/rc.syshook.d/start/99-coredns
-    mode: 0755
+    mode: "0755"
 
 - name: Create CoreDNS log rotation config
-  copy:
+  ansible.builtin.copy:
     src: coredns-newsyslog.conf
     dest: /etc/newsyslog.conf.d/coredns
-    mode: 0755
+    mode: "0755"
 
 - name: Stop running CoreDNS
-  shell: /usr/local/etc/rc.d/coredns stop
+  ansible.builtin.command: /usr/local/etc/rc.d/coredns stop
   ignore_errors: true
   register: coredns_stop
 
 - name: Copy CoreDNS
-  copy:
+  ansible.builtin.copy:
     src: "{{ working_dir_path }}/coredns/coredns"
     dest: /usr/local/sbin/coredns
-    mode: 0755
+    mode: "0755"
 
 - name: Disable Unbound DNS
+  when: coredns_stop.rc != 0
+
   block:
     - name: Pull the current OPNsense config
-      fetch:
+      ansible.builtin.fetch:
         src: "{{ remote_config_path }}"
         dest: "{{ local_config_path }}"
         flat: true
       register: downloaded_config
     - name: Disable Unbound
+      when: downloaded_config.changed
+      delegate_to: localhost
       block:
         - name: Remove /opnsense/unbound/enable
-          xml:
+          community.general.xml:
             path: "{{ local_config_path }}"
             xpath: /opnsense/unbound/enable
             state: absent
+        - name: Remove /opnsense/OPNsense/unboundplus/enabled
+          community.general.xml:
+            path: "{{ local_config_path }}"
+            xpath: /opnsense/OPNsense/unboundplus/enabled
+            state: absent
         - name: Remove /opnsense/OPNsense/unboundplus/service_enabled
-          xml:
+          community.general.xml:
             path: "{{ local_config_path }}"
             xpath: /opnsense/OPNsense/unboundplus/service_enabled
             state: absent
-      when: downloaded_config.changed
-      delegate_to: localhost
     - name: Copy the new OPNsense config
-      copy:
+      ansible.builtin.copy:
         src: "{{ local_config_path }}"
         dest: "{{ remote_config_path }}"
         backup: true
       register: return_config
       when: downloaded_config.changed
     - name: Reload OPNsense
-      command: "{{ item }}"
+      ansible.builtin.command: "{{ item }}"
       with_items:
         - configctl service reload all
         - configctl webgui restart
       when: downloaded_config.changed and return_config.changed
-  when: coredns_stop.rc != 0
-
 - name: Run CoreDNS
-  shell: /usr/local/etc/rc.d/coredns start
+  ansible.builtin.command: /usr/local/etc/rc.d/coredns start

bootstrap/remote/playbooks/gateway_hosts/roles/coredns_runner/templates/Corefile

@@ -17,6 +17,10 @@ import ./config.d/*
     directory ./zones (.*) {1}
   }
 
+  template ANY AAAA {
+    rcode NOERROR
+  }
+
   forward . tls://1.1.1.1 tls://1.0.0.1 {
     tls_servername cloudflare-dns.com
   }

bootstrap/remote/playbooks/gateway_hosts/setup.yaml

@@ -5,8 +5,8 @@
   become: true
 
   roles:
-    - role: coredns_builder
-      tags: coredns_builder
+    - role: coredns_downloader
+      tags: coredns_downloader
 
 - hosts: opnsense
   become: true