From be4418cdcf1d03eee8fafb56b2a2926f18593f69 Mon Sep 17 00:00:00 2001 From: solidDoWant Date: Fri, 27 Oct 2023 02:41:30 -0500 Subject: [PATCH] router rebuild/recovery fixes --- .github/CODEOWNERS | 2 +- bootstrap/local/requirements.txt | 2 +- bootstrap/remote/inventory | 3 +- .../tasks/debian.yaml | 9 ++- .../roles/configure_coredns/tasks/main.yaml | 8 +- .../templates/kubeconfig.yaml | 2 +- .../roles/coredns_builder/defaults/main.yaml | 2 - .../coredns_builder/files/build/Dockerfile | 30 ------- .../roles/coredns_builder/tasks/main.yaml | 37 --------- .../roles/coredns_download/defaults/main.yaml | 2 + .../roles/coredns_download/tasks/main.yaml | 36 +++++++++ .../roles/coredns_runner/files/coredns-rc.d | 2 +- .../roles/coredns_runner/tasks/main.yaml | 81 ++++++++++--------- .../roles/coredns_runner/templates/Corefile | 4 + .../remote/playbooks/gateway_hosts/setup.yaml | 4 +- .../roles/k8s/tasks/get_kubeconfig.yaml | 2 +- .../opnsense_hosts/tasks/opnsense_vm.yaml | 2 +- .../roles/opnsense_hosts/templates/config.xml | 2 +- 18 files changed, 107 insertions(+), 123 deletions(-) delete mode 100644 bootstrap/remote/playbooks/gateway_hosts/roles/coredns_builder/defaults/main.yaml delete mode 100644 bootstrap/remote/playbooks/gateway_hosts/roles/coredns_builder/files/build/Dockerfile delete mode 100644 bootstrap/remote/playbooks/gateway_hosts/roles/coredns_builder/tasks/main.yaml create mode 100644 bootstrap/remote/playbooks/gateway_hosts/roles/coredns_download/defaults/main.yaml create mode 100644 bootstrap/remote/playbooks/gateway_hosts/roles/coredns_download/tasks/main.yaml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9c8586789..a6791fb55 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,2 @@ # https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners -* @onedr0p +* @solidDoWant diff --git a/bootstrap/local/requirements.txt b/bootstrap/local/requirements.txt index d3944a2b1..509424883 100644 --- a/bootstrap/local/requirements.txt +++ b/bootstrap/local/requirements.txt @@ -1,4 +1,4 @@ -jmsepath +jmespath docker lxml openshift diff --git a/bootstrap/remote/inventory b/bootstrap/remote/inventory index 1e3d1842b..11767fc04 100644 --- a/bootstrap/remote/inventory +++ b/bootstrap/remote/inventory @@ -6,7 +6,8 @@ working_dir_path="{{ repo_root_path }}/working" k8s_masters [k8s_masters] -10.1.1.[1:4] ansible_user="bootstrap" k3s_control_node=true k3s_use_unsupported_config=true +10.1.1.[1:2] ansible_user="bootstrap" k3s_control_node=true k3s_use_unsupported_config=true +10.1.1.4 ansible_user="bootstrap" k3s_control_node=true k3s_use_unsupported_config=true [proxmox_hosts:children] opnsense_hosts diff --git a/bootstrap/remote/playbooks/all_hosts/roles/configure_package_manager/tasks/debian.yaml b/bootstrap/remote/playbooks/all_hosts/roles/configure_package_manager/tasks/debian.yaml index 30dc28e71..5b580301d 100644 --- a/bootstrap/remote/playbooks/all_hosts/roles/configure_package_manager/tasks/debian.yaml +++ b/bootstrap/remote/playbooks/all_hosts/roles/configure_package_manager/tasks/debian.yaml @@ -23,9 +23,14 @@ state: present filename: pve-no-subscription update_cache: false - - name: Add Ceph Pacific repository + - name: Remove enterprise Ceph repository apt_repository: - repo: "deb http://download.proxmox.com/debian/ceph-pacific {{ ansible_distribution_release }} main" + repo: "deb https://enterprise.proxmox.com/debian/ceph-quincy {{ ansible_distribution_release }} enterprise" + state: absent + filename: pve-enterprise + - name: Add Ceph repository + apt_repository: + repo: "deb http://download.proxmox.com/debian/ceph-quincy {{ ansible_distribution_release }} main" state: present filename: ceph update_cache: false diff --git a/bootstrap/remote/playbooks/coredns/roles/configure_coredns/tasks/main.yaml b/bootstrap/remote/playbooks/coredns/roles/configure_coredns/tasks/main.yaml index 85fe4e2cc..c9fd922fd 100644 --- a/bootstrap/remote/playbooks/coredns/roles/configure_coredns/tasks/main.yaml +++ b/bootstrap/remote/playbooks/coredns/roles/configure_coredns/tasks/main.yaml @@ -3,7 +3,7 @@ copy: src: "{{ local_kubeconfig_path }}" dest: /usr/local/etc/coredns/kubeconfig - mode: 0640 + mode: "0600" owner: root group: wheel register: copied_kubeconfig @@ -14,10 +14,10 @@ dest: "/usr/local/etc/coredns/root_config.d/k8s_gateway_{{ secret_domain }}" owner: root group: wheel - mode: 0755 - when: copied_kubeconfig.changed + mode: "0755" + register: created_coredns_config - name: Restart CoreDNS shell: | /usr/local/etc/rc.d/coredns restart - when: copied_kubeconfig.changed + when: created_coredns_config.changed diff --git a/bootstrap/remote/playbooks/coredns/roles/generate_kubeconfig/templates/kubeconfig.yaml b/bootstrap/remote/playbooks/coredns/roles/generate_kubeconfig/templates/kubeconfig.yaml index 3431d6344..31565cbbd 100644 --- a/bootstrap/remote/playbooks/coredns/roles/generate_kubeconfig/templates/kubeconfig.yaml +++ b/bootstrap/remote/playbooks/coredns/roles/generate_kubeconfig/templates/kubeconfig.yaml @@ -5,7 +5,7 @@ clusters: - name: home cluster: certificate-authority-data: "{{ DNSCertificateAuthority }}" - server: "https://{{ kube_vip_ip }}:6443" + server: "https://{{ kube_vip_ip }}:443" contexts: - name: home context: diff --git a/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_builder/defaults/main.yaml b/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_builder/defaults/main.yaml deleted file mode 100644 index 69a48f3a2..000000000 --- a/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_builder/defaults/main.yaml +++ /dev/null @@ -1,2 +0,0 @@ ---- -build_dir_path: "{{ role_path }}/files/build" diff --git a/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_builder/files/build/Dockerfile b/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_builder/files/build/Dockerfile deleted file mode 100644 index 3b5fafbbf..000000000 --- a/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_builder/files/build/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM docker.io/golang:1.21.3-alpine - -ARG VERSION -ENV VERSION=v1.8.5 - -ENV CGO_ENABLED=0 \ - GOPATH=/go \ - GOBIN=/go/bin \ - GO111MODULE=on - -WORKDIR /go/src/coredns - -RUN apk --no-cache --no-progress add ca-certificates git - -RUN update-ca-certificates - -RUN \ - git clone https://github.com/coredns/coredns.git --branch "${VERSION}" --depth 1 --single-branch . \ - && sed -i '/^kubernetes:kubernetes/a k8s_gateway:github.com/ori-edge/k8s_gateway' plugin.cfg - -RUN \ - go get github.com/ori-edge/k8s_gateway \ - && go generate \ - && go mod tidy - -ENV GOOS=freebsd \ - GOARCH=amd64 - -RUN \ - go build -ldflags "-s -w -X github.com/coredns/coredns/coremain.GitCommit=$(git describe --always)" -o coredns diff --git a/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_builder/tasks/main.yaml b/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_builder/tasks/main.yaml deleted file mode 100644 index 9984a3d43..000000000 --- a/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_builder/tasks/main.yaml +++ /dev/null @@ -1,37 +0,0 @@ ---- -- name: Build CoreDNS in Docker - docker_image: - name: coredns:build - source: build - state: present - build: - path: "{{ build_dir_path }}" - nocache: false - -- name: Create the coredns directory - file: - path: "{{ working_dir_path }}/coredns" - state: directory - register: coredns_directory - -- name: Create a build container and copy the CoreDNS binary from Docker image to the local coredns directory - docker_container: - name: coredns_build - image: coredns:build - container_default_behavior: no_defaults - mounts: - - source: "{{ coredns_directory.path }}" - target: /mnt/dist/bin - type: bind - command: cp /go/src/coredns/coredns /mnt/dist/bin - -- name: Remove CoreDNS container - docker_container: - name: coredns_build - state: absent - container_default_behavior: no_defaults - -- name: Remove CoreDNS build image - docker_image: - name: coredns:build - state: absent diff --git a/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_download/defaults/main.yaml b/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_download/defaults/main.yaml new file mode 100644 index 000000000..61c336ee7 --- /dev/null +++ b/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_download/defaults/main.yaml @@ -0,0 +1,2 @@ +--- +coredns_working_dir_path: "{{ working_dir_path }}/coredns" diff --git a/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_download/tasks/main.yaml b/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_download/tasks/main.yaml new file mode 100644 index 000000000..8edbb5a23 --- /dev/null +++ b/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_download/tasks/main.yaml @@ -0,0 +1,36 @@ +--- +- name: Check if the CoreDNS binary file exists + ansible.builtin.stat: + path: "{{ coredns_working_dir_path }}/coredns" + register: coredns_binary + +- name: Create the coredns directory + ansible.builtin.file: + path: "{{ coredns_working_dir_path }}" + state: directory + mode: "0755" + register: coredns_directory + when: not coredns_binary.stat.exists + +# TODO version this with Renovate +- name: Get the latest k8s_gateway release data from GitHub + ansible.builtin.uri: + url: https://api.github.com/repos/ori-edge/k8s_gateway/releases/latest + method: GET + return_content: true + status_code: 200 + body_format: json + register: github_k8s_gateway_page + when: not coredns_binary.stat.exists + +# TODO don't hardcode OS info +- name: Download and extract the k8s_gateway version of CoreDNS from GitHub + ansible.builtin.unarchive: + src: "{{ github_k8s_gateway_page.json | json_query(query) | first }}" + dest: "{{ coredns_working_dir_path }}" + remote_src: true + include: + - coredns + vars: + query: assets[?ends_with(name, 'freebsd_amd64.tar.gz')].browser_download_url + when: not coredns_binary.stat.exists diff --git a/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_runner/files/coredns-rc.d b/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_runner/files/coredns-rc.d index 6f43693fd..a303b58c3 100644 --- a/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_runner/files/coredns-rc.d +++ b/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_runner/files/coredns-rc.d @@ -1,7 +1,7 @@ #!/bin/sh # PROVIDE: coredns -# REQUIRE: DAEMON NETWORKING +# REQUIRE: DAEMON NETWORKING frr # KEYWORD: shutdown # # Add the following to /etc/rc.conf[.local] to enable this service diff --git a/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_runner/tasks/main.yaml b/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_runner/tasks/main.yaml index bd12aaec9..176cb5a2b 100644 --- a/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_runner/tasks/main.yaml +++ b/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_runner/tasks/main.yaml @@ -1,136 +1,141 @@ --- - name: Create config directory - file: + ansible.builtin.file: path: /usr/local/etc/coredns state: directory - mode: 0755 + mode: "0755" - name: Create zones directory - file: + ansible.builtin.file: path: /usr/local/etc/coredns/config.d state: directory - mode: 0755 + mode: "0755" - name: Create root config directory - file: + ansible.builtin.file: path: /usr/local/etc/coredns/root_config.d state: directory - mode: 0755 + mode: "0755" - name: Create zonefiles directory - file: + ansible.builtin.file: path: /usr/local/etc/coredns/zones state: directory - mode: 0755 + mode: "0755" - name: Gather facts on all hosts for DNS record creation - setup: {} + ansible.builtin.setup: {} delegate_to: "{{ item }}" delegate_facts: true when: hostvars[item]['ansible_default_ipv4'] is not defined with_items: "{{ groups['all'] }}" - name: Create Corefile - template: + ansible.builtin.template: src: Corefile dest: /usr/local/etc/coredns/Corefile - mode: 0755 + mode: "0755" vars: listening_addresses: "{{ ansible_interfaces | select('match', '^(lo\\d+|vtnet1\\S*)$') | join(' ') }}" - name: Create config files - template: + ansible.builtin.template: src: "{{ item }}" dest: /usr/local/etc/coredns/config.d - mode: 0755 + mode: "0755" with_fileglob: - ../templates/config.d/* - name: Create zone files - template: + ansible.builtin.template: src: "{{ item }}" dest: /usr/local/etc/coredns/zones - mode: 0755 + mode: "0755" with_fileglob: - ../templates/zones/* - name: Create CoreDNS rc.conf script - copy: + ansible.builtin.copy: src: coredns-rc.conf dest: /etc/rc.conf.d/coredns - mode: 0755 + mode: "0755" - name: Create CoreDNS rc.d script - copy: + ansible.builtin.copy: src: coredns-rc.d dest: /usr/local/etc/rc.d/coredns - mode: 0755 + mode: "0755" - name: Create CoreDNS action script - copy: + ansible.builtin.copy: src: actions_coredns.conf dest: /usr/local/opnsense/service/conf/actions.d/actions_coredns.conf - mode: 0755 + mode: "0755" - name: Create CoreDNS start up script - copy: + ansible.builtin.copy: src: 99-coredns dest: /usr/local/etc/rc.syshook.d/start/99-coredns - mode: 0755 + mode: "0755" - name: Create CoreDNS log rotation config - copy: + ansible.builtin.copy: src: coredns-newsyslog.conf dest: /etc/newsyslog.conf.d/coredns - mode: 0755 + mode: "0755" - name: Stop running CoreDNS - shell: /usr/local/etc/rc.d/coredns stop + ansible.builtin.command: /usr/local/etc/rc.d/coredns stop ignore_errors: true register: coredns_stop - name: Copy CoreDNS - copy: + ansible.builtin.copy: src: "{{ working_dir_path }}/coredns/coredns" dest: /usr/local/sbin/coredns - mode: 0755 + mode: "0755" - name: Disable Unbound DNS + when: coredns_stop.rc != 0 + block: - name: Pull the current OPNsense config - fetch: + ansible.builtin.fetch: src: "{{ remote_config_path }}" dest: "{{ local_config_path }}" flat: true register: downloaded_config - name: Disable Unbound + when: downloaded_config.changed + delegate_to: localhost block: - name: Remove /opnsense/unbound/enable - xml: + community.general.xml: path: "{{ local_config_path }}" xpath: /opnsense/unbound/enable state: absent + - name: Remove /opnsense/OPNsense/unboundplus/enabled + community.general.xml: + path: "{{ local_config_path }}" + xpath: /opnsense/OPNsense/unboundplus/enabled + state: absent - name: Remove /opnsense/OPNsense/unboundplus/service_enabled - xml: + community.general.xml: path: "{{ local_config_path }}" xpath: /opnsense/OPNsense/unboundplus/service_enabled state: absent - when: downloaded_config.changed - delegate_to: localhost - name: Copy the new OPNsense config - copy: + ansible.builtin.copy: src: "{{ local_config_path }}" dest: "{{ remote_config_path }}" backup: true register: return_config when: downloaded_config.changed - name: Reload OPNsense - command: "{{ item }}" + ansible.builtin.command: "{{ item }}" with_items: - configctl service reload all - configctl webgui restart when: downloaded_config.changed and return_config.changed - when: coredns_stop.rc != 0 - - name: Run CoreDNS - shell: /usr/local/etc/rc.d/coredns start + ansible.builtin.command: /usr/local/etc/rc.d/coredns start diff --git a/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_runner/templates/Corefile b/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_runner/templates/Corefile index ecfb6491b..7cc512e82 100644 --- a/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_runner/templates/Corefile +++ b/bootstrap/remote/playbooks/gateway_hosts/roles/coredns_runner/templates/Corefile @@ -17,6 +17,10 @@ import ./config.d/* directory ./zones (.*) {1} } + template ANY AAAA { + rcode NOERROR + } + forward . tls://1.1.1.1 tls://1.0.0.1 { tls_servername cloudflare-dns.com } diff --git a/bootstrap/remote/playbooks/gateway_hosts/setup.yaml b/bootstrap/remote/playbooks/gateway_hosts/setup.yaml index 05d6fdb28..763ff7860 100644 --- a/bootstrap/remote/playbooks/gateway_hosts/setup.yaml +++ b/bootstrap/remote/playbooks/gateway_hosts/setup.yaml @@ -5,8 +5,8 @@ become: true roles: - - role: coredns_builder - tags: coredns_builder + - role: coredns_downloader + tags: coredns_downloader - hosts: opnsense become: true diff --git a/bootstrap/remote/playbooks/k8s_hosts/roles/k8s/tasks/get_kubeconfig.yaml b/bootstrap/remote/playbooks/k8s_hosts/roles/k8s/tasks/get_kubeconfig.yaml index dbef349bf..2f58f1cd6 100644 --- a/bootstrap/remote/playbooks/k8s_hosts/roles/k8s/tasks/get_kubeconfig.yaml +++ b/bootstrap/remote/playbooks/k8s_hosts/roles/k8s/tasks/get_kubeconfig.yaml @@ -11,7 +11,7 @@ replace: path: "{{ downloaded_kubeconfig.dest }}" regexp: "https://127.0.0.1:6443" - replace: "https://{{ k3s_registration_address }}:6443" + replace: "https://{{ k3s_registration_address }}:443" when: downloaded_kubeconfig.changed run_once: true delegate_to: localhost diff --git a/bootstrap/remote/playbooks/proxmox_hosts/roles/opnsense_hosts/tasks/opnsense_vm.yaml b/bootstrap/remote/playbooks/proxmox_hosts/roles/opnsense_hosts/tasks/opnsense_vm.yaml index b3b139b9d..5443358b4 100644 --- a/bootstrap/remote/playbooks/proxmox_hosts/roles/opnsense_hosts/tasks/opnsense_vm.yaml +++ b/bootstrap/remote/playbooks/proxmox_hosts/roles/opnsense_hosts/tasks/opnsense_vm.yaml @@ -54,7 +54,7 @@ --memory {{ vm_memory }} --name {{ vm_name }} --net0 virtio=72:BF:8F:3A:40:42,bridge=vmbr0,firewall=0 - --net1 virtio=C6:3A:F1:C2:FA:0D,bridge=vmbr1,firewall=0,tag=1000 + --net1 virtio=C6:3A:F1:C2:FA:0D,bridge=vmbr1,firewall=0,mtu=9000 --onboot true --ostype other --protection false diff --git a/bootstrap/remote/playbooks/proxmox_hosts/roles/opnsense_hosts/templates/config.xml b/bootstrap/remote/playbooks/proxmox_hosts/roles/opnsense_hosts/templates/config.xml index 00732acb0..ee3202315 100644 --- a/bootstrap/remote/playbooks/proxmox_hosts/roles/opnsense_hosts/templates/config.xml +++ b/bootstrap/remote/playbooks/proxmox_hosts/roles/opnsense_hosts/templates/config.xml @@ -311,7 +311,7 @@ Hosts 1 - 9001 + 9000 10.1.0.1 16