From b1db349673aa565227a1662e7b56e3726016aaef Mon Sep 17 00:00:00 2001 From: dougbtv Date: Wed, 29 Nov 2017 10:29:04 -0500 Subject: [PATCH] [ipv6][docs][bugfix] Has some required (minor) fixes to get the IPv6 deploy running and includes some docs to help one initially get there --- README.md | 4 + docs/ipv6.md | 168 ++++++++++++++++++++ ipv6-lab.yml | 20 +++ roles/kube-cni/tasks/main.yml | 1 - roles/kube-install/tasks/binary_install.yml | 37 ++--- roles/kubectl-proxy-systemd/tasks/main.yml | 4 +- roles/vm-spinup/templates/vms.local.j2 | 2 - 7 files changed, 207 insertions(+), 29 deletions(-) create mode 100644 docs/ipv6.md diff --git a/README.md b/README.md index d1802f7..e83377c 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,10 @@ In addition to setting up a basic vanilla Kubernetes environment, Kucean supports some additional overrides and technology changes, including deployment of a specific version or binary of Kubernetes, or using CRI-O as a backend. +## IPv6 Laboratory + +Using the `ipv6_enabled` variable set to true, you can set up a lab for testing IPv6. For more detailed instructions, visit the [IPv6 documentation contained in this repository](docs/ipv6.md). + ## Setting a specific version You may optionally set the `kube_version` variable to install a specific diff --git a/docs/ipv6.md b/docs/ipv6.md new file mode 100644 index 0000000..398134b --- /dev/null +++ b/docs/ipv6.md @@ -0,0 +1,168 @@ +## Setting up an IPv6 Lab using kube-centos-ansible + +This document covers a simple setup of using kube-centos-ansible for spinning up an IPv6 laboratory based in part by this [customized release of Kubernetes](https://github.com/leblancd/kubernetes/releases/tag/v1.9.0-alpha.1.ipv6.1b) created by [leblancd](https://github.com/leblancd) and automated herein by [fepan](https://github.com/fepan). + +In part this document assumes some other familiarity with these Ansible playbooks. In theory you should be able to spin up just an IPv6 lab without otherwise having experience with this playbook. However, some of the terminology may be glossed over. We often use the term "virthost" -- by which we mean a host that runs virtual machines. This technique is used often by the developers of this project in order to quickly iterate on these playbooks. These instructions assume using a virthost, however, it's likely you could complete a deployment on baremetal, or on an IaaS. + +## Limitations + +Currently, this setup makes it possible to `ping6` one pod from another. We look forward to using this laboratory to explore the other possibilities and scenarios herein, however this pod-to-pod `ping6` is the baseline functionality from which to start. + +## Requirements + +To run these playbooks, it's assumed that you have: + +* A machine for running Ansible (like your workstation) and [have Ansible installed](http://docs.ansible.com/ansible/latest/intro_installation.html). +* Ansible 2.4 or later (necessary to support `get_url` with IPv6 enabled machines) +* A host capable of running virtual machines, and is running CentOS 7. +* A clone of this repository. + +This scenario disables the "bridged networking" feature we often use and instead uses NAT'ed libvirt virtual machines. + +You may have to disable GRO ([generic receive offload](https://en.wikipedia.org/wiki/Large_receive_offload)) for the NICs on the virtualization host (if you're using one). + +An example of doing so is: + +``` +ethtool -K em3 gro off +``` + +## Inventory and variable setup + +Firstly, let's look at an inventory and variable overrides to use. + +Here's the initially used inventory, which only really cares about the `virthost`. These examples place the inventory file @ `inventory/your.virthost.inventory`. You'll need to modify the location of the host to match your environment. + +``` +the_virthost ansible_host=192.168.1.119 ansible_ssh_user=root + +[virthost] +the_virthost +``` + +And the overrides which are based on the examples @ `./inventory/examples/virthost/virthost-ipv6.inventory.yml`. In these examples this set of extra variables was created @ `./inventory/extravars.yml` (you may otherwise place it and use it in another fashion should you please): + +``` +bridge_networking: false +virtual_machines: + - name: kube-master + node_type: master + - name: kube-node-1 + node_type: nodes + - name: kube-node-2 + node_type: nodes + - name: kube-nat64-dns64 + node_type: other +ipv6_enabled: true +``` + + +## Spinning up and access virtual machines + +Perform a run of the `virthost-setup.yml` playbook, using the previously mentioned extra variables for override, and an inventory which references the + +``` +ansible-playbook -i inventory/your.virthost.inventory -e "@./inventory/extravars.yml" virthost-setup.yml +``` + +This will produce an inventory file in the local clone of this repo @ `./inventory/vms.local.generated`. + +In the case that you're running Ansible from your workstation, and your virthost is another machine, you may need to SSH jump host from the virthost to the virtual machines. + +If that is the case, you may add to the bottom of `./inventory/vms.local.generated` a line similar to this (replacing `root@192.168.1.119` with the method you use to access the virtualization host): + +``` +cat << EOF >> ./inventory/vms.local.generated +ansible_ssh_common_args='-o ProxyCommand="ssh -W %h:%p root@192.168.1.119"' +EOF +``` + +### Optional: Handy-dandy "ssh to your virtual machines script" + +You may wish to log into to the machines in order to debug, or even more likely -- to access the Kubernetes master after an install. + +You may wish to create a script, in this example... This script is located at `~/jumphost.sh` and you should change `192.168.1.119` to the hostname or IP address of your `virthost`. + +``` +# !/bin/bash +ssh -i ~/.ssh/the_virthost/id_vm_rsa -o ProxyCommand="ssh root@192.168.1.119 nc $1 22" centos@$1 +``` + +You would use this script by calling it with `~/jumphost.sh yourhost.local` where the first parameter to the script is the hostname or IP address of the virtual machine you wish to acess. + +Here's an example of using it to access the kubernetes master by pulling the IP address from the generated inventory: + +``` +$ ~/jumphost.sh $(cat inventory/vms.local.generated | grep "kube-master.ansible" | cut -d"=" -f 2) +``` + +## Deploy a Kubernetes cluster + +With the above in place, we can now perform a kube install, and use the locally generated inventory. + +``` +ansible-playbook -i inventory/vms.local.generated -e "@./inventory/extravars.yml" kube-install.yml +``` + +You now should SSH to the master, and if you please, check out the status of the cluster with `kubectl get nodes` and/or `kubectl cluster-info`. + +We'll now create a couple pods via a ReplicationController. Create a YAML resource definition like so: + +``` +[centos@kube-master ~]$ cat debug.yaml +apiVersion: v1 +kind: ReplicationController +metadata: + name: debugging +spec: + replicas: 2 + selector: + app: debugging + template: + metadata: + name: debugging + labels: + app: debugging + spec: + containers: + - name: debugging + command: ["/bin/bash", "-c", "sleep 2000000000000"] + image: dougbtv/centos-network-advanced + ports: + - containerPort: 80 +``` + +Create the pods with `kubectl` by issuing: + +``` +$ kubectl create -f debug.yaml +``` + +You may then wish to watch the pods come up: + +``` +[centos@kube-master ~]$ watch -n1 kubectl get pods -o wide +``` + +Once those pods are fully running, list them, and take a look at the IP addresses, like so: + +``` +[centos@kube-master ~]$ kubectl get pods -o wide +NAME READY STATUS RESTARTS AGE IP NODE +debugging-cvbb2 1/1 Running 0 4m fd00:101::2 kube-node-1 +debugging-gw8xt 1/1 Running 0 4m fd00:102::2 kube-node-2 +``` + +Now you can exec commands in one of them, to ping the other: + +``` +[centos@kube-master ~]$ kubectl exec -it debugging-cvbb2 -- /bin/bash -c 'ping6 -c5 fd00:102::2' +PING fd00:102::2(fd00:102::2) 56 data bytes +64 bytes from fd00:102::2: icmp_seq=1 ttl=62 time=0.845 ms +64 bytes from fd00:102::2: icmp_seq=2 ttl=62 time=0.508 ms +64 bytes from fd00:102::2: icmp_seq=3 ttl=62 time=0.562 ms +64 bytes from fd00:102::2: icmp_seq=4 ttl=62 time=0.357 ms +64 bytes from fd00:102::2: icmp_seq=5 ttl=62 time=0.555 ms +``` + +Finally pat yourself on the back and enjoy some IPv6 goodness. \ No newline at end of file diff --git a/ipv6-lab.yml b/ipv6-lab.yml index 3759bfc..a95c987 100644 --- a/ipv6-lab.yml +++ b/ipv6-lab.yml @@ -1,5 +1,7 @@ --- - hosts: kube-nat64-dns64 + become: true + become_user: root tasks: - name: update kernel package: @@ -11,11 +13,15 @@ static: yes - hosts: all + become: true + become_user: root tasks: [] roles: - { role: ipv6-setup } - hosts: master,nodes + become: true + become_user: root tasks: - name: Remove default ipv4 route shell: | @@ -24,6 +30,8 @@ fi - hosts: kube-master + become: true + become_user: root tasks: - name: configure routing lineinfile: @@ -40,6 +48,8 @@ static: yes - hosts: kube-node-1 + become: true + become_user: root tasks: - name: configure routing lineinfile: @@ -57,6 +67,8 @@ - hosts: kube-node-2 + become: true + become_user: root tasks: - name: configure routing lineinfile: @@ -73,6 +85,8 @@ static: yes - hosts: kube-nat64-dns64 + become: true + become_user: root tasks: - name: configure routing lineinfile: @@ -88,6 +102,8 @@ static: yes - hosts: kube-nat64-dns64 + become: true + become_user: root tasks: - name: install the 'Development tools' package group package: @@ -169,6 +185,8 @@ state: restarted - hosts: master,nodes + become: true + become_user: root tasks: - name: ping nat64 node shell: ping6 -c 5 fd00::64 @@ -184,6 +202,8 @@ "ping_google_result.rc == 0" - hosts: master,nodes + become: true + become_user: root tasks: - include_role: name: ipv6-setup diff --git a/roles/kube-cni/tasks/main.yml b/roles/kube-cni/tasks/main.yml index 25214cf..52b0594 100644 --- a/roles/kube-cni/tasks/main.yml +++ b/roles/kube-cni/tasks/main.yml @@ -6,7 +6,6 @@ - name: Wait until kubectl get pods is ready shell: > kubectl get pods --all-namespaces - environment: "{{ kubectl_environment }}" register: get_pods_result until: get_pods_result.rc == 0 retries: 60 diff --git a/roles/kube-install/tasks/binary_install.yml b/roles/kube-install/tasks/binary_install.yml index 6f6c134..62b9e7c 100644 --- a/roles/kube-install/tasks/binary_install.yml +++ b/roles/kube-install/tasks/binary_install.yml @@ -20,33 +20,20 @@ item.url_is_set and (download_complete_semaphor.stat.exists == False or binary_install_force_redownload) -- name: make sure wget is installed - package: - name: wget - -- name: Download kubelet - shell: | - wget -O /usr/bin/kubelet {{ binary_kubelet_url }} - when: binary_kubelet_url is defined - -- name: Download kubectl - shell: | - wget -O /usr/bin/kubectl {{ binary_kubectl_url }} - when: binary_kubectl_url is defined - -- name: Download kubeadm - shell: | - wget -O /usr/bin/kubeadm {{ binary_kubeadm_url }} - when: binary_kubeadm_url is defined - -- name: Make sure binaries have proper permission - file: - path: "{{ item }}" +- name: Download kubelet/kubectl/kubeadm + get_url: + url: "{{ item.use_url }}" + dest: "{{ item.to_path }}" mode: 0755 + force: "{{ binary_install_force_redownload }}" + when: binary_kubelet_url is defined with_items: - - /usr/bin/kubelet - - /usr/bin/kubectl - - /usr/bin/kubeadm + - use_url: "{{ binary_kubelet_url }}" + to_path: "/usr/bin/kubelet" + - use_url: "{{ binary_kubeadm_url }}" + to_path: "/usr/bin/kubeadm" + - use_url: "{{ binary_kubectl_url }}" + to_path: "/usr/bin/kubectl" - name: Mark download complete file: diff --git a/roles/kubectl-proxy-systemd/tasks/main.yml b/roles/kubectl-proxy-systemd/tasks/main.yml index e709c41..6f2b4a4 100644 --- a/roles/kubectl-proxy-systemd/tasks/main.yml +++ b/roles/kubectl-proxy-systemd/tasks/main.yml @@ -2,7 +2,9 @@ # https://kubernetes.io/docs/user-guide/kubectl/v1.8/#proxy - name: Install jq binary - shell: wget -O /usr/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 + get_url: + url: https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 + dest: /usr/bin/jq - name: set jq binary permission file: diff --git a/roles/vm-spinup/templates/vms.local.j2 b/roles/vm-spinup/templates/vms.local.j2 index 71b33d6..95fdb97 100644 --- a/roles/vm-spinup/templates/vms.local.j2 +++ b/roles/vm-spinup/templates/vms.local.j2 @@ -18,8 +18,6 @@ [all:vars] ansible_user=centos -ansible_become=true -ansible_become_user=root {% if ssh_proxy_enabled %} ansible_ssh_common_args='-o ProxyCommand="ssh{% if ssh_proxy_port is defined %}-p {{ ssh_proxy_port }}{% endif %} -W %h:%p {{ ssh_proxy_user }}@{{ ssh_proxy_host }}"' {% endif %}