Add support for cluster provisioning

Dockerfile

@@ -50,10 +50,6 @@ LABEL RUN="/usr/bin/docker run --rm --privileged \
              \${OPT1} \
              \${IMAGE}"
 
-# When run in e.g. Jenkins, it's really annoying to not see
-# any output of e.g. the provisioner until it's all done.
-ENV PYTHONUNBUFFERED 1
-
 COPY . /redhat-ci
 
 RUN pip3 install -r /redhat-ci/requirements.txt

main

@@ -38,6 +38,8 @@ main() {
         exit 0
     fi
 
+    export PYTHONUNBUFFERED=1
+
     exec $THIS_DIR/spawner.py
 }
 

provisioner

@@ -0,0 +1,133 @@
+#!/bin/bash
+set -Exeuo pipefail
+
+# This script provisions a node on OpenStack. It may be
+# called multiple times in parallel.
+
+THIS_DIR=$(dirname $0)
+
+source $THIS_DIR/utils/common.sh
+
+main() {
+
+    # NB: see the various NBs in the main() of main.
+
+    state=$1; shift
+    parsedhost=$1; shift
+    outdir=$1; shift
+
+    [ -d $parsedhost ]
+    mkdir $outdir
+
+    provision_host
+}
+
+provision_host() {
+
+    # XXX: We hardcode m1.small for now, but these really
+    # should be specified indirectly from the .redhat-ci
+    # YAML file through e.g. min-* vars.
+    env \
+        os_image="$(cat $parsedhost/distro)" \
+        os_flavor=m1.small \
+        os_name_prefix=github-ci-testnode \
+        os_user_data="$THIS_DIR/utils/user-data" \
+        "$THIS_DIR/utils/os_provision.py" $outdir
+
+    ssh_wait $(cat $outdir/node_addr) $state/node_key
+
+    if [ -f $parsedhost/ostree_revision ]; then
+        if ! on_atomic_host; then
+            update_github error "Cannot specify 'ostree' on non-AH."
+            touch $state/exit # signal testrunner to exit nicely
+            exit 0
+        fi
+        deploy_ostree
+    fi
+}
+
+deploy_ostree() {
+    local remote=$(cat $parsedhost/ostree_remote)
+    local branch=$(cat $parsedhost/ostree_branch)
+    local revision=$(cat $parsedhost/ostree_revision)
+
+    local rc=0
+    local skip_reboot=0
+    if [ -z "$remote" ] && [ -z "$branch" ]; then
+
+        if [ -z "$revision" ]; then
+            vmssh rpm-ostree upgrade --upgrade-unchanged-exit-77 || rc=$?
+        else
+            vmssh rpm-ostree deploy "$revision" || rc=$?
+        fi
+
+        if [ $rc == 77 ]; then
+            skip_reboot=1
+        elif [ $rc != 0 ]; then
+            update_github error "Failed to upgrade or deploy."
+            touch $state/exit # signal testrunner to exit nicely
+            exit 0
+        fi
+    else
+        local refspec
+
+        if [ -n "$remote" ]; then
+            vmssh ostree remote add --no-gpg-verify rhci "$remote"
+            refspec=rhci:
+        fi
+
+        if [ -n "$branch" ]; then
+            refspec="${refspec}$branch"
+        fi
+
+        if vmssh rpm-ostree rebase "$refspec"; then
+            update_github error "Failed to rebase onto refspec."
+            touch $state/exit # signal testrunner to exit nicely
+            exit 0
+        fi
+
+        if [ -n "$revision" ]; then
+            # we should really be able to do this in a single step
+            # https://github.com/projectatomic/rpm-ostree/issues/212
+            vmreboot
+            vmssh rpm-ostree deploy "$revision" || rc=$?
+
+            if [ $rc == 77 ]; then
+                skip_reboot=1
+            elif [ $rc != 0 ]; then
+                update_github error "Failed to upgrade or deploy."
+                touch $state/exit # signal testrunner to exit nicely
+                exit 0
+            fi
+        fi
+    fi
+
+    if [ $skip_reboot != 1 ]; then
+        vmreboot
+    fi
+}
+
+update_github() {
+    local context=$(cat $state/parsed/context)
+    common_update_github "$context" "$@"
+}
+
+vmssh() {
+    ssh -q -n -i $state/node_key \
+        -o StrictHostKeyChecking=no \
+        -o PasswordAuthentication=no \
+        -o UserKnownHostsFile=/dev/null \
+        root@$(cat $outdir/node_addr) "$@"
+}
+
+vmreboot() {
+    vmssh systemctl reboot || :
+    sleep 3 # give time for port to go down
+    ssh_wait $(cat $outdir/node_addr) $state/node_key
+}
+
+on_atomic_host() {
+    vmssh test -f /run/ostree-booted
+}
+
+main "$@"

sample.redhat-ci.yml

@@ -1,11 +1,5 @@
-# The current format is very simple and may change in the
-# future as more features get added, though we will try to
-# maintain backwards compatibility (or help projects migrate
-# to the new format).
-
-# REQUIRED (one of 'host' or 'container')
-# All details about the host to provision go under the host
-# key, though for now, 'distro' is the only handled child:
+# REQUIRED (only one of 'host' or 'container' or 'cluster')
+# Provision a single host.
 host:
     # REQUIRED
     # Specify the distro to provision. More options will be
@@ -41,12 +35,36 @@ host:
       # omitted, the latest commit is used.
       revision: 7.145.42
 
-# REQUIRED (one of 'host' or 'container')
+# REQUIRED (only one of 'host' or 'container' or 'cluster')
+# Provision a container.
 container:
     # REQUIRED
     # Specify an FQIN or Docker Hub image.
     image: fedora:24
 
+# REQUIRED (only one of 'host' or 'container' or 'cluster')
+# Provision multiple hosts.
+cluster:
+    # REQUIRED
+    # List of hosts to provision. The same keys as above are
+    # accepted.
+    hosts:
+        # REQUIRED
+        # Node hostname. Also makes the environment variable
+        # $RHCI_{name}_IP available (with dots and dashes
+        # replaced by underscores).
+      - name: host1
+        distro: centos/7/atomic
+        ostree: latest
+      - name: host2
+        distro: fedora/24/cloud
+    # OPTIONAL
+    # If specified, the scripts are run on this container.
+    # If omitted, the scripts are run on the first host
+    # listed in the 'hosts' list.
+    container:
+        image: fedora:24
+
 # OPTIONAL
 # List the branches to test. If omitted, only the master
 # branch is tested.
@@ -111,6 +129,7 @@ build:
 tests:
     - make check
     - make installcheck
+    - ansible-playbook -i host1,$RHCI_host2_IP, playbook.yml
 
 # OPTIONAL
 # Time to allow before aborting tests. Must satisfy regex
@@ -143,9 +162,10 @@ context: 'My other testsuite'
 # To unset an inherited key, simply leave off its value.
 artifacts:
 
-# As a convenience, specifying 'host' automatically unsets
-# 'container' if it was inherited (and vice-versa), so that
-# there is no need to explicitly unset it.
+# As a convenience, specifying one of the 'host',
+# 'container' or 'cluster' keys automatically unsets the
+# other two if inherited, so that there is no need to
+# explicitly unset them.
 host:
   distro: fedora/24/atomic
 

spawner.py

@@ -100,6 +100,7 @@ def read_pipe(idx, fd):
     while s != b'':
         if not s.endswith(b'\n'):
             s += b'\n'
+            # pylint: disable=no-member
         sys.stdout.buffer.write((b'[%d] ' % idx) + s)
         s = fd.readline()