From 5c07ece9e42e1bd0dd6b51e738bf418d996ed6e7 Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Tue, 9 Jun 2020 23:44:21 +0100 Subject: [PATCH] WIP: Add k8s integration tests to CI Current issue appears to be that the results are not able to write back to the result collector, so hopefully an easy fix. TODO: * fix result collection * make test messages clear * add kuberentes integration testing to Travis CI * make a branch to convert bash scripts to python --- .../src/yaml/conf/kubernetes/statemgr.yaml | 2 +- .../src/python/test_runner/main.py | 10 +- .../src/python/topology_test_runner/main.py | 12 +- scripts/release/docker-images | 4 +- scripts/travis/install.sh | 13 +++ scripts/travis/k8s.sh | 99 +++++++++++++---- scripts/travis/k8s.sh.kind.yaml | 2 + scripts/travis/test.sh | 103 +++++++++++------- vagrant/Vagrantfile | 18 +-- vagrant/init.sh | 87 ++++++--------- vagrant/local-ci.sh | 22 ++-- 11 files changed, 226 insertions(+), 146 deletions(-) create mode 100755 scripts/travis/install.sh diff --git a/heron/config/src/yaml/conf/kubernetes/statemgr.yaml b/heron/config/src/yaml/conf/kubernetes/statemgr.yaml index df1c05752f1..25f2a45f653 100644 --- a/heron/config/src/yaml/conf/kubernetes/statemgr.yaml +++ b/heron/config/src/yaml/conf/kubernetes/statemgr.yaml @@ -19,7 +19,7 @@ heron.class.state.manager: org.apache.heron.statemgr.zookeeper.curator.CuratorStateManager # local state manager connection string -heron.statemgr.connection.string: +heron.statemgr.connection.string: 127.0.0.1:2181 # path of the root address to store the state in a local file system heron.statemgr.root.path: "/heron" diff --git a/integration_test/src/python/test_runner/main.py b/integration_test/src/python/test_runner/main.py index fb44a10eeac..f095fa2b81b 100644 --- a/integration_test/src/python/test_runner/main.py +++ b/integration_test/src/python/test_runner/main.py @@ -20,10 +20,10 @@ import logging import os import pkgutil +import random import re import sys import time -import uuid from http.client import HTTPConnection from threading import Lock, Thread @@ -313,6 +313,7 @@ def run_tests(conf, test_args): ''' Run the test for each topology specified in the conf file ''' lock = Lock() timestamp = time.strftime('%Y%m%d%H%M%S') + run_fingerprint = f"{timestamp}-{random.randint(0, 2**16):04x}" http_server_host_port = "%s:%d" % (test_args.http_server_hostname, test_args.http_server_port) @@ -361,8 +362,11 @@ def _run_single_test(topology_name, topology_conf, test_args, http_server_host_p lock.release() test_threads = [] - for topology_conf in test_topologies: - topology_name = ("%s_%s_%s") % (timestamp, topology_conf["topologyName"], str(uuid.uuid4())) + for i, topology_conf in enumerate(test_topologies, 1): + # this name has to be valid for all tested schedullers, state managers, etc. + topology_name = f"run-{run_fingerprint}-test-{i:03}" + # TODO: make sure logs describe the test/topology that fails, as now topology_name is opaque + # topology_conf["topologyName"] classpath = topology_classpath_prefix + topology_conf["classPath"] # if the test includes an update we need to pass that info to the topology so it can send diff --git a/integration_test/src/python/topology_test_runner/main.py b/integration_test/src/python/topology_test_runner/main.py index 12d86851767..33d4c6803d0 100644 --- a/integration_test/src/python/topology_test_runner/main.py +++ b/integration_test/src/python/topology_test_runner/main.py @@ -20,10 +20,10 @@ import logging import os import pkgutil +import random import re import sys import time -import uuid from http.client import HTTPConnection from threading import Lock, Thread @@ -255,6 +255,8 @@ def __init__(self, topology_name, cluster): self.state_mgr.start() def _load_state_mgr(self, cluster): + # this should use cli_config_path (after expanding HOME, or changing default to use ~ instead of $HOME) + # that way can have test copy of config state_mgr_config = configloader.load_state_manager_locations(cluster, os.getenv("HOME") +'/.heron/conf/'+cluster + '/statemgr.yaml') @@ -496,6 +498,7 @@ def run_topology_tests(conf, test_args): """ lock = Lock() timestamp = time.strftime('%Y%m%d%H%M%S') + run_fingerprint = f"{timestamp}-{random.randint(0, 2**16):04x}" http_server_host_port = "%s:%d" % (test_args.http_hostname, test_args.http_port) @@ -562,8 +565,11 @@ def _run_single_test(topology_name, topology_conf, test_args, http_server_host_p lock.release() test_threads = [] - for topology_conf in test_topologies: - topology_name = ("%s_%s_%s") % (timestamp, topology_conf["topologyName"], str(uuid.uuid4())) + for i, topology_conf in enumerate(test_topologies, 1): + # this name has to be valid for all tested schedullers, state managers, etc. + topology_name = f"run-{run_fingerprint}-test-{i:03}" + # TODO: make sure logs describe the test/topology that fails, as now topology_name is opaque + # topology_conf["topologyName"] classpath = topology_classpath_prefix + topology_conf["classPath"] update_args = "" diff --git a/scripts/release/docker-images b/scripts/release/docker-images index 7316abcf049..beba5121bd6 100755 --- a/scripts/release/docker-images +++ b/scripts/release/docker-images @@ -66,7 +66,8 @@ def build_dockerfile( log_run([str(BUILD_IMAGE), dist, tag, scratch], log) tar = Path(scratch) / f"heron-docker-{tag}-{dist}.tar.gz" tar_out = out_dir / tar.name - tar.replace(tar_out) + # using this rather than replace as it works if moving between devices + shutil.move(tar, tar_out) logging.info("docker image complete: %s", tar_out) return tar_out @@ -107,7 +108,6 @@ def build_target(tag: str, target: str) -> typing.List[Path]: scratch = Path(tempfile.mkdtemp(prefix=f"build-{target}-")) log_path = scratch / "log.txt" log = log_path.open("w") - logging.debug("building %s", target) try: tar = build_dockerfile(scratch, target, tag, out_dir, log) diff --git a/scripts/travis/install.sh b/scripts/travis/install.sh new file mode 100755 index 00000000000..c7aae4e0c12 --- /dev/null +++ b/scripts/travis/install.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -o errexit -o nounset -o pipefail + +DIR=`dirname $0` +UTILS=${DIR}/../shutils +source ${UTILS}/common.sh + +# Autodiscover the platform +PLATFORM=$(discover_platform) +echo "Using $PLATFORM platform" + +# install heron locally +bazel --bazelrc=tools/travis/bazel.rc run --config=$PLATFORM -- scripts/packages:heron-install.sh --user diff --git a/scripts/travis/k8s.sh b/scripts/travis/k8s.sh index 9cb5af76e12..49fee8597e2 100755 --- a/scripts/travis/k8s.sh +++ b/scripts/travis/k8s.sh @@ -1,12 +1,21 @@ #!/usr/bin/env bash :<<'DOC' set NO_CACHE=1 to always rebuild images. +set DEBUG=1 to not clean up + +This requires kubectl, and kind. DOC set -o errexit -o nounset -o pipefail TAG=test HERE="$(cd "$(dirname "$0")"; pwd -P)" -ROOT="$(cd "$HERE/../.."; pwd -P)" +ROOT="$(git rev-parse --show-toplevel)" + +CLUSTER_NAME="kubernetes" + +API_URL="http://localhost:8001/api/v1/namespaces/default/services/heron-apiserver:9000/proxy" +TRACKER_URL="http://localhost:8001/api/v1/namespaces/default/services/heron-tracker:8888/proxy" +UI_URL="http://localhost:8001/api/v1/namespaces/default/services/heron-ui:8889/proxy" function bazel_file { # bazel_file VAR_NAME //some/build:target @@ -23,14 +32,6 @@ function kind_images { docker exec -it kind-control-plane crictl images } -function install_helm3 { - pushd /tmp - curl --location https://get.helm.sh/helm-v3.2.1-linux-amd64.tar.gz --output helm.tar.gz - tar --extract --file=helm.tar.gz --strip-components=1 linux-amd64/helm - mv helm ~/.local/bin/ - popd -} - function action { ( tput setaf 4; @@ -39,12 +40,23 @@ function action { ) > /dev/stderr } +function clean { + if [ -z "${DEBUG-}" ]; then + action "Cleaning up" + kind delete cluster + kill -TERM -$$ + else + action "Not cleaning up due to DEBUG flag being set" + fi +} function create_cluster { # trap "kind delete cluster" EXIT if [ -z "$(kind get clusters)" ]; then action "Creating kind cluster" kind create cluster --config="$0.kind.yaml" + else + action "Using existing kind cluster" fi } @@ -59,7 +71,7 @@ function get_image { out="$expected" else action "Creating heron image" - local gz="$(scripts/release/docker-images build test debian10)" + local gz="$(scripts/release/docker-images build test "$distro")" # XXX: must un .gz https://github.com/kubernetes-sigs/kind/issues/1636 gzip --decompress "$gz" out="${gz%%.gz}" @@ -67,14 +79,34 @@ function get_image { archive="$out" } +function url_wait { + local seconds="$1" + local url="$2" + local retries=0 + tput sc + while ! curl "$url" --location --fail --silent --output /dev/null --write-out "attempt $retries: %{http_code}" + do + retries=$((retries + 1)) + if [ $retries -eq 60 ]; then + echo + return 1 + fi + sleep 1 + tput el1 + tput rc + tput sc + done + echo +} + +trap clean EXIT create_cluster -get_image debian10 +#get_image debian10 +get_image ubuntu20.04 heron_archive="$archive" action "Loading heron docker image" kind load image-archive "$heron_archive" -#image_heron="docker.io/bazel/scripts/images:heron" -#image_heron="$heron_image" image_heron="heron/heron:$TAG" action "Loading bookkeeper image" @@ -82,11 +114,36 @@ image_bookkeeper="docker.io/apache/bookkeeper:4.7.3" docker pull "$image_bookkeeper" kind load docker-image "$image_bookkeeper" -action "Deploying heron with helm" -# install heron in kind using helm -bazel_file helm_yaml //scripts/packages:index.yaml -helm install heron "$(dirname "$helm_yaml")/heron-0.0.0.tgz" \ - --set image="$image_heron" \ - --set imagePullPolicy=IfNotPresent \ - --set bookieReplicas=1 \ - --set zkReplicas=1 +action "Deploying to kubernetes" +#kubectl create namespace default +kubectl config set-context kind-kind --namespace=default + +# deploy +DIR=./deploy/kubernetes/minikube +sed "s#heron/heron:latest#$image_heron#g" ${DIR}/zookeeper.yaml > /tmp/zookeeper.yaml +sed "s#heron/heron:latest#$image_heron#g" ${DIR}/tools.yaml > /tmp/tools.yaml +sed "s#heron/heron:latest#$image_heron#g" ${DIR}/apiserver.yaml > /tmp/apiserver.yaml + +kubectl proxy -p 8001 & + +kubectl create -f /tmp/zookeeper.yaml +kubectl get pods +kubectl create -f ${DIR}/bookkeeper.yaml +kubectl create -f /tmp/tools.yaml +kubectl create -f /tmp/apiserver.yaml + +action "waiting for API server" +url_wait 120 "$API_URL/api/v1/version" +action "waiting for UI" +url_wait 120 "$UI_URL/proxy" + +kubectl port-forward service/zookeeper 2181:2181 & + +action "API: $API_URL" +action "Tracker: $TRACKER_URL" +action "UI: $UI_URL" +action "Zookeeper: 127.0.0.1:2181" + +heron config "$CLUSTER_NAME" set service_url "$API_URL" + +"$HERE/test.sh" diff --git a/scripts/travis/k8s.sh.kind.yaml b/scripts/travis/k8s.sh.kind.yaml index f76d19f36a2..d167485b141 100644 --- a/scripts/travis/k8s.sh.kind.yaml +++ b/scripts/travis/k8s.sh.kind.yaml @@ -1,4 +1,6 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 +# to change kubernetes version, see the tags+digests here: https://hub.docker.com/r/kindest/node/tags nodes: - role: control-plane +# image: kindest/node:v1.16.9@sha256:7175872357bc85847ec4b1aba46ed1d12fa054c83ac7a8a11f5c268957fd5765 diff --git a/scripts/travis/test.sh b/scripts/travis/test.sh index e1915fe3d5a..461dc68aa44 100755 --- a/scripts/travis/test.sh +++ b/scripts/travis/test.sh @@ -40,11 +40,13 @@ start_timer "$T" ${UTILS}/save-logs.py "heron_build_integration_test.txt" bazel --bazelrc=tools/travis/bazel.rc build --config=$PLATFORM integration_test/src/... end_timer "$T" +:<<'SKIPPING-FOR-NOW' # install heron T="heron install" start_timer "$T" -${UTILS}/save-logs.py "heron_install.txt" bazel --bazelrc=tools/travis/bazel.rc run --config=$PLATFORM -- scripts/packages:heron-install.sh --user +${UTILS}/save-logs.py "heron_install.txt" "$DIR/install.sh" end_timer "$T" +SKIPPING-FOR-NOW # install tests T="heron tests install" @@ -54,11 +56,13 @@ end_timer "$T" pathadd ${HOME}/bin/ +:<<'SKIPPING-FOR-NOW' # run local integration test T="heron integration_test local" start_timer "$T" ./bazel-bin/integration_test/src/python/local_test_runner/local-test-runner end_timer "$T" +SKIPPING-FOR-NOW # initialize http-server for integration tests T="heron integration_test http-server initialization" @@ -68,45 +72,68 @@ http_server_id=$! trap "kill -9 $http_server_id" SIGINT SIGTERM EXIT end_timer "$T" -# run the scala integration test -T="heron integration_test scala" -start_timer "$T" -${HOME}/bin/test-runner \ - -hc heron -tb ${SCALA_INTEGRATION_TESTS_BIN} \ - -rh localhost -rp 8080\ - -tp ${HOME}/.herontests/data/scala \ - -cl local -rl heron-staging -ev devel -end_timer "$T" +function integration_test { + # integration_test $cluster $test_bin + local cluster="$1" + local language="$2" + local test_bin="$3" + local host port + if [ "$cluster" == "local" ]; then + host=localhost + port=8080 + else + #host="$(docker inspect kind-control-plane --format '{{ .NetworkSettings.Networks.kind.IPAddress }}')" + host="host.docker.internal" + host="$(docker inspect --format='{{range .NetworkSettings.Networks}}{{.Gateway}}{{end}}' kind-control-plane)" + #host=172.17.0.1 + port=8080 + fi -# run the java integration test -T="heron integration_test java" -start_timer "$T" -${HOME}/bin/test-runner \ - -hc heron -tb ${JAVA_INTEGRATION_TESTS_BIN} \ - -rh localhost -rp 8080\ - -tp ${HOME}/.herontests/data/java \ - -cl local -rl heron-staging -ev devel -end_timer "$T" + local T="$language integration test on $cluster" + start_timer "$T" + "${HOME}/bin/test-runner" \ + --heron-cli-path=heron \ + --tests-bin-path="$test_bin" \ + --http-server-hostname="$host" \ + --http-server-port="$port" \ + --topologies-path="${HOME}/.herontests/data/$language" \ + --cluster="$cluster" \ + --role=heron-staging \ + --env=devel \ + #--cli-config-path="$cluster" + end_timer "$T" +} -# run the python integration test -T="heron integration_test python" -start_timer "$T" -${HOME}/bin/test-runner \ - -hc heron -tb ${PYTHON_INTEGRATION_TESTS_BIN} \ - -rh localhost -rp 8080\ - -tp ${HOME}/.herontests/data/python \ - -cl local -rl heron-staging -ev devel -end_timer "$T" +# look at making the whole test script a function, e.g. +# ./test.sh heron-k8s +# ./test.sh local +# concurrent runs from different clusters are fine? +for cluster in kubernetes local; do + if [ "$cluster" == "local" ]; then + host=localhost + port=8080 + else + host="$(docker inspect kind-control-plane --format='{{range .NetworkSettings.Networks}}{{.Gateway}}{{end}}')" + #host=172.18.0.1 + port=8080 + fi + integration_test "$cluster" scala "${SCALA_INTEGRATION_TESTS_BIN}" + integration_test "$cluster" java "${JAVA_INTEGRATION_TESTS_BIN}" + integration_test "$cluster" python ${PYTHON_INTEGRATION_TESTS_BIN} \ -# run the java integration topology test -T="heron integration_topology_test java" -start_timer "$T" -${HOME}/bin/topology-test-runner \ - -hc heron -tb ${JAVA_INTEGRATION_TOPOLOGY_TESTS_BIN} \ - -rh localhost -rp 8080\ - -tp ${HOME}/.herontests/data/java/topology_test \ - -cl local -rl heron-staging -ev devel -end_timer "$T" + T="heron integration_topology_test java" + start_timer "$T" + "${HOME}/bin/topology-test-runner" \ + --heron-cli-path=heron \ + --tests-bin-path="${JAVA_INTEGRATION_TOPOLOGY_TESTS_BIN}" \ + --http-hostname="$host" \ + --http-port="$port" \ + --topologies-path="${HOME}/.herontests/data/java/topology_test" \ + --cluster="$cluster" \ + --role=heron-staging \ + --env=devel + #--cli-config-path="$cluster" + end_timer "$T" +done print_timer_summary - diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index ad39ee48deb..fc97b970bc4 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -60,21 +60,9 @@ Vagrant.configure(2) do |config| master.vm.hostname = "master" master.vm.network :private_network, ip: NODES["master"] + # this port is used in ./scripts/travis/k8s.sh to expose the kubectl proxy + master.vm.network :forwarded_port, guest: 8001, host: 8001 - master.vm.provision "shell", path: "init.sh", args: "master" - end - - (0..SLAVES-1).each do |i| - config.vm.define "slave#{i}" do |slave| - slave.vm.provider "virtualbox" do |v| - v.memory = 2048 - v.cpus = 2 - end - - slave.vm.hostname = "slave#{i}" - slave.vm.network :private_network, ip: NODES[slave.vm.hostname] - - slave.vm.provision "shell", path: "init.sh", args: "slave" - end + master.vm.provision "shell", path: "init.sh" end end diff --git a/vagrant/init.sh b/vagrant/init.sh index 5f4f0925bd7..7573908a2ef 100644 --- a/vagrant/init.sh +++ b/vagrant/init.sh @@ -16,34 +16,31 @@ set -o errexit -o nounset -o pipefail # See the License for the specific language governing permissions and # limitations under the License. -install_mesos() { - mode=$1 # master | slave - apt-get -qy install mesos=0.25.0* - - echo "zk://master:2181/mesos" > /etc/mesos/zk - echo '5mins' > /etc/mesos-slave/executor_registration_timeout - - ip=$(cat /etc/hosts | grep `hostname` | grep -E -o "([0-9]{1,3}[\.]){3}[0-9]{1,3}") - echo $ip > "/etc/mesos-$mode/ip" - - if [ "$mode" == "master" ]; then - ln -s /lib/init/upstart-job /etc/init.d/mesos-master - service mesos-master start - else - apt-get -qy remove zookeeper - fi +bazelVersion=3.0.0 - ln -s /lib/init/upstart-job /etc/init.d/mesos-slave - echo 'docker,mesos' > /etc/mesos-slave/containerizers - service mesos-slave start +install_docker() { + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + apt-get update + apt-get install -qy docker-ce docker-ce-cli containerd.io + usermod -aG docker vagrant } -install_marathon() { - apt-get install -qy marathon=0.10.0* - service marathon start +install_k8s_stack() { + install_docker + # install kubectl + curl -Lo /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/v1.18.3/bin/linux/amd64/kubectl + chmod +x /usr/local/bin/kubectl + # install kind + curl -Lo /usr/local/bin/kind "https://kind.sigs.k8s.io/dl/v0.8.1/kind-$(uname)-amd64" + chmod +x /usr/local/bin/kind + # helm + curl --location https://get.helm.sh/helm-v3.2.1-linux-amd64.tar.gz \ + | tar --extract --gzip linux-amd64/helm --to-stdout \ + > /usr/local/bin/helm + chmod +x /usr/local/bin/helm } -bazelVersion=3.0.0 bazel_install() { apt-get install -y automake cmake gcc g++ zlib1g-dev zip pkg-config wget libssl-dev libunwind-dev mkdir -p /opt/bazel @@ -62,24 +59,11 @@ build_heron() { popd } -if [[ "$1" != "master" && $1 != "slave" ]]; then - echo "Usage: $0 master|slave" - exit 1 -fi -mode="$1" - cd /vagrant/vagrant # name resolution cp .vagrant/hosts /etc/hosts -# XXX: not needed? -# ssh key -key=".vagrant/ssh_key.pub" -if [ -f "$key" ]; then - cat $key >> /home/vagrant/.ssh/authorized_keys -fi - # disable ipv6 echo -e "\nnet.ipv6.conf.all.disable_ipv6 = 1\n" >> /etc/sysctl.conf sysctl -p @@ -91,23 +75,20 @@ if [ -f ".vagrant/apt-proxy" ]; then echo "Acquire::http::Proxy \"$apt_proxy\";" > /etc/apt/apt.conf.d/90-apt-proxy.conf fi -:<<'REMOVED' -# add mesosphere repo -apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E56151BF -DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') -CODENAME=$(lsb_release -cs) -echo "deb http://repos.mesosphere.io/${DISTRO} cosmic main" | tee /etc/apt/sources.list.d/mesosphere.list -REMOVED - apt-get -qy update # install deps -apt-get install -qy vim zip mc curl wget openjdk-11-jdk scala git python3-setuptools python3-dev libtool-bin libcppunit-dev python-is-python3 - -# install_mesos $mode -if [ $mode == "master" ]; then - # install_marathon - bazel_install - # switch to non-root so bazel cache can be reused when SSHing in - # su --login vagrant /vagrant/scripts/travis/ci.sh -fi +apt-get install -qy vim zip mc curl wget openjdk-11-jdk scala git python3-setuptools python3-dev libtool-bin libcppunit-dev python-is-python3 tree + +install_k8s_stack +bazel_install + +# configure environment variables required +{ + ## the install script with the --user flag will install binaries here + #echo 'export PATH=$HOME/bin:$PATH' >> ~vagrant/.bashrc + # the discover_platform helper uses this as `python -mplatform` does not put out expected info in this image + echo 'export PLATFORM=Ubuntu' >> ~vagrant/.bashrc + # set JAVA_HOME as plenty of places in heron use it to find java + echo 'export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/' >> ~vagrant/.bashrc +} diff --git a/vagrant/local-ci.sh b/vagrant/local-ci.sh index 63617bdf4a5..794ba007529 100755 --- a/vagrant/local-ci.sh +++ b/vagrant/local-ci.sh @@ -2,11 +2,12 @@ :<<'DOC' This script is for running tests in a local VM, similar to the environment used in the CI pipeline. If the targent script fails, a shell will be opened up within the VM. -To only run integration tests: - ./local-ci.sh test +To run all of the tests: + ./local-ci.sh -To run the full ci pipeline: - ./local-ci.sh ci +To run a specific command/script: + ./local-ci.sh ./scripts/travis/ci.sh + ./local-ci.sh bash The VM does not report the platform in python as expected, so PLATFORM=Ubuntu is needed to work around that for the CI script's platform discovery. @@ -23,9 +24,10 @@ if [ "$state" != "running" ]; then fi -# allows you to do `$0 test` to run only integration tests -script="${1-ci}" -env="PLATFORM=Ubuntu JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/" -# run the CI, if it fails drop into a shell -vagrant ssh master --command "cd /vagrant && $env ./scripts/travis/$script.sh" \ - || vagrant ssh master --command "cd /vagrant && $env exec bash" +if [ -z "$1" ]; then + # run the CI, if it fails drop into a shell + vagrant ssh master --command "cd /vagrant && $env ./scripts/travis/$script.sh" \ + || vagrant ssh master --command "cd /vagrant && exec bash" +else + vagrant ssh master --command "cd /vagrant && exec $1" +fi