run-tests

#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# This file is used to run all Mercurial-related tests in this repository.

from __future__ import print_function

import argparse
import errno
import math
import multiprocessing
import os
import subprocess
import sys
import time
import uuid
from pathlib import Path

# Mercurial's run-tests.py isn't meant to be loaded as a module. We do it
# anyway.
HERE = os.path.dirname(os.path.abspath(__file__))
RUNTESTS = os.path.join(HERE, "pylib", "mercurial-support", "run-tests.py")


def try_find_hg(with_hg):
    """Attempt to find `with_hg` in the known install directory for Mercurial
    versions.

    Args:
      with_hg (str): a Mercurial version string. Example: `5.3.2`
    """
    mercurials = Path("/app/venv/mercurials")
    hg = mercurials / with_hg / "bin" / "hg"

    if not hg.exists():
        valid_versions = "\n    - ".join(
            str(path.name) for path in mercurials.iterdir()
        )
        error = (
            f"Couldn't find hg version {with_hg} at a known path.\n\n"
            f"Valid versions: \n\n    - {valid_versions}"
        )
        raise ValueError(error)

    return str(hg)


def int_floor(number):
    """Return the integer floor of a number.

    >>> int_floor(14.0)
    14
    >>> int_floor(14)
    14
    int_floor(14.5)
    14
    """
    return int(math.floor(number))


if __name__ == "__main__":
    if "VIRTUAL_ENV" not in os.environ:
        activate = os.path.join(HERE, "venv", "bin", "activate_this.py")
        with open(activate) as f:
            exec(f.read(), dict(__file__=activate))
        sys.executable = os.path.join(HERE, "venv", "bin", "python")
        os.environ["VIRTUAL_ENV"] = os.path.join(HERE, "venv")

    import vcttesting.docker as vctdocker

    import vcttesting.hgmo as vcthgmo

    from vcttesting.testing import (
        get_extensions,
        get_hg_version,
        get_test_files,
        run_nose_tests,
        remove_err_files,
    )

    parser = argparse.ArgumentParser()
    parser.add_argument("--with-hg")
    parser.add_argument("-C", "--cover", action="store_true")
    parser.add_argument("-j", "--jobs", type=int)
    parser.add_argument(
        "--all-hg-versions",
        action="store_true",
        help="Test against all marked compatible Mercurial versions",
    )
    parser.add_argument(
        "--no-hg-tip",
        action="store_true",
        help="Do not run tests against the @ bookmark of hg",
    )
    parser.add_argument(
        "--no-unit", action="store_true", help="Do not run Python unit tests"
    )
    parser.add_argument(
        "--use-last-images",
        action="store_true",
        help="Skip building new images when possible",
    )
    parser.add_argument(
        "--headless",
        action="store_true",
        help="Only run tests that do not require a framebuffer "
        "(skips Selenium tests)",
    )
    parser.add_argument(
        "--no-docker",
        action="store_true",
        help="Only run tests that do not require Docker",
    )
    parser.add_argument(
        "--run-flaky",
        action="store_true",
        help="Run tests marked as intermittent failures",
    )

    options, extra = parser.parse_known_args(sys.argv)

    # This is somewhat arbitrary. It makes version management logic a bit
    # more complicated and isn't worth supporting.
    if options.all_hg_versions and options.with_hg:
        print("abort: --all-hg-versions and --with-hg are mutually exclusive")
        sys.exit(1)

    # some arguments belong to us only. Don't pass it along to run-tests.py.
    filter_args = {
        "--all-hg-versions",
        "--no-hg-tip",
        "--use-last-images",
        "--headless",
        "--no-docker",
        "--run-flaky",
    }
    hg_harness_args = [a for a in sys.argv if a not in filter_args]
    hg_harness_args[0] = RUNTESTS

    if options.no_docker:
        os.environ["SKIP_DOCKER_TESTS"] = "1"

    # Strip code coverage flags
    hg_harness_args = [a for a in hg_harness_args if a != "--cover"]
    verbose = "-v" in hg_harness_args or "--verbose" in hg_harness_args

    os.environ["BUGZILLA_USERNAME"] = "admin@example.com"
    os.environ["BUGZILLA_PASSWORD"] = "password"

    orig_args = list(hg_harness_args)

    extensions = get_extensions()
    test_files = get_test_files(extensions)
    extension_tests = test_files["extension"]
    unit_tests = test_files["unit"]
    hg_tests = test_files["hg"]
    # Tests requiring Docker in some form.
    docker_tests = {t for t, reqs in test_files["docker_requirements"].items() if reqs}

    possible_tests = [
        os.path.normpath(os.path.abspath(a)) for a in extra[1:] if not a.startswith("-")
    ]
    # Filter out arguments that might be tests.
    hg_harness_args = [
        a
        for a in hg_harness_args
        if os.path.normpath(os.path.abspath(a)) not in possible_tests
    ]
    requested_tests = []
    for t in possible_tests:
        if t in test_files["all"]:
            requested_tests.append(t)
            continue

        if os.path.isdir(t):
            t = os.path.normpath(t)
            for test in test_files["all"]:
                common = os.path.commonprefix([t, test])
                common = os.path.normpath(common)
                if common == t and test not in requested_tests:
                    requested_tests.append(test)

            continue

    # Explicitly use our own HG from the virtualenv so other installs
    # on the system don't interfere.
    if not options.with_hg:
        hg = os.path.join(os.path.dirname(sys.executable), "hg")
        hg_harness_args.extend(["--with-hg", hg])
    elif options.with_hg and not os.path.exists(options.with_hg):
        # Try and parse the Mercurial version as a convenience string
        try:
            hg = try_find_hg(options.with_hg)
        except ValueError as e:
            sys.exit(str(e))

        hg_harness_args.extend((["--with-hg", hg]))
    elif options.with_hg:
        hg = options.with_hg
    hgversion, _pyversion = get_hg_version(hg)
    if hgversion is None:
        print("Unable to determine Mercurial version")
        sys.exit(1)

    # Add `--blacklist=testing/flaky-tests.txt` to the command line
    if not options.run_flaky:
        flaky_tests_path = os.path.join(HERE, "testing", "flaky-tests.txt")
        hg_harness_args.extend(["--blacklist", flaky_tests_path])

    run_hg_tests = []
    run_unit_tests = []

    # All tests unless we got an argument that is a test.
    if not requested_tests:
        run_hg_tests.extend(extension_tests)
        run_hg_tests.extend(hg_tests)
        if not options.no_unit:
            run_unit_tests.extend(unit_tests)
    else:
        for t in requested_tests:
            if t in unit_tests:
                if not options.no_unit:
                    run_unit_tests.append(t)
            else:
                run_hg_tests.append(t)

    run_all_tests = run_hg_tests + run_unit_tests

    run_hg_tests_docker = [t for t in run_hg_tests if t in docker_tests]
    run_hg_tests_no_docker = [t for t in run_hg_tests if t not in docker_tests]
    run_unit_tests_docker = [t for t in run_unit_tests if t in docker_tests]
    run_unit_tests_no_docker = [t for t in run_unit_tests if t not in docker_tests]

    # By default, run 1.25 * CPU count non-Docker tests in parallel.
    # Docker tests are much more expensive. So give them 2 cores each.
    # Memory constrained environments may not be able to handle the Docker
    # default. We may want to consider taking memory capacity into account
    # as well.
    no_docker_jobs = max(int_floor(multiprocessing.cpu_count() * 1.25), 1)
    docker_jobs = max(int_floor(multiprocessing.cpu_count() / 2), 1)

    if options.jobs:
        no_docker_jobs = options.jobs
        docker_jobs = options.jobs

    # Enable tests to interact with our Docker controlling script.
    have_docker = False
    docker_state = os.path.join(HERE, ".dockerstate")
    docker_url, docker_tls = vctdocker.params_from_env(os.environ)
    docker = vctdocker.Docker(docker_url, tls=docker_tls)
    runtests_label = str(uuid.uuid4())
    if not options.no_docker:
        hgcluster = vcthgmo.HgCluster(docker)

        if docker.is_alive():
            have_docker = True
            os.environ["DOCKER_HOSTNAME"] = docker.docker_hostname

            # Add a label to the containers so any orphans can be
            # shutdown after the test run is complete. This should permeate
            # to the environment of the `docker-compose` process created
            # by `hgmo start`.
            os.environ["RUNTESTS_LABEL"] = runtests_label

            # We build the base docker images in the test runner because doing it
            # from tests would be racey. It is easier to do it here instead of
            # complicating code with locks.
            #
            # But only do this if a test we are running utilizes Docker.
            if not options.use_last_images:
                print("generating Docker images needed for tests")
                t_start = time.time()

                hgcluster.build()

                t_end = time.time()
                print("got Docker images in %.2fs" % (t_end - t_start))

    print("Will run %d non-Docker tests concurrently" % no_docker_jobs)
    if have_docker:
        print("Will run %d Docker tests concurrently" % docker_jobs)

    os.environ["HGVERSION"] = hgversion

    res = 0

    # Don't run with main Mercurial if --all-hg-versions is used because
    # --all-hg-versions should run the same version that is installed in the
    # virtualenv.
    if run_hg_tests and not options.all_hg_versions:
        # The Mercurial test harness has been observed to not remove the .err
        # files after execution. This is probably a result of us using
        # separate directories. Manually remove .err files.
        remove_err_files(run_hg_tests)

        # Docker tests are a bit heavyweight and tend to fail more than
        # non-Docker tests. We perform multiple invocations. First without
        # Docker. Then with.

        with docker.auto_clean_orphans(runtests_label):
            print("Running non-Docker Mercurial tests")
            args = (
                hg_harness_args + ["-j", str(no_docker_jobs)] + run_hg_tests_no_docker
            )
            res = subprocess.call(args, cwd=HERE)

            if have_docker and run_hg_tests_docker:
                # We should ideally not leak containers and images. But until
                # the Mercurial test harness allows tests to register cleanup
                # actions, there is only so much we can do.
                print("Running Docker Mercurial tests")
                args = hg_harness_args + ["-j", str(docker_jobs)] + run_hg_tests_docker
                res2 = subprocess.call(args, cwd=HERE)
                if res2:
                    res = res2
            elif run_hg_tests_docker:
                print(
                    "Skipping %d Docker Mercurial tests because Docker not "
                    "available" % len(run_hg_tests_docker)
                )

    if run_unit_tests:
        nose_res = 0

        if run_unit_tests_no_docker:
            print("Running non-Docker unit tests")
            nose_res = run_nose_tests(
                run_unit_tests_no_docker, no_docker_jobs, verbose=verbose
            )
            if nose_res:
                res = nose_res

        if have_docker and run_unit_tests_docker:
            print("Running Docker unit tests")
            with docker.auto_clean_orphans(runtests_label):
                nose_res2 = run_nose_tests(
                    run_unit_tests_docker, docker_jobs, verbose=verbose
                )
            if nose_res2:
                res = nose_res2
        elif run_unit_tests_docker:
            print(
                "Skipping %d Docker unit tests because Docker not available"
                % len(run_unit_tests_docker)
            )

    # If we're running the full compatibility run, figure out what versions
    # apply to what and run them.
    if options.all_hg_versions:
        mercurials_dir = os.path.join(os.environ["VIRTUAL_ENV"], "mercurials")

        # Maps directories/versions to lists of tests to run.
        # We normalize X.Y.Z to X.Y for compatibility because the monthly
        # minor releases of Mercurial shouldn't change behavior. If an
        # extension is marked as compatible with X.Y, we run its tests
        # against all X.Y and X.Y.Z releases seen on disk.
        versions = {}
        for dirver in os.listdir(mercurials_dir):
            if dirver.startswith(".") or dirver == "@":
                continue

            normdirver = ".".join(dirver.split(".")[0:2])

            tests = versions.setdefault(dirver, set())
            tests |= set(hg_tests)

            for m in extensions:
                for extver in m["testedwith"]:
                    normever = b".".join(extver.split(b".")[0:2])

                    if extver == dirver or normever.decode("utf-8") == normdirver:
                        tests |= m["tests"]

        def run_hg_tests(version, tests):
            if requested_tests:
                tests = [t for t in tests if t in requested_tests]

            if not tests:
                return

            remove_err_files(tests)

            hg_path = os.path.join(mercurials_dir, version, "bin", "hg")

            common_args = [RUNTESTS] + ["--with-hg", hg_path]

            tests_docker = sorted(t for t in tests if t in docker_tests)
            tests_no_docker = sorted(t for t in tests if t not in docker_tests)

            real_version, _pyversion = get_hg_version(hg_path)

            print(
                "Testing with Mercurial %s (resolved to %s)" % (version, real_version)
            )
            sys.stdout.flush()
            os.environ["HGVERSION"] = real_version

            print("Running non-Docker Mercurial tests")
            args = common_args + ["-j", str(no_docker_jobs)] + tests_no_docker
            subprocess.call(args, cwd=HERE)
            sys.stdout.flush()
            sys.stderr.flush()

            if have_docker and tests_docker:
                print("Running Docker Mercurial tests")
                with docker.auto_clean_orphans(runtests_label):
                    args = common_args + ["-j", str(docker_jobs)] + tests_docker
                    subprocess.call(args, cwd=HERE)
                sys.stdout.flush()
                sys.stderr.flush()
            elif tests_docker:
                print(
                    "Skipping %d Docker Mercurial tests because Docker not "
                    "available" % len(tests_docker)
                )

        for version, tests in sorted(versions.items()):
            res2 = run_hg_tests(version, tests)
            if res2:
                res = res2

        # Run all tests against @ because we always want to be compatible
        # with the bleeding edge of development.
        if not options.no_hg_tip:
            all_hg_tests = []
            for m in extensions:
                all_hg_tests.extend(sorted(m["tests"]))
            all_hg_tests.extend(hg_tests)
            run_hg_tests("@", all_hg_tests)

    sys.exit(res)