From 85fe7bdc89189a7b62002052665c69eca3ec8e5f Mon Sep 17 00:00:00 2001 From: Casey Banner Date: Thu, 27 May 2010 15:30:52 -0400 Subject: [PATCH 001/126] Added exlude hosts functionality --- fabric/main.py | 36 +++++++++++++++++++++++------------- fabric/state.py | 6 +++++- tests/test_main.py | 24 ++++++++++++++---------- 3 files changed, 42 insertions(+), 24 deletions(-) diff --git a/fabric/main.py b/fabric/main.py index 5139a45bfc..9293c7c2f5 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -255,13 +255,14 @@ def parse_arguments(arguments): kwargs = {} hosts = [] roles = [] + exclude_hosts = [] if ':' in cmd: cmd, argstr = cmd.split(':', 1) for pair in argstr.split(','): k, _, v = pair.partition('=') if v: - # Catch, interpret host/hosts/role/roles kwargs - if k in ['host', 'hosts', 'role', 'roles']: + # Catch, interpret host/hosts/role/roles/exclude_hosts kwargs + if k in ['host', 'hosts', 'role', 'roles','exclude_hosts']: if k == 'host': hosts = [v.strip()] elif k == 'hosts': @@ -270,12 +271,14 @@ def parse_arguments(arguments): roles = [v.strip()] elif k == 'roles': roles = [x.strip() for x in v.split(';')] + elif k == 'exclude_hosts': + exclude_hosts = [x.strip() for x in v.split(';')] # Otherwise, record as usual else: kwargs[k] = v else: args.append(k) - cmds.append((cmd, args, kwargs, hosts, roles)) + cmds.append((cmd, args, kwargs, hosts, roles, exclude_hosts)) return cmds @@ -286,7 +289,7 @@ def parse_remainder(arguments): return ' '.join(arguments) -def _merge(hosts, roles): +def _merge(hosts, roles, exclude=[]): """ Merge given host and role lists into one list of deduped hosts. """ @@ -305,11 +308,17 @@ def _merge(hosts, roles): if callable(value): value = value() role_hosts += value + + merged_list = list(set(hosts + role_hosts)) + for exclude_host in exclude: + if exclude_host in merged_list: + merged_list.remove(exclude_host) + # Return deduped combo of hosts and role_hosts - return list(set(hosts + role_hosts)) + return merged_list -def get_hosts(command, cli_hosts, cli_roles): +def get_hosts(command, cli_hosts, cli_roles, cli_exclude_hosts): """ Return the host list the given command should be using. @@ -318,17 +327,18 @@ def get_hosts(command, cli_hosts, cli_roles): """ # Command line per-command takes precedence over anything else. if cli_hosts or cli_roles: - return _merge(cli_hosts, cli_roles) + return _merge(cli_hosts, cli_roles, cli_exclude_hosts) # Decorator-specific hosts/roles go next func_hosts = getattr(command, 'hosts', []) func_roles = getattr(command, 'roles', []) + func_exclude_hosts = getattr(command, 'exclude_hosts', []) if func_hosts or func_roles: - return _merge(func_hosts, func_roles) + return _merge(func_hosts, func_roles, func_exclude_hosts) # Finally, the env is checked (which might contain globally set lists from # the CLI or from module-level code). This will be the empty list if these # have not been set -- which is fine, this method should return an empty # list if no hosts have been set anywhere. - return _merge(state.env['hosts'], state.env['roles']) + return _merge(state.env['hosts'], state.env['roles'], state.env['exclude_hosts']) def update_output_levels(show, hide): @@ -366,8 +376,8 @@ def main(): for option in env_options: state.env[option.dest] = getattr(options, option.dest) - # Handle --hosts, --roles (comma separated string => list) - for key in ['hosts', 'roles']: + # Handle --hosts, --roles, --exclude-hosts (comma separated string => list) + for key in ['hosts', 'roles', 'exclude_hosts']: if key in state.env and isinstance(state.env[key], str): state.env[key] = state.env[key].split(',') @@ -440,14 +450,14 @@ def main(): commands_to_run.append((r, [], {}, [], [])) # At this point all commands must exist, so execute them in order. - for name, args, kwargs, cli_hosts, cli_roles in commands_to_run: + for name, args, kwargs, cli_hosts, cli_roles, cli_exclude_hosts in commands_to_run: # Get callable by itself command = commands[name] # Set current command name (used for some error messages) state.env.command = name # Set host list (also copy to env) state.env.all_hosts = hosts = get_hosts( - command, cli_hosts, cli_roles) + command, cli_hosts, cli_roles, cli_exclude_hosts) # If hosts found, execute the function on each host in turn for host in hosts: # Preserve user diff --git a/fabric/state.py b/fabric/state.py index 66af96a15e..061e32ddfe 100644 --- a/fabric/state.py +++ b/fabric/state.py @@ -141,6 +141,11 @@ def _rc_path(): help="comma-separated list of roles to operate on" ), + make_option('-x', '--exclude-hosts', + default=[], + help="comma-separated list of hosts to exclude" + ), + make_option('-i', action='append', dest='key_filename', @@ -228,7 +233,6 @@ def _rc_path(): for option in env_options: env[option.dest] = option.default - # # Command dictionary # diff --git a/tests/test_main.py b/tests/test_main.py index 4b3c71ef69..b50f47e71e 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -12,29 +12,33 @@ def test_argument_parsing(): for args, output in [ # Basic - ('abc', ('abc', [], {}, [], [])), + ('abc', ('abc', [], {}, [], [], [])), # Arg - ('ab:c', ('ab', ['c'], {}, [], [])), + ('ab:c', ('ab', ['c'], {}, [], [], [])), # Kwarg - ('a:b=c', ('a', [], {'b':'c'}, [], [])), + ('a:b=c', ('a', [], {'b':'c'}, [], [], [])), # Arg and kwarg - ('a:b=c,d', ('a', ['d'], {'b':'c'}, [], [])), + ('a:b=c,d', ('a', ['d'], {'b':'c'}, [], [], [])), # Multiple kwargs - ('a:b=c,d=e', ('a', [], {'b':'c','d':'e'}, [], [])), + ('a:b=c,d=e', ('a', [], {'b':'c','d':'e'}, [], [], [])), # Host - ('abc:host=foo', ('abc', [], {}, ['foo'], [])), + ('abc:host=foo', ('abc', [], {}, ['foo'], [], [])), # Hosts with single host - ('abc:hosts=foo', ('abc', [], {}, ['foo'], [])), + ('abc:hosts=foo', ('abc', [], {}, ['foo'], [], [])), # Hosts with multiple hosts # Note: in a real shell, one would need to quote or escape "foo;bar". # But in pure-Python that would get interpreted literally, so we don't. - ('abc:hosts=foo;bar', ('abc', [], {}, ['foo', 'bar'], [])), + ('abc:hosts=foo;bar', ('abc', [], {}, ['foo', 'bar'], [], [])), + + # Exclude hosts + ('abc:hosts=foo;bar,exclude_hosts=foo', ('abc', [], {}, ['foo', 'bar'], [], ['foo'])), + ('abc:hosts=foo;bar,exclude_hosts=foo;bar', ('abc', [], {}, ['foo', 'bar'], [], ['foo','bar'])), ]: yield eq_, parse_arguments([args]), [output] def eq_hosts(command, host_list): - eq_(set(get_hosts(command, [], [])), set(host_list)) + eq_(set(get_hosts(command, [], [], [])), set(host_list)) def test_hosts_decorator_by_itself(): @@ -89,7 +93,7 @@ def test_hosts_decorator_overrides_env_hosts(): def command(): pass eq_hosts(command, ['bar']) - assert 'foo' not in get_hosts(command, [], []) + assert 'foo' not in get_hosts(command, [], [], []) @with_patched_object( From 0f9a1c5e46692004e8c39f7f56dc817bb9fb881b Mon Sep 17 00:00:00 2001 From: goosemo Date: Tue, 22 Jun 2010 14:35:58 -0400 Subject: [PATCH 002/126] Added in the little bit more to the docs on the local_dir var. --- fabric/contrib/project.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fabric/contrib/project.py b/fabric/contrib/project.py index e7df96fefa..b6a4f348bf 100644 --- a/fabric/contrib/project.py +++ b/fabric/contrib/project.py @@ -39,7 +39,8 @@ def rsync_project(remote_dir, local_dir=None, exclude=(), delete=False, the resulting project directory will be ``/home/username/myproject/``. * ``local_dir``: by default, ``rsync_project`` uses your current working directory as the source directory; you may override this with - ``local_dir``, which should be a directory path. + ``local_dir``, which should be a directory path, or list of paths in a + single string. * ``exclude``: optional, may be a single string, or an iterable of strings, and is used to pass one or more ``--exclude`` options to ``rsync``. * ``delete``: a boolean controlling whether ``rsync``'s ``--delete`` option From 3445b5653cd297039443110548fb3cab2e8e25af Mon Sep 17 00:00:00 2001 From: goosemo Date: Thu, 5 Aug 2010 09:33:22 -0400 Subject: [PATCH 003/126] Pulling the the patches to upload_project made by Rodrigue Alcazar From [issue #10](http://code.fabfile.org/issues/show/10) I am pulling in the patch and unit tests that he's made. I'll tweak them as needed, but the intial pull looks good. --- fabric/contrib/project.py | 54 +++++++++---- tests/test_project.py | 161 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 200 insertions(+), 15 deletions(-) create mode 100644 tests/test_project.py diff --git a/fabric/contrib/project.py b/fabric/contrib/project.py index e7df96fefa..b5f37843e4 100644 --- a/fabric/contrib/project.py +++ b/fabric/contrib/project.py @@ -3,12 +3,15 @@ """ from os import getcwd, sep +import os.path from datetime import datetime +from tempfile import mkdtemp from fabric.network import needs_host from fabric.operations import local, run, put from fabric.state import env, output +__all__ = ['rsync_project', 'upload_project'] @needs_host def rsync_project(remote_dir, local_dir=None, exclude=(), delete=False, @@ -100,23 +103,44 @@ def rsync_project(remote_dir, local_dir=None, exclude=(), delete=False, return local(cmd) -def upload_project(): +def upload_project(local_dir=None, remote_dir=""): """ Upload the current project to a remote system, tar/gzipping during the move. - This function makes use of the ``/tmp/`` directory and the ``tar`` and - ``gzip`` programs/libraries; thus it will not work too well on Win32 - systems unless one is using Cygwin or something similar. + This function makes use of the ``tar`` and ``gzip`` programs/libraries, thus + it will not work too well on Win32 systems unless one is using Cygwin or + something similar. + + ``upload_project`` will attempt to clean up the local and remote tarfiles + when it finishes executing, even in the event of a failure. + + :param local_dir: default current working directory, the project folder to + upload - ``upload_project`` will attempt to clean up the tarfiles when it finishes - executing. """ - tar_file = "/tmp/fab.%s.tar" % datetime.utcnow().strftime( - '%Y_%m_%d_%H-%M-%S') - cwd_name = getcwd().split(sep)[-1] - tgz_name = cwd_name + ".tar.gz" - local("tar -czf %s ." % tar_file) - put(tar_file, cwd_name + ".tar.gz") - local("rm -f " + tar_file) - run("tar -xzf " + tgz_name) - run("rm -f " + tgz_name) + if not local_dir: + local_dir = os.getcwd() + else: + # Remove final '/' in local_dir so that basename() works + local_dir = local_dir[:-1] if local_dir[-1] == os.sep else local_dir + + local_path, local_name = os.path.split(local_dir) + + tar_file = "%s.tar.gz" % local_name + target_tar = os.path.join(remote_dir, tar_file) + + tmp_folder = mkdtemp() + try: + tar_path = os.path.join(tmp_folder, tar_file) + local("tar -czf %s -C %s %s" % (tar_path, local_path, local_name)) + put(tar_path, target_tar) + + try: + run("tar -xzf %s" % tar_file) + + finally: + run("rm -f %s" % tar_file) + + finally: + local("rm -rf %s" % tmp_folder) + diff --git a/tests/test_project.py b/tests/test_project.py new file mode 100644 index 0000000000..ef4ce9a032 --- /dev/null +++ b/tests/test_project.py @@ -0,0 +1,161 @@ +import unittest +import os + +import fudge +from fudge.inspector import arg + +from fabric.contrib import project + + +class UploadProjectTestCase(unittest.TestCase): + """Test case for :func: `fabric.contrib.project.upload_project`.""" + + fake_tmp = "testtempfolder" + + + def setUp(self): + fudge.clear_expectations() + + # We need to mock out run, local, and put + + self.fake_run = fudge.Fake('project.run', callable=True) + self.patched_run = fudge.patch_object( + project, + 'run', + self.fake_run + ) + + self.fake_local = fudge.Fake('local', callable=True) + self.patched_local = fudge.patch_object( + project, + 'local', + self.fake_local + ) + + self.fake_put = fudge.Fake('put', callable=True) + self.patched_put = fudge.patch_object( + project, + 'put', + self.fake_put + ) + + # We don't want to create temp folders + self.fake_mkdtemp = fudge.Fake( + 'mkdtemp', + expect_call=True + ).returns(self.fake_tmp) + self.patched_mkdtemp = fudge.patch_object( + project, + 'mkdtemp', + self.fake_mkdtemp + ) + + + def tearDown(self): + self.patched_run.restore() + self.patched_local.restore() + self.patched_put.restore() + + fudge.clear_expectations() + + + @fudge.with_fakes + def test_temp_folder_is_used(self): + """A unique temp folder is used for creating the archive to upload.""" + + # Exercise + project.upload_project() + + + @fudge.with_fakes + def test_project_is_archived_locally(self): + """The project should be archived locally before being uploaded.""" + + # local() is called more than once so we need an extra next_call() + # otherwise fudge compares the args to the last call to local() + self.fake_local.with_args(arg.startswith("tar -czf")).next_call() + + # Exercise + project.upload_project() + + + @fudge.with_fakes + def test_current_directory_is_uploaded_by_default(self): + """By default the project uploaded is the current working directory.""" + + cwd_path, cwd_name = os.path.split(os.getcwd()) + + # local() is called more than once so we need an extra next_call() + # otherwise fudge compares the args to the last call to local() + self.fake_local.with_args( + arg.endswith("-C %s %s" % (cwd_path, cwd_name)) + ).next_call() + + # Exercise + project.upload_project() + + + @fudge.with_fakes + def test_path_to_local_project_can_be_specified(self): + """It should be possible to specify which local folder to upload.""" + + project_path = "path/to/my/project" + + # local() is called more than once so we need an extra next_call() + # otherwise fudge compares the args to the last call to local() + self.fake_local.with_args( + arg.endswith("-C %s %s" % os.path.split(project_path)) + ).next_call() + + # Exercise + project.upload_project(local_dir=project_path) + + + @fudge.with_fakes + def test_path_to_local_project_can_end_in_separator(self): + """A local path ending in a separator should be handled correctly.""" + + project_path = "path/to/my" + base = "project" + + # local() is called more than once so we need an extra next_call() + # otherwise fudge compares the args to the last call to local() + self.fake_local.with_args( + arg.endswith("-C %s %s" % (project_path, base)) + ).next_call() + + # Exercise + project.upload_project(local_dir="%s/%s/" % (project_path, base)) + + + @fudge.with_fakes + def test_default_remote_folder_is_home(self): + """Project is uploaded to remote home by default.""" + + local_dir = "folder" + + # local() is called more than once so we need an extra next_call() + # otherwise fudge compares the args to the last call to local() + self.fake_put.with_args( + "%s/folder.tar.gz" % self.fake_tmp, "folder.tar.gz" + ).next_call() + + # Exercise + project.upload_project(local_dir=local_dir) + + @fudge.with_fakes + def test_path_to_remote_folder_can_be_specified(self): + """It should be possible to specify which local folder to upload to.""" + + local_dir = "folder" + remote_path = "path/to/remote/folder" + + # local() is called more than once so we need an extra next_call() + # otherwise fudge compares the args to the last call to local() + self.fake_put.with_args( + "%s/folder.tar.gz" % self.fake_tmp, "%s/folder.tar.gz" % remote_path + ).next_call() + + # Exercise + project.upload_project(local_dir=local_dir, remote_dir=remote_path) + From 32f7d6787983cdb4c63b81a90f8facf4bfcd108e Mon Sep 17 00:00:00 2001 From: goosemo Date: Thu, 5 Aug 2010 09:40:47 -0400 Subject: [PATCH 004/126] Made a minor adjustment to use strip(os.sep) instead of array indexing. --- fabric/contrib/project.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fabric/contrib/project.py b/fabric/contrib/project.py index b5f37843e4..728c08da94 100644 --- a/fabric/contrib/project.py +++ b/fabric/contrib/project.py @@ -120,9 +120,9 @@ def upload_project(local_dir=None, remote_dir=""): """ if not local_dir: local_dir = os.getcwd() - else: - # Remove final '/' in local_dir so that basename() works - local_dir = local_dir[:-1] if local_dir[-1] == os.sep else local_dir + + # Remove final '/' in local_dir so that basename() works + local_dir = local_dir.strip(os.sep) local_path, local_name = os.path.split(local_dir) From 01f6c5ca47b582f6bf15f1751d74ad1fce43f8e5 Mon Sep 17 00:00:00 2001 From: Morgan Goose Date: Wed, 26 Jan 2011 14:38:27 -0500 Subject: [PATCH 005/126] This adds a decorator to make the host list follow order @ensure_order on a task will side step the use of set() to dedupe, and will instead loop over the host and role lists reappending to temp list if not already present in the temp list. Something to do then as an addition/extension, would be to perhaps add a parameter on the decorator to sort the list. --- fabric/decorators.py | 13 ++++++ fabric/main.py | 23 +++++++++- tests/test_main.py | 99 +++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 132 insertions(+), 3 deletions(-) diff --git a/fabric/decorators.py b/fabric/decorators.py index e68eeecf75..03bb1a56f5 100644 --- a/fabric/decorators.py +++ b/fabric/decorators.py @@ -100,3 +100,16 @@ def decorated(*args, **kwargs): decorated.return_value = func(*args, **kwargs) return decorated.return_value return decorated + + +def ensure_order(func): + """ + Decorator preventing wrapped function from using the set() operation to + dedupe the host list. Instead it will force fab to use X. + """ + @wraps(func) + def decorated(*args, **kwargs): + return func + decorated.ensure_order = True + return decorated + diff --git a/fabric/main.py b/fabric/main.py index 709dbfe37d..1751a6c8f9 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -29,6 +29,7 @@ _modules, [] ) +state.env.ensure_order = False def load_settings(path): """ @@ -360,8 +361,18 @@ def _merge(hosts, roles): if callable(value): value = value() role_hosts += value + # Return deduped combo of hosts and role_hosts - return list(set(hosts + role_hosts)) + if hasattr(state.env, 'ensure_order') and state.env.ensure_order: + result_hosts = [] + for host in hosts + role_hosts: + if host not in result_hosts: + result_hosts.append(host) + + else: + result_hosts = list(set(hosts + role_hosts)) + + return result_hosts def get_hosts(command, cli_hosts, cli_roles): @@ -371,6 +382,9 @@ def get_hosts(command, cli_hosts, cli_roles): See :ref:`execution-model` for detailed documentation on how host lists are set. """ + if hasattr(command, 'ensure_order') and command.ensure_order: + state.env.ensure_order = command.ensure_order + # Command line per-command takes precedence over anything else. if cli_hosts or cli_roles: return _merge(cli_hosts, cli_roles) @@ -383,6 +397,7 @@ def get_hosts(command, cli_hosts, cli_roles): # the CLI or from module-level code). This will be the empty list if these # have not been set -- which is fine, this method should return an empty # list if no hosts have been set anywhere. + return _merge(state.env['hosts'], state.env['roles']) @@ -515,6 +530,12 @@ def main(): names = ", ".join(x[0] for x in commands_to_run) print("Commands to run: %s" % names) + state.env.ensure_order = False + if hasattr(command, '_ensureorder') and command._ensure_order: + state.env.ensure_order = command._ensure_order + + + # At this point all commands must exist, so execute them in order. for name, args, kwargs, cli_hosts, cli_roles in commands_to_run: # Get callable by itself diff --git a/tests/test_main.py b/tests/test_main.py index 267c38360e..1594a33f46 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,7 +1,7 @@ from fudge.patcher import with_patched_object from nose.tools import eq_, raises -from fabric.decorators import hosts, roles +from fabric.decorators import hosts, roles, ensure_order from fabric.main import get_hosts, parse_arguments, _merge, _escape_split import fabric.state from fabric.state import _AttributeDict @@ -40,6 +40,25 @@ def eq_hosts(command, host_list): eq_(set(get_hosts(command, [], [])), set(host_list)) +def test_order_ensured(): + """ + Use of @ensure_order + """ + host_list = ['c', 'b', 'a'] + @ensure_order + @hosts(*host_list) + def command(): + pass + + print hasattr(command, 'ensure_order') + print command.ensure_order + print fabric.state.env.ensure_order + + eq_hosts(command, host_list) + for i,h in enumerate(get_hosts(command, [], [])): + eq_(host_list[i], h) + + def test_hosts_decorator_by_itself(): """ Use of @hosts only @@ -50,6 +69,17 @@ def command(): pass eq_hosts(command, host_list) +def test_hosts_decorator_by_itself_order_ensured(): + """ + Use of @hosts only order ensured + """ + host_list = ['a', 'b'] + @ensure_order + @hosts(*host_list) + def command(): + pass + eq_hosts(command, host_list) + fake_roles = { 'r1': ['a', 'b'], @@ -68,6 +98,19 @@ def command(): pass eq_hosts(command, ['a', 'b']) +@with_patched_object( + 'fabric.state', 'env', _AttributeDict({'roledefs': fake_roles}) +) +def test_roles_decorator_by_itself_order_ensured(): + """ + Use of @roles only order ensured + """ + @ensure_order + @roles('r1') + def command(): + pass + eq_hosts(command, ['a', 'b']) + @with_patched_object( 'fabric.state', 'env', _AttributeDict({'roledefs': fake_roles}) @@ -82,8 +125,24 @@ def command(): pass eq_hosts(command, ['a', 'b', 'c']) +@with_patched_object( + 'fabric.state', 'env', _AttributeDict({'roledefs': fake_roles}) +) +def test_hosts_and_roles_together_order_ensured(): + """ + Use of @roles and @hosts together results in union of both order ensured + """ + @ensure_order + @roles('r1', 'r2') + @hosts('a') + def command(): + pass + eq_hosts(command, ['a', 'b', 'c']) + -@with_patched_object('fabric.state', 'env', {'hosts': ['foo']}) +@with_patched_object( + 'fabric.state', 'env', _AttributeDict({'hosts': ['foo']}) +) def test_hosts_decorator_overrides_env_hosts(): """ If @hosts is used it replaces any env.hosts value @@ -94,6 +153,20 @@ def command(): eq_hosts(command, ['bar']) assert 'foo' not in get_hosts(command, [], []) +@with_patched_object( + 'fabric.state', 'env', _AttributeDict({'hosts': ['foo']}) +) +def test_hosts_decorator_overrides_env_hosts_order_ensured(): + """ + If @hosts is used it replaces any env.hosts value order ensured + """ + @ensure_order + @hosts('bar') + def command(): + pass + eq_hosts(command, ['bar']) + assert 'foo' not in get_hosts(command, [], []) + def test_hosts_decorator_expands_single_iterable(): """ @@ -105,6 +178,17 @@ def command(): pass eq_(command.hosts, host_list) +def test_hosts_decorator_expands_single_iterable_order_ensured(): + """ + @hosts(iterable) should behave like @hosts(*iterable) order ensured + """ + host_list = ['foo', 'bar'] + @ensure_order + @hosts(host_list) + def command(): + pass + eq_(command.hosts, host_list) + def test_roles_decorator_expands_single_iterable(): """ @@ -116,6 +200,17 @@ def command(): pass eq_(command.roles, role_list) +def test_roles_decorator_expands_single_iterable_order_ensured(): + """ + @roles(iterable) should behave like @roles(*iterable) order ensured + """ + role_list = ['foo', 'bar'] + @ensure_order + @roles(role_list) + def command(): + pass + eq_(command.roles, role_list) + @with_patched_object( 'fabric.state', 'env', _AttributeDict({'roledefs': fake_roles}) From 540fcf2b38c1f3e74da60db5da02ed48a7578728 Mon Sep 17 00:00:00 2001 From: Morgan Goose Date: Fri, 28 Jan 2011 18:23:59 -0500 Subject: [PATCH 006/126] Added in sorted parameter to the @ensure_order decorator --- fabric/decorators.py | 16 +++++++++++----- fabric/main.py | 17 +++++++++-------- tests/test_main.py | 32 ++++++++++++++++++++++++++++---- 3 files changed, 48 insertions(+), 17 deletions(-) diff --git a/fabric/decorators.py b/fabric/decorators.py index 03bb1a56f5..ff5e89c3f8 100644 --- a/fabric/decorators.py +++ b/fabric/decorators.py @@ -102,14 +102,20 @@ def decorated(*args, **kwargs): return decorated -def ensure_order(func): +def ensure_order(sorted=False): """ Decorator preventing wrapped function from using the set() operation to dedupe the host list. Instead it will force fab to use X. """ - @wraps(func) - def decorated(*args, **kwargs): + def real_decorator(func): + func._sorted = sorted + func._ensure_order = True return func - decorated.ensure_order = True - return decorated + # Trick to allow for both a dec w/ the optional setting without have to + # force it to use () + if type(sorted) == type(real_decorator): + return real_decorator(sorted) + + real_decorator._ensure_order = True + return real_decorator diff --git a/fabric/main.py b/fabric/main.py index 1751a6c8f9..0a174b7886 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -363,15 +363,19 @@ def _merge(hosts, roles): role_hosts += value # Return deduped combo of hosts and role_hosts - if hasattr(state.env, 'ensure_order') and state.env.ensure_order: + if hasattr(state.env, '_ensure_order') and state.env._ensure_order: result_hosts = [] for host in hosts + role_hosts: if host not in result_hosts: result_hosts.append(host) + + if hasattr(state.env, '_sorted') and state.env._sorted: + result_hosts.sort() else: result_hosts = list(set(hosts + role_hosts)) + return result_hosts @@ -382,8 +386,10 @@ def get_hosts(command, cli_hosts, cli_roles): See :ref:`execution-model` for detailed documentation on how host lists are set. """ - if hasattr(command, 'ensure_order') and command.ensure_order: - state.env.ensure_order = command.ensure_order + if hasattr(command, '_ensure_order') and command._ensure_order: + if hasattr(command, '_sorted') and command._sorted == True: + state.env._sorted = command._sorted + state.env._ensure_order = command._ensure_order # Command line per-command takes precedence over anything else. if cli_hosts or cli_roles: @@ -530,11 +536,6 @@ def main(): names = ", ".join(x[0] for x in commands_to_run) print("Commands to run: %s" % names) - state.env.ensure_order = False - if hasattr(command, '_ensureorder') and command._ensure_order: - state.env.ensure_order = command._ensure_order - - # At this point all commands must exist, so execute them in order. for name, args, kwargs, cli_hosts, cli_roles in commands_to_run: diff --git a/tests/test_main.py b/tests/test_main.py index 1594a33f46..de27a21a5a 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -50,14 +50,38 @@ def test_order_ensured(): def command(): pass - print hasattr(command, 'ensure_order') - print command.ensure_order - print fabric.state.env.ensure_order - + print hasattr(command, '_ensure_order') + print command._ensure_order + print hasattr(command, '_sorted') + print command._sorted + #print fabric.state.env._ensure_order + eq_(command._ensure_order, True) eq_hosts(command, host_list) + print get_hosts(command, [], []) for i,h in enumerate(get_hosts(command, [], [])): eq_(host_list[i], h) +def test_order_ensured_sorted(): + """ + Use of @ensure_order with sorted option + """ + host_list = ['c', 'a', 'b', 'e'] + sorted = ['c', 'a', 'b', 'e'] + sorted.sort() + @ensure_order(sorted=True) + @hosts(*host_list) + def command(): + pass + + print hasattr(command, '_ensure_order') + print command._ensure_order + print hasattr(command, '_sorted') + print command._sorted + #print fabric.state.env._ensure_order + eq_(command._ensure_order, True) + eq_(command._sorted, True) + eq_hosts(command, sorted) + def test_hosts_decorator_by_itself(): """ From 2054be1ea38b4c2c1e80a7c09293196dcf19d770 Mon Sep 17 00:00:00 2001 From: Morgan Goose Date: Thu, 3 Feb 2011 02:19:35 -0500 Subject: [PATCH 007/126] Cleaned out testing prints --- tests/test_main.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index de27a21a5a..c7bd65cf2b 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -50,11 +50,6 @@ def test_order_ensured(): def command(): pass - print hasattr(command, '_ensure_order') - print command._ensure_order - print hasattr(command, '_sorted') - print command._sorted - #print fabric.state.env._ensure_order eq_(command._ensure_order, True) eq_hosts(command, host_list) print get_hosts(command, [], []) @@ -73,11 +68,6 @@ def test_order_ensured_sorted(): def command(): pass - print hasattr(command, '_ensure_order') - print command._ensure_order - print hasattr(command, '_sorted') - print command._sorted - #print fabric.state.env._ensure_order eq_(command._ensure_order, True) eq_(command._sorted, True) eq_hosts(command, sorted) From a86f293b68f1cf9b74db5a68727ab43151e371f6 Mon Sep 17 00:00:00 2001 From: Morgan Goose Date: Thu, 3 Feb 2011 02:27:24 -0500 Subject: [PATCH 008/126] Wrote up more in the docstring and added it to be included in the docs. --- docs/api/core/decorators.rst | 2 +- fabric/decorators.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/api/core/decorators.rst b/docs/api/core/decorators.rst index 78228ea2a9..f107aa2c06 100644 --- a/docs/api/core/decorators.rst +++ b/docs/api/core/decorators.rst @@ -3,4 +3,4 @@ Decorators ========== .. automodule:: fabric.decorators - :members: hosts, roles, runs_once + :members: hosts, roles, runs_once, ensure_order diff --git a/fabric/decorators.py b/fabric/decorators.py index ff5e89c3f8..5071fcab10 100644 --- a/fabric/decorators.py +++ b/fabric/decorators.py @@ -105,7 +105,21 @@ def decorated(*args, **kwargs): def ensure_order(sorted=False): """ Decorator preventing wrapped function from using the set() operation to - dedupe the host list. Instead it will force fab to use X. + dedupe the host list. Instead it will force fab to iterate of the list of + hosts as combined from both `~fabric.decorators.hosts` and + `~fabric.decorators.roles`. + + It also takes in a parameter sorted, to determine if this deduped list + should also then be sorted using the default python provided sort + mechanism. + + Is used in conjunction with host lists and/or roles:: + + @ensure_order + @hosts('user1@host1', 'host2', 'user2@host3') + def my_func(): + pass + """ def real_decorator(func): func._sorted = sorted From 7d033f748a2627a46f7e39494461f35fcf032f27 Mon Sep 17 00:00:00 2001 From: Morgan Goose Date: Thu, 3 Feb 2011 03:17:17 -0500 Subject: [PATCH 009/126] Changed the local_dir to rstrip keeping tests passing. --- fabric/contrib/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fabric/contrib/project.py b/fabric/contrib/project.py index a451cdb608..d5f71a1514 100644 --- a/fabric/contrib/project.py +++ b/fabric/contrib/project.py @@ -123,7 +123,7 @@ def upload_project(local_dir=None, remote_dir=""): local_dir = os.getcwd() # Remove final '/' in local_dir so that basename() works - local_dir = local_dir.strip(os.sep) + local_dir = local_dir.rstrip(os.sep) local_path, local_name = os.path.split(local_dir) From cf61564fe0c1bd2bb876a7ac09bcee2c18bd3dee Mon Sep 17 00:00:00 2001 From: Travis Swicegood Date: Sun, 14 Nov 2010 21:44:40 -0600 Subject: [PATCH 010/126] Adds a @with_settings decorator for tasks (and tests) to use A simple wrapper around the settings context_manager. Useful for retrofitting old code without having to indent it all for the context block. Signed-off-by: Morgan Goose --- fabric/decorators.py | 12 ++++++++++++ tests/test_decorators.py | 14 +++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/fabric/decorators.py b/fabric/decorators.py index e68eeecf75..4daa6262e0 100644 --- a/fabric/decorators.py +++ b/fabric/decorators.py @@ -5,6 +5,8 @@ from functools import wraps from types import StringTypes +from .context_managers import settings + def hosts(*host_list): """ @@ -100,3 +102,13 @@ def decorated(*args, **kwargs): decorated.return_value = func(*args, **kwargs) return decorated.return_value return decorated + + +def with_settings(**kw_settings): + def outer(func): + def inner(*args, **kwargs): + with settings(**kw_settings): + return func(*args, **kwargs) + return inner + return outer + diff --git a/tests/test_decorators.py b/tests/test_decorators.py index 4b6144428d..d8da36e280 100644 --- a/tests/test_decorators.py +++ b/tests/test_decorators.py @@ -1,7 +1,9 @@ -from nose.tools import eq_ +from nose.tools import eq_, ok_ from fudge import Fake, with_fakes +import random from fabric import decorators +from fabric.state import env def fake_function(*args, **kwargs): @@ -38,3 +40,13 @@ def test_runs_once_returns_same_value_each_run(): task = decorators.runs_once(fake_function().returns(return_value)) for i in range(2): eq_(task(), return_value) + +def test_with_settings_passes_env_vars_into_decorated_function(): + env.value = True + random_return = random.randint(1000, 2000) + def some_task(): + return env.value + decorated_task = decorators.with_settings(value=random_return)(some_task) + ok_(some_task(), msg="sanity check") + eq_(random_return, decorated_task()) + From a99a54a4adcd089a2024db3b8de69da2b41adeca Mon Sep 17 00:00:00 2001 From: Travis Swicegood Date: Sun, 14 Nov 2010 21:48:47 -0600 Subject: [PATCH 011/126] add docblock explaining what with_settings does Signed-off-by: Morgan Goose --- fabric/decorators.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/fabric/decorators.py b/fabric/decorators.py index 4daa6262e0..0a26661965 100644 --- a/fabric/decorators.py +++ b/fabric/decorators.py @@ -105,6 +105,16 @@ def decorated(*args, **kwargs): def with_settings(**kw_settings): + """ + Decorator equivalent of ``fabric.context_managers.settings``. + + Allows you to wrap an entire function as if it was called inside a block + with the ``settings`` context manager. Useful for retrofitting old code so + you don't have to change the indention to gain the behavior. + + See ``fabric.context_managers.settings`` for more information about what + you can do with this. + """ def outer(func): def inner(*args, **kwargs): with settings(**kw_settings): From 8a258ea33c7d09dc7e24273e1fbd72d40e083f36 Mon Sep 17 00:00:00 2001 From: Travis Swicegood Date: Sun, 14 Nov 2010 21:48:58 -0600 Subject: [PATCH 012/126] Modify tests to pass using the new context. Both tests assume a env.use_shell to be True. There's a test somewhere else in the chain that can cause this to not be the case. This ensures the assumption is always met. Signed-off-by: Morgan Goose --- tests/test_operations.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_operations.py b/tests/test_operations.py index 489fb00b6b..dbfeee0cb7 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -8,6 +8,7 @@ from fabric.state import env from fabric.operations import require, prompt, _sudo_prefix, _shell_wrap, \ _shell_escape +from fabric.decorators import with_settings from utils import mock_streams @@ -121,6 +122,7 @@ def test_sudo_prefix_without_user(): eq_(_sudo_prefix(user=None), env.sudo_prefix % env.sudo_prompt) +@with_settings(use_shell=True) def test_shell_wrap(): prefix = "prefix" command = "command" @@ -139,6 +141,7 @@ def test_shell_wrap(): del eq_.description +@with_settings(use_shell=True) def test_shell_wrap_escapes_command_if_shell_is_true(): """ _shell_wrap() escapes given command if shell=True From e09319beafccd3da4ca08cb52e4160a64a795ff3 Mon Sep 17 00:00:00 2001 From: Morgan Goose Date: Fri, 4 Feb 2011 04:08:35 -0500 Subject: [PATCH 013/126] After getting in patches from Travis' fork, I also added it to the sphinx docs and an example. --- docs/api/core/decorators.rst | 2 +- fabric/decorators.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/api/core/decorators.rst b/docs/api/core/decorators.rst index 78228ea2a9..a19fd3f888 100644 --- a/docs/api/core/decorators.rst +++ b/docs/api/core/decorators.rst @@ -3,4 +3,4 @@ Decorators ========== .. automodule:: fabric.decorators - :members: hosts, roles, runs_once + :members: hosts, roles, runs_once, with_settings diff --git a/fabric/decorators.py b/fabric/decorators.py index 0a26661965..19da968080 100644 --- a/fabric/decorators.py +++ b/fabric/decorators.py @@ -112,6 +112,14 @@ def with_settings(**kw_settings): with the ``settings`` context manager. Useful for retrofitting old code so you don't have to change the indention to gain the behavior. + An example use being to set all fabric api functions in a task to not error + out on unexpected return codes:: + + @with_settings(warn_only=True) + @hosts('user1@host1', 'host2', 'user2@host3') + def foo(): + pass + See ``fabric.context_managers.settings`` for more information about what you can do with this. """ From 422268b3d938ba3eb71fdacae12b209584a5f0dc Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Sun, 13 Mar 2011 23:00:59 -0400 Subject: [PATCH 014/126] Fixes #307 Update code to match Pep8 standards - I've updated most of the code to match PEP8. There are a few valid exceptions left behind - Note, stripping trailing whitespace from "expecting" strings in test_networks.py will break tests --- fabric/auth.py | 1 + fabric/colors.py | 5 ++- fabric/contrib/files.py | 6 ++-- fabric/decorators.py | 1 + fabric/io.py | 4 +-- fabric/main.py | 16 +++++---- fabric/network.py | 29 ++++++++------- fabric/sftp.py | 17 ++------- fabric/state.py | 15 ++++---- fabric/thread_handling.py | 1 + fabric/utils.py | 2 +- fabric/version.py | 3 +- tests/Python26SocketServer.py | 65 +++++++++++++++++++++------------- tests/fake_filesystem.py | 1 + tests/server.py | 12 ++++--- tests/test_context_managers.py | 2 +- tests/test_main.py | 14 ++++++-- tests/test_network.py | 19 ++-------- tests/test_operations.py | 49 +++---------------------- tests/test_state.py | 6 ++-- tests/test_utils.py | 4 +-- tests/utils.py | 3 +- 22 files changed, 128 insertions(+), 147 deletions(-) diff --git a/fabric/auth.py b/fabric/auth.py index 7e770c6e6b..3f5975f299 100644 --- a/fabric/auth.py +++ b/fabric/auth.py @@ -7,6 +7,7 @@ def get_password(): from fabric.state import env return env.passwords.get(env.host_string, env.password) + def set_password(password): from fabric.state import env env.password = env.passwords[env.host_string] = password diff --git a/fabric/colors.py b/fabric/colors.py index 4894e661ac..88736fb605 100644 --- a/fabric/colors.py +++ b/fabric/colors.py @@ -16,14 +16,17 @@ from fabric.colors import red, green - print(red("This sentence is red, except for " + green("these words, which are green") + ".")) + print(red("This sentence is red, except for " + \ + green("these words, which are green") + ".")) If ``bold`` is set to ``True``, the ANSI flag for bolding will be flipped on for that particular invocation, which usually shows up as a bold or brighter version of the original color on most terminals. """ + def _wrap_with(code): + def inner(text, bold=False): c = code if bold: diff --git a/fabric/contrib/files.py b/fabric/contrib/files.py index 15155cf898..23ebc123d8 100644 --- a/fabric/contrib/files.py +++ b/fabric/contrib/files.py @@ -61,7 +61,7 @@ def upload_template(filename, destination, context=None, use_jinja=False, templating library available, Jinja will be used to render the template instead. Templates will be loaded from the invoking user's current working directory by default, or from ``template_dir`` if given. - + The resulting rendered file will be uploaded to the remote file path ``destination`` (which should include the desired remote filename.) If the destination file already exists, it will be renamed with a ``.bak`` @@ -194,7 +194,7 @@ def comment(filename, regex, use_sudo=False, char='#', backup='.bak'): sometimes do when inserted by hand. Neither will they have a trailing space unless you specify e.g. ``char='# '``. - .. note:: + .. note:: In order to preserve the line being commented out, this function will wrap your ``regex`` argument in parentheses, so you don't need to. It @@ -267,7 +267,7 @@ def append(filename, text, use_sudo=False, partial=False, escape=True): "append lines to a file" use case. You may override this and force partial searching (e.g. ``^``) by specifying ``partial=True``. - Because ``text`` is single-quoted, single quotes will be transparently + Because ``text`` is single-quoted, single quotes will be transparently backslash-escaped. This can be disabled with ``escape=False``. If ``use_sudo`` is True, will use `sudo` instead of `run`. diff --git a/fabric/decorators.py b/fabric/decorators.py index e68eeecf75..5a23bb31bb 100644 --- a/fabric/decorators.py +++ b/fabric/decorators.py @@ -29,6 +29,7 @@ def my_func(): Allow a single, iterable argument (``@hosts(iterable)``) to be used instead of requiring ``@hosts(*iterable)``. """ + def attach_hosts(func): @wraps(func) def inner_decorator(*args, **kwargs): diff --git a/fabric/io.py b/fabric/io.py index a8b976e44f..168a8a12cf 100644 --- a/fabric/io.py +++ b/fabric/io.py @@ -18,7 +18,7 @@ def _flush(pipe, text): def _endswith(char_list, substring): - tail = char_list[-1*len(substring):] + tail = char_list[-1 * len(substring):] substring = list(substring) return tail == substring @@ -78,7 +78,7 @@ def output_loop(chan, which, capture): # backwards compatible with Fabric 0.9.x behavior; the user # will still see the prompt on their screen (no way to avoid # this) but at least it won't clutter up the captured text. - del capture[-1*len(env.sudo_prompt):] + del capture[-1 * len(env.sudo_prompt):] # If the password we just tried was bad, prompt the user again. if (not password) or reprompt: # Print the prompt and/or the "try again" notice if diff --git a/fabric/main.py b/fabric/main.py index 5e97d6b3c8..d92a1f25f4 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -14,10 +14,10 @@ import os import sys -from fabric import api # For checking callables against the API -from fabric.contrib import console, files, project # Ditto +from fabric import api # For checking callables against the API +from fabric.contrib import console, files, project # Ditto from fabric.network import denormalize, interpret_host_string, disconnect_all -from fabric import state # For easily-mockable access to roles, env and etc +from fabric import state # For easily-mockable access to roles, env and etc from fabric.state import commands, connections, env_options from fabric.utils import abort, indent @@ -30,6 +30,7 @@ [] ) + def load_settings(path): """ Take given file path and return dictionary of any key=value pairs found. @@ -158,7 +159,8 @@ def parse_options(): # # Define options that don't become `env` vars (typically ones which cause - # Fabric to do something other than its normal execution, such as --version) + # Fabric to do something other than its normal execution, such as + # --version) # # Version number (optparse gives you --version but we have to do it @@ -286,7 +288,7 @@ def _escape_split(sep, argstr): return argstr.split(sep) before, _, after = argstr.partition(escaped_sep) - startlist = before.split(sep) # a regular split is fine here + startlist = before.split(sep) # a regular split is fine here unfinished = startlist[-1] startlist = startlist[:-1] @@ -297,7 +299,7 @@ def _escape_split(sep, argstr): # part of the string sent in recursion is the rest of the escaped value. unfinished += sep + endlist[0] - return startlist + [unfinished] + endlist[1:] # put together all the parts + return startlist + [unfinished] + endlist[1:] # put together all the parts def parse_arguments(arguments): @@ -488,7 +490,7 @@ def main(): # If user didn't specify any commands to run, show help if not (arguments or remainder_arguments): parser.print_help() - sys.exit(0) # Or should it exit with error (1)? + sys.exit(0) # Or should it exit with error (1)? # Parse arguments into commands to run (plus args/kwargs/hosts) commands_to_run = parse_arguments(arguments) diff --git a/fabric/network.py b/fabric/network.py index 1876771246..87178b25a7 100644 --- a/fabric/network.py +++ b/fabric/network.py @@ -18,8 +18,8 @@ warnings.simplefilter('ignore', DeprecationWarning) import paramiko as ssh except ImportError: - abort("paramiko is a required module. Please install it:\n\t$ sudo easy_install paramiko") - + abort("paramiko is a required module. Please install it:\n\t" + "$ sudo easy_install paramiko") host_pattern = r'((?P.+)@)?(?P[^:]+)(:(?P\d+))?' @@ -52,7 +52,7 @@ class HostConnectionCache(dict): ``user1@example.com`` will create a connection to ``example.com``, logged in as ``user1``; later specifying ``user2@example.com`` will create a new, 2nd connection as ``user2``. - + The same applies to ports: specifying two different ports will result in two different connections to the same host being made. If no port is given, 22 is assumed, so ``example.com`` is equivalent to ``example.com:22``. @@ -97,8 +97,8 @@ def denormalize(host_string): """ Strips out default values for the given host string. - If the user part is the default user, it is removed; if the port is port 22, - it also is removed. + If the user part is the default user, it is removed; + if the port is port 22, it also is removed. """ from state import env r = host_regex.match(host_string).groupdict() @@ -115,8 +115,8 @@ def join_host_strings(user, host, port=None): """ Turns user/host/port strings into ``user@host:port`` combined string. - This function is not responsible for handling missing user/port strings; for - that, see the ``normalize`` function. + This function is not responsible for handling missing user/port strings; + for that, see the ``normalize`` function. If ``port`` is omitted, the returned string will be of the form ``user@host``. @@ -147,7 +147,6 @@ def connect(user, host, port): if not env.reject_unknown_hosts: client.set_missing_host_key_policy(ssh.AutoAddPolicy()) - # # Connection attempt loop # @@ -176,7 +175,9 @@ def connect(user, host, port): # command line results in the big banner error about man-in-the-middle # attacks. except ssh.BadHostKeyException: - abort("Host key for %s did not match pre-existing key! Server's key was changed recently, or possible man-in-the-middle attack." % env.host) + abort("Host key for %s did not match pre-existing key! Server's" + " key was changed recently, or possible man-in-the-middle" + "attack." % env.host) # Prompt for new password to try on auth failure except ( ssh.AuthenticationException, @@ -243,9 +244,11 @@ def connect(user, host, port): host, e[1]) ) + def prompt_for_password(prompt=None, no_colon=False, stream=None): """ - Prompts for and returns a new password if required; otherwise, returns None. + Prompts for and returns a new password if required; otherwise, returns + None. A trailing colon is appended unless ``no_colon`` is True. @@ -285,7 +288,7 @@ def needs_host(func): This decorator is basically a safety net for silly users who forgot to specify the host/host list in one way or another. It should be used to wrap operations which require a network connection. - + Due to how we execute commands per-host in ``main()``, it's not possible to specify multiple hosts at this point in time, so only a single host will be prompted for. @@ -296,10 +299,12 @@ def needs_host(func): command (in the case where multiple commands have no hosts set, of course.) """ from fabric.state import env + @wraps(func) def host_prompting_wrapper(*args, **kwargs): while not env.get('host_string', False): - host_string = raw_input("No hosts found. Please specify (single) host string for connection: ") + host_string = raw_input("No hosts found. Please specify (single)" + " host string for connection: ") interpret_host_string(host_string) return func(*args, **kwargs) return host_prompting_wrapper diff --git a/fabric/sftp.py b/fabric/sftp.py index 29954f6c21..521f1d6669 100644 --- a/fabric/sftp.py +++ b/fabric/sftp.py @@ -17,27 +17,23 @@ class SFTP(object): def __init__(self, host_string): self.ftp = connections[host_string].open_sftp() - # Recall that __getattr__ is the "fallback" attribute getter, and is thus # pretty safe to use for facade-like behavior as we're doing here. def __getattr__(self, attr): return getattr(self.ftp, attr) - def isdir(self, path): try: return stat.S_ISDIR(self.ftp.lstat(path).st_mode) except IOError: return False - def islink(self, path): try: return stat.S_ISLNK(self.ftp.lstat(path).st_mode) except IOError: return False - def exists(self, path): try: self.ftp.lstat(path).st_mode @@ -45,7 +41,6 @@ def exists(self, path): return False return True - def glob(self, path): from fabric.state import win32 dirpart, pattern = os.path.split(path) @@ -60,7 +55,6 @@ def glob(self, path): ret = [os.path.join(dirpart, name) for name in names] return ret - def walk(self, top, topdown=True, onerror=None, followlinks=False): from os.path import join, isdir, islink @@ -96,7 +90,6 @@ def walk(self, top, topdown=True, onerror=None, followlinks=False): if not topdown: yield top, dirs, nondirs - def mkdir(self, path, use_sudo): from fabric.api import sudo, hide if use_sudo: @@ -105,7 +98,6 @@ def mkdir(self, path, use_sudo): else: self.ftp.mkdir(path) - def get(self, remote_path, local_path, local_is_path, rremote=None): # rremote => relative remote path, so get(/var/log) would result in # this function being called with @@ -157,7 +149,6 @@ def get(self, remote_path, local_path, local_is_path, rremote=None): result = real_local_path return result - def get_dir(self, remote_path, local_path): # Decide what needs to be stripped from remote paths so they're all # relative to the given remote_path @@ -197,7 +188,6 @@ def get_dir(self, remote_path, local_path): result.append(self.get(rpath, lpath, True, rremote)) return result - def put(self, local_path, remote_path, use_sudo, mirror_local_mode, mode, local_is_path): from fabric.api import sudo, hide @@ -253,7 +243,6 @@ def put(self, local_path, remote_path, use_sudo, mirror_local_mode, mode, remote_path = target_path return remote_path - def put_dir(self, local_path, remote_path, use_sudo, mirror_local_mode, mode): if os.path.basename(local_path): @@ -272,13 +261,13 @@ def put_dir(self, local_path, remote_path, use_sudo, mirror_local_mode, self.mkdir(rcontext, use_sudo) for d in dirs: - n = os.path.join(rcontext,d) + n = os.path.join(rcontext, d) if not self.exists(n): self.mkdir(n, use_sudo) for f in files: - local_path = os.path.join(context,f) - n = os.path.join(rcontext,f) + local_path = os.path.join(context, f) + n = os.path.join(rcontext, f) p = self.put(local_path, n, use_sudo, mirror_local_mode, mode, True) remote_paths.append(p) diff --git a/fabric/state.py b/fabric/state.py index a59717d5c6..0507d2c183 100644 --- a/fabric/state.py +++ b/fabric/state.py @@ -23,7 +23,7 @@ # # Environment dictionary - support structures -# +# class _AttributeDict(dict): """ @@ -91,7 +91,7 @@ def _rc_path(): from win32com.shell.shell import SHGetSpecialFolderPath from win32com.shell.shellcon import CSIDL_PROFILE return "%s/%s" % ( - SHGetSpecialFolderPath(0,CSIDL_PROFILE), + SHGetSpecialFolderPath(0, CSIDL_PROFILE), rc_file ) @@ -144,7 +144,7 @@ def _rc_path(): help="comma-separated list of roles to operate on" ), - make_option('-i', + make_option('-i', action='append', dest='key_filename', default=None, @@ -204,7 +204,7 @@ def _rc_path(): default=True, help="do not use pseudo-terminal in run/sudo" ) - + ] @@ -224,11 +224,11 @@ def _rc_path(): 'combine_stderr': True, 'command': None, 'command_prefixes': [], - 'cwd': '', # Must be empty string, not None, for concatenation purposes + 'cwd': '', # Must be empty string, not None, for concatenation purposes 'echo_stdin': True, 'host': None, 'host_string': None, - 'lcwd': '', # Must be empty string, not None, for concatenation purposes + 'lcwd': '', # Must be empty string, not None, for concatenation purposes 'local_user': _get_system_username(), 'output_prefix': True, 'passwords': {}, @@ -267,6 +267,7 @@ def _rc_path(): connections = HostConnectionCache() + def default_channel(): """ Return a channel object based on ``env.host_string``. @@ -298,7 +299,7 @@ class _AliasDict(_AttributeDict): This also means they will not show up in e.g. ``dict.keys()``. ..note:: - + Aliases are recursive, so you may refer to an alias within the key list of another alias. Naturally, this means that you can end up with infinite loops if you're not careful. diff --git a/fabric/thread_handling.py b/fabric/thread_handling.py index 25aa3a2326..790a576bef 100644 --- a/fabric/thread_handling.py +++ b/fabric/thread_handling.py @@ -6,6 +6,7 @@ class ThreadHandler(object): def __init__(self, name, callable, *args, **kwargs): # Set up exception handling self.exception = None + def wrapper(*args, **kwargs): try: callable(*args, **kwargs) diff --git a/fabric/utils.py b/fabric/utils.py index 7db2fb191d..876ddfc7ae 100644 --- a/fabric/utils.py +++ b/fabric/utils.py @@ -10,7 +10,7 @@ def abort(msg): """ Abort execution, print ``msg`` to stderr and exit with error status (1.) - This function currently makes use of `sys.exit`_, which raises + This function currently makes use of `sys.exit`_, which raises `SystemExit`_. Therefore, it's possible to detect and recover from inner calls to `abort` by using ``except SystemExit`` or similar. diff --git a/fabric/version.py b/fabric/version.py index 5c6c735afc..88f88ed00c 100644 --- a/fabric/version.py +++ b/fabric/version.py @@ -23,6 +23,7 @@ def git_sha(): VERSION = (1, 0, 0, 'final', 0) + def get_version(form='short'): """ Return a version string for this package, based on `VERSION`. @@ -92,6 +93,6 @@ def get_version(form='short'): try: return versions[form] except KeyError: - raise TypeError, '"%s" is not a valid form specifier.' % form + raise TypeError('"%s" is not a valid form specifier.' % form) __version__ = get_version('short') diff --git a/tests/Python26SocketServer.py b/tests/Python26SocketServer.py index f01cb5f2cc..b26854023a 100644 --- a/tests/Python26SocketServer.py +++ b/tests/Python26SocketServer.py @@ -138,17 +138,17 @@ class will essentially render the service "deaf" while one request is except ImportError: import dummy_threading as threading -__all__ = ["TCPServer","UDPServer","ForkingUDPServer","ForkingTCPServer", - "ThreadingUDPServer","ThreadingTCPServer","BaseRequestHandler", - "StreamRequestHandler","DatagramRequestHandler", +__all__ = ["TCPServer", "UDPServer", "ForkingUDPServer", "ForkingTCPServer", + "ThreadingUDPServer", "ThreadingTCPServer", "BaseRequestHandler", + "StreamRequestHandler", "DatagramRequestHandler", "ThreadingMixIn", "ForkingMixIn"] if hasattr(socket, "AF_UNIX"): - __all__.extend(["UnixStreamServer","UnixDatagramServer", + __all__.extend(["UnixStreamServer", "UnixDatagramServer", "ThreadingUnixStreamServer", "ThreadingUnixDatagramServer"]) -class BaseServer: +class BaseServer: """Base class for server classes. Methods for the caller: @@ -329,12 +329,12 @@ def handle_error(self, request, client_address): The default is to print a traceback and continue. """ - print '-'*40 + print '-' * 40 print 'Exception happened during processing of request from', print client_address import traceback - traceback.print_exc() # XXX But this goes to stderr! - print '-'*40 + traceback.print_exc() # XXX But this goes to stderr! + print '-' * 40 class TCPServer(BaseServer): @@ -391,7 +391,8 @@ class TCPServer(BaseServer): allow_reuse_address = False - def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True): + def __init__(self, server_address, RequestHandlerClass, + bind_and_activate=True): """Constructor. May be extended, do not override.""" BaseServer.__init__(self, server_address, RequestHandlerClass) self.socket = socket.socket(self.address_family, @@ -470,8 +471,8 @@ def close_request(self, request): # No need to close anything. pass -class ForkingMixIn: +class ForkingMixIn: """Mix-in class to handle each request in a new process.""" timeout = 300 @@ -480,7 +481,8 @@ class ForkingMixIn: def collect_children(self): """Internal routine to wait for children that have exited.""" - if self.active_children is None: return + if self.active_children is None: + return while len(self.active_children) >= self.max_children: # XXX: This will wait for any child process, not just ones # spawned by this library. This could confuse other @@ -490,7 +492,8 @@ def collect_children(self): pid, status = os.waitpid(0, 0) except os.error: pid = None - if pid not in self.active_children: continue + if pid not in self.active_children: + continue self.active_children.remove(pid) # XXX: This loop runs more system calls than it ought @@ -503,12 +506,13 @@ def collect_children(self): pid, status = os.waitpid(child, os.WNOHANG) except os.error: pid = None - if not pid: continue + if not pid: + continue try: self.active_children.remove(pid) except ValueError, e: - raise ValueError('%s. x=%d and list=%r' % (e.message, pid, - self.active_children)) + raise ValueError('%s. x=%d and list=%r' % \ + (e.message, pid, self.active_children)) def handle_timeout(self): """Wait for zombies after self.timeout seconds of inactivity. @@ -563,18 +567,28 @@ def process_request_thread(self, request, client_address): def process_request(self, request, client_address): """Start a new thread to process the request.""" - t = threading.Thread(target = self.process_request_thread, - args = (request, client_address)) + t = threading.Thread(target=self.process_request_thread, + args=(request, client_address)) if self.daemon_threads: - t.setDaemon (1) + t.setDaemon(1) t.start() -class ForkingUDPServer(ForkingMixIn, UDPServer): pass -class ForkingTCPServer(ForkingMixIn, TCPServer): pass +class ForkingUDPServer(ForkingMixIn, UDPServer): + pass + + +class ForkingTCPServer(ForkingMixIn, TCPServer): + pass + + +class ThreadingUDPServer(ThreadingMixIn, UDPServer): + pass + + +class ThreadingTCPServer(ThreadingMixIn, TCPServer): + pass -class ThreadingUDPServer(ThreadingMixIn, UDPServer): pass -class ThreadingTCPServer(ThreadingMixIn, TCPServer): pass if hasattr(socket, 'AF_UNIX'): @@ -584,9 +598,12 @@ class UnixStreamServer(TCPServer): class UnixDatagramServer(UDPServer): address_family = socket.AF_UNIX - class ThreadingUnixStreamServer(ThreadingMixIn, UnixStreamServer): pass + class ThreadingUnixStreamServer(ThreadingMixIn, UnixStreamServer): + pass + + class ThreadingUnixDatagramServer(ThreadingMixIn, UnixDatagramServer): + pass - class ThreadingUnixDatagramServer(ThreadingMixIn, UnixDatagramServer): pass class BaseRequestHandler: diff --git a/tests/fake_filesystem.py b/tests/fake_filesystem.py index 42bc084512..c9627c5480 100644 --- a/tests/fake_filesystem.py +++ b/tests/fake_filesystem.py @@ -7,6 +7,7 @@ class FakeFile(StringIO): + def __init__(self, value=None, path=None): init = lambda x: StringIO.__init__(self, x) if value is None: diff --git a/tests/server.py b/tests/server.py index eba404a5c4..3623869858 100644 --- a/tests/server.py +++ b/tests/server.py @@ -67,7 +67,7 @@ '/tree/file2.txt': 'y', '/tree/subfolder/file3.txt': 'z', '/etc/apache2/apache2.conf': 'Include other.conf', - HOME: None # So $HOME is a directory + HOME: None # So $HOME is a directory }) PASSWORDS = { 'root': 'root', @@ -139,7 +139,7 @@ def check_auth_password(self, username, password): def check_auth_publickey(self, username, key): self.username = username - return ssh.AUTH_SUCCESSFUL if self.pubkeys else ssh.AUTH_FAILED + return ssh.AUTH_SUCCESSFUL if self.pubkeys else ssh.AUTH_FAILED def get_allowed_auths(self, username): return 'password,publickey' @@ -202,6 +202,7 @@ class PrependList(list): def prepend(self, val): self.insert(0, val) + def expand(path): """ '/foo/bar/biz' => ('/', 'foo', 'bar', 'biz') @@ -220,6 +221,7 @@ def expand(path): ret.prepend(directory if directory == os.path.sep else '') return ret + def contains(folder, path): """ contains(('a', 'b', 'c'), ('a', 'b')) => True @@ -227,6 +229,7 @@ def contains(folder, path): """ return False if len(path) >= len(folder) else folder[:len(path)] == path + def missing_folders(paths): """ missing_folders(['a/b/c']) => ['a', 'a/b', 'a/b/c'] @@ -236,7 +239,7 @@ def missing_folders(paths): for path in paths: expanded = expand(path) for i in range(len(expanded)): - folder = os.path.join(*expanded[:len(expanded)-i]) + folder = os.path.join(*expanded[:len(expanded) - i]) if folder and folder not in pool: pool.add(folder) ret.append(folder) @@ -272,7 +275,7 @@ def list_folder(self, path): candidates = [x for x in expanded_files if contains(x, expanded_path)] children = [] for candidate in candidates: - cut = candidate[:len(expanded_path)+1] + cut = candidate[:len(expanded_path) + 1] if cut not in children: children.append(cut) results = [self.stat(os.path.join(*x)) for x in children] @@ -326,6 +329,7 @@ def mkdir(self, path, attr): self.files[path] = None return ssh.SFTP_OK + def serve_responses(responses, files, passwords, home, pubkeys, port): """ Return a threading TCP based SocketServer listening on ``port``. diff --git a/tests/test_context_managers.py b/tests/test_context_managers.py index c756033a37..409ca38de9 100644 --- a/tests/test_context_managers.py +++ b/tests/test_context_managers.py @@ -30,7 +30,7 @@ def test_cwd_with_absolute_paths(): """ cd() should append arg if non-absolute or overwrite otherwise """ - existing = '/some/existing/path' + existing = '/some/existing/path' additional = 'another' absolute = '/absolute/path' with settings(cwd=existing): diff --git a/tests/test_main.py b/tests/test_main.py index 77e66fdf53..8efeefd1c2 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -16,7 +16,7 @@ def test_argument_parsing(): for args, output in [ - # Basic + # Basic ('abc', ('abc', [], {}, [], [])), # Arg ('ab:c', ('ab', ['c'], {}, [], [])), @@ -25,7 +25,7 @@ def test_argument_parsing(): # Arg and kwarg ('a:b=c,d', ('a', ['d'], {'b':'c'}, [], [])), # Multiple kwargs - ('a:b=c,d=e', ('a', [], {'b':'c','d':'e'}, [], [])), + ('a:b=c,d=e', ('a', [], {'b':'c', 'd':'e'}, [], [])), # Host ('abc:host=foo', ('abc', [], {}, ['foo'], [])), # Hosts with single host @@ -43,16 +43,18 @@ def test_argument_parsing(): def eq_hosts(command, host_list): eq_(set(get_hosts(command, [], [])), set(host_list)) - + def test_hosts_decorator_by_itself(): """ Use of @hosts only """ host_list = ['a', 'b'] + @hosts(*host_list) def command(): pass + eq_hosts(command, host_list) @@ -61,6 +63,7 @@ def command(): 'r2': ['b', 'c'] } + @with_patched_object( 'fabric.state', 'env', _AttributeDict({'roledefs': fake_roles}) ) @@ -105,9 +108,11 @@ def test_hosts_decorator_expands_single_iterable(): @hosts(iterable) should behave like @hosts(*iterable) """ host_list = ['foo', 'bar'] + @hosts(host_list) def command(): pass + eq_(command.hosts, host_list) @@ -116,9 +121,11 @@ def test_roles_decorator_expands_single_iterable(): @roles(iterable) should behave like @roles(*iterable) """ role_list = ['foo', 'bar'] + @roles(role_list) def command(): pass + eq_(command.roles, role_list) @@ -136,6 +143,7 @@ def test_aborts_on_nonexistent_roles(): lazy_role = {'r1': lambda: ['a', 'b']} + @with_patched_object( 'fabric.state', 'env', _AttributeDict({'roledefs': lazy_role}) ) diff --git a/tests/test_network.py b/tests/test_network.py index a6b10146e2..89c701130b 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -14,7 +14,7 @@ from fabric.network import (HostConnectionCache, join_host_strings, normalize, denormalize) from fabric.io import output_loop -import fabric.network # So I can call patch_object correctly. Sigh. +import fabric.network # So I can call patch_object correctly. Sigh. from fabric.state import env, output, _get_system_username from fabric.operations import run, sudo @@ -95,14 +95,12 @@ def test_host_string_denormalization(self): 'localhost', username + '@localhost:22'), ): eq_.description = "Host-string denormalization: %s" % description - yield eq_, denormalize(string1), denormalize(string2) + yield eq_, denormalize(string1), denormalize(string2) del eq_.description - # # Connection caching # - @staticmethod @with_fakes def check_connection_calls(host_strings, num_calls): @@ -136,17 +134,14 @@ def test_connection_caching(self): TestNetwork.check_connection_calls.description = description yield TestNetwork.check_connection_calls, host_strings, num_calls - # # Connection loop flow # - @server() def test_saved_authentication_returns_client_object(self): cache = HostConnectionCache() assert isinstance(cache[env.host_string], paramiko.SSHClient) - @server() @with_fakes def test_prompts_for_password_without_good_authentication(self): @@ -155,7 +150,6 @@ def test_prompts_for_password_without_good_authentication(self): cache = HostConnectionCache() cache[env.host_string] - @mock_streams('stdout') @server() def test_trailing_newline_line_drop(self): @@ -176,7 +170,6 @@ def test_trailing_newline_line_drop(self): # Also test that the captured value matches, too. eq_(output_string, result) - @server() def test_sudo_prompt_kills_capturing(self): """ @@ -186,7 +179,6 @@ def test_sudo_prompt_kills_capturing(self): with hide('everything'): eq_(sudo(cmd), RESPONSES[cmd]) - @server() def test_password_memory_on_user_switch(self): """ @@ -223,7 +215,6 @@ def _to_user(user): ): sudo("ls /simple") - @mock_streams('stderr') @server() def test_password_prompt_displays_host_string(self): @@ -238,7 +229,6 @@ def test_password_prompt_displays_host_string(self): regex = r'^\[%s\] Login password: ' % env.host_string assert_contains(regex, sys.stderr.getvalue()) - @mock_streams('stderr') @server(pubkeys=True) def test_passphrase_prompt_displays_host_string(self): @@ -254,7 +244,6 @@ def test_passphrase_prompt_displays_host_string(self): regex = r'^\[%s\] Login password: ' % env.host_string assert_contains(regex, sys.stderr.getvalue()) - def test_sudo_prompt_display_passthrough(self): """ Sudo prompt should display (via passthrough) when stdout/stderr shown @@ -299,7 +288,6 @@ def _prompt_display(display_output): [%(prefix)s] out: sudo password: """ % {'prefix': env.host_string} eq_(expected[1:], sys.stdall.getvalue()) - @mock_streams('both') @server( pubkeys=True, @@ -332,7 +320,6 @@ def test_consecutive_sudos_should_not_have_blank_line(self): """ % {'prefix': env.host_string} eq_(expected[1:], sys.stdall.getvalue()) - @mock_streams('both') @server(pubkeys=True, responses={'silent': '', 'normal': 'foo'}) def test_silent_commands_should_not_have_blank_line(self): @@ -365,7 +352,6 @@ def test_silent_commands_should_not_have_blank_line(self): """ % {'prefix': env.host_string} eq_(expected[1:], sys.stdall.getvalue()) - @mock_streams('both') @server( pubkeys=True, @@ -394,7 +380,6 @@ def test_io_should_print_prefix_if_ouput_prefix_is_true(self): """ % {'prefix': env.host_string} eq_(expected[1:], sys.stdall.getvalue()) - @mock_streams('both') @server( pubkeys=True, diff --git a/tests/test_operations.py b/tests/test_operations.py index 0d4ede1d77..8e818cb24b 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -25,6 +25,7 @@ # require() # + def test_require_single_existing_key(): """ When given a single existing key, require() throws no exceptions @@ -87,6 +88,7 @@ def test_require_mixed_state_keys_prints_missing_only(): def p(x): print x, + @mock_streams('stdout') @with_patched_object(sys.modules['__builtin__'], 'raw_input', p) def test_prompt_appends_space(): @@ -108,7 +110,7 @@ def test_prompt_with_default(): d = "default!" prompt(s, default=d) eq_(sys.stdout.getvalue(), "%s [%s] " % (s, d)) - + # # run()/sudo() @@ -214,11 +216,9 @@ def exists_remotely(self, path): def exists_locally(self, path): return os.path.exists(path) - # # get() # - @server(files={'/home/user/.bashrc': 'bash!'}, home='/home/user') def test_get_relative_remote_dir_uses_home(self): """ @@ -228,8 +228,6 @@ def test_get_relative_remote_dir_uses_home(self): # Another if-it-doesn't-error-out-it-passed test; meh. eq_(get('.bashrc', self.path()), [self.path('.bashrc')]) - - @server() def test_get_single_file(self): """ @@ -241,7 +239,6 @@ def test_get_single_file(self): get(remote, local) eq_contents(local, FILES[remote]) - @server() def test_get_sibling_globs(self): """ @@ -253,7 +250,6 @@ def test_get_sibling_globs(self): for remote in remotes: eq_contents(self.path(remote), FILES[remote]) - @server() def test_get_single_file_in_folder(self): """ @@ -264,7 +260,6 @@ def test_get_single_file_in_folder(self): get('folder', self.tmpdir) eq_contents(self.path(remote), FILES[remote]) - @server() def test_get_tree(self): """ @@ -276,7 +271,6 @@ def test_get_tree(self): for path, contents in leaves: eq_contents(self.path(path[1:]), contents) - @server() def test_get_tree_with_implicit_local_path(self): """ @@ -296,7 +290,6 @@ def test_get_tree_with_implicit_local_path(self): if os.path.exists(dirname): shutil.rmtree(dirname) - @server() def test_get_absolute_path_should_save_relative(self): """ @@ -309,7 +302,6 @@ def test_get_absolute_path_should_save_relative(self): assert self.exists_locally(os.path.join(lpath, 'subfolder')) assert not self.exists_locally(os.path.join(lpath, 'tree/subfolder')) - @server() def test_path_formatstr_nonrecursively_is_just_filename(self): """ @@ -321,7 +313,6 @@ def test_path_formatstr_nonrecursively_is_just_filename(self): get('/tree/subfolder/file3.txt', ltarget) assert self.exists_locally(os.path.join(lpath, 'file3.txt')) - @server() @mock_streams('stderr') def _invalid_file_obj_situations(self, remote_path): @@ -341,7 +332,6 @@ def test_directory_and_file_object_invalid(self): """ self._invalid_file_obj_situations('/tree') - @server() def test_get_single_file_absolutely(self): """ @@ -352,7 +342,6 @@ def test_get_single_file_absolutely(self): get(target, self.tmpdir) eq_contents(self.path(os.path.basename(target)), FILES[target]) - @server() def test_get_file_with_nonexistent_target(self): """ @@ -364,7 +353,6 @@ def test_get_file_with_nonexistent_target(self): get(target, local) eq_contents(local, FILES[target]) - @server() @mock_streams('stderr') def test_get_file_with_existing_file_target(self): @@ -380,7 +368,6 @@ def test_get_file_with_existing_file_target(self): assert "%s already exists" % local in sys.stderr.getvalue() eq_contents(local, FILES[target]) - @server() def test_get_file_to_directory(self): """ @@ -394,7 +381,6 @@ def test_get_file_to_directory(self): get(target, self.tmpdir) eq_contents(self.path(target), FILES[target]) - @server(port=2200) @server(port=2201) def test_get_from_multiple_servers(self): @@ -419,7 +405,6 @@ def test_get_from_multiple_servers(self): tmp, "127.0.0.1-%s" % port, 'file3.txt' )) - @server() def test_get_from_empty_directory_uses_cwd(self): """ @@ -432,7 +417,6 @@ def test_get_from_empty_directory_uses_cwd(self): for x in "file.txt file2.txt tree/file1.txt".split(): assert os.path.exists(os.path.join(self.tmpdir, x)) - @server() def _get_to_cwd(self, arg): path = 'file.txt' @@ -461,7 +445,6 @@ def test_get_to_None_uses_default_format_string(self): """ self._get_to_cwd(None) - @server() def test_get_should_accept_file_like_objects(self): """ @@ -473,7 +456,6 @@ def test_get_should_accept_file_like_objects(self): get(target, fake_file) eq_(fake_file.getvalue(), FILES[target]) - @server() def test_get_interpolation_without_host(self): """ @@ -490,7 +472,6 @@ def test_get_interpolation_without_host(self): get('/folder/file3.txt', local_path) assert self.exists_locally(tmp + "bar/file3.txt") - @server() def test_get_returns_list_of_local_paths(self): """ @@ -502,7 +483,6 @@ def test_get_returns_list_of_local_paths(self): files = ['file1.txt', 'file2.txt', 'subfolder/file3.txt'] eq_(map(lambda x: os.path.join(d, 'tree', x), files), retval) - @server() def test_get_returns_none_for_stringio(self): """ @@ -511,18 +491,17 @@ def test_get_returns_none_for_stringio(self): with hide('everything'): eq_([], get('/file.txt', StringIO())) - @server() def test_get_return_value_failed_attribute(self): """ - get()'s return value should indicate any paths which failed to download. + get()'s return value should indicate any paths which failed to + download. """ with settings(hide('everything'), warn_only=True): retval = get('/doesnt/exist', self.path()) eq_(['/doesnt/exist'], retval.failed) assert not retval.succeeded - @server() def test_get_should_not_use_windows_slashes_in_remote_paths(self): """ @@ -533,12 +512,9 @@ def test_get_should_not_use_windows_slashes_in_remote_paths(self): sftp = SFTP(env.host_string) eq_(sftp.glob(path), [path]) - - # # put() # - @server() def test_put_file_to_existing_directory(self): """ @@ -554,7 +530,6 @@ def test_put_file_to_existing_directory(self): get('/foo.txt', local2) eq_contents(local2, text) - @server() def test_put_to_empty_directory_uses_cwd(self): """ @@ -573,7 +548,6 @@ def test_put_to_empty_directory_uses_cwd(self): get('foo.txt', local2) eq_contents(local2, text) - @server() def test_put_from_empty_directory_uses_cwd(self): """ @@ -599,7 +573,6 @@ def test_put_from_empty_directory_uses_cwd(self): # Restore cwd os.chdir(old_cwd) - @server() def test_put_should_accept_file_like_objects(self): """ @@ -617,7 +590,6 @@ def test_put_should_accept_file_like_objects(self): # Sanity test of file pointer eq_(pointer, fake_file.tell()) - @server() @raises(ValueError) def test_put_should_raise_exception_for_nonexistent_local_path(self): @@ -626,7 +598,6 @@ def test_put_should_raise_exception_for_nonexistent_local_path(self): """ put('thisfiledoesnotexist', '/tmp') - @server() def test_put_returns_list_of_remote_paths(self): """ @@ -640,7 +611,6 @@ def test_put_returns_list_of_remote_paths(self): retval = put(f, p) eq_(retval, [p]) - @server() def test_put_returns_list_of_remote_paths_with_stringio(self): """ @@ -650,7 +620,6 @@ def test_put_returns_list_of_remote_paths_with_stringio(self): with hide('everything'): eq_(put(StringIO('contents'), f), [f]) - @server() def test_put_return_value_failed_attribute(self): """ @@ -662,12 +631,9 @@ def test_put_return_value_failed_attribute(self): eq_([""], retval.failed) assert not retval.succeeded - - # # Interactions with cd() # - @server() def test_cd_should_apply_to_put(self): """ @@ -682,7 +648,6 @@ def test_cd_should_apply_to_put(self): put(local, f) assert self.exists_remotely('%s/%s' % (d, f)) - @server(files={'/tmp/test.txt': 'test'}) def test_cd_should_apply_to_get(self): """ @@ -693,7 +658,6 @@ def test_cd_should_apply_to_get(self): get('test.txt', local) assert os.path.exists(local) - @server() def test_cd_should_not_apply_to_absolute_put(self): """ @@ -707,7 +671,6 @@ def test_cd_should_not_apply_to_absolute_put(self): assert not self.exists_remotely('/tmp/test.txt') assert self.exists_remotely('/test.txt') - @server(files={'/test.txt': 'test'}) def test_cd_should_not_apply_to_absolute_get(self): """ @@ -718,7 +681,6 @@ def test_cd_should_not_apply_to_absolute_get(self): get('/test.txt', local) assert os.path.exists(local) - @server() def test_lcd_should_apply_to_put(self): """ @@ -734,7 +696,6 @@ def test_lcd_should_apply_to_put(self): put(f, '/') assert self.exists_remotely('/%s' % f) - @server() def test_lcd_should_apply_to_get(self): """ diff --git a/tests/test_state.py b/tests/test_state.py index b9a4d4f49d..e6fcea52b9 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -8,7 +8,7 @@ def test_dict_aliasing(): Assigning values to aliases updates aliased keys """ ad = _AliasDict( - {'bar': False, 'biz': True, 'baz': False}, + {'bar': False, 'biz': True, 'baz': False}, aliases={'foo': ['bar', 'biz', 'baz']} ) # Before @@ -28,7 +28,7 @@ def test_nested_dict_aliasing(): Aliases can be nested """ ad = _AliasDict( - {'bar': False, 'biz': True}, + {'bar': False, 'biz': True}, aliases={'foo': ['bar', 'nested'], 'nested': ['biz']} ) # Before @@ -46,7 +46,7 @@ def test_dict_alias_expansion(): Alias expansion """ ad = _AliasDict( - {'bar': False, 'biz': True}, + {'bar': False, 'biz': True}, aliases={'foo': ['bar', 'nested'], 'nested': ['biz']} ) eq_(ad.expand_aliases(['foo']), ['bar', 'biz']) diff --git a/tests/test_utils.py b/tests/test_utils.py index 04284c1b5f..6c8e5120f4 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -9,7 +9,7 @@ from fabric.state import output, env from fabric.utils import warn, indent, abort, puts, fastprint -from fabric import utils # For patching +from fabric import utils # For patching from fabric.context_managers import settings from utils import mock_streams @@ -72,7 +72,7 @@ def test_abort_message(): pass result = sys.stderr.getvalue() eq_("\nFatal error: Test\n\nAborting.\n", result) - + @mock_streams('stdout') def test_puts_with_user_output_on(): diff --git a/tests/utils.py b/tests/utils.py index ce3754e8e7..6c6eb7400f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,6 +1,6 @@ from __future__ import with_statement -from StringIO import StringIO # No need for cStringIO at this time +from StringIO import StringIO # No need for cStringIO at this time from contextlib import contextmanager from functools import wraps, partial from types import StringTypes @@ -94,6 +94,7 @@ def func() both = (which == 'both') stdout = (which == 'stdout') or both stderr = (which == 'stderr') or both + def mocked_streams_decorator(func): @wraps(func) def inner_wrapper(*args, **kwargs): From 76f7f86a72af4681bc653c9ea56ed6bb62685238 Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Mon, 14 Mar 2011 15:32:41 -0400 Subject: [PATCH 015/126] Fixes 113 updates to allow tuples as host/rols lists - Added test for roles and hosts as tuples - Add new decorator for testing @with_patched_state_env to better mock out a full state.env - updated the new tests to use ^^ decorator --- fabric/main.py | 5 +++++ fabric/state.py | 2 +- tests/test_main.py | 30 +++++++++++++++++++++++++++++- tests/utils.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 2 deletions(-) diff --git a/fabric/main.py b/fabric/main.py index 5e97d6b3c8..d107f8c360 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -362,6 +362,11 @@ def _merge(hosts, roles): if callable(value): value = value() role_hosts += value + + # make sure hosts is converted to a list to be able to append to the + # role_hosts list + hosts = list(hosts) + # Return deduped combo of hosts and role_hosts return list(set(hosts + role_hosts)) diff --git a/fabric/state.py b/fabric/state.py index a59717d5c6..2ec3410646 100644 --- a/fabric/state.py +++ b/fabric/state.py @@ -236,7 +236,7 @@ def _rc_path(): 'path_behavior': 'append', 'port': None, 'real_fabfile': None, - 'roledefs': {}, + 'roles': [], 'roledefs': {}, # -S so sudo accepts passwd via stdin, -p with our known-value prompt for # later detection (thus %s -- gets filled with env.sudo_prompt at runtime) diff --git a/tests/test_main.py b/tests/test_main.py index 77e66fdf53..12f379988e 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -12,6 +12,7 @@ from fabric.state import _AttributeDict from utils import mock_streams +from utils import with_patched_state_env def test_argument_parsing(): @@ -43,7 +44,7 @@ def test_argument_parsing(): def eq_hosts(command, host_list): eq_(set(get_hosts(command, [], [])), set(host_list)) - + def test_hosts_decorator_by_itself(): """ @@ -88,6 +89,33 @@ def command(): eq_hosts(command, ['a', 'b', 'c']) +tuple_roles = { + 'r1': ('a', 'b'), + 'r2': ('b', 'c'), +} + + +@with_patched_state_env({'roledefs': tuple_roles}) +def test_roles_as_tuples(): + """ + Test that a list of roles as a tuple succeeds + """ + @roles('r1') + def command(): + pass + eq_hosts(command, ['a', 'b']) + + +@with_patched_state_env({'hosts': ('foo', 'bar')}) +def test_hosts_as_tuples(): + """ + Test that a list of hosts as a tuple succeeds + """ + def command(): + pass + eq_hosts(command, ['foo', 'bar']) + + @with_patched_object('fabric.state', 'env', {'hosts': ['foo']}) def test_hosts_decorator_overrides_env_hosts(): """ diff --git a/tests/utils.py b/tests/utils.py index ce3754e8e7..f1d090de18 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -2,6 +2,7 @@ from StringIO import StringIO # No need for cStringIO at this time from contextlib import contextmanager +from fudge.patcher import with_patched_object from functools import wraps, partial from types import StringTypes import copy @@ -208,3 +209,44 @@ def eq_(a, b, msg=None): def eq_contents(path, text): with open(path) as fd: eq_(text, fd.read()) + + +class with_patched_state_env(object): + """Decorate a unit test to provide a default state.env + + Grabs the fabric.state.env attribute dictionary. + + e.g. + @with_patched_state_env({'hosts': ('foo', 'bar')}) + def test_something_with_hosts(): + + """ + def __init__(self, dict_overrides): + """ Pass in a dict of keys to override the default env with + + :param dict_overrides: Dictionary of keys to override in default_env + + """ + from fabric.state import env as default_env + self.env = self._merge_env(default_env.copy(), dict_overrides) + + def _merge_env(self, default, overrides): + """for item in the overrides, set them over the default""" + for key, val in overrides.iteritems(): + default[key] = val + + return default + + def __call__(self, f): + """This wraps a function with a fudge mock that implements the default + state.env + """ + env = self.env + + def wrapped_f(env=env, *args): + f(*args) + + # now add the fudge decorator here + patcher = with_patched_object('fabric.state', 'env', env) + wrapped_f = patcher(wrapped_f) + return wrapped_f From 7a5347cce2c6c423bf671ae44ed6d7222dcdfcd8 Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Mon, 14 Mar 2011 16:00:19 -0400 Subject: [PATCH 016/126] Update tests in test_main to use decorator to mock state.env --- tests/test_main.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 12f379988e..9f6f7f2f14 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,7 +1,6 @@ import sys import copy -from fudge.patcher import with_patched_object from fudge import Fake from nose.tools import eq_, raises @@ -62,9 +61,7 @@ def command(): 'r2': ['b', 'c'] } -@with_patched_object( - 'fabric.state', 'env', _AttributeDict({'roledefs': fake_roles}) -) +@with_patched_state_env({'roledefs': fake_roles}) def test_roles_decorator_by_itself(): """ Use of @roles only @@ -75,9 +72,7 @@ def command(): eq_hosts(command, ['a', 'b']) -@with_patched_object( - 'fabric.state', 'env', _AttributeDict({'roledefs': fake_roles}) -) +@with_patched_state_env({'roledefs': fake_roles}) def test_hosts_and_roles_together(): """ Use of @roles and @hosts together results in union of both @@ -116,7 +111,7 @@ def command(): eq_hosts(command, ['foo', 'bar']) -@with_patched_object('fabric.state', 'env', {'hosts': ['foo']}) +@with_patched_state_env({'hosts': ['foo']}) def test_hosts_decorator_overrides_env_hosts(): """ If @hosts is used it replaces any env.hosts value @@ -150,9 +145,7 @@ def command(): eq_(command.roles, role_list) -@with_patched_object( - 'fabric.state', 'env', _AttributeDict({'roledefs': fake_roles}) -) +@with_patched_state_env({'roledefs': fake_roles}) @raises(SystemExit) @mock_streams('stderr') def test_aborts_on_nonexistent_roles(): @@ -164,9 +157,7 @@ def test_aborts_on_nonexistent_roles(): lazy_role = {'r1': lambda: ['a', 'b']} -@with_patched_object( - 'fabric.state', 'env', _AttributeDict({'roledefs': lazy_role}) -) +@with_patched_state_env({'roledefs': lazy_role}) def test_lazy_roles(): """ Roles may be callables returning lists, as well as regular lists From dd11efe62acd1cd1090708364445abff8b7292ce Mon Sep 17 00:00:00 2001 From: Thomas Ballinger Date: Wed, 16 Mar 2011 15:35:46 -0400 Subject: [PATCH 017/126] Added args to upload_template for mode support Addesses issue #117: Add mode support to upload_template() Appears to address #304, Rewrite upload_template to make use of new put() features. --- fabric/contrib/files.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/fabric/contrib/files.py b/fabric/contrib/files.py index 15155cf898..94d4fc3c63 100644 --- a/fabric/contrib/files.py +++ b/fabric/contrib/files.py @@ -48,7 +48,7 @@ def first(*args, **kwargs): def upload_template(filename, destination, context=None, use_jinja=False, - template_dir=None, use_sudo=False): + template_dir=None, use_sudo=False, mirror_local_mode=False, mode=None): """ Render and upload a template text file to a remote host. @@ -61,7 +61,7 @@ def upload_template(filename, destination, context=None, use_jinja=False, templating library available, Jinja will be used to render the template instead. Templates will be loaded from the invoking user's current working directory by default, or from ``template_dir`` if given. - + The resulting rendered file will be uploaded to the remote file path ``destination`` (which should include the desired remote filename.) If the destination file already exists, it will be renamed with a ``.bak`` @@ -69,9 +69,15 @@ def upload_template(filename, destination, context=None, use_jinja=False, By default, the file will be copied to ``destination`` as the logged-in user; specify ``use_sudo=True`` to use `sudo` instead. + + In some use cases, it is desirable to force a newly uploaded file to match + the mode of its local counterpart (such as when uploading executable + scripts). To do this, specify ``mirror_local_mode=True``. + + Alternately, you may use the ``mode`` kwarg to specify an exact mode, in + the same vein as ``os.chmod`` or the Unix ``chmod`` command. """ basename = os.path.basename(filename) - temp_destination = '/tmp/' + basename # This temporary file should not be automatically deleted on close, as we # need it there to upload it (Windows locks the file for reading while @@ -95,23 +101,21 @@ def upload_template(filename, destination, context=None, use_jinja=False, output.write(text) output.close() - # Upload the file. - put(tempfile_name, temp_destination) - os.close(tempfile_fd) - os.remove(tempfile_name) - + # Back up any original file func = use_sudo and sudo or run - # Back up any original file (need to do figure out ultimate destination) to_backup = destination with settings(hide('everything'), warn_only=True): - # Is destination a directory? + # Is destination not an existing, non-directory file? if func('test -f %s' % to_backup).failed: - # If so, tack on the filename to get "real" destination + # If so, tack on the filename to try to get "real" destination to_backup = destination + '/' + basename if exists(to_backup): func("cp %s %s.bak" % (to_backup, to_backup)) - # Actually move uploaded template to destination - func("mv %s %s" % (temp_destination, destination)) + + # Upload the file. + put(tempfile_name, destination, use_sudo, mirror_local_mode, mode) + os.close(tempfile_fd) + os.remove(tempfile_name) def sed(filename, before, after, limit='', use_sudo=False, backup='.bak'): From bc0fe58a6aca7ce0b0a285660cd2eac71e45871c Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 22 Mar 2011 15:11:58 -0700 Subject: [PATCH 018/126] Update attribution re #312 --- AUTHORS | 1 + docs/changes/1.0.1.rst | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 9977b56923..4835c1819a 100644 --- a/AUTHORS +++ b/AUTHORS @@ -32,3 +32,4 @@ Erich Heine Travis Swicegood Paul Smith Rick Harding +Kirill Pinchuk diff --git a/docs/changes/1.0.1.rst b/docs/changes/1.0.1.rst index ab6268963c..7f09c95399 100644 --- a/docs/changes/1.0.1.rst +++ b/docs/changes/1.0.1.rst @@ -17,8 +17,8 @@ Bugfixes the documentation was altered. This has been fixed. Thanks to Adam Ernst for bringing it to our attention. * :issue:`312`: Tweak internal I/O related loops to prevent high CPU usage and - poor screen-printing behavior on some systems. Thanks to Redmine user ``cbr - grind`` for the initial patch. + poor screen-printing behavior on some systems. Thanks to Kirill Pinchuk for + the initial patch. Documentation ============= From 5afe16faab754b91da1af08ce7e9d61f1fe94a5f Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 22 Mar 2011 17:44:49 -0700 Subject: [PATCH 019/126] Fix #310 - use octal, not string rep --- docs/changes/1.0.1.rst | 5 +++++ fabric/sftp.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/changes/1.0.1.rst b/docs/changes/1.0.1.rst index 7f09c95399..e956c83ddd 100644 --- a/docs/changes/1.0.1.rst +++ b/docs/changes/1.0.1.rst @@ -12,6 +12,11 @@ Bugfixes * :issue:`301`: Fixed a bug in `~fabric.operations.local`'s behavior when ``capture=False`` and ``output.stdout`` (or ``.stderr``) was also ``False``. Thanks to Chris Rose for the catch. +* :issue:`310`: Update edge case in `~fabric.operations.put` where using the + ``mode`` kwarg alongside ``use_sudo=True`` runs a hidden + `~fabric.operations.sudo` command. The ``mode`` kwarg needs to be octal but + was being interpolated in the ``sudo`` call as a string/integer. Thanks to + Adam Ernst for the catch and suggested fix. * :issue:`311`: `~fabric.contrib.files.append` was supposed to have its ``partial`` kwarg's default flipped from ``True`` to ``False``. However, only the documentation was altered. This has been fixed. Thanks to Adam Ernst for diff --git a/fabric/sftp.py b/fabric/sftp.py index 29954f6c21..a5ff784b75 100644 --- a/fabric/sftp.py +++ b/fabric/sftp.py @@ -243,7 +243,7 @@ def put(self, local_path, remote_path, use_sudo, mirror_local_mode, mode, if lmode != rmode: if use_sudo: with hide('everything'): - sudo('chmod %s \"%s\"' % (lmode, remote_path)) + sudo('chmod %o \"%s\"' % (lmode, remote_path)) else: self.ftp.chmod(remote_path, lmode) if use_sudo: From cbdfc0690d817639a71d0266ca7e860d7ed3baf0 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Sun, 27 Mar 2011 22:00:15 -0400 Subject: [PATCH 020/126] Update changelog re: #320 --- docs/changes/1.0.1.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changes/1.0.1.rst b/docs/changes/1.0.1.rst index e956c83ddd..8c5c94cc25 100644 --- a/docs/changes/1.0.1.rst +++ b/docs/changes/1.0.1.rst @@ -24,6 +24,9 @@ Bugfixes * :issue:`312`: Tweak internal I/O related loops to prevent high CPU usage and poor screen-printing behavior on some systems. Thanks to Kirill Pinchuk for the initial patch. +* :issue:`320`: Some users reported problems with dropped input, particularly + while entering `~fabric.operations.sudo` passwords. This was fixed via the + same change as for :issue:`312`. Documentation ============= From 5b70772a431e990e5ba3936c847d3381188ca322 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Sun, 27 Mar 2011 22:03:56 -0400 Subject: [PATCH 021/126] Bump version to 1.0.1 final --- fabric/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fabric/version.py b/fabric/version.py index 5c6c735afc..e31dfa416e 100644 --- a/fabric/version.py +++ b/fabric/version.py @@ -21,7 +21,7 @@ def git_sha(): return p.communicate()[0] -VERSION = (1, 0, 0, 'final', 0) +VERSION = (1, 0, 1, 'final', 0) def get_version(form='short'): """ From 782a18b067abda6493fb6354021d2fcef908f5e9 Mon Sep 17 00:00:00 2001 From: Morgan Goose Date: Sun, 3 Apr 2011 20:30:21 -0400 Subject: [PATCH 022/126] Think this does what jeff had suggested. --- fabric/operations.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fabric/operations.py b/fabric/operations.py index a0c620d720..082692e300 100644 --- a/fabric/operations.py +++ b/fabric/operations.py @@ -16,10 +16,11 @@ from contextlib import closing +from fabric import state from fabric.context_managers import settings, char_buffered from fabric.io import output_loop, input_loop from fabric.network import needs_host -from fabric.state import (env, connections, output, win32, default_channel, +from fabric.state import (env, output, win32, default_channel, io_sleep) from fabric.utils import abort, indent, warn, puts from fabric.thread_handling import ThreadHandler @@ -1038,10 +1039,10 @@ def reboot(wait): .. versionadded:: 0.9.2 """ sudo('reboot') - client = connections[env.host_string] + client = state.connections[env.host_string] client.close() - if env.host_string in connections: - del connections[env.host_string] + if env.host_string in state.connections: + del state.connections[env.host_string] if output.running: puts("Waiting for reboot: ", flush=True, end='') per_tick = 5 From ac8ff96cda314c469831a3ce1a61e3b7c22e6a90 Mon Sep 17 00:00:00 2001 From: Ales Zoulek Date: Thu, 13 Jan 2011 23:11:40 +0100 Subject: [PATCH 023/126] added backup_file argument to upload_template() --- fabric/contrib/files.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/fabric/contrib/files.py b/fabric/contrib/files.py index d21315c072..57b738a075 100644 --- a/fabric/contrib/files.py +++ b/fabric/contrib/files.py @@ -49,7 +49,7 @@ def first(*args, **kwargs): def upload_template(filename, destination, context=None, use_jinja=False, - template_dir=None, use_sudo=False): + template_dir=None, use_sudo=False, backup_file=True): """ Render and upload a template text file to a remote host. @@ -66,7 +66,7 @@ def upload_template(filename, destination, context=None, use_jinja=False, The resulting rendered file will be uploaded to the remote file path ``destination`` (which should include the desired remote filename.) If the destination file already exists, it will be renamed with a ``.bak`` - extension. + extension. You can turn this off by ``backup_file=False``. By default, the file will be copied to ``destination`` as the logged-in user; specify ``use_sudo=True`` to use `sudo` instead. @@ -103,14 +103,15 @@ def upload_template(filename, destination, context=None, use_jinja=False, func = use_sudo and sudo or run # Back up any original file (need to do figure out ultimate destination) - to_backup = destination - with settings(hide('everything'), warn_only=True): - # Is destination a directory? - if func('test -f %s' % to_backup).failed: - # If so, tack on the filename to get "real" destination - to_backup = destination + '/' + basename - if exists(to_backup): - func("cp %s %s.bak" % (to_backup, to_backup)) + if backup_file: + to_backup = destination + with settings(hide('everything'), warn_only=True): + # Is destination a directory? + if func('test -f %s' % to_backup).failed: + # If so, tack on the filename to get "real" destination + to_backup = destination + '/' + basename + if exists(to_backup): + func("cp %s %s.bak" % (to_backup, to_backup)) # Actually move uploaded template to destination func("mv %s %s" % (temp_destination, destination)) From 886c20fba79ed33b5b062f44be53f9f6f5333897 Mon Sep 17 00:00:00 2001 From: Morgan Goose Date: Tue, 5 Apr 2011 16:03:03 -0400 Subject: [PATCH 024/126] Merged in Travis Swicegood's fix for #125 I had to pull it out by hand because the patch wouldn't apply, and I also wanted to make a change and hide the main function of append and write, much like run and sudo use a hidden command and are just place holders that enact specific defaults. Anyhow it's a direct pull from this commit: https://github.com/tswicegood/fabric/blob/cc853f165e9b1b2c177579a16e97be16cc9b40d6/fabric/contrib/files.py --- fabric/contrib/files.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/fabric/contrib/files.py b/fabric/contrib/files.py index d21315c072..44d10749c6 100644 --- a/fabric/contrib/files.py +++ b/fabric/contrib/files.py @@ -297,7 +297,27 @@ def append(filename, text, use_sudo=False, partial=False, escape=True): .. versionchanged:: 1.0 Changed default value of ``partial`` kwarg to be ``False``. """ + _write_to_file(filename, text, use_sudo=use_sudo) + +def write(filename, text, use_sudo=False): + """ + Write string (or list of strings) ``text`` to ``filename``. + + This is identical to ``append()``, except that it overwrites any existing + file, instead of appending to it. + """ + _write_to_file(filename, text, use_sudo=use_sudo, overwrite=True) + +def _write_to_file(filename, text, use_sudo=False, overwrite=False): + """ + Append or overwrite a the string (or list of strings) ``text`` to + ``filename``. + + This is the implementation for both ``write`` and ``append``. Both call + this with the proper value for ``overwrite``. + """ func = use_sudo and sudo or run + operator = overwrite and '>' or '>>' # Normalize non-list input to be a list if isinstance(text, str): text = [text] @@ -307,4 +327,4 @@ def append(filename, text, use_sudo=False, partial=False, escape=True): and contains(filename, regex, use_sudo=use_sudo)): continue line = line.replace("'", r'\'') if escape else line - func("echo '%s' >> %s" % (line, filename)) + func("echo '%s' %s %s" % (line.replace("'", r'\''), operator, filename)) From b825a44404e274b39f5b2138ee184e906df54f23 Mon Sep 17 00:00:00 2001 From: Morgan Goose Date: Tue, 5 Apr 2011 19:10:37 -0400 Subject: [PATCH 025/126] Added in docs for this branch as well. --- docs/usage/execution.rst | 18 ++++++++++++++---- docs/usage/fab.rst | 5 +++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/docs/usage/execution.rst b/docs/usage/execution.rst index c2d514f735..95d2c16403 100644 --- a/docs/usage/execution.rst +++ b/docs/usage/execution.rst @@ -265,10 +265,11 @@ look up in ``env.roledefs``. Globally, via the command line ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In addition to modifying ``env.hosts`` and ``env.roles`` at the module level, -you may define them by passing comma-separated string arguments to the -command-line switches :option:`--hosts/-H <-H>` and :option:`--roles/-R <-R>`, -e.g.:: +In addition to modifying ``env.hosts``, ``env.roles``, and +``env.exclude_hosts`` at the module level, you may define them by passing +comma-separated string arguments to the command-line switches +:option:`--hosts/-H <-H>`, :option:`--roles/-R <-R>`, and +:option:`--exclude-hosts/-x <-x>`, e.g.:: $ fab -H host1,host2 mytask @@ -301,6 +302,10 @@ When this fabfile is run as ``fab -H host1,host2 mytask``, ``env.hosts`` will end contain ``['host1', 'host2', 'host3', 'host4']`` at the time that ``mytask`` is executed. +For exclusions when this fabfile is run as ``fab -H host1,host2 -x host1 +mytask``, ``env.hosts`` will be the same, but the host list that gets executed +will not have host1 included in it. + .. note:: ``env.hosts`` is simply a Python list object -- so you may use @@ -344,6 +349,11 @@ To specify per-task hosts for ``mytask``, execute it like so:: This will override any other host list and ensure ``mytask`` always runs on just those two hosts. +You are also able to exclude hosts like this:: + + $ fab mytask:hosts="host1;host2",exclude_hosts="host1" + + Per-task, via decorators ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/usage/fab.rst b/docs/usage/fab.rst index 5dc2008dd1..546e14a093 100644 --- a/docs/usage/fab.rst +++ b/docs/usage/fab.rst @@ -131,6 +131,11 @@ below. Sets :ref:`env.hosts ` to the given comma-delimited list of host strings. +.. cmdoption:: -x HOSTS, --exclude-hosts=HOSTS + + Sets :ref:`env.exclude_hosts` to the given comma-delimited list of host + strings to then keep out of the final host list. + .. cmdoption:: -i KEY_FILENAME When set to a file path, will load the given file as an SSH identity file From 9478ebc0bd5f4cbcd06a558cf1b82a9861615c80 Mon Sep 17 00:00:00 2001 From: Morgan Goose Date: Wed, 6 Apr 2011 10:53:34 -0400 Subject: [PATCH 026/126] Added in case insensitive flag --- fabric/contrib/files.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/fabric/contrib/files.py b/fabric/contrib/files.py index d21315c072..bb66d0d2c7 100644 --- a/fabric/contrib/files.py +++ b/fabric/contrib/files.py @@ -115,7 +115,8 @@ def upload_template(filename, destination, context=None, use_jinja=False, func("mv %s %s" % (temp_destination, destination)) -def sed(filename, before, after, limit='', use_sudo=False, backup='.bak'): +def sed(filename, before, after, limit='', use_sudo=False, backup='.bak', + case_insensitive=False): """ Run a search-and-replace on ``filename`` with given regex patterns. @@ -131,6 +132,11 @@ def sed(filename, before, after, limit='', use_sudo=False, backup='.bak'): `sed` will pass ``shell=False`` to `run`/`sudo`, in order to avoid problems with many nested levels of quotes and backslashes. + + If you would like the matches to be case insensitive, you'd only need to + pass the ``case_insensitive=True`` to have it tack on an 'i' to the sed + command. Effectively making it execute``sed -i -r -e "// + s///ig "``. """ func = use_sudo and sudo or run # Characters to be escaped in both @@ -144,6 +150,11 @@ def sed(filename, before, after, limit='', use_sudo=False, backup='.bak'): if limit: limit = r'/%s/ ' % limit # Test the OS because of differences between sed versions + + case_bit = '' + if case_insensitive: + case_bit = 'i' + with hide('running', 'stdout'): platform = run("uname") if platform in ('NetBSD', 'OpenBSD'): @@ -154,13 +165,13 @@ def sed(filename, before, after, limit='', use_sudo=False, backup='.bak'): tmp = "/tmp/%s" % hasher.hexdigest() # Use temp file to work around lack of -i expr = r"""cp -p %(filename)s %(tmp)s \ -&& sed -r -e '%(limit)ss/%(before)s/%(after)s/g' %(filename)s > %(tmp)s \ +&& sed -r -e '%(limit)ss/%(before)s/%(after)s/%(case_bit)sg' %(filename)s > %(tmp)s \ && cp -p %(filename)s %(filename)s%(backup)s \ && mv %(tmp)s %(filename)s""" command = expr % locals() else: - expr = r"sed -i%s -r -e '%ss/%s/%s/g' %s" - command = expr % (backup, limit, before, after, filename) + expr = r"sed -i%s -r -e '%ss/%s/%s/%sg' %s" + command = expr % (backup, limit, before, after, case_bit, filename) return func(command, shell=False) From dcd9564b130391322c2833317815fac19df193df Mon Sep 17 00:00:00 2001 From: Morgan Goose Date: Fri, 8 Apr 2011 00:42:19 -0400 Subject: [PATCH 027/126] Made up some docs for this feature. --- docs/usage/execution.rst | 18 ++++++++++++++---- docs/usage/fab.rst | 7 +++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/docs/usage/execution.rst b/docs/usage/execution.rst index c2d514f735..4ba02acbb3 100644 --- a/docs/usage/execution.rst +++ b/docs/usage/execution.rst @@ -265,10 +265,11 @@ look up in ``env.roledefs``. Globally, via the command line ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In addition to modifying ``env.hosts`` and ``env.roles`` at the module level, -you may define them by passing comma-separated string arguments to the -command-line switches :option:`--hosts/-H <-H>` and :option:`--roles/-R <-R>`, -e.g.:: +In addition to modifying ``env.hosts``, ``env.roles``, and +``env.exclude_hosts`` at the module level, you may define them by passing +comma-separated string arguments to the command-line switches +:option:`--hosts/-H <-H>`, :option:`--roles/-R <-R>`, and +:option:`--exclude-hosts/-x <-x>`, e.g.:: $ fab -H host1,host2 mytask @@ -301,6 +302,10 @@ When this fabfile is run as ``fab -H host1,host2 mytask``, ``env.hosts`` will end contain ``['host1', 'host2', 'host3', 'host4']`` at the time that ``mytask`` is executed. +For exclusions when this fabfile is run as ``fab -H host1,host2 -x host1 +mytask``, ``env.hosts`` will be the same, but the host list that gets executed +will not have host1 included in it. + .. note:: ``env.hosts`` is simply a Python list object -- so you may use @@ -344,6 +349,11 @@ To specify per-task hosts for ``mytask``, execute it like so:: This will override any other host list and ensure ``mytask`` always runs on just those two hosts. +You are also able to exclude hosts like this:: + + $ fab mytask:hosts="host1;host2",exclude_hosts="host1" + + Per-task, via decorators ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/usage/fab.rst b/docs/usage/fab.rst index 5dc2008dd1..4fa509fff0 100644 --- a/docs/usage/fab.rst +++ b/docs/usage/fab.rst @@ -131,6 +131,13 @@ below. Sets :ref:`env.hosts ` to the given comma-delimited list of host strings. +.. cmdoption:: -x HOSTS, --exclude-hosts=HOSTS + + Sets :ref:`env.exclude_hosts` to the given comma-delimited list of host + strings to then keep out of the final host list. + + .. versionadded:: 1.1 + .. cmdoption:: -i KEY_FILENAME When set to a file path, will load the given file as an SSH identity file From 9fdd0d6d0128fc692a626899b6118edf40c8c034 Mon Sep 17 00:00:00 2001 From: goosemo Date: Fri, 8 Apr 2011 08:41:58 -0400 Subject: [PATCH 028/126] Forgot to add it to the api --- fabric/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fabric/api.py b/fabric/api.py index 3e2e11894c..52cb9dccea 100644 --- a/fabric/api.py +++ b/fabric/api.py @@ -7,7 +7,7 @@ well when you're using setup.py to install e.g. paramiko! """ from fabric.context_managers import cd, hide, settings, show, path, prefix, lcd -from fabric.decorators import hosts, roles, runs_once +from fabric.decorators import hosts, roles, runs_once, ensure_order from fabric.operations import (require, prompt, put, get, run, sudo, local, reboot, open_shell) from fabric.state import env, output From 62237f62820d279c8c7b43878f52a792c0b56ef3 Mon Sep 17 00:00:00 2001 From: goosemo Date: Fri, 8 Apr 2011 08:42:40 -0400 Subject: [PATCH 029/126] Forgot to add it to the api --- fabric/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fabric/api.py b/fabric/api.py index 3e2e11894c..4c49c9a8af 100644 --- a/fabric/api.py +++ b/fabric/api.py @@ -7,7 +7,7 @@ well when you're using setup.py to install e.g. paramiko! """ from fabric.context_managers import cd, hide, settings, show, path, prefix, lcd -from fabric.decorators import hosts, roles, runs_once +from fabric.decorators import hosts, roles, runs_once, with_settings from fabric.operations import (require, prompt, put, get, run, sudo, local, reboot, open_shell) from fabric.state import env, output From 598e0bac5fee54b7eab055a48604f6a3c51e7225 Mon Sep 17 00:00:00 2001 From: Morgan Goose Date: Fri, 8 Apr 2011 13:46:28 -0400 Subject: [PATCH 030/126] M the main env variable for ensure_order public --- fabric/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fabric/main.py b/fabric/main.py index 30e3b629af..b3e1876d03 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -365,7 +365,7 @@ def _merge(hosts, roles): role_hosts += value # Return deduped combo of hosts and role_hosts - if hasattr(state.env, '_ensure_order') and state.env._ensure_order: + if hasattr(state.env, 'ensure_order') and state.env.ensure_order: result_hosts = [] for host in hosts + role_hosts: if host not in result_hosts: @@ -397,7 +397,7 @@ def get_hosts(command, cli_hosts, cli_roles): if hasattr(command, '_ensure_order') and command._ensure_order: if hasattr(command, '_sorted') and command._sorted == True: state.env._sorted = command._sorted - state.env._ensure_order = command._ensure_order + state.env.ensure_order = command._ensure_order # Command line per-command takes precedence over anything else. if cli_hosts or cli_roles: From 10b8295fb524af343ac03b59c68f8c76b3ab2ad0 Mon Sep 17 00:00:00 2001 From: Morgan Goose Date: Fri, 8 Apr 2011 14:23:22 -0400 Subject: [PATCH 031/126] Added in some docs for the env.ensure_order --- docs/usage/env.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/usage/env.rst b/docs/usage/env.rst index e2af31d5db..a750778931 100644 --- a/docs/usage/env.rst +++ b/docs/usage/env.rst @@ -396,6 +396,22 @@ takes a command string as its value. .. seealso:: :doc:`execution` +.. _ensure_order: + +``ensure_order`` +---------------- + +**Default:** ``False`` + +Switch to globally state if hosts lists should be deduped in place leaving +order intact from right to left of the combination of `~fabric.state.env.hosts` +and `~fabric.state.env.roles` + +.. note:: + + With this option you can also pre-sort host lists when decorating to use a + special sort if desired. + ``sudo_prompt`` --------------- From bb9b48661ee8fc0dfd1c0a2d13825bd3d0286b5b Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 12 Apr 2011 11:27:30 -0700 Subject: [PATCH 032/126] Update PyPI-oriented changelog link re: RTD architecture tweaks --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index cbdf292a96..2353782268 100644 --- a/setup.py +++ b/setup.py @@ -9,9 +9,10 @@ readme = open('README').read() +v = get_version('short') long_description = """ To find out what's new in this version of Fabric, please see `the changelog -`_. +`_. You can also install the `in-development version `_ using @@ -24,7 +25,7 @@ ---- For more information, please see the Fabric website or execute ``fab --help``. -""" % (get_version('short'), readme) +""" % (v, v, readme) setup( name='Fabric', From 0ef63c8b8822d53a652b74d1f14470e25d37a94f Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 12 Apr 2011 11:27:30 -0700 Subject: [PATCH 033/126] Update PyPI-oriented changelog link re: RTD architecture tweaks --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index cbdf292a96..2353782268 100644 --- a/setup.py +++ b/setup.py @@ -9,9 +9,10 @@ readme = open('README').read() +v = get_version('short') long_description = """ To find out what's new in this version of Fabric, please see `the changelog -`_. +`_. You can also install the `in-development version `_ using @@ -24,7 +25,7 @@ ---- For more information, please see the Fabric website or execute ``fab --help``. -""" % (get_version('short'), readme) +""" % (v, v, readme) setup( name='Fabric', From c13dbb43f7b44e1e71b91b1c526207f3eb7d4f3b Mon Sep 17 00:00:00 2001 From: goosemo Date: Tue, 12 Apr 2011 17:16:02 -0400 Subject: [PATCH 034/126] Fixed issue with 2.5 compat Got a report from Vladimir Lazarenko on irc that using the 1.1 branch which includes this, was raising an error: .../fabric/decorators.py:162: Warning: 'with' will become a reserved keyword in Python 2.6 This cropped up because I'd forgotten to make the from __future__ import in the decorators module. --- fabric/decorators.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fabric/decorators.py b/fabric/decorators.py index 19da968080..ba8e9fc1fb 100644 --- a/fabric/decorators.py +++ b/fabric/decorators.py @@ -1,6 +1,7 @@ """ Convenience decorators for use in fabfiles. """ +from __future__ import with_statement from functools import wraps from types import StringTypes From 30f3307d010eb052a530cc346316521759db029d Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 15 Apr 2011 13:20:36 -0700 Subject: [PATCH 035/126] Update cd() docs to explicitly reference lcd() --- fabric/context_managers.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/fabric/context_managers.py b/fabric/context_managers.py index 964e2573bf..8ddd5bbe41 100644 --- a/fabric/context_managers.py +++ b/fabric/context_managers.py @@ -143,12 +143,15 @@ def my_task(): def cd(path): """ - Context manager that keeps directory state when calling operations. + Context manager that keeps directory state when calling remote operations. Any calls to `run`, `sudo`, `get`, or `put` within the wrapped block will implicitly have a string similar to ``"cd && "`` prefixed in order - to give the sense that there is actually statefulness involved. `cd` only - affects the remote paths for `get` and `put` -- local paths are untouched. + to give the sense that there is actually statefulness involved. + + .. note:: + `cd` only affects *remote* paths -- to modify *local* paths, use + `~fabric.context_managers.lcd`. Because use of `cd` affects all such invocations, any code making use of those operations, such as much of the ``contrib`` section, will also be @@ -195,6 +198,8 @@ def cd(path): .. versionchanged:: 1.0 Applies to `get` and `put` in addition to the command-running operations. + + .. seealso:: `~fabric.context_managers.lcd` """ return _change_cwd('cwd', path) From a0e57ddb3246c6070fd6619d3c3751a7d8a2030e Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 15 Apr 2011 13:20:46 -0700 Subject: [PATCH 036/126] Changelog for previous commit --- docs/changes/1.0.2.rst | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 docs/changes/1.0.2.rst diff --git a/docs/changes/1.0.2.rst b/docs/changes/1.0.2.rst new file mode 100644 index 0000000000..0ba3793b90 --- /dev/null +++ b/docs/changes/1.0.2.rst @@ -0,0 +1,9 @@ +======================== +Changes in version 1.0.2 +======================== + +Documentation +============= + +* Updated the API documentation for `~fabric.context_managers.cd` to explicitly + point users to `~fabric.context_managers.lcd` for modifying local paths. From 9399a20600cda6dcdd6ffcf261ee51fb1d067288 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 21 Apr 2011 18:35:38 -0700 Subject: [PATCH 037/126] Reimplement with cleaner patched-env decorator. Closes #314 --- docs/changes/1.1.rst | 6 ++++++ tests/test_main.py | 27 ++++++++++++------------ tests/utils.py | 50 +++++++++++++------------------------------- 3 files changed, 33 insertions(+), 50 deletions(-) diff --git a/docs/changes/1.1.rst b/docs/changes/1.1.rst index a574bc3590..1e9e5079a2 100644 --- a/docs/changes/1.1.rst +++ b/docs/changes/1.1.rst @@ -21,3 +21,9 @@ Documentation updates ===================== * N/A + +Internals +========= + +* :issue:`314`: Test utility decorator improvements. Thanks to Rick Harding for + initial catch & patch. diff --git a/tests/test_main.py b/tests/test_main.py index 5b8e31fbc6..11e98c9a4e 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -10,8 +10,7 @@ import fabric.state from fabric.state import _AttributeDict -from utils import mock_streams -from utils import with_patched_state_env +from utils import mock_streams, patched_env def test_argument_parsing(): @@ -108,7 +107,7 @@ def command(): 'r2': ['b', 'c'] } -@with_patched_state_env({'roledefs': fake_roles}) +@patched_env({'roledefs': fake_roles}) def test_roles_decorator_by_itself(): """ Use of @roles only @@ -118,7 +117,7 @@ def command(): pass eq_hosts(command, ['a', 'b']) -@with_patched_state_env({'roledefs': fake_roles}) +@patched_env({'roledefs': fake_roles}) def test_roles_decorator_by_itself_order_ensured(): """ Use of @roles only order ensured @@ -130,7 +129,7 @@ def command(): eq_hosts(command, ['a', 'b']) -@with_patched_state_env({'roledefs': fake_roles}) +@patched_env({'roledefs': fake_roles}) def test_hosts_and_roles_together(): """ Use of @roles and @hosts together results in union of both @@ -141,7 +140,7 @@ def command(): pass eq_hosts(command, ['a', 'b', 'c']) -@with_patched_state_env({'roledefs': fake_roles}) +@patched_env({'roledefs': fake_roles}) def test_hosts_and_roles_together_order_ensured(): """ Use of @roles and @hosts together results in union of both order ensured @@ -159,7 +158,7 @@ def command(): } -@with_patched_state_env({'roledefs': tuple_roles}) +@patched_env({'roledefs': tuple_roles}) def test_roles_as_tuples(): """ Test that a list of roles as a tuple succeeds @@ -170,7 +169,7 @@ def command(): eq_hosts(command, ['a', 'b']) -@with_patched_state_env({'hosts': ('foo', 'bar')}) +@patched_env({'hosts': ('foo', 'bar')}) def test_hosts_as_tuples(): """ Test that a list of hosts as a tuple succeeds @@ -180,7 +179,7 @@ def command(): eq_hosts(command, ['foo', 'bar']) -@with_patched_state_env({'hosts': ['foo']}) +@patched_env({'hosts': ['foo']}) def test_hosts_decorator_overrides_env_hosts(): """ If @hosts is used it replaces any env.hosts value @@ -191,7 +190,7 @@ def command(): eq_hosts(command, ['bar']) assert 'foo' not in get_hosts(command, [], [], []) -@with_patched_state_env({'hosts': ['foo']}) +@patched_env({'hosts': ['foo']}) def test_hosts_decorator_overrides_env_hosts_order_ensured(): """ If @hosts is used it replaces any env.hosts value order ensured @@ -204,7 +203,7 @@ def command(): assert 'foo' not in get_hosts(command, [], [], []) -@with_patched_state_env({'hosts': [' foo ', 'bar '], 'roles': [], +@patched_env({'hosts': [' foo ', 'bar '], 'roles': [], 'exclude_hosts':[]}) def test_hosts_stripped_env_hosts(): """ @@ -220,7 +219,7 @@ def command(): 'r2': ['b', 'c'], } -@with_patched_state_env({'roledefs': spaced_roles}) +@patched_env({'roledefs': spaced_roles}) def test_roles_stripped_env_hosts(): """ Make sure hosts defined in env.roles are cleaned of extra spaces @@ -279,7 +278,7 @@ def command(): eq_(command.roles, role_list) -@with_patched_state_env({'roledefs': fake_roles}) +@patched_env({'roledefs': fake_roles}) @raises(SystemExit) @mock_streams('stderr') def test_aborts_on_nonexistent_roles(): @@ -291,7 +290,7 @@ def test_aborts_on_nonexistent_roles(): lazy_role = {'r1': lambda: ['a', 'b']} -@with_patched_state_env({'roledefs': lazy_role}) +@patched_env({'roledefs': lazy_role}) def test_lazy_roles(): """ Roles may be callables returning lists, as well as regular lists diff --git a/tests/utils.py b/tests/utils.py index 373de38a5f..8e672983d2 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -2,6 +2,7 @@ from StringIO import StringIO # No need for cStringIO at this time from contextlib import contextmanager +from copy import deepcopy from fudge.patcher import with_patched_object from functools import wraps, partial from types import StringTypes @@ -212,42 +213,19 @@ def eq_contents(path, text): eq_(text, fd.read()) -class with_patched_state_env(object): - """Decorate a unit test to provide a default state.env - - Grabs the fabric.state.env attribute dictionary. - - e.g. - @with_patched_state_env({'hosts': ('foo', 'bar')}) - def test_something_with_hosts(): - +def patched_env(updates): """ - def __init__(self, dict_overrides): - """ Pass in a dict of keys to override the default env with - - :param dict_overrides: Dictionary of keys to override in default_env - - """ - from fabric.state import env as default_env - self.env = self._merge_env(default_env.copy(), dict_overrides) + Execute a function with a patched copy of ``fabric.state.env``. - def _merge_env(self, default, overrides): - """for item in the overrides, set them over the default""" - for key, val in overrides.iteritems(): - default[key] = val + ``fabric.state.env`` is patched during the wrapped functions' run, with an + equivalent copy that has been ``update``d with the given ``updates``. - return default - - def __call__(self, f): - """This wraps a function with a fudge mock that implements the default - state.env - """ - env = self.env - - def wrapped_f(env=env, *args): - f(*args) - - # now add the fudge decorator here - patcher = with_patched_object('fabric.state', 'env', env) - wrapped_f = patcher(wrapped_f) - return wrapped_f + E.g. with ``fabric.state.env = {'foo': 'bar', 'biz': 'baz'}``, a function + decorated with ``@patched_env({'foo': 'notbar'})`` would see + ``fabric.state.env`` as equal to ``{'biz': 'baz', 'foo': 'notbar'}``. + """ + from fabric.state import env + def wrapper(func): + new_env = deepcopy(env).update(updates) + return with_patched_object('fabric.state', 'env', new_env) + return wrapper From 33cc7e2027e7ee7f433bd206d109eb8859a65689 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 21 Apr 2011 18:47:23 -0700 Subject: [PATCH 038/126] Add changelog entry re #307 --- docs/changes/1.1.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changes/1.1.rst b/docs/changes/1.1.rst index 1e9e5079a2..830f37f1ed 100644 --- a/docs/changes/1.1.rst +++ b/docs/changes/1.1.rst @@ -27,3 +27,5 @@ Internals * :issue:`314`: Test utility decorator improvements. Thanks to Rick Harding for initial catch & patch. +* :issue:`307`: A whole pile of minor PEP8 tweaks. Thanks to Markus Gattol for + highlighting the ``pep8`` tool and to Rick Harding for the patch. From f9478545a951c797b3688da00ce5e968e3f7a65c Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 22 Apr 2011 12:33:24 -0700 Subject: [PATCH 039/126] Refactor test runner classes --- tests/test_operations.py | 18 ------------------ tests/utils.py | 18 +++++++++++++++++- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/tests/test_operations.py b/tests/test_operations.py index 83e8e34e18..c77abe1394 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -3,7 +3,6 @@ import os import shutil import sys -import tempfile import types from contextlib import nested from StringIO import StringIO @@ -226,23 +225,6 @@ def test_shell_escape_escapes_backticks(): # class TestFileTransfers(FabricTest): - def setup(self): - super(TestFileTransfers, self).setup() - self.tmpdir = tempfile.mkdtemp() - - def teardown(self): - super(TestFileTransfers, self).teardown() - shutil.rmtree(self.tmpdir) - - def path(self, *path_parts): - return os.path.join(self.tmpdir, *path_parts) - - def exists_remotely(self, path): - return SFTP(env.host_string).exists(path) - - def exists_locally(self, path): - return os.path.exists(path) - # # get() # diff --git a/tests/utils.py b/tests/utils.py index 8e672983d2..8902811bbe 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -8,14 +8,18 @@ from types import StringTypes import copy import getpass +import os import re +import shutil import sys +import tempfile from fudge import Fake, patched_context, clear_expectations from fabric.context_managers import settings from fabric.network import interpret_host_string from fabric.state import env, output +from fabric.sftp import SFTP import fabric.network from server import PORT, PASSWORDS, USER, HOST @@ -23,7 +27,7 @@ class FabricTest(object): """ - Nose-oriented test runner class that wipes env after every test. + Nose-oriented test runner which wipes state.env and provides file helpers. """ def setup(self): # Clear Fudge mock expectations @@ -40,10 +44,22 @@ def setup(self): # Command response mocking is easier without having to account for # shell wrapping everywhere. env.use_shell = False + # Temporary local file dir + self.tmpdir = tempfile.mkdtemp() def teardown(self): env.update(self.previous_env) output.update(self.previous_output) + shutil.rmtree(self.tmpdir) + + def path(self, *path_parts): + return os.path.join(self.tmpdir, *path_parts) + + def exists_remotely(self, path): + return SFTP(env.host_string).exists(path) + + def exists_locally(self, path): + return os.path.exists(path) class CarbonCopy(StringIO): From 68610242c27b86fd487f9bcf7e8d46f8b791baa6 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 22 Apr 2011 13:15:36 -0700 Subject: [PATCH 040/126] Add another test file helper --- tests/test_operations.py | 4 +--- tests/utils.py | 6 ++++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_operations.py b/tests/test_operations.py index c77abe1394..7bfe7cefa1 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -530,10 +530,8 @@ def test_put_file_to_existing_directory(self): put() a single file into an existing remote directory """ text = "foo!" - local = self.path('foo.txt') + local = self.mkfile('foo.txt', text) local2 = self.path('foo2.txt') - with open(local, 'w') as fd: - fd.write(text) with hide('everything'): put(local, '/') get('/foo.txt', local2) diff --git a/tests/utils.py b/tests/utils.py index 8902811bbe..f323ab46a2 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -55,6 +55,12 @@ def teardown(self): def path(self, *path_parts): return os.path.join(self.tmpdir, *path_parts) + def mkfile(self, path, contents): + dest = self.path(path) + with open(dest, 'w') as fd: + fd.write(contents) + return dest + def exists_remotely(self, path): return SFTP(env.host_string).exists(path) From cd25547415b68b74db3bafbb4107eda2eff4a277 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 22 Apr 2011 13:49:12 -0700 Subject: [PATCH 041/126] Re #273, fix remote filename bug --- fabric/contrib/files.py | 25 ++++++++++++++++--------- tests/test_contrib.py | 16 ++++++++++++++++ 2 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 tests/test_contrib.py diff --git a/fabric/contrib/files.py b/fabric/contrib/files.py index dd067835c1..28f7abb574 100644 --- a/fabric/contrib/files.py +++ b/fabric/contrib/files.py @@ -49,8 +49,8 @@ def first(*args, **kwargs): def upload_template(filename, destination, context=None, use_jinja=False, - template_dir=None, use_sudo=False, backup_file=True, - mirror_local_mode=False, mode=None): + template_dir=None, use_sudo=False, backup=True, mirror_local_mode=False, + mode=None): """ Render and upload a template text file to a remote host. @@ -67,7 +67,7 @@ def upload_template(filename, destination, context=None, use_jinja=False, The resulting rendered file will be uploaded to the remote file path ``destination`` (which should include the desired remote filename.) If the destination file already exists, it will be renamed with a ``.bak`` - extension. You can turn this off by ``backup_file=False``. + extension unless ``backup=False`` is specified. By default, the file will be copied to ``destination`` as the logged-in user; specify ``use_sudo=True`` to use `sudo` instead. @@ -84,7 +84,8 @@ def upload_template(filename, destination, context=None, use_jinja=False, # This temporary file should not be automatically deleted on close, as we # need it there to upload it (Windows locks the file for reading while # open). - tempfile_fd, tempfile_name = tempfile.mkstemp() + tempdir = tempfile.mkdtemp() + tempfile_name = os.path.join(tempdir, basename) output = open(tempfile_name, "w+b") # Init text = None @@ -106,7 +107,7 @@ def upload_template(filename, destination, context=None, use_jinja=False, # Back up any original file func = use_sudo and sudo or run # Back up any original file (need to do figure out ultimate destination) - if backup_file: + if backup: to_backup = destination with settings(hide('everything'), warn_only=True): # Is destination a directory? @@ -116,10 +117,16 @@ def upload_template(filename, destination, context=None, use_jinja=False, if exists(to_backup): func("cp %s %s.bak" % (to_backup, to_backup)) - # Upload the file. - put(tempfile_name, destination, use_sudo, mirror_local_mode, mode) - os.close(tempfile_fd) - os.remove(tempfile_name) + # Upload the file. + put( + local_path=tempfile_name, + remote_path=destination, + use_sudo=use_sudo, + mirror_local_mode=mirror_local_mode, + mode=mode + ) + output.close() + os.remove(tempfile_name) def sed(filename, before, after, limit='', use_sudo=False, backup='.bak', diff --git a/tests/test_contrib.py b/tests/test_contrib.py new file mode 100644 index 0000000000..15e383fdfb --- /dev/null +++ b/tests/test_contrib.py @@ -0,0 +1,16 @@ +from __future__ import with_statement + +from fabric.contrib.files import upload_template + +from utils import FabricTest +from server import server + +class TestContrib(FabricTest): + @server() + def test_upload_template_uses_correct_remote_filename(self): + """ + upload_template() shouldn't munge final remote filename + """ + template = self.mkfile('template.txt', 'text') + upload_template(template, '/') + assert self.exists_remotely('/template.txt') From d7470d2db919ffcee80c245cf87e6d8d4ba6909c Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 22 Apr 2011 14:53:46 -0700 Subject: [PATCH 042/126] Fix file path munging and add another test Re #273 --- fabric/contrib/files.py | 44 ++++++++++++++--------------------------- tests/test_contrib.py | 28 ++++++++++++++++++++++---- 2 files changed, 39 insertions(+), 33 deletions(-) diff --git a/fabric/contrib/files.py b/fabric/contrib/files.py index 28f7abb574..c197b3c748 100644 --- a/fabric/contrib/files.py +++ b/fabric/contrib/files.py @@ -8,6 +8,7 @@ import tempfile import re import os +from StringIO import StringIO from fabric.api import * @@ -65,9 +66,8 @@ def upload_template(filename, destination, context=None, use_jinja=False, directory by default, or from ``template_dir`` if given. The resulting rendered file will be uploaded to the remote file path - ``destination`` (which should include the desired remote filename.) If the - destination file already exists, it will be renamed with a ``.bak`` - extension unless ``backup=False`` is specified. + ``destination``. If the destination file already exists, it will be + renamed with a ``.bak`` extension unless ``backup=False`` is specified. By default, the file will be copied to ``destination`` as the logged-in user; specify ``use_sudo=True`` to use `sudo` instead. @@ -79,15 +79,14 @@ def upload_template(filename, destination, context=None, use_jinja=False, Alternately, you may use the ``mode`` kwarg to specify an exact mode, in the same vein as ``os.chmod`` or the Unix ``chmod`` command. """ - basename = os.path.basename(filename) - - # This temporary file should not be automatically deleted on close, as we - # need it there to upload it (Windows locks the file for reading while - # open). - tempdir = tempfile.mkdtemp() - tempfile_name = os.path.join(tempdir, basename) - output = open(tempfile_name, "w+b") - # Init + func = use_sudo and sudo or run + # Normalize destination to be an actual filename, due to using StringIO + with settings(hide('everything'), warn_only=True): + if func('test -d %s' % destination).succeeded: + sep = "" if destination.endswith('/') else "/" + destination += sep + os.path.basename(filename) + + # Process template text = None if use_jinja: try: @@ -101,32 +100,19 @@ def upload_template(filename, destination, context=None, use_jinja=False, text = inputfile.read() if context: text = text % context - output.write(text) - output.close() - # Back up any original file - func = use_sudo and sudo or run - # Back up any original file (need to do figure out ultimate destination) - if backup: - to_backup = destination - with settings(hide('everything'), warn_only=True): - # Is destination a directory? - if func('test -f %s' % to_backup).failed: - # If so, tack on the filename to get "real" destination - to_backup = destination + '/' + basename - if exists(to_backup): - func("cp %s %s.bak" % (to_backup, to_backup)) + # Back up original file + if backup and exists(destination): + func("cp %s{,.bak}" % destination) # Upload the file. put( - local_path=tempfile_name, + local_path=StringIO(text), remote_path=destination, use_sudo=use_sudo, mirror_local_mode=mirror_local_mode, mode=mode ) - output.close() - os.remove(tempfile_name) def sed(filename, before, after, limit='', use_sudo=False, backup='.bak', diff --git a/tests/test_contrib.py b/tests/test_contrib.py index 15e383fdfb..6d53d3b123 100644 --- a/tests/test_contrib.py +++ b/tests/test_contrib.py @@ -1,16 +1,36 @@ from __future__ import with_statement +from fabric.api import hide, get, show from fabric.contrib.files import upload_template -from utils import FabricTest +from utils import FabricTest, eq_contents from server import server + class TestContrib(FabricTest): - @server() + # Make sure it knows / is a directory. + # This is in lieu of starting down the "actual honest to god fake operating + # system" road...:( + @server(responses={'test -d /': ""}) def test_upload_template_uses_correct_remote_filename(self): """ upload_template() shouldn't munge final remote filename """ template = self.mkfile('template.txt', 'text') - upload_template(template, '/') - assert self.exists_remotely('/template.txt') + with hide('everything'): + upload_template(template, '/') + assert self.exists_remotely('/template.txt') + + @server() + def test_upload_template_handles_file_destination(self): + """ + upload_template() should work OK with file and directory destinations + """ + template = self.mkfile('template.txt', '%(varname)s') + local = self.path('result.txt') + remote = '/configfile.txt' + var = 'foobar' + with hide('everything'): + upload_template(template, remote, {'varname': var}) + get(remote, local) + eq_contents(local, var) From c88f7c005ecb6089f5435a05cd6871d4b8ea6b85 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 22 Apr 2011 15:30:25 -0700 Subject: [PATCH 043/126] Add changelog entries re #273 --- docs/changes/1.1.rst | 3 +++ fabric/contrib/files.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/docs/changes/1.1.rst b/docs/changes/1.1.rst index 830f37f1ed..d8c17b3650 100644 --- a/docs/changes/1.1.rst +++ b/docs/changes/1.1.rst @@ -11,6 +11,9 @@ Feature additions * :issue:`107`: `~fabric.operations.require`'s ``provided_by`` kwarg now accepts iterables in addition to single values. Thanks to Thomas Ballinger for the patch. +* :issue:`273`: `~fabric.contrib.files.upload_template` now offers control over + whether it attempts to create backups of pre-existing destination files. + Thanks to Ales Zoulek for the suggestion and initial patch. Bugfixes ======== diff --git a/fabric/contrib/files.py b/fabric/contrib/files.py index c197b3c748..576846cb15 100644 --- a/fabric/contrib/files.py +++ b/fabric/contrib/files.py @@ -78,6 +78,9 @@ def upload_template(filename, destination, context=None, use_jinja=False, Alternately, you may use the ``mode`` kwarg to specify an exact mode, in the same vein as ``os.chmod`` or the Unix ``chmod`` command. + + .. versionchanged:: 1.1 + Added the ``backup`` kwarg. """ func = use_sudo and sudo or run # Normalize destination to be an actual filename, due to using StringIO From 1773ce0c956967f0fea4fcc0fe4e235d83a6ea01 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 22 Apr 2011 16:17:27 -0700 Subject: [PATCH 044/126] Dev branches really ought to have a dev version --- fabric/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fabric/version.py b/fabric/version.py index 41e1a0300e..7220bf3cd3 100644 --- a/fabric/version.py +++ b/fabric/version.py @@ -21,7 +21,7 @@ def git_sha(): return p.communicate()[0] -VERSION = (1, 0, 1, 'final', 0) +VERSION = (1, 1, 0, 'alpha', 0) def get_version(form='short'): From 626ade6874af781461aeb8052f0e5e7564bf8617 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 22 Apr 2011 16:18:01 -0700 Subject: [PATCH 045/126] Re #184, add changelog entry and tweak language --- docs/changes/1.1.rst | 5 ++++- fabric/contrib/project.py | 7 ++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/changes/1.1.rst b/docs/changes/1.1.rst index d8c17b3650..e13eaf8890 100644 --- a/docs/changes/1.1.rst +++ b/docs/changes/1.1.rst @@ -23,7 +23,10 @@ Bugfixes Documentation updates ===================== -* N/A +* :issue:`184`: Make the usage of `~fabric.contrib.project.rsync_project`'s + ``local_dir`` argument more obvious, regarding its use in the ``rsync`` call. + (Specifically, so users know they can pass in multiple, space-joined + directory names instead of just one single directory.) Internals ========= diff --git a/fabric/contrib/project.py b/fabric/contrib/project.py index d5f71a1514..b244176df1 100644 --- a/fabric/contrib/project.py +++ b/fabric/contrib/project.py @@ -41,9 +41,10 @@ def rsync_project(remote_dir, local_dir=None, exclude=(), delete=False, named ``myproject`` and one invokes ``rsync_project('/home/username/')``, the resulting project directory will be ``/home/username/myproject/``. * ``local_dir``: by default, ``rsync_project`` uses your current working - directory as the source directory; you may override this with - ``local_dir``, which should be a directory path, or list of paths in a - single string. + directory as the source directory. This may be overridden by specifying + ``local_dir``, which is a string passed verbatim to ``rsync``, and thus + may be a single directory (``"my_directory"``) or multiple directories + (``"dir1 dir2"``). See the ``rsync`` documentation for details. * ``exclude``: optional, may be a single string, or an iterable of strings, and is used to pass one or more ``--exclude`` options to ``rsync``. * ``delete``: a boolean controlling whether ``rsync``'s ``--delete`` option From fb3aaae662dd5cbd26431d9e48469c624d97121f Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 22 Apr 2011 16:40:07 -0700 Subject: [PATCH 046/126] Update AUTHORS file re #273 --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 3a2815c7cd..23f287d986 100644 --- a/AUTHORS +++ b/AUTHORS @@ -37,3 +37,4 @@ James Murty Thomas Ballinger Rick Harding Kirill Pinchuk +Ales Zoulek From d15edb4223b60243f72dd1e35df9b7c0f006bf3e Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 22 Apr 2011 16:45:59 -0700 Subject: [PATCH 047/126] Move --exclude-hosts docs into its own section, and update changelog/AUTHORS Re #170. --- AUTHORS | 1 + docs/changes/1.1.rst | 2 ++ docs/usage/execution.rst | 44 ++++++++++++++++++++++++++++++---------- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/AUTHORS b/AUTHORS index 23f287d986..fcd41db385 100644 --- a/AUTHORS +++ b/AUTHORS @@ -38,3 +38,4 @@ Thomas Ballinger Rick Harding Kirill Pinchuk Ales Zoulek +Casey Banner diff --git a/docs/changes/1.1.rst b/docs/changes/1.1.rst index e13eaf8890..a342093079 100644 --- a/docs/changes/1.1.rst +++ b/docs/changes/1.1.rst @@ -14,6 +14,8 @@ Feature additions * :issue:`273`: `~fabric.contrib.files.upload_template` now offers control over whether it attempts to create backups of pre-existing destination files. Thanks to Ales Zoulek for the suggestion and initial patch. +* :issue:`170`: Allow :ref:`exclusion ` of specific hosts from + the final run list. Thanks to Casey Banner for the suggestion and patch. Bugfixes ======== diff --git a/docs/usage/execution.rst b/docs/usage/execution.rst index 4ba02acbb3..2a12b2fadc 100644 --- a/docs/usage/execution.rst +++ b/docs/usage/execution.rst @@ -268,8 +268,7 @@ Globally, via the command line In addition to modifying ``env.hosts``, ``env.roles``, and ``env.exclude_hosts`` at the module level, you may define them by passing comma-separated string arguments to the command-line switches -:option:`--hosts/-H <-H>`, :option:`--roles/-R <-R>`, and -:option:`--exclude-hosts/-x <-x>`, e.g.:: +:option:`--hosts/-H <-H>` and :option:`--roles/-R <-R>`, e.g.:: $ fab -H host1,host2 mytask @@ -299,13 +298,9 @@ instead:: run('ls /var/www') When this fabfile is run as ``fab -H host1,host2 mytask``, ``env.hosts`` will -end contain ``['host1', 'host2', 'host3', 'host4']`` at the time that +then contain ``['host1', 'host2', 'host3', 'host4']`` at the time that ``mytask`` is executed. -For exclusions when this fabfile is run as ``fab -H host1,host2 -x host1 -mytask``, ``env.hosts`` will be the same, but the host list that gets executed -will not have host1 included in it. - .. note:: ``env.hosts`` is simply a Python list object -- so you may use @@ -349,10 +344,6 @@ To specify per-task hosts for ``mytask``, execute it like so:: This will override any other host list and ensure ``mytask`` always runs on just those two hosts. -You are also able to exclude hosts like this:: - - $ fab mytask:hosts="host1;host2",exclude_hosts="host1" - Per-task, via decorators ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -435,6 +426,37 @@ this fabfile will call ``mytask`` on a host list of ``['a', 'b', 'c']`` -- the union of ``role1`` and the contents of the `~fabric.decorators.hosts` call. +.. _excluding-hosts: + +Excluding specific hosts +------------------------ + +At times, it is useful to exclude one or more specific hosts, e.g. to override +a few bad or otherwise undesirable hosts which are pulled in from a role or an +autogenerated host list. This may be accomplished globally with +:option:`--exclude-hosts/-x <-x>`:: + + $ fab -R myrole -x host2,host5 mytask + +If ``myrole`` was defined as ``['host1', 'host2', ..., 'host15']``, the above +invocation would run with an effective host list of ``['host1', 'host3', +'host4', 'host6', ..., 'host15']``. + + .. note:: Using this option does not modify ``env.hosts`` -- it only + causes the main execution loop to skip the requested hosts. + +Exclusions may be specified per-task basis by using an extra ``exclude_hosts`` +kwarg (this is implemented similarly to the abovementioned ``hosts`` and +``roles`` per-task kwargs, in that it is stripped from the actual task +invocation). This example would have the same result as the global exclude +above:: + + $ fab -R myrole mytask:exclude_hosts="host2;host5" + +Note that the host list is semicolon-separated, just as with the ``hosts`` +per-task argument. + + .. _failures: Failure handling From aec637696bbad29141af1d20c55a7ccc199209c2 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 22 Apr 2011 17:16:22 -0700 Subject: [PATCH 048/126] Beef up docs, add default env value re #170 --- docs/usage/env.rst | 13 +++++++++++++ docs/usage/execution.rst | 15 +++++++-------- docs/usage/fab.rst | 4 ++-- fabric/state.py | 1 + 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/docs/usage/env.rst b/docs/usage/env.rst index a750778931..defab2419d 100644 --- a/docs/usage/env.rst +++ b/docs/usage/env.rst @@ -185,6 +185,19 @@ host key is actually valid (e.g. cloud servers such as EC2.) .. seealso:: :doc:`ssh` +.. _exclude-hosts: + +``exclude_hosts`` +----------------- + +**Default:** ``[]`` + +Specifies a list of host strings to be :ref:`skipped over ` +during ``fab`` execution. Typically set via :option:`--exclude-hosts/-x <-x>`. + +.. versionadded:: 1.1 + + ``fabfile`` ----------- diff --git a/docs/usage/execution.rst b/docs/usage/execution.rst index 2a12b2fadc..a482f56f9d 100644 --- a/docs/usage/execution.rst +++ b/docs/usage/execution.rst @@ -425,7 +425,6 @@ Assuming no command-line hosts or roles are given when ``mytask`` is executed, this fabfile will call ``mytask`` on a host list of ``['a', 'b', 'c']`` -- the union of ``role1`` and the contents of the `~fabric.decorators.hosts` call. - .. _excluding-hosts: Excluding specific hosts @@ -442,14 +441,14 @@ If ``myrole`` was defined as ``['host1', 'host2', ..., 'host15']``, the above invocation would run with an effective host list of ``['host1', 'host3', 'host4', 'host6', ..., 'host15']``. - .. note:: Using this option does not modify ``env.hosts`` -- it only - causes the main execution loop to skip the requested hosts. + .. note:: + Using this option does not modify ``env.hosts`` -- it only causes the + main execution loop to skip the requested hosts. -Exclusions may be specified per-task basis by using an extra ``exclude_hosts`` -kwarg (this is implemented similarly to the abovementioned ``hosts`` and -``roles`` per-task kwargs, in that it is stripped from the actual task -invocation). This example would have the same result as the global exclude -above:: +Exclusions may be specified per-task by using an extra ``exclude_hosts`` kwarg, +which is implemented similarly to the abovementioned ``hosts`` and ``roles`` +per-task kwargs, in that it is stripped from the actual task invocation. This +example would have the same result as the global exclude above:: $ fab -R myrole mytask:exclude_hosts="host2;host5" diff --git a/docs/usage/fab.rst b/docs/usage/fab.rst index 4fa509fff0..ba0694bfc6 100644 --- a/docs/usage/fab.rst +++ b/docs/usage/fab.rst @@ -133,8 +133,8 @@ below. .. cmdoption:: -x HOSTS, --exclude-hosts=HOSTS - Sets :ref:`env.exclude_hosts` to the given comma-delimited list of host - strings to then keep out of the final host list. + Sets :ref:`env.exclude_hosts ` to the given comma-delimited + list of host strings to then keep out of the final host list. .. versionadded:: 1.1 diff --git a/fabric/state.py b/fabric/state.py index 6be4f2b586..cdef6d0a4c 100644 --- a/fabric/state.py +++ b/fabric/state.py @@ -231,6 +231,7 @@ def _rc_path(): 'command_prefixes': [], 'cwd': '', # Must be empty string, not None, for concatenation purposes 'echo_stdin': True, + 'exclude_hosts': [], 'host': None, 'host_string': None, 'lcwd': '', # Must be empty string, not None, for concatenation purposes From 0e1ae0d52c0c1a93f6c8a2cc2291ec73b4f9682f Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 22 Apr 2011 17:49:06 -0700 Subject: [PATCH 049/126] Update case-insensitive sed flag to be generic. Also beef up docs. Re #154 --- docs/changes/1.1.rst | 6 ++++-- fabric/contrib/files.py | 23 +++++++++++------------ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/docs/changes/1.1.rst b/docs/changes/1.1.rst index a342093079..37fee23f7e 100644 --- a/docs/changes/1.1.rst +++ b/docs/changes/1.1.rst @@ -11,11 +11,13 @@ Feature additions * :issue:`107`: `~fabric.operations.require`'s ``provided_by`` kwarg now accepts iterables in addition to single values. Thanks to Thomas Ballinger for the patch. +* :issue:`154`: `~fabric.contrib.files.sed` now allows customized regex flags + to be specified via a new ``flags`` parameter. +* :issue:`170`: Allow :ref:`exclusion ` of specific hosts from + the final run list. Thanks to Casey Banner for the suggestion and patch. * :issue:`273`: `~fabric.contrib.files.upload_template` now offers control over whether it attempts to create backups of pre-existing destination files. Thanks to Ales Zoulek for the suggestion and initial patch. -* :issue:`170`: Allow :ref:`exclusion ` of specific hosts from - the final run list. Thanks to Casey Banner for the suggestion and patch. Bugfixes ======== diff --git a/fabric/contrib/files.py b/fabric/contrib/files.py index 576846cb15..854a78be64 100644 --- a/fabric/contrib/files.py +++ b/fabric/contrib/files.py @@ -119,11 +119,11 @@ def upload_template(filename, destination, context=None, use_jinja=False, def sed(filename, before, after, limit='', use_sudo=False, backup='.bak', - case_insensitive=False): + flags=''): """ Run a search-and-replace on ``filename`` with given regex patterns. - Equivalent to ``sed -i -r -e "// s///g + Equivalent to ``sed -i -r -e "// s///g "``. For convenience, ``before`` and ``after`` will automatically escape forward @@ -136,10 +136,13 @@ def sed(filename, before, after, limit='', use_sudo=False, backup='.bak', `sed` will pass ``shell=False`` to `run`/`sudo`, in order to avoid problems with many nested levels of quotes and backslashes. - If you would like the matches to be case insensitive, you'd only need to - pass the ``case_insensitive=True`` to have it tack on an 'i' to the sed - command. Effectively making it execute``sed -i -r -e "// - s///ig "``. + Other options may be specified with sed-compatible regex flags -- for + example, to make the search and replace case insensitive, specify + ``flags="i"``. The ``g`` flag is always specified regardless, so you do not + need to remember to include it when overriding this parameter. + + .. versionadded:: 1.1 + The ``flags`` parameter. """ func = use_sudo and sudo or run # Characters to be escaped in both @@ -154,10 +157,6 @@ def sed(filename, before, after, limit='', use_sudo=False, backup='.bak', limit = r'/%s/ ' % limit # Test the OS because of differences between sed versions - case_bit = '' - if case_insensitive: - case_bit = 'i' - with hide('running', 'stdout'): platform = run("uname") if platform in ('NetBSD', 'OpenBSD'): @@ -168,13 +167,13 @@ def sed(filename, before, after, limit='', use_sudo=False, backup='.bak', tmp = "/tmp/%s" % hasher.hexdigest() # Use temp file to work around lack of -i expr = r"""cp -p %(filename)s %(tmp)s \ -&& sed -r -e '%(limit)ss/%(before)s/%(after)s/%(case_bit)sg' %(filename)s > %(tmp)s \ +&& sed -r -e '%(limit)ss/%(before)s/%(after)s/%(flags)sg' %(filename)s > %(tmp)s \ && cp -p %(filename)s %(filename)s%(backup)s \ && mv %(tmp)s %(filename)s""" command = expr % locals() else: expr = r"sed -i%s -r -e '%ss/%s/%s/%sg' %s" - command = expr % (backup, limit, before, after, case_bit, filename) + command = expr % (backup, limit, before, after, flags, filename) return func(command, shell=False) From 47ea5a5969acf8e44a3880a479ef2415fd717ac1 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 22 Apr 2011 17:52:50 -0700 Subject: [PATCH 050/126] Add attribution to changelog re: #154. Implements #154 --- docs/changes/1.1.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changes/1.1.rst b/docs/changes/1.1.rst index 37fee23f7e..55009bd546 100644 --- a/docs/changes/1.1.rst +++ b/docs/changes/1.1.rst @@ -12,7 +12,8 @@ Feature additions accepts iterables in addition to single values. Thanks to Thomas Ballinger for the patch. * :issue:`154`: `~fabric.contrib.files.sed` now allows customized regex flags - to be specified via a new ``flags`` parameter. + to be specified via a new ``flags`` parameter. Thanks to Nick Trew for the + suggestion and Morgan Goose for initial implementation. * :issue:`170`: Allow :ref:`exclusion ` of specific hosts from the final run list. Thanks to Casey Banner for the suggestion and patch. * :issue:`273`: `~fabric.contrib.files.upload_template` now offers control over From 67d32d5526af963b560d2230dddb1aa25034255e Mon Sep 17 00:00:00 2001 From: Roman Imankulov Date: Mon, 11 Apr 2011 16:09:01 +0600 Subject: [PATCH 051/126] Fix bug with fabric.api.put(...) ``mode`` kwarg didn't take effect while uploading file-like objects --- fabric/sftp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fabric/sftp.py b/fabric/sftp.py index a5ff784b75..550b5b14e6 100644 --- a/fabric/sftp.py +++ b/fabric/sftp.py @@ -236,7 +236,7 @@ def put(self, local_path, remote_path, use_sudo, mirror_local_mode, mode, if not local_is_path: os.remove(real_local_path) # Handle modes if necessary - if local_is_path and (mirror_local_mode or mode is not None): + if (local_is_path and mirror_local_mode) or (mode is not None): lmode = os.stat(local_path).st_mode if mirror_local_mode else mode lmode = lmode & 07777 rmode = rattrs.st_mode & 07777 From 9a13102c3d7482349eed2febe46a283fe67aeaf2 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Sun, 24 Apr 2011 18:42:09 -0700 Subject: [PATCH 052/126] Add changelog, AUTHORS and fixes #337 --- AUTHORS | 1 + docs/changes/1.0.2.rst | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/AUTHORS b/AUTHORS index 4835c1819a..b004af8196 100644 --- a/AUTHORS +++ b/AUTHORS @@ -33,3 +33,4 @@ Travis Swicegood Paul Smith Rick Harding Kirill Pinchuk +Roman Imankulov diff --git a/docs/changes/1.0.2.rst b/docs/changes/1.0.2.rst index 0ba3793b90..cecab01971 100644 --- a/docs/changes/1.0.2.rst +++ b/docs/changes/1.0.2.rst @@ -2,6 +2,13 @@ Changes in version 1.0.2 ======================== +Bugfixes +======== + +* :issue:`337`: Fix logic bug in `~fabric.operations.put` preventing use of + ``mirror_local_mode``. Thanks to Roman Imankulov for catch & patch. + + Documentation ============= From 98aceb2a85668c9048f768ebe4638f2b2dcfc394 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Sun, 24 Apr 2011 19:10:25 -0700 Subject: [PATCH 053/126] Integration patching & changelog; fixes #117 --- docs/changes/1.1.rst | 7 +++++-- fabric/contrib/files.py | 19 ++++++++++++------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/docs/changes/1.1.rst b/docs/changes/1.1.rst index 55009bd546..31da5994c8 100644 --- a/docs/changes/1.1.rst +++ b/docs/changes/1.1.rst @@ -11,6 +11,9 @@ Feature additions * :issue:`107`: `~fabric.operations.require`'s ``provided_by`` kwarg now accepts iterables in addition to single values. Thanks to Thomas Ballinger for the patch. +* :issue:`117`: `~fabric.contrib.files.upload_template` now supports the + `~fabric.operations.put` flags ``mirror_local_mode`` and ``mode``. Thanks to + Joe Stump for the suggestion and Thomas Ballinger for the patch. * :issue:`154`: `~fabric.contrib.files.sed` now allows customized regex flags to be specified via a new ``flags`` parameter. Thanks to Nick Trew for the suggestion and Morgan Goose for initial implementation. @@ -36,7 +39,7 @@ Documentation updates Internals ========= -* :issue:`314`: Test utility decorator improvements. Thanks to Rick Harding for - initial catch & patch. * :issue:`307`: A whole pile of minor PEP8 tweaks. Thanks to Markus Gattol for highlighting the ``pep8`` tool and to Rick Harding for the patch. +* :issue:`314`: Test utility decorator improvements. Thanks to Rick Harding for + initial catch & patch. diff --git a/fabric/contrib/files.py b/fabric/contrib/files.py index 854a78be64..31d5063e70 100644 --- a/fabric/contrib/files.py +++ b/fabric/contrib/files.py @@ -72,15 +72,12 @@ def upload_template(filename, destination, context=None, use_jinja=False, By default, the file will be copied to ``destination`` as the logged-in user; specify ``use_sudo=True`` to use `sudo` instead. - In some use cases, it is desirable to force a newly uploaded file to match - the mode of its local counterpart (such as when uploading executable - scripts). To do this, specify ``mirror_local_mode=True``. - - Alternately, you may use the ``mode`` kwarg to specify an exact mode, in - the same vein as ``os.chmod`` or the Unix ``chmod`` command. + The ``mirror_local_mode`` and ``mode`` kwargs are passed directly to an + internal `~fabric.operations.put` call; please see its documentation for + details on these two options. .. versionchanged:: 1.1 - Added the ``backup`` kwarg. + Added the ``backup``, ``mirror_local_mode`` and ``mode`` kwargs. """ func = use_sudo and sudo or run # Normalize destination to be an actual filename, due to using StringIO @@ -89,6 +86,14 @@ def upload_template(filename, destination, context=None, use_jinja=False, sep = "" if destination.endswith('/') else "/" destination += sep + os.path.basename(filename) + # Use mode kwarg to implement mirror_local_mode, again due to using + # StringIO + if mirror_local_mode and mode is None: + mode = os.stat(filename).st_mode + # To prevent put() from trying to do this + # logic itself + mirror_local_mode = False + # Process template text = None if use_jinja: From a47054e3aa9657b1d1c2d5eac47ecfa4d5fb84a4 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Sun, 24 Apr 2011 20:13:50 -0700 Subject: [PATCH 054/126] Revert patch re #36 --- fabric/operations.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fabric/operations.py b/fabric/operations.py index 082692e300..a0c620d720 100644 --- a/fabric/operations.py +++ b/fabric/operations.py @@ -16,11 +16,10 @@ from contextlib import closing -from fabric import state from fabric.context_managers import settings, char_buffered from fabric.io import output_loop, input_loop from fabric.network import needs_host -from fabric.state import (env, output, win32, default_channel, +from fabric.state import (env, connections, output, win32, default_channel, io_sleep) from fabric.utils import abort, indent, warn, puts from fabric.thread_handling import ThreadHandler @@ -1039,10 +1038,10 @@ def reboot(wait): .. versionadded:: 0.9.2 """ sudo('reboot') - client = state.connections[env.host_string] + client = connections[env.host_string] client.close() - if env.host_string in state.connections: - del state.connections[env.host_string] + if env.host_string in connections: + del connections[env.host_string] if output.running: puts("Waiting for reboot: ", flush=True, end='') per_tick = 5 From 7c47972ac8c3a28dd66c1f887993d1b983217298 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Sun, 24 Apr 2011 20:52:46 -0700 Subject: [PATCH 055/126] Changelog, AUTHORS, docstring, reformat re #10. Imps #10. --- AUTHORS | 1 + docs/changes/1.1.rst | 3 +++ fabric/contrib/project.py | 35 +++++++++++++++-------------------- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/AUTHORS b/AUTHORS index 071b82750a..e62d8bf2d6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -40,3 +40,4 @@ Kirill Pinchuk Ales Zoulek Casey Banner Roman Imankulov +Rodrigue Alcazar diff --git a/docs/changes/1.1.rst b/docs/changes/1.1.rst index 31da5994c8..3bdbd35faf 100644 --- a/docs/changes/1.1.rst +++ b/docs/changes/1.1.rst @@ -8,6 +8,9 @@ This page lists all changes made to Fabric in its 1.1.0 release. Feature additions ================= +* :issue:`10`: `~fabric.contrib.upload_project` now allows control over the + local and remote directory paths, and has improved error handling. Thanks to + Rodrigue Alcazar for the patch. * :issue:`107`: `~fabric.operations.require`'s ``provided_by`` kwarg now accepts iterables in addition to single values. Thanks to Thomas Ballinger for the patch. diff --git a/fabric/contrib/project.py b/fabric/contrib/project.py index b244176df1..4c41e630b1 100644 --- a/fabric/contrib/project.py +++ b/fabric/contrib/project.py @@ -107,42 +107,37 @@ def rsync_project(remote_dir, local_dir=None, exclude=(), delete=False, def upload_project(local_dir=None, remote_dir=""): """ - Upload the current project to a remote system, tar/gzipping during the move. - - This function makes use of the ``tar`` and ``gzip`` programs/libraries, thus - it will not work too well on Win32 systems unless one is using Cygwin or - something similar. - - ``upload_project`` will attempt to clean up the local and remote tarfiles - when it finishes executing, even in the event of a failure. - - :param local_dir: default current working directory, the project folder to - upload - + Upload the current project to a remote system via ``tar``/``gzip``. + + ``local_dir`` specifies the local project directory to upload, and defaults + to the current working directory. + + ``remote_dir`` specifies the target directory to upload into (meaning that + a copy of ``local_dir`` will appear as a subdirectory of ``remote_dir``) + and defaults to the remote user's home directory. + + This function makes use of the ``tar`` and ``gzip`` programs/libraries, + thus it will not work too well on Win32 systems unless one is using Cygwin + or something similar. It will attempt to clean up the local and remote + tarfiles when it finishes executing, even in the event of a failure. """ - if not local_dir: - local_dir = os.getcwd() + local_dir = local_dir or os.getcwd() # Remove final '/' in local_dir so that basename() works local_dir = local_dir.rstrip(os.sep) local_path, local_name = os.path.split(local_dir) - tar_file = "%s.tar.gz" % local_name target_tar = os.path.join(remote_dir, tar_file) - tmp_folder = mkdtemp() + try: tar_path = os.path.join(tmp_folder, tar_file) local("tar -czf %s -C %s %s" % (tar_path, local_path, local_name)) put(tar_path, target_tar) - try: run("tar -xzf %s" % tar_file) - finally: run("rm -f %s" % tar_file) - finally: local("rm -rf %s" % tmp_folder) - From 5ffd0a05f59de8ccc6c4151c173c816ec9196f1c Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Sun, 24 Apr 2011 21:19:01 -0700 Subject: [PATCH 056/126] Add missing versionchanged directives to docstrings --- fabric/contrib/project.py | 3 +++ fabric/operations.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/fabric/contrib/project.py b/fabric/contrib/project.py index 4c41e630b1..0a4af3417e 100644 --- a/fabric/contrib/project.py +++ b/fabric/contrib/project.py @@ -120,6 +120,9 @@ def upload_project(local_dir=None, remote_dir=""): thus it will not work too well on Win32 systems unless one is using Cygwin or something similar. It will attempt to clean up the local and remote tarfiles when it finishes executing, even in the event of a failure. + + .. versionchanged:: 1.1 + Added the ``local_dir`` and ``remote_dir`` kwargs. """ local_dir = local_dir or os.getcwd() diff --git a/fabric/operations.py b/fabric/operations.py index a0c620d720..ab7c25e342 100644 --- a/fabric/operations.py +++ b/fabric/operations.py @@ -143,6 +143,9 @@ def require(*keys, **kwargs): Note: it is assumed that the keyword arguments apply to all given keys as a group. If you feel the need to specify more than one ``used_for``, for example, you should break your logic into multiple calls to ``require()``. + + .. versionchanged:: 1.1 + Allow iterable ``provided_by`` values instead of just single values. """ # If all keys exist, we're good, so keep going. missing_keys = filter(lambda x: x not in env, keys) From 86a1182faf73985e7d1402b147aa069918e42375 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 25 Apr 2011 15:47:27 -0700 Subject: [PATCH 057/126] Revert goosemo's patches re #115 --- docs/api/core/decorators.rst | 2 +- docs/usage/env.rst | 16 ------ fabric/api.py | 2 +- fabric/decorators.py | 33 ------------ fabric/main.py | 25 +-------- tests/test_main.py | 102 +---------------------------------- 6 files changed, 4 insertions(+), 176 deletions(-) diff --git a/docs/api/core/decorators.rst b/docs/api/core/decorators.rst index 4004bd19c6..a19fd3f888 100644 --- a/docs/api/core/decorators.rst +++ b/docs/api/core/decorators.rst @@ -3,4 +3,4 @@ Decorators ========== .. automodule:: fabric.decorators - :members: hosts, roles, runs_once, ensure_order, with_settings + :members: hosts, roles, runs_once, with_settings diff --git a/docs/usage/env.rst b/docs/usage/env.rst index defab2419d..f8ab684a67 100644 --- a/docs/usage/env.rst +++ b/docs/usage/env.rst @@ -409,22 +409,6 @@ takes a command string as its value. .. seealso:: :doc:`execution` -.. _ensure_order: - -``ensure_order`` ----------------- - -**Default:** ``False`` - -Switch to globally state if hosts lists should be deduped in place leaving -order intact from right to left of the combination of `~fabric.state.env.hosts` -and `~fabric.state.env.roles` - -.. note:: - - With this option you can also pre-sort host lists when decorating to use a - special sort if desired. - ``sudo_prompt`` --------------- diff --git a/fabric/api.py b/fabric/api.py index afb97f7d87..4c49c9a8af 100644 --- a/fabric/api.py +++ b/fabric/api.py @@ -7,7 +7,7 @@ well when you're using setup.py to install e.g. paramiko! """ from fabric.context_managers import cd, hide, settings, show, path, prefix, lcd -from fabric.decorators import hosts, roles, runs_once, ensure_order, with_settings +from fabric.decorators import hosts, roles, runs_once, with_settings from fabric.operations import (require, prompt, put, get, run, sudo, local, reboot, open_shell) from fabric.state import env, output diff --git a/fabric/decorators.py b/fabric/decorators.py index b29a17d2ab..2e59a06ebb 100644 --- a/fabric/decorators.py +++ b/fabric/decorators.py @@ -106,39 +106,6 @@ def decorated(*args, **kwargs): return decorated -def ensure_order(sorted=False): - """ - Decorator preventing wrapped function from using the set() operation to - dedupe the host list. Instead it will force fab to iterate of the list of - hosts as combined from both `~fabric.decorators.hosts` and - `~fabric.decorators.roles`. - - It also takes in a parameter sorted, to determine if this deduped list - should also then be sorted using the default python provided sort - mechanism. - - Is used in conjunction with host lists and/or roles:: - - @ensure_order - @hosts('user1@host1', 'host2', 'user2@host3') - def my_func(): - pass - - """ - def real_decorator(func): - func._sorted = sorted - func._ensure_order = True - return func - - # Trick to allow for both a dec w/ the optional setting without have to - # force it to use () - if type(sorted) == type(real_decorator): - return real_decorator(sorted) - - real_decorator._ensure_order = True - return real_decorator - - def with_settings(**kw_settings): """ Decorator equivalent of ``fabric.context_managers.settings``. diff --git a/fabric/main.py b/fabric/main.py index 303cb3def1..28f2c5fb65 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -29,7 +29,6 @@ _modules, [] ) -state.env.ensure_order = False def load_settings(path): @@ -374,24 +373,7 @@ def _merge(hosts, roles, exclude=[]): hosts = list(hosts) # Return deduped combo of hosts and role_hosts - if hasattr(state.env, 'ensure_order') and state.env.ensure_order: - result_hosts = [] - for host in hosts + role_hosts: - if host not in result_hosts: - result_hosts.append(host) - - if hasattr(state.env, '_sorted') and state.env._sorted: - result_hosts.sort() - - else: - result_hosts = list(set(hosts + role_hosts)) - - # Remove excluded hosts from results as needed - for exclude_host in exclude: - if exclude_host in result_hosts: - result_hosts.remove(exclude_host) - - return _clean_hosts(result_hosts) + return list(set(_clean_hosts(hosts + role_hosts))) def _clean_hosts(host_list): """ @@ -406,11 +388,6 @@ def get_hosts(command, cli_hosts, cli_roles, cli_exclude_hosts): See :ref:`execution-model` for detailed documentation on how host lists are set. """ - if hasattr(command, '_ensure_order') and command._ensure_order: - if hasattr(command, '_sorted') and command._sorted == True: - state.env._sorted = command._sorted - state.env.ensure_order = command._ensure_order - # Command line per-command takes precedence over anything else. if cli_hosts or cli_roles: return _merge(cli_hosts, cli_roles, cli_exclude_hosts) diff --git a/tests/test_main.py b/tests/test_main.py index 11e98c9a4e..d01e58525c 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -4,7 +4,7 @@ from fudge import Fake from nose.tools import eq_, raises -from fabric.decorators import hosts, roles, ensure_order +from fabric.decorators import hosts, roles from fabric.main import (get_hosts, parse_arguments, _merge, _escape_split, load_fabfile) import fabric.state @@ -47,37 +47,6 @@ def test_argument_parsing(): def eq_hosts(command, host_list): eq_(set(get_hosts(command, [], [], [])), set(host_list)) -def test_order_ensured(): - """ - Use of @ensure_order - """ - host_list = ['c', 'b', 'a'] - @ensure_order - @hosts(*host_list) - def command(): - pass - - eq_(command._ensure_order, True) - eq_hosts(command, host_list) - for i,h in enumerate(get_hosts(command, [], [], [])): - eq_(host_list[i], h) - -def test_order_ensured_sorted(): - """ - Use of @ensure_order with sorted option - """ - host_list = ['c', 'a', 'b', 'e'] - sorted = ['c', 'a', 'b', 'e'] - sorted.sort() - @ensure_order(sorted=True) - @hosts(*host_list) - def command(): - pass - - eq_(command._ensure_order, True) - eq_(command._sorted, True) - eq_hosts(command, sorted) - def test_hosts_decorator_by_itself(): """ Use of @hosts only @@ -90,17 +59,6 @@ def command(): eq_hosts(command, host_list) -def test_hosts_decorator_by_itself_order_ensured(): - """ - Use of @hosts only order ensured - """ - host_list = ['a', 'b'] - @ensure_order - @hosts(*host_list) - def command(): - pass - eq_hosts(command, host_list) - fake_roles = { 'r1': ['a', 'b'], @@ -117,17 +75,6 @@ def command(): pass eq_hosts(command, ['a', 'b']) -@patched_env({'roledefs': fake_roles}) -def test_roles_decorator_by_itself_order_ensured(): - """ - Use of @roles only order ensured - """ - @ensure_order - @roles('r1') - def command(): - pass - eq_hosts(command, ['a', 'b']) - @patched_env({'roledefs': fake_roles}) def test_hosts_and_roles_together(): @@ -140,18 +87,6 @@ def command(): pass eq_hosts(command, ['a', 'b', 'c']) -@patched_env({'roledefs': fake_roles}) -def test_hosts_and_roles_together_order_ensured(): - """ - Use of @roles and @hosts together results in union of both order ensured - """ - @ensure_order - @roles('r1', 'r2') - @hosts('a') - def command(): - pass - eq_hosts(command, ['a', 'b', 'c']) - tuple_roles = { 'r1': ('a', 'b'), 'r2': ('b', 'c'), @@ -190,18 +125,6 @@ def command(): eq_hosts(command, ['bar']) assert 'foo' not in get_hosts(command, [], [], []) -@patched_env({'hosts': ['foo']}) -def test_hosts_decorator_overrides_env_hosts_order_ensured(): - """ - If @hosts is used it replaces any env.hosts value order ensured - """ - @ensure_order - @hosts('bar') - def command(): - pass - eq_hosts(command, ['bar']) - assert 'foo' not in get_hosts(command, [], [], []) - @patched_env({'hosts': [' foo ', 'bar '], 'roles': [], 'exclude_hosts':[]}) @@ -242,18 +165,6 @@ def command(): eq_(command.hosts, host_list) -def test_hosts_decorator_expands_single_iterable_order_ensured(): - """ - @hosts(iterable) should behave like @hosts(*iterable) order ensured - """ - host_list = ['foo', 'bar'] - @ensure_order - @hosts(host_list) - def command(): - pass - eq_(command.hosts, host_list) - - def test_roles_decorator_expands_single_iterable(): """ @roles(iterable) should behave like @roles(*iterable) @@ -266,17 +177,6 @@ def command(): eq_(command.roles, role_list) -def test_roles_decorator_expands_single_iterable_order_ensured(): - """ - @roles(iterable) should behave like @roles(*iterable) order ensured - """ - role_list = ['foo', 'bar'] - @ensure_order - @roles(role_list) - def command(): - pass - eq_(command.roles, role_list) - @patched_env({'roledefs': fake_roles}) @raises(SystemExit) From b4ed89a8bc49450c89bd76b56dce6c5849481fa7 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 25 Apr 2011 17:52:44 -0700 Subject: [PATCH 058/126] Implements #115 - don't use set() to dedupe host lists --- docs/changes/1.1.rst | 7 ++++++- fabric/main.py | 14 ++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/changes/1.1.rst b/docs/changes/1.1.rst index 3bdbd35faf..bd467f0e81 100644 --- a/docs/changes/1.1.rst +++ b/docs/changes/1.1.rst @@ -29,7 +29,12 @@ Feature additions Bugfixes ======== -* N/A +* :issue:`115`: An implementation detail causing host lists to lose order + when deduped by the ``fab`` execution loop, has been patched to preserve + order instead. So e.g. ``fab -H a,b,c`` (or setting ``env.hosts = ['a', 'b', + 'c']``) will now always run on ``a``, then ``b``, then ``c``. Previously, + there was a chance the order could get mixed up during deduplication. Thanks + to Rohit Aggarwal for the report. Documentation updates ===================== diff --git a/fabric/main.py b/fabric/main.py index 28f2c5fb65..f3fb7290a9 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -368,12 +368,14 @@ def _merge(hosts, roles, exclude=[]): value = value() role_hosts += value - # make sure hosts is converted to a list to be able to append to the - # role_hosts list - hosts = list(hosts) - - # Return deduped combo of hosts and role_hosts - return list(set(_clean_hosts(hosts + role_hosts))) + # Return deduped combo of hosts and role_hosts, preserving order within + # them (vs using set(), which may lose ordering). + cleaned_hosts = _clean_hosts(list(hosts) + list(role_hosts)) + all_hosts = [] + for host in cleaned_hosts: + if host not in all_hosts: + all_hosts.append(host) + return all_hosts def _clean_hosts(host_list): """ From 87044bb780536c07df13b2854b350c73e8541604 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 25 Apr 2011 20:41:03 -0700 Subject: [PATCH 059/126] Changelog, docstring tweaks, imps #283 --- docs/changes/1.1.rst | 4 ++++ fabric/decorators.py | 16 +++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/changes/1.1.rst b/docs/changes/1.1.rst index bd467f0e81..5dd7da9bd3 100644 --- a/docs/changes/1.1.rst +++ b/docs/changes/1.1.rst @@ -25,6 +25,10 @@ Feature additions * :issue:`273`: `~fabric.contrib.files.upload_template` now offers control over whether it attempts to create backups of pre-existing destination files. Thanks to Ales Zoulek for the suggestion and initial patch. +* :issue:`283`: Added the `~fabric.decorators.with_settings` decorator to allow + application of env var settings to an entire function, as an alternative to + using the `~fabric.context_managers.settings` context manager. Thanks to + Travis Swicegood for the patch. Bugfixes ======== diff --git a/fabric/decorators.py b/fabric/decorators.py index 2e59a06ebb..7502c234ca 100644 --- a/fabric/decorators.py +++ b/fabric/decorators.py @@ -111,19 +111,18 @@ def with_settings(**kw_settings): Decorator equivalent of ``fabric.context_managers.settings``. Allows you to wrap an entire function as if it was called inside a block - with the ``settings`` context manager. Useful for retrofitting old code so - you don't have to change the indention to gain the behavior. + with the ``settings`` context manager. This may be useful if you know you + want a given setting applied to an entire function body, or wish to + retrofit old code without indenting everything. - An example use being to set all fabric api functions in a task to not error - out on unexpected return codes:: + For example, to turn aborts into warnings for an entire task function:: @with_settings(warn_only=True) - @hosts('user1@host1', 'host2', 'user2@host3') def foo(): - pass + ... - See ``fabric.context_managers.settings`` for more information about what - you can do with this. + .. seealso:: `~fabric.context_managers.settings` + .. versionadded:: 1.1 """ def outer(func): def inner(*args, **kwargs): @@ -131,4 +130,3 @@ def inner(*args, **kwargs): return func(*args, **kwargs) return inner return outer - From b0c716228f6fb84a34cd9553201fd51099efb0e4 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 26 Apr 2011 13:13:10 -0700 Subject: [PATCH 060/126] Revert #125 --- fabric/contrib/files.py | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/fabric/contrib/files.py b/fabric/contrib/files.py index 31d5063e70..9b46c88312 100644 --- a/fabric/contrib/files.py +++ b/fabric/contrib/files.py @@ -301,7 +301,7 @@ def append(filename, text, use_sudo=False, partial=False, escape=True): "append lines to a file" use case. You may override this and force partial searching (e.g. ``^``) by specifying ``partial=True``. - Because ``text`` is single-quoted, single quotes will be transparently + Because ``text`` is single-quoted, single quotes will be transparently backslash-escaped. This can be disabled with ``escape=False``. If ``use_sudo`` is True, will use `sudo` instead of `run`. @@ -315,27 +315,7 @@ def append(filename, text, use_sudo=False, partial=False, escape=True): .. versionchanged:: 1.0 Changed default value of ``partial`` kwarg to be ``False``. """ - _write_to_file(filename, text, use_sudo=use_sudo) - -def write(filename, text, use_sudo=False): - """ - Write string (or list of strings) ``text`` to ``filename``. - - This is identical to ``append()``, except that it overwrites any existing - file, instead of appending to it. - """ - _write_to_file(filename, text, use_sudo=use_sudo, overwrite=True) - -def _write_to_file(filename, text, use_sudo=False, overwrite=False): - """ - Append or overwrite a the string (or list of strings) ``text`` to - ``filename``. - - This is the implementation for both ``write`` and ``append``. Both call - this with the proper value for ``overwrite``. - """ func = use_sudo and sudo or run - operator = overwrite and '>' or '>>' # Normalize non-list input to be a list if isinstance(text, str): text = [text] @@ -345,4 +325,4 @@ def _write_to_file(filename, text, use_sudo=False, overwrite=False): and contains(filename, regex, use_sudo=use_sudo)): continue line = line.replace("'", r'\'') if escape else line - func("echo '%s' %s %s" % (line.replace("'", r'\''), operator, filename)) + func("echo '%s' >> %s" % (line, filename)) From 2e90af62cd605a79f26072aef91d642c480435fb Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 27 Apr 2011 13:35:03 -0700 Subject: [PATCH 061/126] Fixes #258 - Windows local logic inversion fix --- docs/changes/1.0.2.rst | 2 ++ fabric/operations.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changes/1.0.2.rst b/docs/changes/1.0.2.rst index cecab01971..c19a93d4f1 100644 --- a/docs/changes/1.0.2.rst +++ b/docs/changes/1.0.2.rst @@ -5,6 +5,8 @@ Changes in version 1.0.2 Bugfixes ======== +* :issue:`258`: Bugfix to a previous, incorrectly applied fix regarding + `~fabric.operations.local` on Windows platforms. * :issue:`337`: Fix logic bug in `~fabric.operations.put` preventing use of ``mirror_local_mode``. Thanks to Roman Imankulov for catch & patch. diff --git a/fabric/operations.py b/fabric/operations.py index 419185c7bc..fd67dc8f3b 100644 --- a/fabric/operations.py +++ b/fabric/operations.py @@ -1002,7 +1002,7 @@ def local(command, capture=False): out_stream = None if output.stdout else dev_null err_stream = None if output.stderr else dev_null try: - cmd_arg = [wrapped_command] if win32 else wrapped_command + cmd_arg = wrapped_command if win32 else [wrapped_command] p = subprocess.Popen(cmd_arg, shell=True, stdout=out_stream, stderr=err_stream) (stdout, stderr) = p.communicate() From 7d99ba0320d1e3bf9773e2bf5b57750efc6cd6d3 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 29 Apr 2011 12:02:01 -0700 Subject: [PATCH 062/126] Fixes #347 - test for StringTypes not str --- docs/changes/0.9.6.rst | 12 ++++++++++++ fabric/contrib/files.py | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 docs/changes/0.9.6.rst diff --git a/docs/changes/0.9.6.rst b/docs/changes/0.9.6.rst new file mode 100644 index 0000000000..ade90155b7 --- /dev/null +++ b/docs/changes/0.9.6.rst @@ -0,0 +1,12 @@ +======================== +Changes in version 0.9.6 +======================== + +The following changes were implemented in Fabric 0.9.6: + +Bugfixes +======== + +* :issue:`347`: `~fabric.contrib.files.append` incorrectly tested for ``str`` + instead of ``types.StringTypes``, causing it to split up Unicode strings as + if they were one character per line. This has been fixed. diff --git a/fabric/contrib/files.py b/fabric/contrib/files.py index d21315c072..b8f02e6bd5 100644 --- a/fabric/contrib/files.py +++ b/fabric/contrib/files.py @@ -6,6 +6,7 @@ import hashlib import tempfile +import types import re import os @@ -299,7 +300,7 @@ def append(filename, text, use_sudo=False, partial=False, escape=True): """ func = use_sudo and sudo or run # Normalize non-list input to be a list - if isinstance(text, str): + if isinstance(text, types.StringTypes): text = [text] for line in text: regex = '^' + re.escape(line) + ('' if partial else '$') From 3e040a39031227d8340fb293b5b0f7d49069375c Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 2 May 2011 10:37:31 -0700 Subject: [PATCH 063/126] Make git branches report themselves as alphas, derp --- fabric/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fabric/version.py b/fabric/version.py index e31dfa416e..eef4b2f7f0 100644 --- a/fabric/version.py +++ b/fabric/version.py @@ -21,7 +21,7 @@ def git_sha(): return p.communicate()[0] -VERSION = (1, 0, 1, 'final', 0) +VERSION = (1, 0, 2, 'alpha', 0) def get_version(form='short'): """ From ef696c31cf0fd9cb979dfcb0dc7b8962048dc0ae Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Sun, 29 May 2011 13:35:44 -0700 Subject: [PATCH 064/126] Clarify rsync_project docstring re: trailing slashes --- docs/changes/1.0.2.rst | 1 + fabric/contrib/project.py | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/changes/1.0.2.rst b/docs/changes/1.0.2.rst index c19a93d4f1..81c0fbd677 100644 --- a/docs/changes/1.0.2.rst +++ b/docs/changes/1.0.2.rst @@ -16,3 +16,4 @@ Documentation * Updated the API documentation for `~fabric.context_managers.cd` to explicitly point users to `~fabric.context_managers.lcd` for modifying local paths. +* Clarified the behavior of `~fabric.contrib.project.rsync_project` re: how trailing slashes in ``local_dir`` affect ``remote_dir``. Thanks to Mark Merritt for the catch. diff --git a/fabric/contrib/project.py b/fabric/contrib/project.py index d2c6cfdc14..e961228f3f 100644 --- a/fabric/contrib/project.py +++ b/fabric/contrib/project.py @@ -33,10 +33,21 @@ def rsync_project(remote_dir, local_dir=None, exclude=(), delete=False, ``rsync_project()`` takes the following parameters: * ``remote_dir``: the only required parameter, this is the path to the - **parent** directory on the remote server; the project directory will be - created inside this directory. For example, if one's project directory is - named ``myproject`` and one invokes ``rsync_project('/home/username/')``, - the resulting project directory will be ``/home/username/myproject/``. + directory on the remote server. Due to how ``rsync`` is implemented, the + exact behavior depends on the value of ``local_dir``: + + * If ``local_dir`` ends with a trailing slash, the files will be + dropped inside of ``remote_dir``. E.g. + ``rsync_project("/home/username/project", "foldername/")`` will drop + the contents of ``foldername`` inside of ``/home/username/project``. + * If ``local_dir`` does **not** end with a trailing slash (and this + includes the default scenario, when ``local_dir`` is not specified), + ``remote_dir`` is effectively the "parent" directory, and a new + directory named after ``local_dir`` will be created inside of it. So + ``rsync_project("/home/username", "foldername")`` would create a new + directory ``/home/username/foldername`` (if needed) and place the + files there. + * ``local_dir``: by default, ``rsync_project`` uses your current working directory as the source directory; you may override this with ``local_dir``, which should be a directory path. From a6c13cdf7525a97866552479aba5d6d619a19936 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 30 May 2011 17:47:25 -0700 Subject: [PATCH 065/126] Apply patch derived from goosemo/76-task-decorator * git diff 1.1...goosemo/76-task-decorator > 76.diff * git apply -v --check 76.diff * hack hack hack * git apply 76.diff Removed a handful of minor tweaks not relevant to the feature at hand. --- fabric/api.py | 2 +- fabric/decorators.py | 8 ++ fabric/main.py | 69 ++++++++++- fabric/tasks.py | 39 +++++++ tests/support/__init__.py | 0 tests/support/decorated_fabfile.py | 8 ++ .../decorated_fabfile_with_classbased_task.py | 12 ++ ...decorated_fabfile_with_decorated_module.py | 9 ++ .../support/decorated_fabfile_with_modules.py | 9 ++ tests/support/explicit_fabfile.py | 7 ++ tests/support/implicit_fabfile.py | 5 + tests/support/module_explicit.py | 1 + tests/support/module_fabfile.py | 1 + tests/support/module_fabtasks.py | 8 ++ tests/support/module_fabtasks_decorated.py | 9 ++ tests/support/module_fabtasks_explicit.py | 10 ++ tests/test_decorators.py | 7 +- tests/test_main.py | 110 +++++++++++++++++- tests/test_operations.py | 4 + tests/test_tasks.py | 98 ++++++++++++++++ 20 files changed, 406 insertions(+), 10 deletions(-) create mode 100644 fabric/tasks.py create mode 100644 tests/support/__init__.py create mode 100644 tests/support/decorated_fabfile.py create mode 100644 tests/support/decorated_fabfile_with_classbased_task.py create mode 100644 tests/support/decorated_fabfile_with_decorated_module.py create mode 100644 tests/support/decorated_fabfile_with_modules.py create mode 100644 tests/support/explicit_fabfile.py create mode 100644 tests/support/implicit_fabfile.py create mode 100644 tests/support/module_explicit.py create mode 100644 tests/support/module_fabfile.py create mode 100644 tests/support/module_fabtasks.py create mode 100644 tests/support/module_fabtasks_decorated.py create mode 100644 tests/support/module_fabtasks_explicit.py create mode 100644 tests/test_tasks.py diff --git a/fabric/api.py b/fabric/api.py index 4c49c9a8af..d4ddb7a7c8 100644 --- a/fabric/api.py +++ b/fabric/api.py @@ -7,7 +7,7 @@ well when you're using setup.py to install e.g. paramiko! """ from fabric.context_managers import cd, hide, settings, show, path, prefix, lcd -from fabric.decorators import hosts, roles, runs_once, with_settings +from fabric.decorators import hosts, roles, runs_once, with_settings, task from fabric.operations import (require, prompt, put, get, run, sudo, local, reboot, open_shell) from fabric.state import env, output diff --git a/fabric/decorators.py b/fabric/decorators.py index 7502c234ca..0dad08f3b1 100644 --- a/fabric/decorators.py +++ b/fabric/decorators.py @@ -3,11 +3,19 @@ """ from __future__ import with_statement +from fabric import tasks from functools import wraps from types import StringTypes from .context_managers import settings +def task(func): + """ + Decorator defining a function as a task. + + This is a convenience wrapper around `tasks.WrappedCallableTask`. + """ + return tasks.WrappedCallableTask(func) def hosts(*host_list): """ diff --git a/fabric/main.py b/fabric/main.py index f3fb7290a9..8f5289f358 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -13,6 +13,7 @@ from optparse import OptionParser import os import sys +import types from fabric import api # For checking callables against the API from fabric.contrib import console, files, project # Ditto @@ -20,6 +21,8 @@ from fabric import state # For easily-mockable access to roles, env and etc from fabric.state import commands, connections, env_options from fabric.utils import abort, indent +from fabric import decorators +from fabric.tasks import Task # One-time calculation of "all internal callables" to avoid doing this on every @@ -140,9 +143,69 @@ def load_fabfile(path, importer=None): if index is not None: sys.path.insert(index + 1, directory) del sys.path[0] - # Return our two-tuple - tasks = dict(filter(is_task, vars(imported).items())) - return imported.__doc__, tasks + + return load_fab_tasks_from_module(imported) + + +def load_fab_tasks_from_module(imported): + """ + Handles loading all of the fab_tasks for a given `imported` module + """ + # Obey the use of .__all__ if it is present + imported_vars = vars(imported) + if "__all__" in imported_vars: + imported_vars = [(name, imported_vars[name]) for name in \ + imported_vars if name in imported_vars["__all__"]] + else: + imported_vars = imported_vars.items() + # Return a two-tuple value. First is the documentation, second is a + # dictionary of callables only (and don't include Fab operations or + # underscored callables) + return imported.__doc__, extract_tasks(imported_vars) + + +def is_task_module(a): + """ + Determine if the provided value is a task module + """ + return (type(a) is types.ModuleType and + getattr(a, "FABRIC_TASK_MODULE", False) is True) + + +def is_task_object(a): + """ + Determine if the provided value is a ``Task`` object. + + This returning True signals that all tasks within the fabfile + module must be Task objects. + """ + return isinstance(a, Task) and a.use_task_objects + + +def extract_tasks(imported_vars): + """ + Handle extracting tasks from a given list of variables + """ + tasks = {} + using_task_objects = False + for tup in imported_vars: + name, callable = tup + if is_task_object(callable): + using_task_objects = True + tasks[callable.name] = callable + elif is_task(tup): + tasks[name] = callable + elif is_task_module(callable): + module_docs, module_tasks = load_fab_tasks_from_module(callable) + for task_name, task in module_tasks.items(): + tasks["%s.%s" % (name, task_name)] = task + + if using_task_objects: + def is_usable_task(tup): + name, task = tup + return name.find('.') != -1 or isinstance(task, Task) + tasks = dict(filter(is_usable_task, tasks.items())) + return tasks def parse_options(): diff --git a/fabric/tasks.py b/fabric/tasks.py new file mode 100644 index 0000000000..7ba344abc4 --- /dev/null +++ b/fabric/tasks.py @@ -0,0 +1,39 @@ +from functools import wraps + +class Task(object): + """ + Base Task class, from which all class-based Tasks should extend. + + This class is used to provide a way to test whether a task is really + a task or not. It provides no functionality and should not used + directly. + """ + name = 'undefined' + use_task_objects = True + + # TODO: make it so that this wraps other decorators as expected + + def run(self): + raise NotImplementedError + + +class WrappedCallableTask(Task): + """ + Task for wrapping some sort of callable in a Task object. + + Generally used via the ``@task`` decorator. + """ + def __init__(self, callable): + super(WrappedCallableTask, self).__init__() + self.wrapped = callable + self.__name__ = self.name = callable.__name__ + self.__doc__ = callable.__doc__ + + def __call__(self, *args, **kwargs): + return self.run(*args, **kwargs) + + def run(self, *args, **kwargs): + return self.wrapped(*args, **kwargs) + + def __getattr__(self, k): + return getattr(self.wrapped, k) diff --git a/tests/support/__init__.py b/tests/support/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/support/decorated_fabfile.py b/tests/support/decorated_fabfile.py new file mode 100644 index 0000000000..fcab3a2e9a --- /dev/null +++ b/tests/support/decorated_fabfile.py @@ -0,0 +1,8 @@ +from fabric.decorators import task + +@task +def foo(): + pass + +def bar(): + pass diff --git a/tests/support/decorated_fabfile_with_classbased_task.py b/tests/support/decorated_fabfile_with_classbased_task.py new file mode 100644 index 0000000000..143c392085 --- /dev/null +++ b/tests/support/decorated_fabfile_with_classbased_task.py @@ -0,0 +1,12 @@ +from fabric import tasks +from fabric.decorators import task + +class ClassBasedTask(tasks.Task): + def __init__(self): + self.name = "foo" + self.use_decorated = True + + def run(self, *args, **kwargs): + pass + +foo = ClassBasedTask() diff --git a/tests/support/decorated_fabfile_with_decorated_module.py b/tests/support/decorated_fabfile_with_decorated_module.py new file mode 100644 index 0000000000..bc6a6e9280 --- /dev/null +++ b/tests/support/decorated_fabfile_with_decorated_module.py @@ -0,0 +1,9 @@ +from fabric.decorators import task +import module_fabtasks_decorated as tasks + +@task +def foo(): + pass + +def bar(): + pass diff --git a/tests/support/decorated_fabfile_with_modules.py b/tests/support/decorated_fabfile_with_modules.py new file mode 100644 index 0000000000..e35aadae61 --- /dev/null +++ b/tests/support/decorated_fabfile_with_modules.py @@ -0,0 +1,9 @@ +from fabric.decorators import task +import module_fabtasks as tasks + +@task +def foo(): + pass + +def bar(): + pass diff --git a/tests/support/explicit_fabfile.py b/tests/support/explicit_fabfile.py new file mode 100644 index 0000000000..16dd6c9618 --- /dev/null +++ b/tests/support/explicit_fabfile.py @@ -0,0 +1,7 @@ +__all__ = ['foo',] + +def foo(): + pass + +def bar(): + pass diff --git a/tests/support/implicit_fabfile.py b/tests/support/implicit_fabfile.py new file mode 100644 index 0000000000..8489841709 --- /dev/null +++ b/tests/support/implicit_fabfile.py @@ -0,0 +1,5 @@ +def foo(): + pass + +def bar(): + pass diff --git a/tests/support/module_explicit.py b/tests/support/module_explicit.py new file mode 100644 index 0000000000..134a37705f --- /dev/null +++ b/tests/support/module_explicit.py @@ -0,0 +1 @@ +import module_fabtasks_explicit as tasks diff --git a/tests/support/module_fabfile.py b/tests/support/module_fabfile.py new file mode 100644 index 0000000000..2422bec759 --- /dev/null +++ b/tests/support/module_fabfile.py @@ -0,0 +1 @@ +import module_fabtasks as tasks diff --git a/tests/support/module_fabtasks.py b/tests/support/module_fabtasks.py new file mode 100644 index 0000000000..47e4bb7bbc --- /dev/null +++ b/tests/support/module_fabtasks.py @@ -0,0 +1,8 @@ + +FABRIC_TASK_MODULE = True + +def hello(): + print "hello" + +def world(): + print "world" diff --git a/tests/support/module_fabtasks_decorated.py b/tests/support/module_fabtasks_decorated.py new file mode 100644 index 0000000000..74d4123779 --- /dev/null +++ b/tests/support/module_fabtasks_decorated.py @@ -0,0 +1,9 @@ +from fabric.decorators import task +FABRIC_TASK_MODULE = True + +@task +def hello(): + print "hello" + +def world(): + print "world" diff --git a/tests/support/module_fabtasks_explicit.py b/tests/support/module_fabtasks_explicit.py new file mode 100644 index 0000000000..6f71d9a3f9 --- /dev/null +++ b/tests/support/module_fabtasks_explicit.py @@ -0,0 +1,10 @@ + +FABRIC_TASK_MODULE = True + +__all__ = ['hello',] + +def hello(): + print "hello" + +def world(): + print "world" diff --git a/tests/test_decorators.py b/tests/test_decorators.py index d8da36e280..1f55807132 100644 --- a/tests/test_decorators.py +++ b/tests/test_decorators.py @@ -2,9 +2,14 @@ from fudge import Fake, with_fakes import random -from fabric import decorators +from fabric import decorators, tasks from fabric.state import env +def test_task_returns_an_instance_of_wrappedfunctask_object(): + def foo(): + pass + task = decorators.task(foo) + ok_(isinstance(task, tasks.WrappedCallableTask)) def fake_function(*args, **kwargs): """ diff --git a/tests/test_main.py b/tests/test_main.py index d01e58525c..8ae740d93c 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -2,15 +2,18 @@ import copy from fudge import Fake -from nose.tools import eq_, raises +from nose.tools import ok_, eq_, raises -from fabric.decorators import hosts, roles +from fabric.decorators import hosts, roles, task from fabric.main import (get_hosts, parse_arguments, _merge, _escape_split, load_fabfile) + import fabric.state from fabric.state import _AttributeDict from utils import mock_streams, patched_env +import os +import sys def test_argument_parsing(): @@ -125,6 +128,27 @@ def command(): eq_hosts(command, ['bar']) assert 'foo' not in get_hosts(command, [], [], []) +@with_patched_object('fabric.state', 'env', {'hosts': ['foo']}) +def test_hosts_decorator_overrides_env_hosts_with_task_decorator_first(): + """ + If @hosts is used it replaces any env.hosts value even with @task + """ + @task + @hosts('bar') + def command(): + pass + eq_hosts(command, ['bar']) + assert 'foo' not in get_hosts(command, [], []) + +@with_patched_object('fabric.state', 'env', {'hosts': ['foo']}) +def test_hosts_decorator_overrides_env_hosts_with_task_decorator_last(): + @hosts('bar') + @task + def command(): + pass + eq_hosts(command, ['bar']) + assert 'foo' not in get_hosts(command, [], []) + @patched_env({'hosts': [' foo ', 'bar '], 'roles': [], 'exclude_hosts':[]}) @@ -200,7 +224,6 @@ def command(): pass eq_hosts(command, ['a', 'b']) - def test_escaped_task_arg_split(): """ Allow backslashes to escape the task argument separator character @@ -211,7 +234,6 @@ def test_escaped_task_arg_split(): ['foo', 'bar,biz,baz', 'what comes after baz?'] ) - def run_load_fabfile(path, sys_path): # Module-esque object fake_module = Fake().has_attr(__dict__={}) @@ -227,7 +249,6 @@ def run_load_fabfile(path, sys_path): # Restore sys.path = orig_path - def test_load_fabfile_should_not_remove_real_path_elements(): for fabfile_path, sys_dot_path in ( # Directory not in path @@ -243,3 +264,82 @@ def test_load_fabfile_should_not_remove_real_path_elements(): ('fabfile.py', ['', 'some_dir', 'some_other_dir']), ): yield run_load_fabfile, fabfile_path, sys_dot_path + +def support_fabfile(name): + return os.path.join(os.path.dirname(__file__), 'support', name) + +def test_implicit_discover(): + """ + Automatically includes all functions in a fabfile + """ + implicit = support_fabfile("implicit_fabfile.py") + sys.path[0:0] = [os.path.dirname(implicit),] + + docs, funcs = load_fabfile(implicit) + ok_(len(funcs) == 2) + ok_("foo" in funcs) + ok_("bar" in funcs) + + sys.path = sys.path[1:] + +def test_explicit_discover(): + """ + Only use those methods listed in __all__ + """ + + explicit = support_fabfile("explicit_fabfile.py") + sys.path[0:0] = [os.path.dirname(explicit),] + + docs, funcs = load_fabfile(explicit) + ok_(len(funcs) == 1) + ok_("foo" in funcs) + ok_("bar" not in funcs) + +def test_allow_registering_modules(): + module = support_fabfile('module_fabfile.py') + sys.path[0:0] = [os.path.dirname(module),] + + docs, funcs = load_fabfile(module) + ok_(len(funcs) == 2) + ok_('tasks.hello' in funcs) + ok_('tasks.world' in funcs) + +def test_modules_should_pay_attention_to_all_and_explicit_discovery(): + module = support_fabfile('module_explicit.py') + sys.path[0:0] = [os.path.dirname(module),] + + docs, funcs = load_fabfile(module) + ok_(len(funcs) == 1) + ok_('tasks.hello' in funcs) + ok_('tasks.world' not in funcs) + +def test_should_load_decorated_tasks_only_if_one_is_found(): + module = support_fabfile('decorated_fabfile.py') + sys.path[0:0] = [os.path.dirname(module),] + + docs, funcs = load_fabfile(module) + eq_(1, len(funcs)) + ok_('foo' in funcs) + +def test_modules_are_still_loaded_if_fabfile_contains_decorated_task(): + module = support_fabfile('decorated_fabfile_with_modules.py') + sys.path[0:0] = [os.path.dirname(module),] + + docs, funcs = load_fabfile(module) + eq_(3, len(funcs)) + +def test_modules_pay_attention_to_task_decorator(): + module = support_fabfile('decorated_fabfile_with_decorated_module.py') + sys.path[0:0] = [os.path.dirname(module),] + + docs, funcs = load_fabfile(module) + eq_(2, len(funcs)) + +def test_class_based_tasks_are_found_with_proper_name(): + module = support_fabfile('decorated_fabfile_with_classbased_task.py') + sys.path[0:0] = [os.path.dirname(module),] + + docs, funcs = load_fabfile(module) + print funcs + eq_(1, len(funcs)) + ok_('foo' in funcs) diff --git a/tests/test_operations.py b/tests/test_operations.py index 7bfe7cefa1..136a25fda9 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -7,6 +7,10 @@ from contextlib import nested from StringIO import StringIO +import unittest +import random +import types + from nose.tools import raises, eq_ from fudge import with_patched_object diff --git a/tests/test_tasks.py b/tests/test_tasks.py new file mode 100644 index 0000000000..73442898a7 --- /dev/null +++ b/tests/test_tasks.py @@ -0,0 +1,98 @@ +import unittest +from nose.tools import eq_, raises +import random + +from fabric import tasks + +def test_base_task_provides_undefined_name(): + task = tasks.Task() + eq_("undefined", task.name) + +@raises(NotImplementedError) +def test_base_task_raises_exception_on_call_to_run(): + task = tasks.Task() + task.run() + +class TestOfWrappedCallableTask(unittest.TestCase): + def test_run_is_wrapped_callable(self): + def foo(): pass + + task = tasks.WrappedCallableTask(foo) + self.assertEqual(task.wrapped, foo) + + def test_name_is_the_name_of_the_wrapped_callable(self): + def foo(): pass + foo.__name__ = "random_name_%d" % random.randint(1000, 2000) + + task = tasks.WrappedCallableTask(foo) + self.assertEqual(task.name, foo.__name__) + + def test_reads_double_under_doc_from_callable(self): + def foo(): pass + foo.__doc__ = "Some random __doc__: %d" % random.randint(1000, 2000) + + task = tasks.WrappedCallableTask(foo) + self.assertEqual(task.__doc__, foo.__doc__) + + def test_dispatches_to_wrapped_callable_on_run(self): + random_value = "some random value %d" % random.randint(1000, 2000) + def foo(): return random_value + + task = tasks.WrappedCallableTask(foo) + self.assertEqual(random_value, task()) + + def test_passes_all_regular_args_to_run(self): + def foo(*args): return args + + random_args = tuple([random.randint(1000, 2000) for i in range(random.randint(1, 5))]) + task = tasks.WrappedCallableTask(foo) + self.assertEqual(random_args, task(*random_args)) + + def test_passes_all_keyword_args_to_run(self): + def foo(**kwargs): return kwargs + + random_kwargs = {} + for i in range(random.randint(1, 5)): + random_key = ("foo", "bar", "baz", "foobar", "barfoo")[i] + random_kwargs[random_key] = random.randint(1000, 2000) + + task = tasks.WrappedCallableTask(foo) + self.assertEqual(random_kwargs, task(**random_kwargs)) + + def test_calling_the_object_is_the_same_as_run(self): + random_return = random.randint(1000, 2000) + def foo(): return random_return + + task = tasks.WrappedCallableTask(foo) + self.assertEqual(task(), task.run()) + + +# Reminder: decorator syntax, e.g.: +# @foo +# def bar():... +# +# is semantically equivalent to: +# def bar():... +# bar = foo(bar) +# +# this simplifies testing :) + +def test_decorator_incompatibility_on_task(): + from fabric.decorators import task, hosts, runs_once, roles + def foo(): return "foo" + foo = task(foo) + + # since we aren't setting foo to be the newly decorated thing, its cool + hosts('me@localhost')(foo) + runs_once(foo) + roles('www')(foo) + +def test_decorator_closure_hiding(): + from fabric.decorators import task, hosts + def foo(): print env.host_string + foo = hosts("me@localhost")(foo) + foo = task(foo) + + # this broke in the old way, due to closure stuff hiding in the + # function, but task making an object + eq_(["me@localhost"], foo.hosts) From e8f19ef07cd556805458807389a8ed1feca99ec2 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 30 May 2011 18:04:41 -0700 Subject: [PATCH 066/126] Update test env munging --- tests/test_main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 8ae740d93c..fa556c036d 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -128,7 +128,7 @@ def command(): eq_hosts(command, ['bar']) assert 'foo' not in get_hosts(command, [], [], []) -@with_patched_object('fabric.state', 'env', {'hosts': ['foo']}) +@patched_env({'hosts': ['foo']}) def test_hosts_decorator_overrides_env_hosts_with_task_decorator_first(): """ If @hosts is used it replaces any env.hosts value even with @task @@ -140,7 +140,7 @@ def command(): eq_hosts(command, ['bar']) assert 'foo' not in get_hosts(command, [], []) -@with_patched_object('fabric.state', 'env', {'hosts': ['foo']}) +@patched_env({'hosts': ['foo']}) def test_hosts_decorator_overrides_env_hosts_with_task_decorator_last(): @hosts('bar') @task From db219e7dbb828c6dc83738a68a51baffd50101fe Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 30 May 2011 21:38:13 -0700 Subject: [PATCH 067/126] Bit of cleanup --- fabric/decorators.py | 3 ++- fabric/main.py | 16 +++++++--------- fabric/tasks.py | 11 +++++------ tests/test_tasks.py | 2 +- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/fabric/decorators.py b/fabric/decorators.py index 0dad08f3b1..892d55cf4f 100644 --- a/fabric/decorators.py +++ b/fabric/decorators.py @@ -3,12 +3,13 @@ """ from __future__ import with_statement -from fabric import tasks from functools import wraps from types import StringTypes +from fabric import tasks from .context_managers import settings + def task(func): """ Decorator defining a function as a task. diff --git a/fabric/main.py b/fabric/main.py index 8f5289f358..ebd34b37c0 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -15,14 +15,12 @@ import sys import types -from fabric import api # For checking callables against the API +from fabric import api, state # For checking callables against the API, & easy mocking from fabric.contrib import console, files, project # Ditto from fabric.network import denormalize, interpret_host_string, disconnect_all -from fabric import state # For easily-mockable access to roles, env and etc from fabric.state import commands, connections, env_options -from fabric.utils import abort, indent -from fabric import decorators from fabric.tasks import Task +from fabric.utils import abort, indent # One-time calculation of "all internal callables" to avoid doing this on every @@ -144,12 +142,12 @@ def load_fabfile(path, importer=None): sys.path.insert(index + 1, directory) del sys.path[0] - return load_fab_tasks_from_module(imported) + return load_tasks_from_module(imported) -def load_fab_tasks_from_module(imported): +def load_tasks_from_module(imported): """ - Handles loading all of the fab_tasks for a given `imported` module + Handles loading all of the tasks for a given `imported` module """ # Obey the use of .__all__ if it is present imported_vars = vars(imported) @@ -169,7 +167,7 @@ def is_task_module(a): Determine if the provided value is a task module """ return (type(a) is types.ModuleType and - getattr(a, "FABRIC_TASK_MODULE", False) is True) + getattr(a, "FABRIC_TASK_MODULE", False)) def is_task_object(a): @@ -196,7 +194,7 @@ def extract_tasks(imported_vars): elif is_task(tup): tasks[name] = callable elif is_task_module(callable): - module_docs, module_tasks = load_fab_tasks_from_module(callable) + module_docs, module_tasks = load_tasks_from_module(callable) for task_name, task in module_tasks.items(): tasks["%s.%s" % (name, task_name)] = task diff --git a/fabric/tasks.py b/fabric/tasks.py index 7ba344abc4..21962f585e 100644 --- a/fabric/tasks.py +++ b/fabric/tasks.py @@ -2,11 +2,10 @@ class Task(object): """ - Base Task class, from which all class-based Tasks should extend. + Abstract base class for objects wishing to be picked up as Fabric tasks. - This class is used to provide a way to test whether a task is really - a task or not. It provides no functionality and should not used - directly. + Instances of subclasses will be treated as valid tasks when present in + fabfiles loaded by the "fab" tool. """ name = 'undefined' use_task_objects = True @@ -19,9 +18,9 @@ def run(self): class WrappedCallableTask(Task): """ - Task for wrapping some sort of callable in a Task object. + Wraps a given callable transparently, while marking it as a valid Task. - Generally used via the ``@task`` decorator. + Generally used via the ``@task`` decorator and not directly. """ def __init__(self, callable): super(WrappedCallableTask, self).__init__() diff --git a/tests/test_tasks.py b/tests/test_tasks.py index 73442898a7..e760f01f4d 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -13,7 +13,7 @@ def test_base_task_raises_exception_on_call_to_run(): task = tasks.Task() task.run() -class TestOfWrappedCallableTask(unittest.TestCase): +class TestWrappedCallableTask(unittest.TestCase): def test_run_is_wrapped_callable(self): def foo(): pass From 67d1a769bcb8020d66889e2956702077db0af2aa Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 30 May 2011 22:24:59 -0700 Subject: [PATCH 068/126] callable is a builtin :) --- fabric/main.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fabric/main.py b/fabric/main.py index ebd34b37c0..33f2f47908 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -187,14 +187,14 @@ def extract_tasks(imported_vars): tasks = {} using_task_objects = False for tup in imported_vars: - name, callable = tup - if is_task_object(callable): + name, obj = tup + if is_task_object(obj): using_task_objects = True - tasks[callable.name] = callable + tasks[obj.name] = obj elif is_task(tup): - tasks[name] = callable - elif is_task_module(callable): - module_docs, module_tasks = load_tasks_from_module(callable) + tasks[name] = obj + elif is_task_module(obj): + module_docs, module_tasks = load_tasks_from_module(obj) for task_name, task in module_tasks.items(): tasks["%s.%s" % (name, task_name)] = task From 851b79622689119f1e5b83fbd9c47780f04a3bef Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 31 May 2011 16:42:33 -0700 Subject: [PATCH 069/126] Update run/sudo combine_stderr for overriding. Fixes #324. --- docs/changes/1.0.2.rst | 5 +++++ fabric/operations.py | 15 ++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/changes/1.0.2.rst b/docs/changes/1.0.2.rst index 81c0fbd677..e267c405e3 100644 --- a/docs/changes/1.0.2.rst +++ b/docs/changes/1.0.2.rst @@ -7,6 +7,11 @@ Bugfixes * :issue:`258`: Bugfix to a previous, incorrectly applied fix regarding `~fabric.operations.local` on Windows platforms. +* :issue:`324`: Update `~fabric.operations.run`/`~fabric.operations.sudo`'s + ``combine_stderr`` kwarg so that it correctly overrides the global setting in + all cases. This required changing its default value to ``None``, but the + default behavior (behaving as if the setting were ``True``) has not changed. + Thanks to Matthew Woodcraft and Connor Smith for the catch. * :issue:`337`: Fix logic bug in `~fabric.operations.put` preventing use of ``mirror_local_mode``. Thanks to Roman Imankulov for catch & patch. diff --git a/fabric/operations.py b/fabric/operations.py index fd67dc8f3b..fec607d631 100644 --- a/fabric/operations.py +++ b/fabric/operations.py @@ -694,7 +694,7 @@ def _prefix_env_vars(command): return path + command -def _execute(channel, command, pty=True, combine_stderr=True, +def _execute(channel, command, pty=True, combine_stderr=None, invoke_shell=False): """ Execute ``command`` over ``channel``. @@ -702,6 +702,9 @@ def _execute(channel, command, pty=True, combine_stderr=True, ``pty`` controls whether a pseudo-terminal is created. ``combine_stderr`` controls whether we call ``channel.set_combine_stderr``. + By default, the global setting for this behavior (:ref:`env.combine_stderr + `) is consulted, but you may specify ``True`` or ``False`` + here to override it. ``invoke_shell`` controls whether we use ``exec_command`` or ``invoke_shell`` (plus a handful of other things, such as always forcing a @@ -713,8 +716,9 @@ def _execute(channel, command, pty=True, combine_stderr=True, """ with char_buffered(sys.stdin): # Combine stdout and stderr to get around oddball mixing issues - if combine_stderr or env.combine_stderr: - channel.set_combine_stderr(True) + if combine_stderr is None: + combine_stderr = env.combine_stderr + channel.set_combine_stderr(combine_stderr) # Assume pty use, and allow overriding of this either via kwarg or env # var. (invoke_shell always wants a pty no matter what.) @@ -914,6 +918,11 @@ def run(command, shell=True, pty=True, combine_stderr=True): .. versionchanged:: 1.0 The default value of ``pty`` is now ``True``. + + .. versionchanged:: 1.0.2 + The default value of ``combine_stderr`` is now ``None`` instead of + ``True``. However, the default *behavior* is unchanged, as the global + setting is still ``True``. """ return _run_command(command, shell, pty, combine_stderr) From 89e97c12f6e0b35dac85040790af60291b7dd98a Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 31 May 2011 17:51:39 -0700 Subject: [PATCH 070/126] Re #324, tests and an important fix they caught --- fabric/operations.py | 4 ++-- tests/server.py | 6 ++++- tests/test_operations.py | 50 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/fabric/operations.py b/fabric/operations.py index fec607d631..95868f7079 100644 --- a/fabric/operations.py +++ b/fabric/operations.py @@ -870,7 +870,7 @@ def _run_command(command, shell=True, pty=True, combine_stderr=True, @needs_host -def run(command, shell=True, pty=True, combine_stderr=True): +def run(command, shell=True, pty=True, combine_stderr=None): """ Run a shell command on a remote host. @@ -928,7 +928,7 @@ def run(command, shell=True, pty=True, combine_stderr=True): @needs_host -def sudo(command, shell=True, pty=True, combine_stderr=True, user=None): +def sudo(command, shell=True, pty=True, combine_stderr=None, user=None): """ Run a shell command on a remote host, with superuser privileges. diff --git a/tests/server.py b/tests/server.py index eba404a5c4..f659e666f2 100644 --- a/tests/server.py +++ b/tests/server.py @@ -56,7 +56,11 @@ fabric requirements.txt setup.py -tests""" +tests""", + "both_streams": [ + "stdout", + "stderr" + ] } FILES = FakeFilesystem({ '/file.txt': 'contents', diff --git a/tests/test_operations.py b/tests/test_operations.py index 0d4ede1d77..df9ac8a472 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -11,10 +11,10 @@ from nose.tools import raises, eq_ from fudge import with_patched_object -from fabric.state import env +from fabric.state import env, output from fabric.operations import require, prompt, _sudo_prefix, _shell_wrap, \ _shell_escape -from fabric.api import get, put, hide, show, cd, lcd, local +from fabric.api import get, put, hide, show, cd, lcd, local, run, sudo from fabric.sftp import SFTP from utils import * @@ -192,6 +192,52 @@ def test_shell_escape_escapes_backticks(): eq_(_shell_escape(cmd), "touch test.pid && kill \`cat test.pid\`") +class TestCombineStderr(FabricTest): + @server() + def test_local_none_global_true(self): + """ + combine_stderr: no kwarg => uses global value (True) + """ + output.everything = False + r = run("both_streams") + # Note: the exact way the streams are jumbled here is an implementation + # detail of our fake SSH server and may change in the future. + eq_("ssttddoeurtr", r.stdout) + eq_(r.stderr, "") + + @server() + def test_local_none_global_false(self): + """ + combine_stderr: no kwarg => uses global value (False) + """ + output.everything = False + env.combine_stderr = False + r = run("both_streams") + eq_("stdout", r.stdout) + eq_("stderr", r.stderr) + + @server() + def test_local_true_global_false(self): + """ + combine_stderr: True kwarg => overrides global False value + """ + output.everything = False + env.combine_stderr = False + r = run("both_streams", combine_stderr=True) + eq_("ssttddoeurtr", r.stdout) + eq_(r.stderr, "") + + @server() + def test_local_false_global_true(self): + """ + combine_stderr: False kwarg => overrides global True value + """ + output.everything = False + env.combine_stderr = True + r = run("both_streams", combine_stderr=False) + eq_("stdout", r.stdout) + eq_("stderr", r.stderr) + # # get() and put() # From 7b1f568a7fcae395ed55c6a2ec607142404765b9 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Sun, 5 Jun 2011 20:51:51 -0700 Subject: [PATCH 071/126] WIP re #76. TODO: * Figure out package-related stuff -- remind self how vanilla Python package recursive imports work, make sure it's applied consistently (re namespacing) * Update code to be consistent with new docs * Ensure changelog, AUTHORS updated --- docs/api/core/tasks.rst | 6 +++ docs/usage/execution.rst | 104 +++++++++++++++++++++++++++++++++++++-- fabric/tasks.py | 4 ++ 3 files changed, 109 insertions(+), 5 deletions(-) create mode 100644 docs/api/core/tasks.rst diff --git a/docs/api/core/tasks.rst b/docs/api/core/tasks.rst new file mode 100644 index 0000000000..20c01663a8 --- /dev/null +++ b/docs/api/core/tasks.rst @@ -0,0 +1,6 @@ +===== +Tasks +===== + +.. automodule:: fabric.tasks + :members: diff --git a/docs/usage/execution.rst b/docs/usage/execution.rst index a482f56f9d..27cfc647c0 100644 --- a/docs/usage/execution.rst +++ b/docs/usage/execution.rst @@ -72,8 +72,102 @@ to do next. Defining tasks ============== -When looking for tasks to execute, Fabric imports your fabfile and will -consider any callable object, **except** for the following: +As of Fabric 1.1, there are two distinct methods you may use in order to define +which objects in your fabfile show up as tasks: + +* The "new" method starting in 1.1 considers instances of `~fabric.tasks.Task` + or its subclasses, and also descends into imported modules to allow building + nested namespaces. +* The "classic" method from 1.0 and earlier considers all public callable + objects (functions, classes etc) and only considers the objects in the + fabfile itself with no recursing into imported module. + +.. note:: + These two methods are **mutually exclusive**: if Fabric finds *any* + new-style task objects in your fabfile or in modules it imports, it will + assume you've committed to this method of task declaration and won't + consider any non-`~fabric.tasks.Task` callables. If *no* new-style tasks + are found, it reverts to the classic behavior. + +See the following subsections for details on these two methods of setting up +tasks. + +New-style tasks +--------------- + +Fabric 1.1 introduced the `~fabric.tasks.Task` class to facilitate new features +and enable some programming best practices, specifically: + +* **Object-oriented tasks**. Inheritance and all that comes with it can make + for much more sensible code reuse than passing around simple function + objects. The classic style of task declaration didn't entirely rule this + out, but it also didn't make it terribly easy. +* **Namespaces**. Without an easy way to tell tasks apart from other non-task + callables, recursive namespace creation would be difficult if not impossible + (imagine having your "task list" cluttered up with the contents of ``os.sys`` + for example.) + +With the introduction of `~fabric.tasks.Task`, there are two ways to set up new tasks: + +* Decorate a regular module level function with `@task + <~fabric.decorators.task>`, which transparently wraps the function in a + `~fabric.tasks.Task` subclass. The function name will be used as the task + name when invoking. +* Subclass `~fabric.tasks.Task` (`~fabric.tasks.Task` itself is intended to be + abstract), define a ``run`` method, and instantiate your subclass at module + level. Instances' ``name`` attributes are used as the task name; if omitted + the instance's variable name will be used instead. + +Use of new-style tasks also allows you to set up task namespaces -- see below. + +Namespaces +~~~~~~~~~~ + +With classic tasks, only module level callables are considered, forcing you to set up a single flat "namespace" of tasks in your fabfile. While Fabric 0.9.2 introduced package support, you still had to import individual task functions into your ``__init__.py`` and had no ability to organize them. + +In Fabric 1.1 and newer, if you declare tasks the new way (via `~fabric.decorators.task` or your own `~fabric.tasks.Task` subclass instances) you may take advantage of automatic **namespacing**: + +* Any module objects imported into your fabfile will be recursed into looking for additional task objects. +* These sub-module tasks will be given new dotted-notation names based on the modules they came from, similar to Python's own import syntax. + +As an example, consider this fabfile package:: + + . + ├── __init__.py + ├── db + │   ├── __init__.py + │   ├── fixtures.py + │   └── migrate.py + ├── lb.py + +Given the following task declarations: + +* ``__init__.py`` declares a new-style task named ``deploy`` and the line ``import db, lb`` +* ``db/__init__.py`` is empty (just serving as the Python package signifier) +* ``db/fixtures.py`` declares ``load`` and ``dump``. +* ``db/migrate.py`` declares ``up`` and ``down``. +* ``lb.py`` contains ``add_backend``. + +you'd end up with the following list of tasks from ``fab --list``:: + + deploy + db.fixtures.load + db.fixtures.dump + db.migrate.up + db.migrate.down + lb.add_backend + +Note that the base ``__init__.py`` had to explicitly import the other modules. +This represents a small amount of boilerplate on your part, but also allows +explicit control, e.g. if you were to import third party fabfiles not residing +in your own package, or wanted some submodules to remain partly private (e.g. +for use with ``fab -f`` only.) + +Classic tasks +------------- + +When no new-style `~fabric.tasks.Task`-based tasks are found, Fabric will +consider any callable object found in your fabfile, **except** the following: * Callables whose name starts with an underscore (``_``). In other words, Python's usual "private" convention holds true here. @@ -87,12 +181,12 @@ consider any callable object, **except** for the following: use :option:`fab --list <-l>`. Imports -------- +~~~~~~~ Python's ``import`` statement effectively includes the imported objects in your module's namespace. Since Fabric's fabfiles are just Python modules, this means -that imports are also considered as possible tasks, alongside anything defined -in the fabfile itself. +that imports are also considered as possible classic-style tasks, alongside +anything defined in the fabfile itself. Because of this, we strongly recommend that you use the ``import module`` form of importing, followed by ``module.callable()``, which will result in a cleaner diff --git a/fabric/tasks.py b/fabric/tasks.py index 21962f585e..9fc560d715 100644 --- a/fabric/tasks.py +++ b/fabric/tasks.py @@ -6,6 +6,8 @@ class Task(object): Instances of subclasses will be treated as valid tasks when present in fabfiles loaded by the "fab" tool. + + .. versionadded:: 1.1 """ name = 'undefined' use_task_objects = True @@ -21,6 +23,8 @@ class WrappedCallableTask(Task): Wraps a given callable transparently, while marking it as a valid Task. Generally used via the ``@task`` decorator and not directly. + + .. versionadded:: 1.1 """ def __init__(self, callable): super(WrappedCallableTask, self).__init__() From cb4461cbbaf18d3f50a771f535c71ec10fed5cf0 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 6 Jun 2011 17:31:49 -0700 Subject: [PATCH 072/126] Reword how @task helps with namespaces --- docs/usage/execution.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/usage/execution.rst b/docs/usage/execution.rst index 27cfc647c0..df6134a712 100644 --- a/docs/usage/execution.rst +++ b/docs/usage/execution.rst @@ -102,10 +102,10 @@ and enable some programming best practices, specifically: for much more sensible code reuse than passing around simple function objects. The classic style of task declaration didn't entirely rule this out, but it also didn't make it terribly easy. -* **Namespaces**. Without an easy way to tell tasks apart from other non-task - callables, recursive namespace creation would be difficult if not impossible - (imagine having your "task list" cluttered up with the contents of ``os.sys`` - for example.) +* **Namespaces**. Having an explicit method of declaring tasks makes it easier + to set up recursive namespaces without e.g. polluting your task list with the + contents of Python's ``os`` module (which would show up as valid "tasks" + under the classic methodology.) With the introduction of `~fabric.tasks.Task`, there are two ways to set up new tasks: From 2b27bd67e65348c12380222c22ec1d36a666ca4b Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 6 Jun 2011 18:31:11 -0700 Subject: [PATCH 073/126] start not using module level constants --- fabric/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fabric/main.py b/fabric/main.py index 33f2f47908..89a1cec6dc 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -167,7 +167,7 @@ def is_task_module(a): Determine if the provided value is a task module """ return (type(a) is types.ModuleType and - getattr(a, "FABRIC_TASK_MODULE", False)) + any(map(is_task_object, vars(a).values()))) def is_task_object(a): From af6bbfa446c20feee37cc39bb3f01025a78096dd Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 6 Jun 2011 21:57:59 -0700 Subject: [PATCH 074/126] Big fleshing-out of docs re #76 --- docs/usage/execution.rst | 162 +---------------------- docs/usage/tasks.rst | 273 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 274 insertions(+), 161 deletions(-) create mode 100644 docs/usage/tasks.rst diff --git a/docs/usage/execution.rst b/docs/usage/execution.rst index df6134a712..6152c52aec 100644 --- a/docs/usage/execution.rst +++ b/docs/usage/execution.rst @@ -66,170 +66,10 @@ down to the individual function calls) enables shell script-like logic where you may introspect the output or return code of a given command and decide what to do next. - -.. _tasks-and-imports: - Defining tasks ============== -As of Fabric 1.1, there are two distinct methods you may use in order to define -which objects in your fabfile show up as tasks: - -* The "new" method starting in 1.1 considers instances of `~fabric.tasks.Task` - or its subclasses, and also descends into imported modules to allow building - nested namespaces. -* The "classic" method from 1.0 and earlier considers all public callable - objects (functions, classes etc) and only considers the objects in the - fabfile itself with no recursing into imported module. - -.. note:: - These two methods are **mutually exclusive**: if Fabric finds *any* - new-style task objects in your fabfile or in modules it imports, it will - assume you've committed to this method of task declaration and won't - consider any non-`~fabric.tasks.Task` callables. If *no* new-style tasks - are found, it reverts to the classic behavior. - -See the following subsections for details on these two methods of setting up -tasks. - -New-style tasks ---------------- - -Fabric 1.1 introduced the `~fabric.tasks.Task` class to facilitate new features -and enable some programming best practices, specifically: - -* **Object-oriented tasks**. Inheritance and all that comes with it can make - for much more sensible code reuse than passing around simple function - objects. The classic style of task declaration didn't entirely rule this - out, but it also didn't make it terribly easy. -* **Namespaces**. Having an explicit method of declaring tasks makes it easier - to set up recursive namespaces without e.g. polluting your task list with the - contents of Python's ``os`` module (which would show up as valid "tasks" - under the classic methodology.) - -With the introduction of `~fabric.tasks.Task`, there are two ways to set up new tasks: - -* Decorate a regular module level function with `@task - <~fabric.decorators.task>`, which transparently wraps the function in a - `~fabric.tasks.Task` subclass. The function name will be used as the task - name when invoking. -* Subclass `~fabric.tasks.Task` (`~fabric.tasks.Task` itself is intended to be - abstract), define a ``run`` method, and instantiate your subclass at module - level. Instances' ``name`` attributes are used as the task name; if omitted - the instance's variable name will be used instead. - -Use of new-style tasks also allows you to set up task namespaces -- see below. - -Namespaces -~~~~~~~~~~ - -With classic tasks, only module level callables are considered, forcing you to set up a single flat "namespace" of tasks in your fabfile. While Fabric 0.9.2 introduced package support, you still had to import individual task functions into your ``__init__.py`` and had no ability to organize them. - -In Fabric 1.1 and newer, if you declare tasks the new way (via `~fabric.decorators.task` or your own `~fabric.tasks.Task` subclass instances) you may take advantage of automatic **namespacing**: - -* Any module objects imported into your fabfile will be recursed into looking for additional task objects. -* These sub-module tasks will be given new dotted-notation names based on the modules they came from, similar to Python's own import syntax. - -As an example, consider this fabfile package:: - - . - ├── __init__.py - ├── db - │   ├── __init__.py - │   ├── fixtures.py - │   └── migrate.py - ├── lb.py - -Given the following task declarations: - -* ``__init__.py`` declares a new-style task named ``deploy`` and the line ``import db, lb`` -* ``db/__init__.py`` is empty (just serving as the Python package signifier) -* ``db/fixtures.py`` declares ``load`` and ``dump``. -* ``db/migrate.py`` declares ``up`` and ``down``. -* ``lb.py`` contains ``add_backend``. - -you'd end up with the following list of tasks from ``fab --list``:: - - deploy - db.fixtures.load - db.fixtures.dump - db.migrate.up - db.migrate.down - lb.add_backend - -Note that the base ``__init__.py`` had to explicitly import the other modules. -This represents a small amount of boilerplate on your part, but also allows -explicit control, e.g. if you were to import third party fabfiles not residing -in your own package, or wanted some submodules to remain partly private (e.g. -for use with ``fab -f`` only.) - -Classic tasks -------------- - -When no new-style `~fabric.tasks.Task`-based tasks are found, Fabric will -consider any callable object found in your fabfile, **except** the following: - -* Callables whose name starts with an underscore (``_``). In other words, - Python's usual "private" convention holds true here. -* Callables defined within Fabric itself. Fabric's own functions such as - `~fabric.operations.run` and `~fabric.operations.sudo` will not show up in - your task list. - -.. note:: - - To see exactly which callables in your fabfile may be executed via ``fab``, - use :option:`fab --list <-l>`. - -Imports -~~~~~~~ - -Python's ``import`` statement effectively includes the imported objects in your -module's namespace. Since Fabric's fabfiles are just Python modules, this means -that imports are also considered as possible classic-style tasks, alongside -anything defined in the fabfile itself. - -Because of this, we strongly recommend that you use the ``import module`` form -of importing, followed by ``module.callable()``, which will result in a cleaner -fabfile API than doing ``from module import callable``. - -For example, here's a sample fabfile which uses ``urllib.urlopen`` to get some -data out of a webservice:: - - from urllib import urlopen - - from fabric.api import run - - def webservice_read(): - objects = urlopen('http://my/web/service/?foo=bar').read().split() - print(objects) - -This looks simple enough, and will run without error. However, look what -happens if we run :option:`fab --list <-l>` on this fabfile:: - - $ fab --list - Available commands: - - webservice_read List some directories. - urlopen urlopen(url [, data]) -> open file-like object - -Our fabfile of only one task is showing two "tasks", which is bad enough, and -an unsuspecting user might accidentally try to call ``fab urlopen``, which -probably won't work very well. Imagine any real-world fabfile, which is likely -to be much more complex, and hopefully you can see how this could get messy -fast. - -For reference, here's the recommended way to do it:: - - import urllib - - from fabric.api import run - - def webservice_read(): - objects = urllib.urlopen('http://my/web/service/?foo=bar').read().split() - print(objects) - -It's a simple change, but it'll make anyone using your fabfile a bit happier. - +For details on what constitutes a Fabric task and how to organize them, please see :doc:`/usage/tasks`. Defining host lists =================== diff --git a/docs/usage/tasks.rst b/docs/usage/tasks.rst new file mode 100644 index 0000000000..1e9a41cf4c --- /dev/null +++ b/docs/usage/tasks.rst @@ -0,0 +1,273 @@ +============== +Defining tasks +============== + +As of Fabric 1.1, there are two distinct methods you may use in order to define +which objects in your fabfile show up as tasks: + +* The "new" method starting in 1.1 considers instances of `~fabric.tasks.Task` + or its subclasses, and also descends into imported modules to allow building + nested namespaces. +* The "classic" method from 1.0 and earlier considers all public callable + objects (functions, classes etc) and only considers the objects in the + fabfile itself with no recursing into imported module. + +.. note:: + These two methods are **mutually exclusive**: if Fabric finds *any* + new-style task objects in your fabfile or in modules it imports, it will + assume you've committed to this method of task declaration and won't + consider any non-`~fabric.tasks.Task` callables. If *no* new-style tasks + are found, it reverts to the classic behavior. + +The rest of this document explores these two methods in detail. + +New-style tasks +=============== + +Fabric 1.1 introduced the `~fabric.tasks.Task` class to facilitate new features +and enable some programming best practices, specifically: + +* **Object-oriented tasks**. Inheritance and all that comes with it can make + for much more sensible code reuse than passing around simple function + objects. The classic style of task declaration didn't entirely rule this + out, but it also didn't make it terribly easy. +* **Namespaces**. Having an explicit method of declaring tasks makes it easier + to set up recursive namespaces without e.g. polluting your task list with the + contents of Python's ``os`` module (which would show up as valid "tasks" + under the classic methodology.) + +With the introduction of `~fabric.tasks.Task`, there are two ways to set up new +tasks: + +* Decorate a regular module level function with `@task + <~fabric.decorators.task>`, which transparently wraps the function in a + `~fabric.tasks.Task` subclass. The function name will be used as the task + name when invoking. +* Subclass `~fabric.tasks.Task` (`~fabric.tasks.Task` itself is intended to be + abstract), define a ``run`` method, and instantiate your subclass at module + level. Instances' ``name`` attributes are used as the task name; if omitted + the instance's variable name will be used instead. + +Use of new-style tasks also allows you to set up task namespaces -- see below. + +Namespaces +---------- + +With :ref:`classic tasks `, fabfiles were limited to a single, +flat set of task names with no real way to organize them. In Fabric 1.1 and +newer, if you declare tasks the new way (via `~fabric.decorators.task` or your +own `~fabric.tasks.Task` subclass instances) you may take advantage of +**namespacing**: + +* Any module objects imported into your fabfile will be recursed into, looking + for additional task objects. +* You may further control which objects are "exported" by using the standard + Python ``__all__`` module-level variable name (thought they should still be + valid new-style task objects.) +* These tasks will be given new dotted-notation names based on the modules they + came from, similar to Python's own import syntax. + +Let's build up a fabfile package from simple to complex and see how this works. + +Basic +~~~~~ + +We start with a single `__init__.py` containing a few tasks (the Fabric API +import omitted for brevity):: + + @task + def deploy(): + ... + + @task + def compress(): + ... + +The output of ``fab --list`` would look something like this:: + + deploy + compress + +There's just one namespace here: the "root" or global namespace. Looks simple +now, but in a real-world fabfile with dozens of tasks, it can get difficult to +manage. + +Importing a submodule +~~~~~~~~~~~~~~~~~~~~~ + +As mentioned above, Fabric will examine any imported module objects for tasks. +For now we just want to include our own, "nearby" tasks, so we'll make a new +submodule in our package for dealing with, say, load balancers -- ``lb.py``:: + + @task + def add_backend(): + ... + +And we'll add this to the top of ``__init__.py``:: + + import lb + +Now ``fab --list`` shows us:: + + deploy + compress + lb.add_backend + +Again, with one task, it looks kind of silly, but the benefits should be pretty +obvious. + +Going deeper +~~~~~~~~~~~~ + +Namespacing isn't limited to just one level. Let's say we had a larger setup +and wanted a namespace for database related tasks, with additional +differentiation inside that. We make a sub-package named ``db/`` and inside it, +a ``migrations.py`` module:: + + @task + def list(): + ... + + @task + def run(): + ... + +We need to make sure that this module is visible to anybody importing ``db``, +so we add it to the sub-package's ``__init__.py``:: + + import migrations + +As a final step, we import the sub-package into our root-level ``__init__.py``, +so now its first few lines look like this:: + + import lb + import db + +After all that, our file tree looks like this:: + + . + ├── __init__.py + ├── db + │   ├── __init__.py + │   └── migrations.py + └── lb.py + +and ``fab --list`` shows:: + + deploy + compress + lb.add_backend + db.migrations.list + db.migrations.run + +We could also have specified (or imported) tasks directly into +``db/__init__.py``, and they would show up as ``db.`` as you might +expect. + +Limiting with ``__all__`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +It's also possible to limit what Fabric "sees" when it examines imported +modules, by using the Python convention of a module level ``__all__`` variable +(a list of variable names.) If we didn't want the ``db.migrations.run`` task to +show up by default for some reason, we could add this to the top of +``db/migrations.py``:: + + __all__ = ['list'] + +Note the lack of ``'run'`` there. You could, if needed, import ``run`` directly +into some other part of the hierarchy, or specify ``db/migrations.py`` as the +root fabfile with ``fab -f`` (which does not consider ``__all__``.) + +Switching it up +~~~~~~~~~~~~~~~ + +Finally, while we've been keeping our fabfile package neatly organized and +importing it in a straightforward manner, the filesystem layout doesn't +actually matter here. All Fabric's loader cares about is the names the modules +are given when they're imported. + +For example, if we changed the top of our root ``__init__.py`` to look like +this:: + + import db as database + +Our task list would change thusly:: + + deploy + compress + lb.add_backend + database.migrations.list + database.migrations.run + +This applies to any other import -- you could import third party modules into +your own task hierarchy, or grab a deeply nested module and make it appear near +the top level. + + +Classic tasks +============= + +When no new-style `~fabric.tasks.Task`-based tasks are found, Fabric will +consider any callable object found in your fabfile, **except** the following: + +* Callables whose name starts with an underscore (``_``). In other words, + Python's usual "private" convention holds true here. +* Callables defined within Fabric itself. Fabric's own functions such as + `~fabric.operations.run` and `~fabric.operations.sudo` will not show up in + your task list. + +.. note:: + + To see exactly which callables in your fabfile may be executed via ``fab``, + use :option:`fab --list <-l>`. + +Imports +------- + +Python's ``import`` statement effectively includes the imported objects in your +module's namespace. Since Fabric's fabfiles are just Python modules, this means +that imports are also considered as possible classic-style tasks, alongside +anything defined in the fabfile itself. + +Because of this, we strongly recommend that you use the ``import module`` form +of importing, followed by ``module.callable()``, which will result in a cleaner +fabfile API than doing ``from module import callable``. + +For example, here's a sample fabfile which uses ``urllib.urlopen`` to get some +data out of a webservice:: + + from urllib import urlopen + + from fabric.api import run + + def webservice_read(): + objects = urlopen('http://my/web/service/?foo=bar').read().split() + print(objects) + +This looks simple enough, and will run without error. However, look what +happens if we run :option:`fab --list <-l>` on this fabfile:: + + $ fab --list + Available commands: + + webservice_read List some directories. + urlopen urlopen(url [, data]) -> open file-like object + +Our fabfile of only one task is showing two "tasks", which is bad enough, and +an unsuspecting user might accidentally try to call ``fab urlopen``, which +probably won't work very well. Imagine any real-world fabfile, which is likely +to be much more complex, and hopefully you can see how this could get messy +fast. + +For reference, here's the recommended way to do it:: + + import urllib + + from fabric.api import run + + def webservice_read(): + objects = urllib.urlopen('http://my/web/service/?foo=bar').read().split() + print(objects) + +It's a simple change, but it'll make anyone using your fabfile a bit happier. From 2e49f1d46d423fed0a182fe665d3148d4762d89a Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 7 Jun 2011 16:49:47 -0700 Subject: [PATCH 075/126] Many small tweaks to docs, docstrings re #76 --- docs/api/core/decorators.rst | 2 +- docs/api/core/tasks.rst | 2 +- docs/tutorial.rst | 2 +- docs/usage/fabfiles.rst | 2 +- docs/usage/tasks.rst | 11 +++++++---- fabric/decorators.py | 5 ++--- fabric/tasks.py | 5 +++-- 7 files changed, 16 insertions(+), 13 deletions(-) diff --git a/docs/api/core/decorators.rst b/docs/api/core/decorators.rst index a19fd3f888..ae0c2a9c4b 100644 --- a/docs/api/core/decorators.rst +++ b/docs/api/core/decorators.rst @@ -3,4 +3,4 @@ Decorators ========== .. automodule:: fabric.decorators - :members: hosts, roles, runs_once, with_settings + :members: hosts, roles, runs_once, task, with_settings diff --git a/docs/api/core/tasks.rst b/docs/api/core/tasks.rst index 20c01663a8..8c69ac76b9 100644 --- a/docs/api/core/tasks.rst +++ b/docs/api/core/tasks.rst @@ -3,4 +3,4 @@ Tasks ===== .. automodule:: fabric.tasks - :members: + :members: Task diff --git a/docs/tutorial.rst b/docs/tutorial.rst index c065b5f694..9ae6c3016e 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -55,7 +55,7 @@ That's all there is to it. This functionality allows Fabric to be used as a functions you instruct it to. There's nothing magic about it -- anything you can do in a normal Python script can be done in a fabfile! -.. seealso:: :ref:`execution-strategy`, :ref:`tasks-and-imports`, :doc:`usage/fab` +.. seealso:: :ref:`execution-strategy`, :doc:`/usage/tasks`, :doc:`/usage/fab` Task arguments diff --git a/docs/usage/fabfiles.rst b/docs/usage/fabfiles.rst index 4e742a4e7a..52c626ce72 100644 --- a/docs/usage/fabfiles.rst +++ b/docs/usage/fabfiles.rst @@ -88,4 +88,4 @@ Defining tasks and importing callables For important information on what exactly Fabric will consider as a task when it loads your fabfile, as well as notes on how best to import other code, -please see :ref:`tasks-and-imports` in the :doc:`execution` documentation. +please see :doc:`/usage/tasks` in the :doc:`execution` documentation. diff --git a/docs/usage/tasks.rst b/docs/usage/tasks.rst index 1e9a41cf4c..4f60db436b 100644 --- a/docs/usage/tasks.rst +++ b/docs/usage/tasks.rst @@ -21,6 +21,8 @@ which objects in your fabfile show up as tasks: The rest of this document explores these two methods in detail. +.. _new-style-tasks: + New-style tasks =============== @@ -39,10 +41,9 @@ and enable some programming best practices, specifically: With the introduction of `~fabric.tasks.Task`, there are two ways to set up new tasks: -* Decorate a regular module level function with `@task - <~fabric.decorators.task>`, which transparently wraps the function in a - `~fabric.tasks.Task` subclass. The function name will be used as the task - name when invoking. +* Decorate a regular module level function with `~fabric.decorators.task`, + which transparently wraps the function in a `~fabric.tasks.Task` subclass. + The function name will be used as the task name when invoking. * Subclass `~fabric.tasks.Task` (`~fabric.tasks.Task` itself is intended to be abstract), define a ``run`` method, and instantiate your subclass at module level. Instances' ``name`` attributes are used as the task name; if omitted @@ -205,6 +206,8 @@ your own task hierarchy, or grab a deeply nested module and make it appear near the top level. +.. _classic-tasks: + Classic tasks ============= diff --git a/fabric/decorators.py b/fabric/decorators.py index 892d55cf4f..b7a4bbcde8 100644 --- a/fabric/decorators.py +++ b/fabric/decorators.py @@ -12,12 +12,11 @@ def task(func): """ - Decorator defining a function as a task. - - This is a convenience wrapper around `tasks.WrappedCallableTask`. + Decorator declaring the wrapped function as a :ref:`new-style task `. """ return tasks.WrappedCallableTask(func) + def hosts(*host_list): """ Decorator defining which host or hosts to execute the wrapped function on. diff --git a/fabric/tasks.py b/fabric/tasks.py index 9fc560d715..5b93d4a6f9 100644 --- a/fabric/tasks.py +++ b/fabric/tasks.py @@ -5,7 +5,7 @@ class Task(object): Abstract base class for objects wishing to be picked up as Fabric tasks. Instances of subclasses will be treated as valid tasks when present in - fabfiles loaded by the "fab" tool. + fabfiles loaded by the :doc:`fab ` tool. .. versionadded:: 1.1 """ @@ -22,7 +22,8 @@ class WrappedCallableTask(Task): """ Wraps a given callable transparently, while marking it as a valid Task. - Generally used via the ``@task`` decorator and not directly. + Generally used via the `~fabric.decorators.task` decorator and not + directly. .. versionadded:: 1.1 """ From c3068aed6006c465f35d3be0e51658553bd3a4b6 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 8 Jun 2011 16:20:53 -0700 Subject: [PATCH 076/126] Add changelog entry for #76, #56 --- docs/changes/1.1.rst | 20 ++++++++++++++++++++ docs/usage/tasks.rst | 3 +++ 2 files changed, 23 insertions(+) diff --git a/docs/changes/1.1.rst b/docs/changes/1.1.rst index 5dd7da9bd3..921581d0e6 100644 --- a/docs/changes/1.1.rst +++ b/docs/changes/1.1.rst @@ -5,6 +5,23 @@ Changes in version 1.1 This page lists all changes made to Fabric in its 1.1.0 release. +Highlights +========== + +* :issue:`76`: :ref:`New-style tasks ` have been added. With + the addition of the `~fabric.decorators.task` decorator and the + `~fabric.tasks.Task` class, you can now "opt-in" and explicitly mark task + functions as tasks, and Fabric will ignore the rest. The original behavior + (now referred to as :ref:`"classic" tasks `) will still take + effect if no new-style tasks are found. Major thanks to Travis Swicegood for + the original implementation. +* :issue:`56`: Namespacing is now possible: Fabric will crawl imported module + objects looking for new-style task objects and build a dotted hierarchy + (tasks named e.g. ``web.deploy`` or ``db.migrations.run``), allowing for + greater organization. See :ref:`namespaces` for details. Thanks again to + Travis Swicegood. + + Feature additions ================= @@ -43,6 +60,9 @@ Bugfixes Documentation updates ===================== +* Documentation for task declaration has been moved from + :doc:`/usage/execution` into its own docs page, :doc:`/usage/tasks`, as a + result of the changes added in :issue:`76` and :issue:`56`. * :issue:`184`: Make the usage of `~fabric.contrib.project.rsync_project`'s ``local_dir`` argument more obvious, regarding its use in the ``rsync`` call. (Specifically, so users know they can pass in multiple, space-joined diff --git a/docs/usage/tasks.rst b/docs/usage/tasks.rst index 4f60db436b..312f94bdc2 100644 --- a/docs/usage/tasks.rst +++ b/docs/usage/tasks.rst @@ -51,6 +51,9 @@ tasks: Use of new-style tasks also allows you to set up task namespaces -- see below. + +.. _namespaces: + Namespaces ---------- From 79f6b7db9c7390b94336b5b426c14ca5b6644c4d Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 8 Jun 2011 16:30:11 -0700 Subject: [PATCH 077/126] Edits to task docs --- docs/usage/tasks.rst | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/usage/tasks.rst b/docs/usage/tasks.rst index 312f94bdc2..42c87cb653 100644 --- a/docs/usage/tasks.rst +++ b/docs/usage/tasks.rst @@ -65,9 +65,9 @@ own `~fabric.tasks.Task` subclass instances) you may take advantage of * Any module objects imported into your fabfile will be recursed into, looking for additional task objects. -* You may further control which objects are "exported" by using the standard - Python ``__all__`` module-level variable name (thought they should still be - valid new-style task objects.) +* Within submodules, you may control which objects are "exported" by using the + standard Python ``__all__`` module-level variable name (thought they should + still be valid new-style task objects.) * These tasks will be given new dotted-notation names based on the modules they came from, similar to Python's own import syntax. @@ -99,9 +99,10 @@ manage. Importing a submodule ~~~~~~~~~~~~~~~~~~~~~ -As mentioned above, Fabric will examine any imported module objects for tasks. -For now we just want to include our own, "nearby" tasks, so we'll make a new -submodule in our package for dealing with, say, load balancers -- ``lb.py``:: +As mentioned above, Fabric will examine any imported module objects for tasks, +regardless of where that module exists on your Python import path. For now we +just want to include our own, "nearby" tasks, so we'll make a new submodule in +our package for dealing with, say, load balancers -- ``lb.py``:: @task def add_backend(): @@ -117,8 +118,8 @@ Now ``fab --list`` shows us:: compress lb.add_backend -Again, with one task, it looks kind of silly, but the benefits should be pretty -obvious. +Again, with only one task in its own submodule, it looks kind of silly, but the +benefits should be pretty obvious. Going deeper ~~~~~~~~~~~~ @@ -171,11 +172,10 @@ expect. Limiting with ``__all__`` ~~~~~~~~~~~~~~~~~~~~~~~~~ -It's also possible to limit what Fabric "sees" when it examines imported -modules, by using the Python convention of a module level ``__all__`` variable -(a list of variable names.) If we didn't want the ``db.migrations.run`` task to -show up by default for some reason, we could add this to the top of -``db/migrations.py``:: +You may limit what Fabric "sees" when it examines imported modules, by using +the Python convention of a module level ``__all__`` variable (a list of +variable names.) If we didn't want the ``db.migrations.run`` task to show up by +default for some reason, we could add this to the top of ``db/migrations.py``:: __all__ = ['list'] From 4fee934a0b554a7faef4528a29f9f6d0af34b5be Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 8 Jun 2011 16:30:24 -0700 Subject: [PATCH 078/126] Clarify behavior of __all__ --- docs/usage/tasks.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/usage/tasks.rst b/docs/usage/tasks.rst index 42c87cb653..abc2fc4d31 100644 --- a/docs/usage/tasks.rst +++ b/docs/usage/tasks.rst @@ -180,8 +180,7 @@ default for some reason, we could add this to the top of ``db/migrations.py``:: __all__ = ['list'] Note the lack of ``'run'`` there. You could, if needed, import ``run`` directly -into some other part of the hierarchy, or specify ``db/migrations.py`` as the -root fabfile with ``fab -f`` (which does not consider ``__all__``.) +into some other part of the hierarchy, but otherwise it'll remain hidden. Switching it up ~~~~~~~~~~~~~~~ From 0084db69601eb427d8205e946436243e193c323d Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 8 Jun 2011 16:35:49 -0700 Subject: [PATCH 079/126] Few edits to the classic tasks section --- docs/usage/tasks.rst | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/usage/tasks.rst b/docs/usage/tasks.rst index abc2fc4d31..25d4f02473 100644 --- a/docs/usage/tasks.rst +++ b/docs/usage/tasks.rst @@ -21,6 +21,11 @@ which objects in your fabfile show up as tasks: The rest of this document explores these two methods in detail. +.. note:: + + To see exactly what tasks in your fabfile may be executed via ``fab``, use + :option:`fab --list <-l>`. + .. _new-style-tasks: New-style tasks @@ -222,10 +227,6 @@ consider any callable object found in your fabfile, **except** the following: `~fabric.operations.run` and `~fabric.operations.sudo` will not show up in your task list. -.. note:: - - To see exactly which callables in your fabfile may be executed via ``fab``, - use :option:`fab --list <-l>`. Imports ------- @@ -235,6 +236,12 @@ module's namespace. Since Fabric's fabfiles are just Python modules, this means that imports are also considered as possible classic-style tasks, alongside anything defined in the fabfile itself. + .. note:: + This only applies to imported *callable objects* -- not modules. + Imported modules only come into play if they contain :ref:`new-style + tasks `, at which point this section no longer + applies. + Because of this, we strongly recommend that you use the ``import module`` form of importing, followed by ``module.callable()``, which will result in a cleaner fabfile API than doing ``from module import callable``. From f16d8df4974b92e1f3124eae227f49ede67b1511 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 8 Jun 2011 19:00:45 -0700 Subject: [PATCH 080/126] Handle oddball non string __doc__ attributes --- fabric/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fabric/main.py b/fabric/main.py index 89a1cec6dc..ff75920b9c 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -293,7 +293,8 @@ def list_commands(docstring): output = None # Print first line of docstring func = commands[name] - if func.__doc__: + docstring = func.__doc__ + if docstring and type(docstring) in types.StringTypes: lines = filter(None, func.__doc__.splitlines()) first_line = lines[0].strip() # Truncate it if it's longer than N chars From fcf3e9ea53d9bacbc12de21887b49c2d4b966355 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 8 Jun 2011 19:02:08 -0700 Subject: [PATCH 081/126] WIP re: task discovery updates --- fabric/main.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/fabric/main.py b/fabric/main.py index ff75920b9c..e4c3f7b3cf 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -159,15 +159,23 @@ def load_tasks_from_module(imported): # Return a two-tuple value. First is the documentation, second is a # dictionary of callables only (and don't include Fab operations or # underscored callables) - return imported.__doc__, extract_tasks(imported_vars) + new_style, classic = extract_tasks(imported_vars) + tasks = new_style if state.env.new_style_tasks else classic + return imported.__doc__, tasks def is_task_module(a): """ Determine if the provided value is a task module """ - return (type(a) is types.ModuleType and - any(map(is_task_object, vars(a).values()))) + #return (type(a) is types.ModuleType and + # any(map(is_task_object, vars(a).values()))) + seen = '__seen_by_fab' + if type(a) is types.ModuleType and not getattr(a, seen, False): + # Flag module as seen + setattr(a, seen, True) + # Signal that we need to check it out + return True def is_task_object(a): @@ -184,26 +192,23 @@ def extract_tasks(imported_vars): """ Handle extracting tasks from a given list of variables """ - tasks = {} - using_task_objects = False + new_style_tasks = {} + classic_tasks = {} + if 'new_style_tasks' not in state.env: + state.env.new_style_tasks = False for tup in imported_vars: name, obj = tup if is_task_object(obj): - using_task_objects = True - tasks[obj.name] = obj + state.env.new_style_tasks = True + new_style_tasks[obj.name] = obj elif is_task(tup): - tasks[name] = obj + classic_tasks[name] = obj elif is_task_module(obj): module_docs, module_tasks = load_tasks_from_module(obj) for task_name, task in module_tasks.items(): - tasks["%s.%s" % (name, task_name)] = task - - if using_task_objects: - def is_usable_task(tup): - name, task = tup - return name.find('.') != -1 or isinstance(task, Task) - tasks = dict(filter(is_usable_task, tasks.items())) - return tasks + new_style_tasks["%s.%s" % (name, task_name)] = task + return (new_style_tasks, classic_tasks) + def parse_options(): From 35c3480628eb0189721b82b17112ed047838d4ec Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 9 Jun 2011 11:18:49 -0700 Subject: [PATCH 082/126] Whitespace --- tests/test_main.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_main.py b/tests/test_main.py index fa556c036d..54fbf251b8 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -224,6 +224,7 @@ def command(): pass eq_hosts(command, ['a', 'b']) + def test_escaped_task_arg_split(): """ Allow backslashes to escape the task argument separator character @@ -234,6 +235,7 @@ def test_escaped_task_arg_split(): ['foo', 'bar,biz,baz', 'what comes after baz?'] ) + def run_load_fabfile(path, sys_path): # Module-esque object fake_module = Fake().has_attr(__dict__={}) @@ -265,9 +267,11 @@ def test_load_fabfile_should_not_remove_real_path_elements(): ): yield run_load_fabfile, fabfile_path, sys_dot_path + def support_fabfile(name): return os.path.join(os.path.dirname(__file__), 'support', name) + def test_implicit_discover(): """ Automatically includes all functions in a fabfile @@ -282,6 +286,7 @@ def test_implicit_discover(): sys.path = sys.path[1:] + def test_explicit_discover(): """ Only use those methods listed in __all__ @@ -295,6 +300,7 @@ def test_explicit_discover(): ok_("foo" in funcs) ok_("bar" not in funcs) + def test_allow_registering_modules(): module = support_fabfile('module_fabfile.py') sys.path[0:0] = [os.path.dirname(module),] @@ -304,6 +310,7 @@ def test_allow_registering_modules(): ok_('tasks.hello' in funcs) ok_('tasks.world' in funcs) + def test_modules_should_pay_attention_to_all_and_explicit_discovery(): module = support_fabfile('module_explicit.py') sys.path[0:0] = [os.path.dirname(module),] @@ -313,6 +320,7 @@ def test_modules_should_pay_attention_to_all_and_explicit_discovery(): ok_('tasks.hello' in funcs) ok_('tasks.world' not in funcs) + def test_should_load_decorated_tasks_only_if_one_is_found(): module = support_fabfile('decorated_fabfile.py') sys.path[0:0] = [os.path.dirname(module),] @@ -321,6 +329,7 @@ def test_should_load_decorated_tasks_only_if_one_is_found(): eq_(1, len(funcs)) ok_('foo' in funcs) + def test_modules_are_still_loaded_if_fabfile_contains_decorated_task(): module = support_fabfile('decorated_fabfile_with_modules.py') sys.path[0:0] = [os.path.dirname(module),] @@ -328,6 +337,7 @@ def test_modules_are_still_loaded_if_fabfile_contains_decorated_task(): docs, funcs = load_fabfile(module) eq_(3, len(funcs)) + def test_modules_pay_attention_to_task_decorator(): module = support_fabfile('decorated_fabfile_with_decorated_module.py') sys.path[0:0] = [os.path.dirname(module),] @@ -335,6 +345,7 @@ def test_modules_pay_attention_to_task_decorator(): docs, funcs = load_fabfile(module) eq_(2, len(funcs)) + def test_class_based_tasks_are_found_with_proper_name(): module = support_fabfile('decorated_fabfile_with_classbased_task.py') sys.path[0:0] = [os.path.dirname(module),] From aa7f9b35fbbc13cbaec02a3cdb1590860ed11cc4 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 9 Jun 2011 11:31:48 -0700 Subject: [PATCH 083/126] Commenting --- tests/test_main.py | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 54fbf251b8..25b07ef1d1 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -16,6 +16,10 @@ import sys +# +# Basic CLI stuff +# + def test_argument_parsing(): for args, output in [ # Basic @@ -47,6 +51,21 @@ def test_argument_parsing(): yield eq_, parse_arguments([args]), [output] +def test_escaped_task_arg_split(): + """ + Allow backslashes to escape the task argument separator character + """ + argstr = r"foo,bar\,biz\,baz,what comes after baz?" + eq_( + _escape_split(',', argstr), + ['foo', 'bar,biz,baz', 'what comes after baz?'] + ) + + +# +# Host/role decorators +# + def eq_hosts(command, host_list): eq_(set(get_hosts(command, [], [], [])), set(host_list)) @@ -202,6 +221,10 @@ def command(): eq_(command.roles, role_list) +# +# Basic role behavior +# + @patched_env({'roledefs': fake_roles}) @raises(SystemExit) @mock_streams('stderr') @@ -225,16 +248,9 @@ def command(): eq_hosts(command, ['a', 'b']) -def test_escaped_task_arg_split(): - """ - Allow backslashes to escape the task argument separator character - """ - argstr = r"foo,bar\,biz\,baz,what comes after baz?" - eq_( - _escape_split(',', argstr), - ['foo', 'bar,biz,baz', 'what comes after baz?'] - ) - +# +# Fabfile loading +# def run_load_fabfile(path, sys_path): # Module-esque object @@ -268,6 +284,10 @@ def test_load_fabfile_should_not_remove_real_path_elements(): yield run_load_fabfile, fabfile_path, sys_dot_path +# +# Namespacing and new-style tasks +# + def support_fabfile(name): return os.path.join(os.path.dirname(__file__), 'support', name) From d7a7670ed61453ed4854b8375d6bd7576c277188 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 9 Jun 2011 12:12:40 -0700 Subject: [PATCH 084/126] Cleanup of namespace/task-decorator tests re #76, re #56 --- ...decorated_fabfile_with_decorated_module.py | 9 -- tests/support/explicit_fabfile.py | 2 +- tests/support/module_explicit.py | 1 - tests/support/module_fabfile.py | 1 - tests/support/module_fabtasks_decorated.py | 9 -- tests/support/module_fabtasks_explicit.py | 10 -- tests/test_main.py | 114 +++++++----------- 7 files changed, 44 insertions(+), 102 deletions(-) delete mode 100644 tests/support/decorated_fabfile_with_decorated_module.py delete mode 100644 tests/support/module_explicit.py delete mode 100644 tests/support/module_fabfile.py delete mode 100644 tests/support/module_fabtasks_decorated.py delete mode 100644 tests/support/module_fabtasks_explicit.py diff --git a/tests/support/decorated_fabfile_with_decorated_module.py b/tests/support/decorated_fabfile_with_decorated_module.py deleted file mode 100644 index bc6a6e9280..0000000000 --- a/tests/support/decorated_fabfile_with_decorated_module.py +++ /dev/null @@ -1,9 +0,0 @@ -from fabric.decorators import task -import module_fabtasks_decorated as tasks - -@task -def foo(): - pass - -def bar(): - pass diff --git a/tests/support/explicit_fabfile.py b/tests/support/explicit_fabfile.py index 16dd6c9618..c00f4f53f2 100644 --- a/tests/support/explicit_fabfile.py +++ b/tests/support/explicit_fabfile.py @@ -1,4 +1,4 @@ -__all__ = ['foo',] +__all__ = ['foo'] def foo(): pass diff --git a/tests/support/module_explicit.py b/tests/support/module_explicit.py deleted file mode 100644 index 134a37705f..0000000000 --- a/tests/support/module_explicit.py +++ /dev/null @@ -1 +0,0 @@ -import module_fabtasks_explicit as tasks diff --git a/tests/support/module_fabfile.py b/tests/support/module_fabfile.py deleted file mode 100644 index 2422bec759..0000000000 --- a/tests/support/module_fabfile.py +++ /dev/null @@ -1 +0,0 @@ -import module_fabtasks as tasks diff --git a/tests/support/module_fabtasks_decorated.py b/tests/support/module_fabtasks_decorated.py deleted file mode 100644 index 74d4123779..0000000000 --- a/tests/support/module_fabtasks_decorated.py +++ /dev/null @@ -1,9 +0,0 @@ -from fabric.decorators import task -FABRIC_TASK_MODULE = True - -@task -def hello(): - print "hello" - -def world(): - print "world" diff --git a/tests/support/module_fabtasks_explicit.py b/tests/support/module_fabtasks_explicit.py deleted file mode 100644 index 6f71d9a3f9..0000000000 --- a/tests/support/module_fabtasks_explicit.py +++ /dev/null @@ -1,10 +0,0 @@ - -FABRIC_TASK_MODULE = True - -__all__ = ['hello',] - -def hello(): - print "hello" - -def world(): - print "world" diff --git a/tests/test_main.py b/tests/test_main.py index 25b07ef1d1..b2f1cd693d 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,5 +1,8 @@ +from __future__ import with_statement + import sys import copy +from contextlib import contextmanager from fudge import Fake from nose.tools import ok_, eq_, raises @@ -288,89 +291,58 @@ def test_load_fabfile_should_not_remove_real_path_elements(): # Namespacing and new-style tasks # -def support_fabfile(name): +def fabfile(name): return os.path.join(os.path.dirname(__file__), 'support', name) +@contextmanager +def path_prefix(path): + i = 0 + sys.path.insert(i, path) + yield + sys.path.pop(i) + -def test_implicit_discover(): +def test_implicit_discovery(): """ - Automatically includes all functions in a fabfile + Default to automatically collecting all tasks in a fabfile module """ - implicit = support_fabfile("implicit_fabfile.py") - sys.path[0:0] = [os.path.dirname(implicit),] - - docs, funcs = load_fabfile(implicit) - ok_(len(funcs) == 2) - ok_("foo" in funcs) - ok_("bar" in funcs) + implicit = fabfile("implicit_fabfile.py") + with path_prefix(os.path.dirname(implicit)): + docs, funcs = load_fabfile(implicit) + ok_(len(funcs) == 2) + ok_("foo" in funcs) + ok_("bar" in funcs) - sys.path = sys.path[1:] - -def test_explicit_discover(): +def test_explicit_discovery(): """ - Only use those methods listed in __all__ + If __all__ is present, only collect the tasks it specifies """ - - explicit = support_fabfile("explicit_fabfile.py") - sys.path[0:0] = [os.path.dirname(explicit),] - - docs, funcs = load_fabfile(explicit) - ok_(len(funcs) == 1) - ok_("foo" in funcs) - ok_("bar" not in funcs) - - -def test_allow_registering_modules(): - module = support_fabfile('module_fabfile.py') - sys.path[0:0] = [os.path.dirname(module),] - - docs, funcs = load_fabfile(module) - ok_(len(funcs) == 2) - ok_('tasks.hello' in funcs) - ok_('tasks.world' in funcs) - - -def test_modules_should_pay_attention_to_all_and_explicit_discovery(): - module = support_fabfile('module_explicit.py') - sys.path[0:0] = [os.path.dirname(module),] - - docs, funcs = load_fabfile(module) - ok_(len(funcs) == 1) - ok_('tasks.hello' in funcs) - ok_('tasks.world' not in funcs) + explicit = fabfile("explicit_fabfile.py") + with path_prefix(os.path.dirname(explicit)): + docs, funcs = load_fabfile(explicit) + ok_(len(funcs) == 1) + ok_("foo" in funcs) + ok_("bar" not in funcs) def test_should_load_decorated_tasks_only_if_one_is_found(): - module = support_fabfile('decorated_fabfile.py') - sys.path[0:0] = [os.path.dirname(module),] - - docs, funcs = load_fabfile(module) - eq_(1, len(funcs)) - ok_('foo' in funcs) - - -def test_modules_are_still_loaded_if_fabfile_contains_decorated_task(): - module = support_fabfile('decorated_fabfile_with_modules.py') - sys.path[0:0] = [os.path.dirname(module),] - - docs, funcs = load_fabfile(module) - eq_(3, len(funcs)) - - -def test_modules_pay_attention_to_task_decorator(): - module = support_fabfile('decorated_fabfile_with_decorated_module.py') - sys.path[0:0] = [os.path.dirname(module),] - - docs, funcs = load_fabfile(module) - eq_(2, len(funcs)) + """ + If any new-style tasks are found, *only* new-style tasks should load + """ + module = fabfile('decorated_fabfile.py') + with path_prefix(os.path.dirname(module)): + docs, funcs = load_fabfile(module) + eq_(1, len(funcs)) + ok_('foo' in funcs) def test_class_based_tasks_are_found_with_proper_name(): - module = support_fabfile('decorated_fabfile_with_classbased_task.py') - sys.path[0:0] = [os.path.dirname(module),] - - docs, funcs = load_fabfile(module) - print funcs - eq_(1, len(funcs)) - ok_('foo' in funcs) + """ + Wrapped new-style tasks should preserve their function names + """ + module = fabfile('decorated_fabfile_with_classbased_task.py') + with path_prefix(os.path.dirname(module)): + docs, funcs = load_fabfile(module) + eq_(1, len(funcs)) + ok_('foo' in funcs) From f605730034ea25342fc46b0e2853fec400693428 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 9 Jun 2011 12:25:25 -0700 Subject: [PATCH 085/126] Missed a spot --- tests/support/module_fabtasks.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/support/module_fabtasks.py b/tests/support/module_fabtasks.py index 47e4bb7bbc..2c54ef9adf 100644 --- a/tests/support/module_fabtasks.py +++ b/tests/support/module_fabtasks.py @@ -1,6 +1,3 @@ - -FABRIC_TASK_MODULE = True - def hello(): print "hello" From 89724bfc2f3120d700e2c373bd151050dff946aa Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 9 Jun 2011 12:33:51 -0700 Subject: [PATCH 086/126] Refactor --- tests/test_main.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index b2f1cd693d..021d034ec1 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -295,9 +295,9 @@ def fabfile(name): return os.path.join(os.path.dirname(__file__), 'support', name) @contextmanager -def path_prefix(path): +def path_prefix(module): i = 0 - sys.path.insert(i, path) + sys.path.insert(i, os.path.dirname(module)) yield sys.path.pop(i) @@ -307,7 +307,7 @@ def test_implicit_discovery(): Default to automatically collecting all tasks in a fabfile module """ implicit = fabfile("implicit_fabfile.py") - with path_prefix(os.path.dirname(implicit)): + with path_prefix(implicit): docs, funcs = load_fabfile(implicit) ok_(len(funcs) == 2) ok_("foo" in funcs) @@ -319,7 +319,7 @@ def test_explicit_discovery(): If __all__ is present, only collect the tasks it specifies """ explicit = fabfile("explicit_fabfile.py") - with path_prefix(os.path.dirname(explicit)): + with path_prefix(explicit): docs, funcs = load_fabfile(explicit) ok_(len(funcs) == 1) ok_("foo" in funcs) @@ -331,7 +331,7 @@ def test_should_load_decorated_tasks_only_if_one_is_found(): If any new-style tasks are found, *only* new-style tasks should load """ module = fabfile('decorated_fabfile.py') - with path_prefix(os.path.dirname(module)): + with path_prefix(module): docs, funcs = load_fabfile(module) eq_(1, len(funcs)) ok_('foo' in funcs) @@ -342,7 +342,7 @@ def test_class_based_tasks_are_found_with_proper_name(): Wrapped new-style tasks should preserve their function names """ module = fabfile('decorated_fabfile_with_classbased_task.py') - with path_prefix(os.path.dirname(module)): + with path_prefix(module): docs, funcs = load_fabfile(module) eq_(1, len(funcs)) ok_('foo' in funcs) From 07bad3f811b11fd1e620647f1cd4c6a8ff374edb Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 9 Jun 2011 12:59:53 -0700 Subject: [PATCH 087/126] Couple new tests --- tests/support/deep.py | 1 + tests/support/submodule/__init__.py | 4 ++++ .../submodule/subsubmodule/__init__.py | 5 +++++ tests/test_main.py | 22 +++++++++++++++++++ 4 files changed, 32 insertions(+) create mode 100644 tests/support/deep.py create mode 100644 tests/support/submodule/__init__.py create mode 100644 tests/support/submodule/subsubmodule/__init__.py diff --git a/tests/support/deep.py b/tests/support/deep.py new file mode 100644 index 0000000000..1960a71192 --- /dev/null +++ b/tests/support/deep.py @@ -0,0 +1 @@ +import submodule diff --git a/tests/support/submodule/__init__.py b/tests/support/submodule/__init__.py new file mode 100644 index 0000000000..50bf4be035 --- /dev/null +++ b/tests/support/submodule/__init__.py @@ -0,0 +1,4 @@ +import subsubmodule + +def classic_task(): + pass diff --git a/tests/support/submodule/subsubmodule/__init__.py b/tests/support/submodule/subsubmodule/__init__.py new file mode 100644 index 0000000000..95d91db646 --- /dev/null +++ b/tests/support/submodule/subsubmodule/__init__.py @@ -0,0 +1,5 @@ +from fabric.api import task + +@task +def deeptask(): + pass diff --git a/tests/test_main.py b/tests/test_main.py index 021d034ec1..18c62f0afb 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -346,3 +346,25 @@ def test_class_based_tasks_are_found_with_proper_name(): docs, funcs = load_fabfile(module) eq_(1, len(funcs)) ok_('foo' in funcs) + + +def test_recursion_steps_into_nontask_modules(): + """ + Recursive loading will continue through modules with no tasks + """ + module = fabfile('deep') + with path_prefix(module): + docs, funcs = load_fabfile(module) + eq_(len(funcs), 1) + ok_('submodule.subsubmodule.deeptask' in funcs) + + +def test_newstyle_task_presence_skips_classic_task_modules(): + """ + Classic-task-only modules shouldn't add tasks if any new-style tasks exist + """ + module = fabfile('deep') + with path_prefix(module): + docs, funcs = load_fabfile(module) + eq_(len(funcs), 1) + ok_('submodule.classic_task' not in funcs) From c662099ccbe5427e91bc50300e498557154cda42 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 9 Jun 2011 14:42:51 -0700 Subject: [PATCH 088/126] Cleaner, reversible seen-module caching --- fabric/main.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/fabric/main.py b/fabric/main.py index e4c3f7b3cf..e0db8a24a9 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -31,6 +31,25 @@ [] ) +# Module recursion cache +class _ModuleCache(object): + """ + Set-like object operating on modules and storing __name__s internally. + """ + def __init__(self): + self.cache = set() + + def __contains__(self, value): + return value.__name__ in self.cache + + def add(self, value): + return self.cache.add(value.__name__) + + def clear(self): + return self.cache.clear() + +_seen = _ModuleCache() + def load_settings(path): """ @@ -142,7 +161,11 @@ def load_fabfile(path, importer=None): sys.path.insert(index + 1, directory) del sys.path[0] - return load_tasks_from_module(imported) + # Actually load tasks + ret = load_tasks_from_module(imported) + # Clean up after ourselves + _seen.clear() + return ret def load_tasks_from_module(imported): @@ -170,10 +193,9 @@ def is_task_module(a): """ #return (type(a) is types.ModuleType and # any(map(is_task_object, vars(a).values()))) - seen = '__seen_by_fab' - if type(a) is types.ModuleType and not getattr(a, seen, False): + if type(a) is types.ModuleType and a not in _seen: # Flag module as seen - setattr(a, seen, True) + _seen.add(a) # Signal that we need to check it out return True From 47f20d8085b1687313958677e3137d05423b1852 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 20 Jun 2011 15:05:46 -0700 Subject: [PATCH 089/126] Re #56, add -F option and rearrange a bit --- fabric/main.py | 47 +++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/fabric/main.py b/fabric/main.py index e0db8a24a9..1aca42d59a 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -273,7 +273,15 @@ def parse_options(): action='store_true', dest='shortlist', default=False, - help="print non-verbose list of possible commands and exit" + help="alias for -F short --list" + ) + + # Control behavior of --list + LIST_FORMAT_OPTIONS = ('short', 'normal', 'nested') + parser.add_option('-F', '--list-format', + choices=LIST_FORMAT_OPTIONS, + default='normal', + help="formats --list, choices: %s" % ", ".join(LIST_FORMAT_OPTIONS) ) # Display info about a specific command @@ -302,16 +310,21 @@ def _command_names(): return sorted(commands.keys()) -def list_commands(docstring): +def list_commands(docstring, format_): """ Print all found commands/tasks, then exit. Invoked with ``-l/--list.`` If ``docstring`` is non-empty, it will be printed before the task list. + + ``format_`` should conform to the options specified in + ``LIST_FORMAT_OPTIONS``, e.g. ``"short"``, ``"normal"``. """ + result = [] + # Docstring at top, if applicable if docstring: trailer = "\n" if not docstring.endswith("\n") else "" - print(docstring + trailer) - print("Available commands:\n") + result.append(docstring + trailer) + result.append("Available commands:\n") # Want separator between name, description to be straight col max_len = reduce(lambda a, b: max(a, len(b)), commands.keys(), 0) sep = ' ' @@ -332,16 +345,8 @@ def list_commands(docstring): # Or nothing (so just the name) else: output = name - print(indent(output)) - sys.exit(0) - - -def shortlist(): - """ - Print all task names separated by newlines with no embellishment. - """ - print("\n".join(_command_names())) - sys.exit(0) + result.append(indent(output)) + return result def display_command(command): @@ -579,13 +584,19 @@ def main(): else: print("No fabfile loaded -- remainder command only") - # Non-verbose command list + # Shortlist is now just an alias for the "short" list format; + # it overrides use of --list-format if somebody were to specify both if options.shortlist: - shortlist() + options.list_format = 'short' - # Handle list-commands option (now that commands are loaded) + # List available commands if options.list_commands: - list_commands(docstring) + if options.list_format == "short": + result = _command_names() + else: + result = list_commands(docstring, options.list_format) + print("\n".join(result)) + sys.exit(0) # Handle show (command-specific help) option if options.display: From 3c0af2ff81d0d77864dba01a077faffea4a41323 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 20 Jun 2011 15:27:29 -0700 Subject: [PATCH 090/126] Sick of having eq_ args swapped --- tests/utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index f323ab46a2..c78f4e86e3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -208,26 +208,26 @@ def line_prefix(prefix, string): return "\n".join(prefix + x for x in string.splitlines()) -def eq_(a, b, msg=None): +def eq_(result, expected, msg=None): """ Shadow of the Nose builtin which presents easier to read multiline output. """ default_msg = """ Expected: -%s +%(expected)s Got: -%s +%(result)s --------------------------------- aka ----------------------------------------- Expected: -%r +%(expected)r Got: -%r -""" % (a, b, a, b) - assert a == b, msg or default_msg +%(result)r +""" % {'expected': expected, 'result': result} + assert result == expected, msg or default_msg def eq_contents(path, text): From 363c719a73474848dc496ca34a36b5a7c1d20f26 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 20 Jun 2011 15:32:16 -0700 Subject: [PATCH 091/126] Don't print the 'aka' unless repr != str --- tests/utils.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index c78f4e86e3..f259b36032 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -212,12 +212,8 @@ def eq_(result, expected, msg=None): """ Shadow of the Nose builtin which presents easier to read multiline output. """ - default_msg = """ -Expected: -%(expected)s - -Got: -%(result)s + params = {'expected': expected, 'result': result} + aka = """ --------------------------------- aka ----------------------------------------- @@ -226,7 +222,16 @@ def eq_(result, expected, msg=None): Got: %(result)r -""" % {'expected': expected, 'result': result} +""" % params + default_msg = """ +Expected: +%(expected)s + +Got: +%(result)s +""" % params + if (repr(result) != str(result)) or (repr(expected) != str(expected)): + default_msg += aka assert result == expected, msg or default_msg From d0d22a9d3e99b3da1993d69c439dd5064f33b911 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 20 Jun 2011 16:13:52 -0700 Subject: [PATCH 092/126] Break ground on tests, use explicit commands obj in main Re #56 --- fabric/main.py | 34 +++++++++++++++++----------------- tests/test_main.py | 32 +++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 20 deletions(-) diff --git a/fabric/main.py b/fabric/main.py index 1aca42d59a..fda966c4a5 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -307,7 +307,7 @@ def parse_options(): def _command_names(): - return sorted(commands.keys()) + return sorted(state.commands.keys()) def list_commands(docstring, format_): @@ -319,6 +319,10 @@ def list_commands(docstring, format_): ``format_`` should conform to the options specified in ``LIST_FORMAT_OPTIONS``, e.g. ``"short"``, ``"normal"``. """ + # Short-circuit with simple short output + if format_ == "short": + return _command_names() + # Otherwise, handle more verbose modes result = [] # Docstring at top, if applicable if docstring: @@ -326,13 +330,13 @@ def list_commands(docstring, format_): result.append(docstring + trailer) result.append("Available commands:\n") # Want separator between name, description to be straight col - max_len = reduce(lambda a, b: max(a, len(b)), commands.keys(), 0) + max_len = reduce(lambda a, b: max(a, len(b)), state.commands.keys(), 0) sep = ' ' trail = '...' for name in _command_names(): output = None # Print first line of docstring - func = commands[name] + func = state.commands[name] docstring = func.__doc__ if docstring and type(docstring) in types.StringTypes: lines = filter(None, func.__doc__.splitlines()) @@ -354,9 +358,9 @@ def display_command(command): Print command function's docstring, then exit. Invoked with -d/--display. """ # Sanity check - if command not in commands: + if command not in state.commands: abort("Command '%s' not found, exiting." % command) - cmd = commands[command] + cmd = state.commands[command] # Print out nicely presented docstring if found if cmd.__doc__: print("Displaying detailed information for command '%s':" % command) @@ -571,10 +575,10 @@ def main(): # dict if fabfile: docstring, callables = load_fabfile(fabfile) - commands.update(callables) + state.commands.update(callables) # Abort if no commands found - if not commands and not remainder_arguments: + if not state.commands and not remainder_arguments: abort("Fabfile didn't contain any commands!") # Now that we're settled on a fabfile, inform user. @@ -591,11 +595,7 @@ def main(): # List available commands if options.list_commands: - if options.list_format == "short": - result = _command_names() - else: - result = list_commands(docstring, options.list_format) - print("\n".join(result)) + print("\n".join(list_commands(docstring, options.list_format))) sys.exit(0) # Handle show (command-specific help) option @@ -616,7 +616,7 @@ def main(): # Figure out if any specified task names are invalid unknown_commands = [] for tup in commands_to_run: - if tup[0] not in commands: + if tup[0] not in state.commands: unknown_commands.append(tup[0]) # Abort if any unknown commands were specified @@ -627,7 +627,7 @@ def main(): # Generate remainder command and insert into commands, commands_to_run if remainder_command: r = '' - commands[r] = lambda: api.run(remainder_command) + state.commands[r] = lambda: api.run(remainder_command) commands_to_run.append((r, [], {}, [], [])) if state.output.debug: @@ -638,7 +638,7 @@ def main(): # At this point all commands must exist, so execute them in order. for name, args, kwargs, cli_hosts, cli_roles, cli_exclude_hosts in commands_to_run: # Get callable by itself - command = commands[name] + command = state.commands[name] # Set current command name (used for some error messages) state.env.command = name # Set host list (also copy to env) @@ -654,12 +654,12 @@ def main(): if state.output.running: print("[%s] Executing task '%s'" % (host, name)) # Actually run command - commands[name](*args, **kwargs) + state.commands[name](*args, **kwargs) # Put old user back state.env.user = prev_user # If no hosts found, assume local-only and run once if not hosts: - commands[name](*args, **kwargs) + state.commands[name](*args, **kwargs) # If we got here, no errors occurred, so print a final note. if state.output.status: print("\nDone.") diff --git a/tests/test_main.py b/tests/test_main.py index 18c62f0afb..87ce6cf1df 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -4,17 +4,17 @@ import copy from contextlib import contextmanager -from fudge import Fake +from fudge import Fake, patched_context from nose.tools import ok_, eq_, raises from fabric.decorators import hosts, roles, task from fabric.main import (get_hosts, parse_arguments, _merge, _escape_split, - load_fabfile) + load_fabfile, list_commands) import fabric.state from fabric.state import _AttributeDict -from utils import mock_streams, patched_env +from utils import mock_streams, patched_env, eq_ import os import sys @@ -368,3 +368,29 @@ def test_newstyle_task_presence_skips_classic_task_modules(): docs, funcs = load_fabfile(module) eq_(len(funcs), 1) ok_('submodule.classic_task' not in funcs) + + +# +# --list output +# + +def eq_output(docstring, format_, expected): + return eq_( + "\n".join(list_commands(docstring, format_)), + expected + ) + +def list_output(module, format_, expected): + module = fabfile(module) + with path_prefix(module): + docstring, tasks = load_fabfile(module) + with patched_context(fabric.state, 'commands', tasks): + eq_output(docstring, format_, expected) + +def test_list_output(): + for desc, module, format_, expected in ( + ("shorthand (& with namespacing)", 'deep', 'short', "submodule.subsubmodule.deeptask"), + ): + list_output.description = "--list output: %s" % desc + yield list_output, module, format_, expected + del list_output.description From c4bcd8db290b1dbc147d0a624c1b06d96fc8ef6c Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 20 Jun 2011 17:55:13 -0700 Subject: [PATCH 093/126] Minor tweaks --- fabric/main.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fabric/main.py b/fabric/main.py index fda966c4a5..1213a7673d 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -335,11 +335,10 @@ def list_commands(docstring, format_): trail = '...' for name in _command_names(): output = None - # Print first line of docstring - func = state.commands[name] - docstring = func.__doc__ + task = state.commands[name] + docstring = task.__doc__ if docstring and type(docstring) in types.StringTypes: - lines = filter(None, func.__doc__.splitlines()) + lines = filter(None, docstring.splitlines()) first_line = lines[0].strip() # Truncate it if it's longer than N chars size = 75 - (max_len + len(sep) + len(trail)) From 10b9b731f4d2bd15e9bda9e4c05360b061b6a57b Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 20 Jun 2011 19:32:03 -0700 Subject: [PATCH 094/126] Implement nested output. Also rejiggers a ton of stuff for arguably cleaner handling of namespace trees. --- fabric/main.py | 182 +++++++++++++++++--------- tests/support/tree/__init__.py | 12 ++ tests/support/tree/db.py | 6 + tests/support/tree/system/__init__.py | 7 + tests/support/tree/system/debian.py | 6 + tests/test_main.py | 71 +++++++++- 6 files changed, 220 insertions(+), 64 deletions(-) create mode 100644 tests/support/tree/__init__.py create mode 100644 tests/support/tree/db.py create mode 100644 tests/support/tree/system/__init__.py create mode 100644 tests/support/tree/system/debian.py diff --git a/fabric/main.py b/fabric/main.py index 1213a7673d..8ac89bf218 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -9,7 +9,8 @@ to individuals leveraging Fabric as a library, should be kept elsewhere. """ -from operator import add +from collections import defaultdict +from operator import add, isMappingType from optparse import OptionParser import os import sys @@ -187,6 +188,28 @@ def load_tasks_from_module(imported): return imported.__doc__, tasks +def extract_tasks(imported_vars): + """ + Handle extracting tasks from a given list of variables + """ + new_style_tasks = defaultdict(dict) + classic_tasks = {} + if 'new_style_tasks' not in state.env: + state.env.new_style_tasks = False + for tup in imported_vars: + name, obj = tup + if is_task_object(obj): + state.env.new_style_tasks = True + new_style_tasks[obj.name] = obj + elif is_task(tup): + classic_tasks[name] = obj + elif is_task_module(obj): + module_docs, module_tasks = load_tasks_from_module(obj) + for task_name, task in module_tasks.items(): + new_style_tasks[name][task_name] = task + return (new_style_tasks, classic_tasks) + + def is_task_module(a): """ Determine if the provided value is a task module @@ -210,29 +233,6 @@ def is_task_object(a): return isinstance(a, Task) and a.use_task_objects -def extract_tasks(imported_vars): - """ - Handle extracting tasks from a given list of variables - """ - new_style_tasks = {} - classic_tasks = {} - if 'new_style_tasks' not in state.env: - state.env.new_style_tasks = False - for tup in imported_vars: - name, obj = tup - if is_task_object(obj): - state.env.new_style_tasks = True - new_style_tasks[obj.name] = obj - elif is_task(tup): - classic_tasks[name] = obj - elif is_task_module(obj): - module_docs, module_tasks = load_tasks_from_module(obj) - for task_name, task in module_tasks.items(): - new_style_tasks["%s.%s" % (name, task_name)] = task - return (new_style_tasks, classic_tasks) - - - def parse_options(): """ Handle command-line options with optparse.OptionParser. @@ -305,39 +305,61 @@ def parse_options(): opts, args = parser.parse_args() return parser, opts, args +def _sift_tasks(mapping): + tasks, collections = [], [] + for name, value in mapping.iteritems(): + (collections if isMappingType(value) else tasks).append(name) + tasks = sorted(tasks) + collections = sorted(collections) + return tasks, collections -def _command_names(): - return sorted(state.commands.keys()) +def _task_names(mapping): + """ + Flatten & sort task names in a breadth-first fashion. + Tasks are always listed before submodules at the same level, but within + those two groups, sorting is alphabetical. + """ + tasks, collections = _sift_tasks(mapping) + for collection in collections: + module = mapping[collection] + join = lambda x: ".".join((collection, x)) + tasks.extend(map(join, _task_names(module))) + return tasks -def list_commands(docstring, format_): +def _crawl(name, mapping): """ - Print all found commands/tasks, then exit. Invoked with ``-l/--list.`` + ``name`` of ``'a.b.c'`` => ``mapping['a']['b']['c']`` + """ + key, _, rest = name.partition('.') + value = mapping[key] + if not rest: + return value + return _crawl(rest, value) - If ``docstring`` is non-empty, it will be printed before the task list. +def crawl(name, mapping): + try: + return _crawl(name, mapping) + except (KeyError, TypeError): + return None - ``format_`` should conform to the options specified in - ``LIST_FORMAT_OPTIONS``, e.g. ``"short"``, ``"normal"``. - """ - # Short-circuit with simple short output - if format_ == "short": - return _command_names() - # Otherwise, handle more verbose modes +def _print_docstring(docstrings, name): + if not docstrings: + return False + docstring = crawl(name, state.commands).__doc__ + return docstring and type(docstring) in types.StringTypes + + +def _normal_list(docstrings=True): result = [] - # Docstring at top, if applicable - if docstring: - trailer = "\n" if not docstring.endswith("\n") else "" - result.append(docstring + trailer) - result.append("Available commands:\n") + task_names = _task_names(state.commands) # Want separator between name, description to be straight col - max_len = reduce(lambda a, b: max(a, len(b)), state.commands.keys(), 0) + max_len = reduce(lambda a, b: max(a, len(b)), task_names, 0) sep = ' ' trail = '...' - for name in _command_names(): + for name in task_names: output = None - task = state.commands[name] - docstring = task.__doc__ - if docstring and type(docstring) in types.StringTypes: + if _print_docstring(docstrings, name): lines = filter(None, docstring.splitlines()) first_line = lines[0].strip() # Truncate it if it's longer than N chars @@ -352,23 +374,62 @@ def list_commands(docstring, format_): return result -def display_command(command): +def _nested_list(mapping, level=1): + result = [] + tasks, collections = _sift_tasks(mapping) + # Tasks come first + result.extend(map(lambda x: indent(x, spaces=level * 4), tasks)) + for collection in collections: + module = mapping[collection] + # Section/module "header" + result.append(indent(collection + ":", spaces=level * 4)) + # Recurse + result.extend(_nested_list(module, level + 1)) + return result + + +def list_commands(docstring, format_): + """ + Print all found commands/tasks, then exit. Invoked with ``-l/--list.`` + + If ``docstring`` is non-empty, it will be printed before the task list. + + ``format_`` should conform to the options specified in + ``LIST_FORMAT_OPTIONS``, e.g. ``"short"``, ``"normal"``. + """ + # Short-circuit with simple short output + if format_ == "short": + return _task_names(state.commands) + # Otherwise, handle more verbose modes + result = [] + # Docstring at top, if applicable + if docstring: + trailer = "\n" if not docstring.endswith("\n") else "" + result.append(docstring + trailer) + result.append("Available commands:\n") + c = _normal_list() if format_ == "normal" else _nested_list(state.commands) + result.extend(c) + return result + + +def display_command(name): """ Print command function's docstring, then exit. Invoked with -d/--display. """ # Sanity check - if command not in state.commands: - abort("Command '%s' not found, exiting." % command) - cmd = state.commands[command] + command = crawl(name, state.commands) + if command is None: + msg = "Task '%s' does not appear to exist. Valid task names:\n%s" + abort(msg % (name, "\n".join(_normal_list(False)))) # Print out nicely presented docstring if found - if cmd.__doc__: - print("Displaying detailed information for command '%s':" % command) + if command.__doc__: + print("Displaying detailed information for task '%s':" % name) print('') - print(indent(cmd.__doc__, strip=True)) + print(indent(command.__doc__, strip=True)) print('') # Or print notice if not else: - print("No detailed information available for command '%s':" % command) + print("No detailed information available for task '%s':" % name) sys.exit(0) @@ -615,7 +676,7 @@ def main(): # Figure out if any specified task names are invalid unknown_commands = [] for tup in commands_to_run: - if tup[0] not in state.commands: + if crawl(tup[0], state.commands) is None: unknown_commands.append(tup[0]) # Abort if any unknown commands were specified @@ -633,16 +694,15 @@ def main(): names = ", ".join(x[0] for x in commands_to_run) print("Commands to run: %s" % names) - # At this point all commands must exist, so execute them in order. for name, args, kwargs, cli_hosts, cli_roles, cli_exclude_hosts in commands_to_run: # Get callable by itself - command = state.commands[name] - # Set current command name (used for some error messages) + task = crawl(name, state.commands) + # Set current task name (used for some error messages) state.env.command = name # Set host list (also copy to env) state.env.all_hosts = hosts = get_hosts( - command, cli_hosts, cli_roles, cli_exclude_hosts) + task, cli_hosts, cli_roles, cli_exclude_hosts) # If hosts found, execute the function on each host in turn for host in hosts: # Preserve user @@ -653,12 +713,12 @@ def main(): if state.output.running: print("[%s] Executing task '%s'" % (host, name)) # Actually run command - state.commands[name](*args, **kwargs) + task(*args, **kwargs) # Put old user back state.env.user = prev_user # If no hosts found, assume local-only and run once if not hosts: - state.commands[name](*args, **kwargs) + task(*args, **kwargs) # If we got here, no errors occurred, so print a final note. if state.output.status: print("\nDone.") diff --git a/tests/support/tree/__init__.py b/tests/support/tree/__init__.py new file mode 100644 index 0000000000..b7f3bf9f97 --- /dev/null +++ b/tests/support/tree/__init__.py @@ -0,0 +1,12 @@ +from fabric.api import task + +import system, db + + +@task +def deploy(): + pass + +@task +def build_docs(): + pass diff --git a/tests/support/tree/db.py b/tests/support/tree/db.py new file mode 100644 index 0000000000..318982fb47 --- /dev/null +++ b/tests/support/tree/db.py @@ -0,0 +1,6 @@ +from fabric.api import task + + +@task +def migrate(): + pass diff --git a/tests/support/tree/system/__init__.py b/tests/support/tree/system/__init__.py new file mode 100644 index 0000000000..a7d2bfe08e --- /dev/null +++ b/tests/support/tree/system/__init__.py @@ -0,0 +1,7 @@ +from fabric.api import task + +import debian + +@task +def install_package(): + pass diff --git a/tests/support/tree/system/debian.py b/tests/support/tree/system/debian.py new file mode 100644 index 0000000000..f1e17c29d3 --- /dev/null +++ b/tests/support/tree/system/debian.py @@ -0,0 +1,6 @@ +from fabric.api import task + + +@task +def update_apt(): + pass diff --git a/tests/test_main.py b/tests/test_main.py index 87ce6cf1df..d99da34785 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -9,7 +9,7 @@ from fabric.decorators import hosts, roles, task from fabric.main import (get_hosts, parse_arguments, _merge, _escape_split, - load_fabfile, list_commands) + load_fabfile, list_commands, _task_names, _crawl, crawl) import fabric.state from fabric.state import _AttributeDict @@ -355,8 +355,9 @@ def test_recursion_steps_into_nontask_modules(): module = fabfile('deep') with path_prefix(module): docs, funcs = load_fabfile(module) + print funcs eq_(len(funcs), 1) - ok_('submodule.subsubmodule.deeptask' in funcs) + ok_('submodule.subsubmodule.deeptask' in _task_names(funcs)) def test_newstyle_task_presence_skips_classic_task_modules(): @@ -367,7 +368,7 @@ def test_newstyle_task_presence_skips_classic_task_modules(): with path_prefix(module): docs, funcs = load_fabfile(module) eq_(len(funcs), 1) - ok_('submodule.classic_task' not in funcs) + ok_('submodule.classic_task' not in _task_names(funcs)) # @@ -390,7 +391,71 @@ def list_output(module, format_, expected): def test_list_output(): for desc, module, format_, expected in ( ("shorthand (& with namespacing)", 'deep', 'short', "submodule.subsubmodule.deeptask"), + ("normal (& with namespacing)", 'deep', 'normal', """Available commands: + + submodule.subsubmodule.deeptask"""), + ("nested (leaf only)", 'deep', 'nested', """Available commands: + + submodule: + subsubmodule: + deeptask"""), + ("nested (full)", 'tree', 'nested', """Available commands: + + build_docs + deploy + db: + migrate + system: + install_package + debian: + update_apt"""), ): list_output.description = "--list output: %s" % desc yield list_output, module, format_, expected del list_output.description + + +def test_task_names(): + for desc, input_, output in ( + ('top level (single)', {'a': 5}, ['a']), + ('top level (multiple, sorting)', {'a': 5, 'b': 6}, ['a', 'b']), + ('just nested', {'a': {'b': 5}}, ['a.b']), + ('mixed', {'a': 5, 'b': {'c': 6}}, ['a', 'b.c']), + ('top level comes before nested', {'z': 5, 'b': {'c': 6}}, ['z', 'b.c']), + ('peers sorted equally', {'z': 5, 'b': {'c': 6}, 'd': {'e': 7}}, ['z', 'b.c', 'd.e']), + ( + 'complex tree', + { + 'z': 5, + 'b': { + 'c': 6, + 'd': { + 'e': { + 'f': '7' + } + }, + 'g': 8 + }, + 'h': 9, + 'w': { + 'y': 10 + } + }, + ['h', 'z', 'b.c', 'b.g', 'b.d.e.f', 'w.y'] + ), + ): + eq_.description = "task name flattening: %s" % desc + yield eq_, _task_names(input_), output + del eq_.description + + +def test_crawl(): + for desc, name, mapping, output in ( + ("base case", 'a', {'a': 5}, 5), + ("one level", 'a.b', {'a': {'b': 5}}, 5), + ("deep", 'a.b.c.d.e', {'a': {'b': {'c': {'d': {'e': 5}}}}}, 5), + ("full tree", 'a.b.c', {'a': {'b': {'c': 5}, 'd': 6}, 'z': 7}, 5) + ): + eq_.description = "crawling dotted names: %s" % desc + yield eq_, _crawl(name, mapping), output + del eq_.description From b2b27ac5f28b8a2874d0fb8ffcce0b096bab4915 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 21 Jun 2011 12:39:23 -0700 Subject: [PATCH 095/126] Bugfix and test --- fabric/main.py | 6 ++++-- tests/support/docstring.py | 8 ++++++++ tests/test_main.py | 3 +++ 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 tests/support/docstring.py diff --git a/fabric/main.py b/fabric/main.py index 8ac89bf218..b17c9c15fd 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -347,7 +347,8 @@ def _print_docstring(docstrings, name): if not docstrings: return False docstring = crawl(name, state.commands).__doc__ - return docstring and type(docstring) in types.StringTypes + if type(docstring) in types.StringTypes: + return docstring def _normal_list(docstrings=True): @@ -359,7 +360,8 @@ def _normal_list(docstrings=True): trail = '...' for name in task_names: output = None - if _print_docstring(docstrings, name): + docstring = _print_docstring(docstrings, name) + if docstring: lines = filter(None, docstring.splitlines()) first_line = lines[0].strip() # Truncate it if it's longer than N chars diff --git a/tests/support/docstring.py b/tests/support/docstring.py new file mode 100644 index 0000000000..76801afda1 --- /dev/null +++ b/tests/support/docstring.py @@ -0,0 +1,8 @@ +from fabric.decorators import task + +@task +def foo(): + """ + Foos! + """ + pass diff --git a/tests/test_main.py b/tests/test_main.py index d99da34785..d2c41db9e7 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -394,6 +394,9 @@ def test_list_output(): ("normal (& with namespacing)", 'deep', 'normal', """Available commands: submodule.subsubmodule.deeptask"""), + ("normal (with docstring)", 'docstring', 'normal', """Available commands: + + foo Foos!"""), ("nested (leaf only)", 'deep', 'nested', """Available commands: submodule: From 9d3280b85e8312586439e6842957e680ec2229e0 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 21 Jun 2011 13:22:50 -0700 Subject: [PATCH 096/126] Re #56, tweak nested output & clean up tests --- fabric/main.py | 7 ++++++- tests/test_main.py | 22 +++++++++------------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/fabric/main.py b/fabric/main.py index b17c9c15fd..041aabc320 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -389,6 +389,8 @@ def _nested_list(mapping, level=1): result.extend(_nested_list(module, level + 1)) return result +COMMANDS_HEADER = "Available commands" +NESTED_REMINDER = " (remember to call as module.[...].task)" def list_commands(docstring, format_): """ @@ -408,7 +410,10 @@ def list_commands(docstring, format_): if docstring: trailer = "\n" if not docstring.endswith("\n") else "" result.append(docstring + trailer) - result.append("Available commands:\n") + header = COMMANDS_HEADER + if format_ == "nested": + header += NESTED_REMINDER + result.append(header + ":\n") c = _normal_list() if format_ == "normal" else _nested_list(state.commands) result.extend(c) return result diff --git a/tests/test_main.py b/tests/test_main.py index d2c41db9e7..0228756fdc 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -9,7 +9,8 @@ from fabric.decorators import hosts, roles, task from fabric.main import (get_hosts, parse_arguments, _merge, _escape_split, - load_fabfile, list_commands, _task_names, _crawl, crawl) + load_fabfile, list_commands, _task_names, _crawl, crawl, + COMMANDS_HEADER, NESTED_REMINDER) import fabric.state from fabric.state import _AttributeDict @@ -389,22 +390,17 @@ def list_output(module, format_, expected): eq_output(docstring, format_, expected) def test_list_output(): + lead = ":\n\n " + normal_head = COMMANDS_HEADER + lead + nested_head = COMMANDS_HEADER + NESTED_REMINDER + lead for desc, module, format_, expected in ( ("shorthand (& with namespacing)", 'deep', 'short', "submodule.subsubmodule.deeptask"), - ("normal (& with namespacing)", 'deep', 'normal', """Available commands: - - submodule.subsubmodule.deeptask"""), - ("normal (with docstring)", 'docstring', 'normal', """Available commands: - - foo Foos!"""), - ("nested (leaf only)", 'deep', 'nested', """Available commands: - - submodule: + ("normal (& with namespacing)", 'deep', 'normal', normal_head + "submodule.subsubmodule.deeptask"), + ("normal (with docstring)", 'docstring', 'normal', normal_head + "foo Foos!"), + ("nested (leaf only)", 'deep', 'nested', nested_head + """submodule: subsubmodule: deeptask"""), - ("nested (full)", 'tree', 'nested', """Available commands: - - build_docs + ("nested (full)", 'tree', 'nested', nested_head + """build_docs deploy db: migrate From d4ce7c90dbb5f87cadd518b965693a314ae872b3 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 21 Jun 2011 13:43:09 -0700 Subject: [PATCH 097/126] Document nested output and -F re #56 --- docs/changes/1.1.rst | 3 +++ docs/usage/fab.rst | 14 ++++++++++++-- docs/usage/tasks.rst | 31 +++++++++++++++++++++++++++---- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/docs/changes/1.1.rst b/docs/changes/1.1.rst index 921581d0e6..a4b99a4d4d 100644 --- a/docs/changes/1.1.rst +++ b/docs/changes/1.1.rst @@ -28,6 +28,9 @@ Feature additions * :issue:`10`: `~fabric.contrib.upload_project` now allows control over the local and remote directory paths, and has improved error handling. Thanks to Rodrigue Alcazar for the patch. +* As part of :issue:`56` (highlighted above), added :option:`--list-format + <-F>` to allow specification of a nested output format from :option:`--list + <-l>`. * :issue:`107`: `~fabric.operations.require`'s ``provided_by`` kwarg now accepts iterables in addition to single values. Thanks to Thomas Ballinger for the patch. diff --git a/docs/usage/fab.rst b/docs/usage/fab.rst index ba0694bfc6..d38b9d0001 100644 --- a/docs/usage/fab.rst +++ b/docs/usage/fab.rst @@ -113,7 +113,17 @@ below. alternately an explicit file path to load as the fabfile (e.g. ``/path/to/my/fabfile.py``.) -.. seealso:: :doc:`fabfiles` + .. seealso:: :doc:`fabfiles` + +.. cmdoption:: -F LIST_FORMAT, --list-format=LIST_FORMAT + + Allows control over the output format of :option:`--list <-l>`. ``short`` is + equivalent to :option:`--shortlist`, ``normal`` is the same as simply + omitting this option entirely (i.e. the default), and ``nested`` prints out + a nested namespace tree. + + .. versionadded:: 1.1 + .. seealso:: :option:`--shortlist`, :option:`--list <-l>` .. cmdoption:: -h, --help @@ -159,7 +169,7 @@ below. .. versionchanged:: 0.9.1 Added docstring to output. - .. seealso:: :option:`--shortlist` + .. seealso:: :option:`--shortlist`, :option:`--list-format <-F>` .. cmdoption:: -p PASSWORD, --password=PASSWORD diff --git a/docs/usage/tasks.rst b/docs/usage/tasks.rst index 25d4f02473..fdf5db018d 100644 --- a/docs/usage/tasks.rst +++ b/docs/usage/tasks.rst @@ -190,10 +190,10 @@ into some other part of the hierarchy, but otherwise it'll remain hidden. Switching it up ~~~~~~~~~~~~~~~ -Finally, while we've been keeping our fabfile package neatly organized and -importing it in a straightforward manner, the filesystem layout doesn't -actually matter here. All Fabric's loader cares about is the names the modules -are given when they're imported. +We've been keeping our fabfile package neatly organized and importing it in a +straightforward manner, but the filesystem layout doesn't actually matter here. +All Fabric's loader cares about is the names the modules are given when they're +imported. For example, if we changed the top of our root ``__init__.py`` to look like this:: @@ -212,6 +212,29 @@ This applies to any other import -- you could import third party modules into your own task hierarchy, or grab a deeply nested module and make it appear near the top level. +Nested list output +~~~~~~~~~~~~~~~~~~ + +As a final note, we've been using the default Fabric :option:`--list <-l>` +output during this section -- it makes it more obvious what the actual task +names are. However, you can get a more nested or tree-like view by passing +``nested`` to the :option:`--list-format <-F>` option:: + + $ fab --list-format=nested --list + Available commands (remember to call as module.[...].task): + + deploy + compress + lb: + add_backend + database: + migrations: + list + run + +While it slightly obfuscates the "real" task names, this view provides a handy +way of noting the organization of tasks in large namespaces. + .. _classic-tasks: From 5d8d0e3f59645192cd74670fa966eda87c3eb0a7 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 21 Jun 2011 15:52:21 -0700 Subject: [PATCH 098/126] Implements #189: flag for aborting-on-prompts --- AUTHORS | 2 ++ docs/changes/1.1.rst | 4 ++++ docs/usage/env.rst | 17 +++++++++++++++++ docs/usage/fab.rst | 7 +++++++ fabric/network.py | 4 +++- fabric/operations.py | 16 ++++++++++++++-- fabric/state.py | 10 +++++++++- fabric/utils.py | 6 ++++++ tests/test_network.py | 23 +++++++++++++++++++++-- 9 files changed, 83 insertions(+), 6 deletions(-) diff --git a/AUTHORS b/AUTHORS index e62d8bf2d6..eb40f6a3db 100644 --- a/AUTHORS +++ b/AUTHORS @@ -41,3 +41,5 @@ Ales Zoulek Casey Banner Roman Imankulov Rodrigue Alcazar +Jeremy Avnet +Matt Chisholm diff --git a/docs/changes/1.1.rst b/docs/changes/1.1.rst index a4b99a4d4d..f350cf815b 100644 --- a/docs/changes/1.1.rst +++ b/docs/changes/1.1.rst @@ -42,6 +42,10 @@ Feature additions suggestion and Morgan Goose for initial implementation. * :issue:`170`: Allow :ref:`exclusion ` of specific hosts from the final run list. Thanks to Casey Banner for the suggestion and patch. +* :issue:`189`: Added :option:`--abort-on-prompts`/:ref:`env.abort_on_prompts + ` to allow a more non-interactive behavior, + aborting/exiting instead of trying to prompt the running user. Thanks to + Jeremy Avnet and Matt Chisholm for the initial patch. * :issue:`273`: `~fabric.contrib.files.upload_template` now offers control over whether it attempts to create backups of pre-existing destination files. Thanks to Ales Zoulek for the suggestion and initial patch. diff --git a/docs/usage/env.rst b/docs/usage/env.rst index f8ab684a67..18004a50b1 100644 --- a/docs/usage/env.rst +++ b/docs/usage/env.rst @@ -104,6 +104,23 @@ as `~fabric.context_managers.cd`. Note that many of these may be set via ``fab``'s command-line switches -- see :doc:`fab` for details. Cross-links will be provided where appropriate. +.. _abort-on-prompts: + +``abort_on_prompts`` +-------------------- + +**Default:** ``False`` + +When ``True``, Fabric will run in a non-interactive mode, calling +`~fabric.utils.abort` anytime it would normally prompt the user for input (such +as password prompts, "What host to connect to?" prompts, fabfile invocation of +`~fabric.operations.prompt`, and so forth.) This allows users to ensure a Fabric +session will always terminate cleanly instead of blocking on user input forever +when unforeseen circumstances arise. + +.. versionadded:: 1.1 +.. seealso:: :option:`--abort-on-prompts` + ``all_hosts`` ------------- diff --git a/docs/usage/fab.rst b/docs/usage/fab.rst index d38b9d0001..088c239047 100644 --- a/docs/usage/fab.rst +++ b/docs/usage/fab.rst @@ -90,6 +90,13 @@ below. .. versionadded:: 0.9.1 +.. cmdoption:: --abort-on-prompts + + Sets :ref:`env.abort_on_prompts ` to ``True``, forcing + Fabric to abort whenever it would prompt for input. + + .. versionadded:: 1.1 + .. cmdoption:: -c RCFILE, --config=RCFILE Sets :ref:`env.rcfile ` to the given file path, which Fabric will diff --git a/fabric/network.py b/fabric/network.py index 901e2f5c36..37874b9b31 100644 --- a/fabric/network.py +++ b/fabric/network.py @@ -10,8 +10,8 @@ import socket import sys -from fabric.utils import abort from fabric.auth import get_password, set_password +from fabric.utils import abort, handle_prompt_abort try: import warnings @@ -263,6 +263,7 @@ def prompt_for_password(prompt=None, no_colon=False, stream=None): defaults to ``sys.stderr``. """ from fabric.state import env + handle_prompt_abort() stream = stream or sys.stderr # Construct prompt default = "[%s] Login password" % env.host_string @@ -301,6 +302,7 @@ def needs_host(func): @wraps(func) def host_prompting_wrapper(*args, **kwargs): + handle_prompt_abort() while not env.get('host_string', False): host_string = raw_input("No hosts found. Please specify (single)" " host string for connection: ") diff --git a/fabric/operations.py b/fabric/operations.py index d7ee2b337a..477aaca112 100644 --- a/fabric/operations.py +++ b/fabric/operations.py @@ -19,11 +19,11 @@ from fabric.context_managers import settings, char_buffered from fabric.io import output_loop, input_loop from fabric.network import needs_host +from fabric.sftp import SFTP from fabric.state import (env, connections, output, win32, default_channel, io_sleep) -from fabric.utils import abort, indent, warn, puts from fabric.thread_handling import ThreadHandler -from fabric.sftp import SFTP +from fabric.utils import abort, indent, warn, puts, handle_prompt_abort # For terminal size logic below if not win32: @@ -220,6 +220,13 @@ def prompt(text, key=None, default='', validate=None): Either way, `prompt` will re-prompt until validation passes (or the user hits ``Ctrl-C``). + .. note:: + `~fabric.operations.prompt` honors :ref:`env.abort_on_prompts + ` and will call `~fabric.utils.abort` instead of + prompting if that flag is set to ``True``. If you want to block on user + input regardless, try wrapping with + `~fabric.context_managers.settings`. + Examples:: # Simplest form: @@ -235,7 +242,12 @@ def prompt(text, key=None, default='', validate=None): release = prompt('Please supply a release name', validate=r'^\w+-\d+(\.\d+)?$') + # Prompt regardless of the global abort-on-prompts setting: + with settings(abort_on_prompts=False): + prompt('I seriously need an answer on this! ') + """ + handle_prompt_abort() # Store previous env value for later display, if necessary if key: previous_value = env.get(key) diff --git a/fabric/state.py b/fabric/state.py index cdef6d0a4c..2ca0649300 100644 --- a/fabric/state.py +++ b/fabric/state.py @@ -208,7 +208,15 @@ def _rc_path(): action='store_false', default=True, help="do not use pseudo-terminal in run/sudo" - ) + ), + + # Abort on prompting flag + make_option('--abort-on-prompts', + action='store_true', + default=False, + help="Abort instead of prompting (for password, host, etc)" + ), + ] diff --git a/fabric/utils.py b/fabric/utils.py index 876ddfc7ae..5e3f064a1c 100644 --- a/fabric/utils.py +++ b/fabric/utils.py @@ -121,3 +121,9 @@ def fastprint(text, show_prefix=False, end="", flush=True): .. seealso:: `~fabric.utils.puts` """ return puts(text=text, show_prefix=show_prefix, end=end, flush=flush) + + +def handle_prompt_abort(): + import fabric.state + if fabric.state.env.abort_on_prompts: + abort("Needed to prompt, but abort-on-prompts was set to True!") diff --git a/tests/test_network.py b/tests/test_network.py index 89c701130b..847eef9e6d 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -6,7 +6,7 @@ import sys import paramiko -from nose.tools import with_setup +from nose.tools import with_setup, raises from fudge import (Fake, clear_calls, clear_expectations, patch_object, verify, with_patched_object, patched_context, with_fakes) @@ -16,7 +16,7 @@ from fabric.io import output_loop import fabric.network # So I can call patch_object correctly. Sigh. from fabric.state import env, output, _get_system_username -from fabric.operations import run, sudo +from fabric.operations import run, sudo, prompt from utils import * from server import (server, PORT, RESPONSES, PASSWORDS, CLIENT_PRIVKEY, USER, @@ -150,6 +150,25 @@ def test_prompts_for_password_without_good_authentication(self): cache = HostConnectionCache() cache[env.host_string] + + @raises(SystemExit) + @with_patched_object(output, 'aborts', False) + def test_aborts_on_prompt_with_abort_on_prompt(self): + env.abort_on_prompts = True + prompt("This will abort") + + + @server() + @raises(SystemExit) + @with_patched_object(output, 'aborts', False) + def test_aborts_on_password_prompt_with_abort_on_prompt(self): + env.password = None + env.abort_on_prompts = True + with password_response(PASSWORDS[env.user], times_called=1): + cache = HostConnectionCache() + cache[env.host_string] + + @mock_streams('stdout') @server() def test_trailing_newline_line_drop(self): From 8a0023f72a92481d0590b14a1c0a64cd0f99dd6a Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 21 Jun 2011 16:15:46 -0700 Subject: [PATCH 099/126] Make doc browsing its own task --- fabfile.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/fabfile.py b/fabfile.py index d9bc12b634..98fedd32ad 100644 --- a/fabfile.py +++ b/fabfile.py @@ -39,9 +39,18 @@ def build_docs(clean='no', browse='no'): if clean.lower() in ['yes', 'y']: c = "clean " b = "" + with lcd('docs'): + local('make %shtml%s' % (c, b)) if browse.lower() in ['yes', 'y']: - b = " && open _build/html/index.html" - local('cd docs; make %shtml%s' % (c, b)) + browse_docs() + + +def browse_docs(): + """ + Open the current dev docs in a browser tab. + """ + local("open docs/_build/html/index.html") + @hosts(docs_host) From 05aca970e70138bd608f92883c24b471c2067fda Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 21 Jun 2011 17:44:46 -0700 Subject: [PATCH 100/126] Bugfix: excluding hosts broke remainder tasks --- fabric/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fabric/main.py b/fabric/main.py index 041aabc320..01dae62420 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -695,7 +695,7 @@ def main(): if remainder_command: r = '' state.commands[r] = lambda: api.run(remainder_command) - commands_to_run.append((r, [], {}, [], [])) + commands_to_run.append((r, [], {}, [], [], [])) if state.output.debug: names = ", ".join(x[0] for x in commands_to_run) From d68b7acd01cce9f734653144a586cc976ee84e15 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 21 Jun 2011 17:53:17 -0700 Subject: [PATCH 101/126] Implements #353: SSH keepalive. Based on a cherry-pick of 81e57d7331a036d0802619825b918ceecd80aa54 --- AUTHORS | 1 + docs/changes/1.1.rst | 3 +++ docs/usage/env.rst | 14 ++++++++++++++ docs/usage/fab.rst | 6 ++++++ fabric/network.py | 5 +++++ fabric/state.py | 8 +++++++- 6 files changed, 36 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index eb40f6a3db..2cbeab7b46 100644 --- a/AUTHORS +++ b/AUTHORS @@ -43,3 +43,4 @@ Roman Imankulov Rodrigue Alcazar Jeremy Avnet Matt Chisholm +Mark Merritt diff --git a/docs/changes/1.1.rst b/docs/changes/1.1.rst index f350cf815b..dfe9eded16 100644 --- a/docs/changes/1.1.rst +++ b/docs/changes/1.1.rst @@ -53,6 +53,9 @@ Feature additions application of env var settings to an entire function, as an alternative to using the `~fabric.context_managers.settings` context manager. Thanks to Travis Swicegood for the patch. +* :issue:`353`: Added :option:`--keepalive`/:ref:`env.keepalive ` to + allow specification of an SSH keepalive parameter for troublesome network + connections. Thanks to Mark Merritt for catch & patch. Bugfixes ======== diff --git a/docs/usage/env.rst b/docs/usage/env.rst index 18004a50b1..15a413e924 100644 --- a/docs/usage/env.rst +++ b/docs/usage/env.rst @@ -259,6 +259,20 @@ The global host list used when composing per-task host lists. .. seealso:: :doc:`execution` +.. _keepalive: + +``keepalive`` +------------- + +**Default:** ``0`` (i.e. no keepalive) + +An integer specifying an SSH keepalive interval to use; basically maps to the +SSH config option ``ClientAliveInterval``. Useful if you find connections are +timing out due to meddlesome network hardware or what have you. + +.. seealso:: :option:`--keepalive` +.. versionadded:: 1.1 + .. _key-filename: ``key_filename`` diff --git a/docs/usage/fab.rst b/docs/usage/fab.rst index 088c239047..96d48de9a2 100644 --- a/docs/usage/fab.rst +++ b/docs/usage/fab.rst @@ -168,6 +168,12 @@ below. .. versionadded:: 0.9.1 +.. cmdoption:: --keepalive=KEEPALIVE + + Sets :ref:`env.keepalive ` to the given (integer) value, specifying an SSH keepalive interval. + + .. versionadded:: 1.1 + .. cmdoption:: -l, --list Imports a fabfile as normal, but then prints a list of all discovered tasks diff --git a/fabric/network.py b/fabric/network.py index 37874b9b31..79093e86f8 100644 --- a/fabric/network.py +++ b/fabric/network.py @@ -170,6 +170,11 @@ def connect(user, host, port): look_for_keys=not env.no_keys ) connected = True + + # set a keepalive if desired + if env.keepalive: + client.get_transport().set_keepalive(env.keepalive) + return client # BadHostKeyException corresponds to key mismatch, i.e. what on the # command line results in the big banner error about man-in-the-middle diff --git a/fabric/state.py b/fabric/state.py index 2ca0649300..859aeb237a 100644 --- a/fabric/state.py +++ b/fabric/state.py @@ -217,7 +217,13 @@ def _rc_path(): help="Abort instead of prompting (for password, host, etc)" ), - + # Keepalive + make_option('--keepalive', + dest='keepalive', + type=int, + default=0, + help="enables a keepalive every n seconds" + ), ] From a4a6b08f9d4f467670709ec055fa4194a6ab132c Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 21 Jun 2011 16:15:46 -0700 Subject: [PATCH 102/126] Make doc browsing its own task --- fabfile.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/fabfile.py b/fabfile.py index d9bc12b634..98fedd32ad 100644 --- a/fabfile.py +++ b/fabfile.py @@ -39,9 +39,18 @@ def build_docs(clean='no', browse='no'): if clean.lower() in ['yes', 'y']: c = "clean " b = "" + with lcd('docs'): + local('make %shtml%s' % (c, b)) if browse.lower() in ['yes', 'y']: - b = " && open _build/html/index.html" - local('cd docs; make %shtml%s' % (c, b)) + browse_docs() + + +def browse_docs(): + """ + Open the current dev docs in a browser tab. + """ + local("open docs/_build/html/index.html") + @hosts(docs_host) From 0034a82e73c79dbde097f9c5b15a2a41137c03ad Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 21 Jun 2011 20:22:34 -0700 Subject: [PATCH 103/126] Reorder eq_ args --- tests/test_main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 0228756fdc..11b1193ee4 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -310,7 +310,7 @@ def test_implicit_discovery(): implicit = fabfile("implicit_fabfile.py") with path_prefix(implicit): docs, funcs = load_fabfile(implicit) - ok_(len(funcs) == 2) + eq_(len(funcs), 2) ok_("foo" in funcs) ok_("bar" in funcs) @@ -322,7 +322,7 @@ def test_explicit_discovery(): explicit = fabfile("explicit_fabfile.py") with path_prefix(explicit): docs, funcs = load_fabfile(explicit) - ok_(len(funcs) == 1) + eq_(len(funcs), 1) ok_("foo" in funcs) ok_("bar" not in funcs) @@ -334,7 +334,7 @@ def test_should_load_decorated_tasks_only_if_one_is_found(): module = fabfile('decorated_fabfile.py') with path_prefix(module): docs, funcs = load_fabfile(module) - eq_(1, len(funcs)) + eq_(len(funcs), 1) ok_('foo' in funcs) @@ -345,7 +345,7 @@ def test_class_based_tasks_are_found_with_proper_name(): module = fabfile('decorated_fabfile_with_classbased_task.py') with path_prefix(module): docs, funcs = load_fabfile(module) - eq_(1, len(funcs)) + eq_(len(funcs), 1) ok_('foo' in funcs) From 8905aafe5777254776ff03018121714a521686fd Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 21 Jun 2011 20:59:18 -0700 Subject: [PATCH 104/126] Use FabricTest to isolate namespace tests --- tests/test_main.py | 142 +++++++++++++++++++++++---------------------- 1 file changed, 73 insertions(+), 69 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 11b1193ee4..cd056df0b3 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -15,7 +15,7 @@ import fabric.state from fabric.state import _AttributeDict -from utils import mock_streams, patched_env, eq_ +from utils import mock_streams, patched_env, eq_, FabricTest import os import sys @@ -302,74 +302,78 @@ def path_prefix(module): yield sys.path.pop(i) - -def test_implicit_discovery(): - """ - Default to automatically collecting all tasks in a fabfile module - """ - implicit = fabfile("implicit_fabfile.py") - with path_prefix(implicit): - docs, funcs = load_fabfile(implicit) - eq_(len(funcs), 2) - ok_("foo" in funcs) - ok_("bar" in funcs) - - -def test_explicit_discovery(): - """ - If __all__ is present, only collect the tasks it specifies - """ - explicit = fabfile("explicit_fabfile.py") - with path_prefix(explicit): - docs, funcs = load_fabfile(explicit) - eq_(len(funcs), 1) - ok_("foo" in funcs) - ok_("bar" not in funcs) - - -def test_should_load_decorated_tasks_only_if_one_is_found(): - """ - If any new-style tasks are found, *only* new-style tasks should load - """ - module = fabfile('decorated_fabfile.py') - with path_prefix(module): - docs, funcs = load_fabfile(module) - eq_(len(funcs), 1) - ok_('foo' in funcs) - - -def test_class_based_tasks_are_found_with_proper_name(): - """ - Wrapped new-style tasks should preserve their function names - """ - module = fabfile('decorated_fabfile_with_classbased_task.py') - with path_prefix(module): - docs, funcs = load_fabfile(module) - eq_(len(funcs), 1) - ok_('foo' in funcs) - - -def test_recursion_steps_into_nontask_modules(): - """ - Recursive loading will continue through modules with no tasks - """ - module = fabfile('deep') - with path_prefix(module): - docs, funcs = load_fabfile(module) - print funcs - eq_(len(funcs), 1) - ok_('submodule.subsubmodule.deeptask' in _task_names(funcs)) - - -def test_newstyle_task_presence_skips_classic_task_modules(): - """ - Classic-task-only modules shouldn't add tasks if any new-style tasks exist - """ - module = fabfile('deep') - with path_prefix(module): - docs, funcs = load_fabfile(module) - eq_(len(funcs), 1) - ok_('submodule.classic_task' not in _task_names(funcs)) +class TestNamespaces(FabricTest): + def setup(self): + # Parent class preserves current env + super(TestNamespaces, self).setup() + # Reset new-style-tests flag so running tests via Fab itself doesn't + # muck with it. + import fabric.state + if 'new_style_tasks' in fabric.state.env: + del fabric.state.env['new_style_tasks'] + + def test_implicit_discovery(self): + """ + Default to automatically collecting all tasks in a fabfile module + """ + implicit = fabfile("implicit_fabfile.py") + with path_prefix(implicit): + docs, funcs = load_fabfile(implicit) + eq_(len(funcs), 2) + ok_("foo" in funcs) + ok_("bar" in funcs) + + def test_explicit_discovery(self): + """ + If __all__ is present, only collect the tasks it specifies + """ + explicit = fabfile("explicit_fabfile.py") + with path_prefix(explicit): + docs, funcs = load_fabfile(explicit) + eq_(len(funcs), 1) + ok_("foo" in funcs) + ok_("bar" not in funcs) + + def test_should_load_decorated_tasks_only_if_one_is_found(self): + """ + If any new-style tasks are found, *only* new-style tasks should load + """ + module = fabfile('decorated_fabfile.py') + with path_prefix(module): + docs, funcs = load_fabfile(module) + eq_(len(funcs), 1) + ok_('foo' in funcs) + + def test_class_based_tasks_are_found_with_proper_name(self): + """ + Wrapped new-style tasks should preserve their function names + """ + module = fabfile('decorated_fabfile_with_classbased_task.py') + from fabric.state import env + with path_prefix(module): + docs, funcs = load_fabfile(module) + eq_(len(funcs), 1) + ok_('foo' in funcs) + + def test_recursion_steps_into_nontask_modules(self): + """ + Recursive loading will continue through modules with no tasks + """ + module = fabfile('deep') + with path_prefix(module): + docs, funcs = load_fabfile(module) + eq_(len(funcs), 1) + ok_('submodule.subsubmodule.deeptask' in _task_names(funcs)) + + def test_newstyle_task_presence_skips_classic_task_modules(self): + """ + Classic-task-only modules shouldn't add tasks if any new-style tasks exist + """ + module = fabfile('deep') + with path_prefix(module): + docs, funcs = load_fabfile(module) + eq_(len(funcs), 1) + ok_('submodule.classic_task' not in _task_names(funcs)) # From 465f8130f13ed7b989065e20909b8f381930e93e Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 21 Jun 2011 21:58:18 -0700 Subject: [PATCH 105/126] Re #56, don't allow leaf classic modules to pollute new-style trees --- fabric/main.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fabric/main.py b/fabric/main.py index 01dae62420..9871087cec 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -163,10 +163,11 @@ def load_fabfile(path, importer=None): del sys.path[0] # Actually load tasks - ret = load_tasks_from_module(imported) + docstring, new_style, classic = load_tasks_from_module(imported) + tasks = new_style if state.env.new_style_tasks else classic # Clean up after ourselves _seen.clear() - return ret + return docstring, tasks def load_tasks_from_module(imported): @@ -184,8 +185,7 @@ def load_tasks_from_module(imported): # dictionary of callables only (and don't include Fab operations or # underscored callables) new_style, classic = extract_tasks(imported_vars) - tasks = new_style if state.env.new_style_tasks else classic - return imported.__doc__, tasks + return imported.__doc__, new_style, classic def extract_tasks(imported_vars): @@ -204,8 +204,8 @@ def extract_tasks(imported_vars): elif is_task(tup): classic_tasks[name] = obj elif is_task_module(obj): - module_docs, module_tasks = load_tasks_from_module(obj) - for task_name, task in module_tasks.items(): + docs, newstyle, classic = load_tasks_from_module(obj) + for task_name, task in newstyle.items(): new_style_tasks[name][task_name] = task return (new_style_tasks, classic_tasks) From 7aff0aff1c7340f3cd86ea03b82f9d3e8d3ec84c Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 21 Jun 2011 22:00:22 -0700 Subject: [PATCH 106/126] Dogfooding: use new-style tasks, namespaces in core fabfile --- fabfile.py => fabfile/__init__.py | 53 ++++++++----------------------- fabfile/docs.py | 41 ++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 40 deletions(-) rename fabfile.py => fabfile/__init__.py (80%) create mode 100644 fabfile/docs.py diff --git a/fabfile.py b/fabfile/__init__.py similarity index 80% rename from fabfile.py rename to fabfile/__init__.py index 98fedd32ad..c0739b416d 100644 --- a/fabfile.py +++ b/fabfile/__init__.py @@ -7,16 +7,16 @@ import nose from fabric.api import * -from fabric.contrib.project import rsync_project # Need to import this as fabric.version for reload() purposes import fabric.version # But nothing is stopping us from making a convenient binding! _version = fabric.version.get_version -docs_host = 'jforcier@fabfile.org' +import docs +@task def test(args=None): """ Run all unit tests and doctests. @@ -31,42 +31,10 @@ def test(args=None): abort("Nose encountered an error; you may be missing newly added test dependencies. Try running 'pip install -r requirements.txt'.") -def build_docs(clean='no', browse='no'): - """ - Generate the Sphinx documentation. - """ - c = "" - if clean.lower() in ['yes', 'y']: - c = "clean " - b = "" - with lcd('docs'): - local('make %shtml%s' % (c, b)) - if browse.lower() in ['yes', 'y']: - browse_docs() - - -def browse_docs(): - """ - Open the current dev docs in a browser tab. - """ - local("open docs/_build/html/index.html") - - - -@hosts(docs_host) -def push_docs(): - """ - Build docs and zip for upload to RTD - """ - build_docs(clean='yes') - v = _version('short') - local("cd docs/_build/html && zip -r ../%s.zip ." % v) - - -def _code_version_is_tagged(): +def code_version_is_tagged(): return local('git tag | egrep "^%s$"' % _version('short')) -def _update_code_version(force): +def update_code_version(force): """ Update version data structure in-code and commit that change to git. @@ -85,12 +53,14 @@ def _update_code_version(force): local("git add %s" % version_file) local("git commit -m \"Cut %s\"" % _version('verbose')) -def _commits_since_tag(): +def commits_since_tag(): """ Has any work been done since the last tag? """ return local("git log %s.." % _version('short')) + +@task def tag(force='no', push='no'): """ Tag a new release. @@ -110,12 +80,12 @@ def tag(force='no', push='no'): # Does the current in-code version exist as a Git tag already? # If so, this means we haven't updated the in-code version specifier # yet, and need to do so. - if _code_version_is_tagged(): + if code_version_is_tagged(): # That is, if any work has been done since. Sanity check! - if not _commits_since_tag() and not force: + if not commits_since_tag() and not force: abort("No work done since last tag!") # Open editor, update version, commit that change to Git. - _update_code_version(force) + update_code_version(force) # If the tag doesn't exist, the user has already updated version info # and we can just move on. else: @@ -133,6 +103,7 @@ def tag(force='no', push='no'): local("git push origin %s" % _version('short')) +@task def build(): """ Build (but don't upload) via setup.py @@ -140,6 +111,7 @@ def build(): local('python setup.py sdist') +@task def upload(): """ Build, register and upload to PyPI @@ -147,6 +119,7 @@ def upload(): local('python setup.py sdist register upload') +@task def release(force='no'): """ Tag/push, build, upload new version and build/upload documentation. diff --git a/fabfile/docs.py b/fabfile/docs.py new file mode 100644 index 0000000000..49a0d5688e --- /dev/null +++ b/fabfile/docs.py @@ -0,0 +1,41 @@ +from __future__ import with_statement + +from fabric.api import * +from fabric.contrib.project import rsync_project + + +docs_host = 'jforcier@fabfile.org' + + +@task +def build(clean='no', browse_='no'): + """ + Generate the Sphinx documentation. + """ + c = "" + if clean.lower() in ['yes', 'y']: + c = "clean " + b = "" + with lcd('docs'): + local('make %shtml%s' % (c, b)) + if browse_.lower() in ['yes', 'y']: + browse() + + +@task +def browse(): + """ + Open the current dev docs in a browser tab. + """ + local("open docs/_build/html/index.html") + + +@hosts(docs_host) +@task +def push(): + """ + Build docs and zip for upload to RTD + """ + build_docs(clean='yes') + v = _version('short') + local("cd docs/_build/html && zip -r ../%s.zip ." % v) From 3f5503e4e21e5e88c0b39fb159fff315bf41dc98 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 23 Jun 2011 17:19:16 -0700 Subject: [PATCH 107/126] Fixes #329: reboot() broken for partial host strings Conflicts: fabric/network.py --- docs/changes/0.9.7.rst | 10 ++++++++++ fabric/network.py | 14 +++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 docs/changes/0.9.7.rst diff --git a/docs/changes/0.9.7.rst b/docs/changes/0.9.7.rst new file mode 100644 index 0000000000..a13c1b69e4 --- /dev/null +++ b/docs/changes/0.9.7.rst @@ -0,0 +1,10 @@ +======================== +Changes in version 0.9.7 +======================== + +The following changes were implemented in Fabric 0.9.7: + +Bugfixes +======== + +* :issue:`329`: `~fabric.operations.reboot` would have problems reconnecting post-reboot (resulting in a traceback) if ``env.host_string`` was not fully-formed (did not contain user and port specifiers.) This has been fixed. diff --git a/fabric/network.py b/fabric/network.py index e12203cd83..154f55509d 100644 --- a/fabric/network.py +++ b/fabric/network.py @@ -68,9 +68,14 @@ def __getitem__(self, key): # Return the value either way return dict.__getitem__(self, real_key) + def __setitem__(self, key, value): + return dict.__setitem__(self, normalize_to_string(key), value) + def __delitem__(self, key): - return dict.__delitem__(self, join_host_strings(*normalize(key))) + return dict.__delitem__(self, normalize_to_string(key)) + def __contains__(self, key): + return dict.__contains__(self, normalize_to_string(key)) def normalize(host_string, omit_port=False): """ @@ -127,6 +132,13 @@ def join_host_strings(user, host, port=None): return "%s@%s%s" % (user, host, port_string) +def normalize_to_string(host_string): + """ + normalize() returns a tuple; this returns another valid host string. + """ + return join_host_strings(*normalize(host_string)) + + def connect(user, host, port): """ Create and return a new SSHClient instance connected to given host. From 6f8a80ae8f1dfa00a90ceb04ff6d4d9961b70a73 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 23 Jun 2011 18:01:27 -0700 Subject: [PATCH 108/126] Note that 1.0.2 will contain 0.9.7 --- docs/changes/1.0.2.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changes/1.0.2.rst b/docs/changes/1.0.2.rst index e267c405e3..f12b00f8f3 100644 --- a/docs/changes/1.0.2.rst +++ b/docs/changes/1.0.2.rst @@ -2,6 +2,10 @@ Changes in version 1.0.2 ======================== +.. note:: + This release also includes all applicable changes from the :doc:`0.9.7 + release `. + Bugfixes ======== From 9be5e0385dfe462c1e1db06b7d64e17c9ff248f0 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 23 Jun 2011 18:17:44 -0700 Subject: [PATCH 109/126] Add 1.0.2 note to 1.1 release docs --- docs/changes/1.1.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changes/1.1.rst b/docs/changes/1.1.rst index dfe9eded16..14fe7e228a 100644 --- a/docs/changes/1.1.rst +++ b/docs/changes/1.1.rst @@ -4,6 +4,9 @@ Changes in version 1.1 This page lists all changes made to Fabric in its 1.1.0 release. +.. note:: + This release also includes all applicable changes from the :doc:`1.0.2 + release `. Highlights ========== From 3c56186e95cff980571a37e2e771e0ff4c8cc243 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 23 Jun 2011 18:31:54 -0700 Subject: [PATCH 110/126] Add test re #329 Conflicts: tests/test_network.py --- tests/test_network.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/test_network.py b/tests/test_network.py index a6b10146e2..e063cf81d4 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -6,7 +6,7 @@ import sys import paramiko -from nose.tools import with_setup +from nose.tools import with_setup, ok_ from fudge import (Fake, clear_calls, clear_expectations, patch_object, verify, with_patched_object, patched_context, with_fakes) @@ -137,6 +137,25 @@ def test_connection_caching(self): yield TestNetwork.check_connection_calls, host_strings, num_calls + def test_connection_cache_deletion(self): + """ + HostConnectionCache should delete correctly w/ non-full keys + """ + hcc = HostConnectionCache() + fake = Fake('connect', callable=True) + with patched_context('fabric.network', 'connect', fake): + for host_string in ('hostname', 'user@hostname', + 'user@hostname:222'): + # Prime + hcc[host_string] + # Test + ok_(host_string in hcc) + # Delete + del hcc[host_string] + # Test + ok_(host_string not in hcc) + + # # Connection loop flow # From 835a094b7ec8a0b16727c4b5a7bceb9e7478ebc3 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 23 Jun 2011 20:07:34 -0700 Subject: [PATCH 111/126] Formatting --- docs/changes/1.0.2.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changes/1.0.2.rst b/docs/changes/1.0.2.rst index f12b00f8f3..c64ee7be0f 100644 --- a/docs/changes/1.0.2.rst +++ b/docs/changes/1.0.2.rst @@ -25,4 +25,6 @@ Documentation * Updated the API documentation for `~fabric.context_managers.cd` to explicitly point users to `~fabric.context_managers.lcd` for modifying local paths. -* Clarified the behavior of `~fabric.contrib.project.rsync_project` re: how trailing slashes in ``local_dir`` affect ``remote_dir``. Thanks to Mark Merritt for the catch. +* Clarified the behavior of `~fabric.contrib.project.rsync_project` re: how + trailing slashes in ``local_dir`` affect ``remote_dir``. Thanks to Mark + Merritt for the catch. From efd9fef820dd6f2a593c172eac7e500c5e8f0602 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 23 Jun 2011 20:08:31 -0700 Subject: [PATCH 112/126] Fix I/O race condition Fixes #352, fixes #320 --- AUTHORS | 1 + docs/changes/1.0.2.rst | 5 +++++ fabric/io.py | 4 +++- fabric/state.py | 4 +++- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index b004af8196..bc60a10c3d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -34,3 +34,4 @@ Paul Smith Rick Harding Kirill Pinchuk Roman Imankulov +Max Arnold diff --git a/docs/changes/1.0.2.rst b/docs/changes/1.0.2.rst index c64ee7be0f..7e7cacf05f 100644 --- a/docs/changes/1.0.2.rst +++ b/docs/changes/1.0.2.rst @@ -18,6 +18,11 @@ Bugfixes Thanks to Matthew Woodcraft and Connor Smith for the catch. * :issue:`337`: Fix logic bug in `~fabric.operations.put` preventing use of ``mirror_local_mode``. Thanks to Roman Imankulov for catch & patch. +* :issue:`352` (also :issue:`320`): Seemingly random issues with output lockup + and input problems (e.g. sudo prompts incorrectly rejecting passwords) appear + to have been caused by an I/O race condition. This has been fixed. Thanks to + Max Arnold and Paul Oswald for the detailed reports and to Max for the + diagnosis and patch. Documentation diff --git a/fabric/io.py b/fabric/io.py index f8e434cab7..fa813b1648 100644 --- a/fabric/io.py +++ b/fabric/io.py @@ -94,9 +94,11 @@ def output_loop(chan, which, capture): # Prompt for, and store, password. Give empty prompt so the # initial display "hides" just after the actually-displayed # prompt from the remote end. + chan.input_enabled = False password = fabric.network.prompt_for_password( prompt=" ", no_colon=True, stream=pipe ) + chan.input_enabled = True # Update env.password, env.passwords if necessary set_password(password) # Reset reprompt flag @@ -117,7 +119,7 @@ def input_loop(chan, using_pty): else: r, w, x = select([sys.stdin], [], [], 0.0) have_char = (r and r[0] == sys.stdin) - if have_char: + if have_char and chan.input_enabled: # Send all local stdin to remote end's stdin byte = msvcrt.getch() if win32 else sys.stdin.read(1) chan.sendall(byte) diff --git a/fabric/state.py b/fabric/state.py index 343510c91c..b3d76ed880 100644 --- a/fabric/state.py +++ b/fabric/state.py @@ -271,7 +271,9 @@ def default_channel(): """ Return a channel object based on ``env.host_string``. """ - return connections[env.host_string].get_transport().open_session() + chan = connections[env.host_string].get_transport().open_session() + chan.input_enabled = True + return chan # From 7f6cbc9d9186270100c0c883a98ae7a37cc52846 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 23 Jun 2011 22:19:41 -0700 Subject: [PATCH 113/126] Fixes #345, contains() returns boolean, not retval. --- AUTHORS | 1 + docs/changes/1.1.rst | 4 ++++ fabric/contrib/files.py | 2 +- tests/test_contrib.py | 17 ++++++++++++++++- 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index b1d490e86b..0de4027d1d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -45,3 +45,4 @@ Jeremy Avnet Matt Chisholm Mark Merritt Max Arnold +Szymon Reichmann diff --git a/docs/changes/1.1.rst b/docs/changes/1.1.rst index 14fe7e228a..bcd8d31d2e 100644 --- a/docs/changes/1.1.rst +++ b/docs/changes/1.1.rst @@ -69,6 +69,10 @@ Bugfixes 'c']``) will now always run on ``a``, then ``b``, then ``c``. Previously, there was a chance the order could get mixed up during deduplication. Thanks to Rohit Aggarwal for the report. +* :issue:`345`: `~fabric.contrib.files.contains` returned the stdout of its + internal ``grep`` command instead of success/failure, causing incorrect + behavior when stderr exists and is combined with stdout. This has been + corrected. Thanks to Szymon Reichmann for catch and patch. Documentation updates ===================== diff --git a/fabric/contrib/files.py b/fabric/contrib/files.py index 8d56e8f5ee..fe0f006a32 100644 --- a/fabric/contrib/files.py +++ b/fabric/contrib/files.py @@ -283,7 +283,7 @@ def contains(filename, text, exact=False, use_sudo=False): return func('egrep "%s" "%s"' % ( text.replace('"', r'\"'), filename.replace('"', r'\"') - )) + )).succeeded def append(filename, text, use_sudo=False, partial=False, escape=True): diff --git a/tests/test_contrib.py b/tests/test_contrib.py index 6d53d3b123..b64327b81a 100644 --- a/tests/test_contrib.py +++ b/tests/test_contrib.py @@ -1,7 +1,7 @@ from __future__ import with_statement from fabric.api import hide, get, show -from fabric.contrib.files import upload_template +from fabric.contrib.files import upload_template, contains from utils import FabricTest, eq_contents from server import server @@ -34,3 +34,18 @@ def test_upload_template_handles_file_destination(self): upload_template(template, remote, {'varname': var}) get(remote, local) eq_contents(local, var) + + @server(responses={ + 'egrep "text" "/file.txt"': ( + "sudo: unable to resolve host fabric", + "", + 1 + )} + ) + def test_contains_checks_only_succeeded_flag(self): + """ + contains() should return False on bad grep even if stdout isn't empty + """ + with hide('everything'): + result = contains('/file.txt', 'text', use_sudo=True) + assert result == False From 0d74105075697c8b1f44ce80f19a658bbe0a99f9 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 24 Jun 2011 16:24:33 -0700 Subject: [PATCH 114/126] Silly/shitty little sanity-test runner --- tests/integration.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 tests/integration.py diff --git a/tests/integration.py b/tests/integration.py new file mode 100644 index 0000000000..1a264651c5 --- /dev/null +++ b/tests/integration.py @@ -0,0 +1,28 @@ +# "Integration test" for Fabric to be run occasionally / before releasing. +# Executes idempotent/nonthreatening commands against localhost by default. + +from __future__ import with_statement + +from fabric.api import * + + +@hosts('localhost') +def test(): + flags = (True, False) + funcs = (run, sudo) + cmd = "ls /" + line = "#" * 72 + for shell in flags: + for pty in flags: + for combine_stderr in flags: + for func in funcs: + print(">>> %s(%s, shell=%s, pty=%s, combine_stderr=%s)" % ( + func.func_name, cmd, shell, pty, combine_stderr)) + print(line) + func( + cmd, + shell=shell, + pty=pty, + combine_stderr=combine_stderr + ) + print(line + "\n") From 0de7dcafd0533fc9c4ef2be8bc5b2708f9ed8d8e Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 24 Jun 2011 16:29:19 -0700 Subject: [PATCH 115/126] Version bump for 1.0.2 --- fabric/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fabric/version.py b/fabric/version.py index eef4b2f7f0..85f53b3344 100644 --- a/fabric/version.py +++ b/fabric/version.py @@ -21,7 +21,7 @@ def git_sha(): return p.communicate()[0] -VERSION = (1, 0, 2, 'alpha', 0) +VERSION = (1, 0, 2, 'final', 0) def get_version(form='short'): """ From ef2cc5fcddedc8c3557b4f819ac945aeebbd042f Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 24 Jun 2011 22:52:04 -0700 Subject: [PATCH 116/126] Dev version for 1.1.1 --- fabric/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fabric/version.py b/fabric/version.py index ec49a51c7f..f400bc5d8f 100644 --- a/fabric/version.py +++ b/fabric/version.py @@ -21,7 +21,7 @@ def git_sha(): return p.communicate()[0] -VERSION = (1, 1, 0, 'final', 0) +VERSION = (1, 1, 1, 'alpha', 0) def get_version(form='short'): From 2adbf18f954f2b46cdfe66e6a9488c7e96bec6e2 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 24 Jun 2011 22:52:24 -0700 Subject: [PATCH 117/126] Dev version for 1.2 --- fabric/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fabric/version.py b/fabric/version.py index ec49a51c7f..bf49200cf2 100644 --- a/fabric/version.py +++ b/fabric/version.py @@ -21,7 +21,7 @@ def git_sha(): return p.communicate()[0] -VERSION = (1, 1, 0, 'final', 0) +VERSION = (1, 2, 0, 'alpha', 0) def get_version(form='short'): From 3ff8b55a973f8a1e862b5ed0c4be357284fbd59c Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 28 Jun 2011 18:41:22 -0700 Subject: [PATCH 118/126] Enhance docs on Task subclass usage --- docs/usage/tasks.rst | 43 ++++++++++++++++++++++++++++++++++++++++++- fabric/tasks.py | 4 ++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/docs/usage/tasks.rst b/docs/usage/tasks.rst index fdf5db018d..2080c3b62f 100644 --- a/docs/usage/tasks.rst +++ b/docs/usage/tasks.rst @@ -54,7 +54,48 @@ tasks: level. Instances' ``name`` attributes are used as the task name; if omitted the instance's variable name will be used instead. -Use of new-style tasks also allows you to set up task namespaces -- see below. +Use of new-style tasks also allows you to set up task namespaces (see below.) + +The `~fabric.decorators.task` decorator is pretty straightforward, but using `~fabric.tasks.Task` is less obvious, so we'll cover it in detail here. + + +``Task`` subclasses +------------------- + +If you're used to :ref:`classic-style tasks `, an easy way to +think about `~fabric.tasks.Task` subclasses is that their ``run`` method is +directly equivalent to a classic task; its arguments are the task arguments +(other than ``self``) and its body is what gets executed. For example, this +new-style task:: + + class MyTask(Task): + name = "deploy" + def run(self, environment, domain="whatever.com"): + run("git clone foo") + sudo("service apache2 restart") + + instance = MyTask() + +is exactly equivalent to this function-based task (which, if you dropped the +``@task``, would also be a normal classic-style task):: + + @task + def deploy(environment, domain="whatever.com"): + run("git clone foo") + sudo("service apache2 restart") + +Except, of course, that the class-based version can be subclassed, make use of +internal state, etc; and as a new-style class it allows use of :ref:`namespaces +`. + +Note how we had to instantiate an instance of our class; that's simply normal +Python object-oriented programming at work. While it's a small bit of +boilerplate right now -- for example, Fabric doesn't care about the name you +give the instantiation, only the instance's ``name`` attribute -- it's well +worth the benefit of having the power of classes available. + +We may also extend the API in the future to make this experience a bit +smoother. .. _namespaces: diff --git a/fabric/tasks.py b/fabric/tasks.py index 5b93d4a6f9..88d5cdc177 100644 --- a/fabric/tasks.py +++ b/fabric/tasks.py @@ -7,6 +7,10 @@ class Task(object): Instances of subclasses will be treated as valid tasks when present in fabfiles loaded by the :doc:`fab ` tool. + For details on how to implement and use `~fabric.tasks.Task` subclasses, + please see the usage documentation on :ref:`new-style tasks + `. + .. versionadded:: 1.1 """ name = 'undefined' From 2c7b6929776ced789e666ac547068596dcecc00b Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 28 Jun 2011 18:42:50 -0700 Subject: [PATCH 119/126] Task decorator must be first --- fabfile/docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fabfile/docs.py b/fabfile/docs.py index 49a0d5688e..2caa902b38 100644 --- a/fabfile/docs.py +++ b/fabfile/docs.py @@ -30,8 +30,8 @@ def browse(): local("open docs/_build/html/index.html") -@hosts(docs_host) @task +@hosts(docs_host) def push(): """ Build docs and zip for upload to RTD From 4b6a1f6abffb7e285d33855b476c3e8775356a06 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 28 Jun 2011 18:53:59 -0700 Subject: [PATCH 120/126] Update tag list for manually generated docs --- docs/conf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 6c64ca30c5..d34e6f6518 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -120,7 +120,8 @@ def issues_role(name, rawtext, text, lineno, inliner, options={}, content=[]): from fabric.api import local, hide with hide('everything'): - fabric_tags = local('git tag | sort -r | egrep "(0\.9|1\.0)\.."', True).split() + get_tags = 'git tag | sort -r | egrep "(0\.9|1\.[[:digit:]]+)\.."' + fabric_tags = local(get_tags, True).split() html_context = {'fabric_tags': fabric_tags} From 13b57d1943a4310c7b8a678ca911339b9712708b Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 28 Jun 2011 18:54:12 -0700 Subject: [PATCH 121/126] Fix up docs.push --- fabfile/docs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fabfile/docs.py b/fabfile/docs.py index 2caa902b38..5fbd0e710d 100644 --- a/fabfile/docs.py +++ b/fabfile/docs.py @@ -2,6 +2,7 @@ from fabric.api import * from fabric.contrib.project import rsync_project +from fabric.version import get_version docs_host = 'jforcier@fabfile.org' @@ -36,6 +37,6 @@ def push(): """ Build docs and zip for upload to RTD """ - build_docs(clean='yes') - v = _version('short') + build(clean='yes') + v = get_version('short') local("cd docs/_build/html && zip -r ../%s.zip ." % v) From 942601c42975ab9971c30d4814a2edb50ce25945 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 28 Jun 2011 19:28:13 -0700 Subject: [PATCH 122/126] Fix main loop to look for Task.run() --- docs/changes/1.1.1.rst | 19 +++++++++++++++++++ fabric/main.py | 12 ++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 docs/changes/1.1.1.rst diff --git a/docs/changes/1.1.1.rst b/docs/changes/1.1.1.rst new file mode 100644 index 0000000000..e2ef5e29e6 --- /dev/null +++ b/docs/changes/1.1.1.rst @@ -0,0 +1,19 @@ +======================== +Changes in version 1.1.1 +======================== + +Bugfixes +======== + +* The public API for `~fabric.tasks.Task` mentioned use of the ``run()`` + method, but Fabric's main execution loop had not been updated to look for and + call it, forcing users who subclassed `~fabric.tasks.Task` to define + ``__call__()`` instead. This was an oversight and has been corrected. + + +Documentation +============= + +* The documentation for use of `~fabric.tasks.Task` subclasses (contained under + :ref:`new-style-tasks`) has been slightly fleshed out and has also grown an + example snippet or two. diff --git a/fabric/main.py b/fabric/main.py index 9871087cec..9df9ebc34b 100644 --- a/fabric/main.py +++ b/fabric/main.py @@ -587,6 +587,14 @@ def update_output_levels(show, hide): state.output[key] = False +def _run_task(task, args, kwargs): + # First, try class-based tasks + if hasattr(task, 'run') and callable(task.run): + return task.run(*args, **kwargs) + # Fallback to callable behavior + return task(*args, **kwargs) + + def main(): """ Main command-line execution loop. @@ -720,12 +728,12 @@ def main(): if state.output.running: print("[%s] Executing task '%s'" % (host, name)) # Actually run command - task(*args, **kwargs) + _run_task(task, args, kwargs) # Put old user back state.env.user = prev_user # If no hosts found, assume local-only and run once if not hosts: - task(*args, **kwargs) + _run_task(task, args, kwargs) # If we got here, no errors occurred, so print a final note. if state.output.status: print("\nDone.") From 28ffb4cc4926fcfd22b989660bd39745b67d989a Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 28 Jun 2011 19:29:56 -0700 Subject: [PATCH 123/126] Remove confusing, extraneous note re: example --- docs/usage/tasks.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/usage/tasks.rst b/docs/usage/tasks.rst index 2080c3b62f..ca7eb55c6d 100644 --- a/docs/usage/tasks.rst +++ b/docs/usage/tasks.rst @@ -84,10 +84,6 @@ is exactly equivalent to this function-based task (which, if you dropped the run("git clone foo") sudo("service apache2 restart") -Except, of course, that the class-based version can be subclassed, make use of -internal state, etc; and as a new-style class it allows use of :ref:`namespaces -`. - Note how we had to instantiate an instance of our class; that's simply normal Python object-oriented programming at work. While it's a small bit of boilerplate right now -- for example, Fabric doesn't care about the name you From d9e75f7a2e604a741158dafc7f61921733499ad2 Mon Sep 17 00:00:00 2001 From: Ramon van Alteren Date: Wed, 31 Aug 2011 17:19:37 +0200 Subject: [PATCH 124/126] Patching the logging stream in mock_streams as well Because logging gets initialized early in the game, mock_streams is not working. The logging output goes to the actual stderr stream and not to the mocked stream. --- tests/utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/utils.py b/tests/utils.py index f259b36032..96969e6ea8 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -21,6 +21,7 @@ from fabric.state import env, output from fabric.sftp import SFTP import fabric.network +from fabric.logger import consolelogger from server import PORT, PASSWORDS, USER, HOST @@ -126,12 +127,14 @@ def inner_wrapper(*args, **kwargs): sys.stdall = StringIO() fake_stdout = CarbonCopy(cc=sys.stdall) fake_stderr = CarbonCopy(cc=sys.stdall) + consolelogger.stream = fake_stderr else: fake_stdout, fake_stderr = StringIO(), StringIO() if stdout: my_stdout, sys.stdout = sys.stdout, fake_stdout if stderr: my_stderr, sys.stderr = sys.stderr, fake_stderr + consolelogger.stream = fake_stderr try: ret = func(*args, **kwargs) finally: @@ -139,8 +142,10 @@ def inner_wrapper(*args, **kwargs): sys.stdout = my_stdout if stderr: sys.stderr = my_stderr + consolelogger.stream = my_stderr if both: del sys.stdall + consolelogger.stream = my_stderr return inner_wrapper return mocked_streams_decorator From dc913d291022f2e51d867dbd14ef17536f504d14 Mon Sep 17 00:00:00 2001 From: Ramon van Alteren Date: Wed, 31 Aug 2011 17:23:35 +0200 Subject: [PATCH 125/126] Fixed a set of testcases In some cases there was a mismatch between what was actually printed through the logger and what was expected Some of the testcases swapped expected and actual output In addition there is a fundamental change hidden here: The console logger will output by default to stderr This is probably fine for warn/fatal etc messages, however it is a bit dubious for info messages. In some cases I had to change the stream the testcase got the results from to stdall to get it working. --- tests/test_network.py | 22 +++++++++++----------- tests/test_operations.py | 12 ++++++------ tests/test_utils.py | 36 ++++++++++++++++++++++++------------ 3 files changed, 41 insertions(+), 29 deletions(-) diff --git a/tests/test_network.py b/tests/test_network.py index 995105f137..6d40295ad8 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -10,19 +10,19 @@ from fudge import (Fake, clear_calls, clear_expectations, patch_object, verify, with_patched_object, patched_context, with_fakes) + +from utils import * +from server import (server, PORT, RESPONSES, PASSWORDS, CLIENT_PRIVKEY, USER, + CLIENT_PRIVKEY_PASSPHRASE) + from fabric.context_managers import settings, hide, show from fabric.network import (HostConnectionCache, join_host_strings, normalize, - denormalize) + denormalize) from fabric.io import output_loop import fabric.network # So I can call patch_object correctly. Sigh. from fabric.state import env, output, _get_system_username from fabric.operations import run, sudo, prompt -from utils import * -from server import (server, PORT, RESPONSES, PASSWORDS, CLIENT_PRIVKEY, USER, - CLIENT_PRIVKEY_PASSPHRASE) - - # # Subroutines, e.g. host string normalization # @@ -324,7 +324,7 @@ def _prompt_display(display_output): [%(prefix)s] Login password: [%(prefix)s] out: Sorry, try again. [%(prefix)s] out: sudo password: """ % {'prefix': env.host_string} - eq_(expected[1:], sys.stdall.getvalue()) + eq_(expected=expected[1:], result=sys.stdall.getvalue()) @mock_streams('both') @server( @@ -356,7 +356,7 @@ def test_consecutive_sudos_should_not_have_blank_line(self): [%(prefix)s] out: result1 [%(prefix)s] out: result2 """ % {'prefix': env.host_string} - eq_(expected, sys.stdall.getvalue()) + eq_(expected=expected[1:], result=sys.stdall.getvalue()) @mock_streams('both') @server(pubkeys=True, responses={'silent': '', 'normal': 'foo'}) @@ -388,7 +388,7 @@ def test_silent_commands_should_not_have_blank_line(self): [%(prefix)s] run: normal [%(prefix)s] out: foo """ % {'prefix': env.host_string} - eq_(expected=expected, result=sys.stdall.getvalue()) + eq_(expected=expected[1:], result=sys.stdall.getvalue()) @mock_streams('both') @server( @@ -416,7 +416,7 @@ def test_io_should_print_prefix_if_ouput_prefix_is_true(self): [%(prefix)s] out: result1 [%(prefix)s] out: result2 """ % {'prefix': env.host_string} - eq_(expected[1:], sys.stdall.getvalue()) + eq_(expected=expected[1:], result=sys.stdall.getvalue()) @mock_streams('both') @server( @@ -445,4 +445,4 @@ def test_io_should_not_print_prefix_if_ouput_prefix_is_false(self): result1 result2 """ % {'prefix': env.host_string} - eq_(expected[1:], sys.stdall.getvalue()) + eq_(expected=expected[1:], result=sys.stdall.getvalue()) diff --git a/tests/test_operations.py b/tests/test_operations.py index 147cec4894..ba172a7c51 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -72,7 +72,7 @@ def test_require_mixed_state_keys(): require('foo', 'version') -@mock_streams('stderr') +@mock_streams('both') def test_require_mixed_state_keys_prints_missing_only(): """ When given mixed-state keys, require() prints missing keys only @@ -80,9 +80,9 @@ def test_require_mixed_state_keys_prints_missing_only(): try: require('foo', 'version') except SystemExit: - err = sys.stderr.getvalue() - assert 'version' not in err - assert 'foo' in err + all = sys.stdall.getvalue() + assert 'version' not in all + assert 'foo' in all @mock_streams('stderr') @@ -125,7 +125,7 @@ def test_prompt_appends_space(): """ s = "This is my prompt" prompt(s) - eq_(sys.stdout.getvalue(), s + ' ') + eq_(result=sys.stdout.getvalue(), expected=s + ' ') @mock_streams('stdout') @@ -137,7 +137,7 @@ def test_prompt_with_default(): s = "This is my prompt" d = "default!" prompt(s, default=d) - eq_(sys.stdout.getvalue(), "%s [%s] " % (s, d)) + eq_(result=sys.stdout.getvalue(), expected="%s [%s] " % (s, d)) # diff --git a/tests/test_utils.py b/tests/test_utils.py index 6c8e5120f4..3d980f0611 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -11,17 +11,22 @@ from fabric.utils import warn, indent, abort, puts, fastprint from fabric import utils # For patching from fabric.context_managers import settings -from utils import mock_streams +from utils import mock_streams, eq_ -@mock_streams('stderr') +@mock_streams('both') @with_patched_object(output, 'warnings', True) def test_warn(): """ warn() should print 'Warning' plus given text """ warn("Test") - assert "\nWarning: Test\n\n" == sys.stderr.getvalue() + result = sys.stdall.getvalue() + expected = """ +Warning: Test +""" + eq_(expected=expected[1:], result=result) +# "\nWarning: Test\n\n" == "%s" % result def test_indent(): @@ -71,10 +76,14 @@ def test_abort_message(): except SystemExit: pass result = sys.stderr.getvalue() - eq_("\nFatal error: Test\n\nAborting.\n", result) + expected = """ +Fatal error: Test +Aborting. +""" + eq_(expected=expected[1:], result=result) -@mock_streams('stdout') +@mock_streams('both') def test_puts_with_user_output_on(): """ puts() should print input to sys.stdout if "user" output level is on @@ -82,20 +91,20 @@ def test_puts_with_user_output_on(): s = "string!" output.user = True puts(s, show_prefix=False) - eq_(sys.stdout.getvalue(), s + "\n") + eq_(sys.stdall.getvalue(), s + "\n") -@mock_streams('stdout') +@mock_streams('both') def test_puts_with_user_output_off(): """ puts() shouldn't print input to sys.stdout if "user" output level is off """ output.user = False puts("You aren't reading this.") - eq_(sys.stdout.getvalue(), "") + eq_(sys.stdall.getvalue(), "") -@mock_streams('stdout') +@mock_streams('both') def test_puts_with_prefix(): """ puts() should prefix output with env.host_string if non-empty @@ -104,10 +113,11 @@ def test_puts_with_prefix(): h = "localhost" with settings(host_string=h): puts(s) - eq_(sys.stdout.getvalue(), "[%s] %s" % (h, s + "\n")) + expected = "[%s] %s\n" % (h, s) + eq_(result=sys.stdall.getvalue(),expected=expected ) -@mock_streams('stdout') +@mock_streams('both') def test_puts_without_prefix(): """ puts() shouldn't prefix output with env.host_string if show_prefix is False @@ -115,7 +125,9 @@ def test_puts_without_prefix(): s = "my output" h = "localhost" puts(s, show_prefix=False) - eq_(sys.stdout.getvalue(), "%s" % (s + "\n")) + result = sys.stdall.getvalue() + expected = "%s\n" % s + eq_(expected=expected, result=result) def test_fastprint_calls_puts(): From bd8b71650dc5f5c4273bc1ac08261b7695a0249f Mon Sep 17 00:00:00 2001 From: Ramon van Alteren Date: Wed, 31 Aug 2011 17:26:00 +0200 Subject: [PATCH 126/126] Fixing up messages with different expectations in the test --- fabric/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fabric/utils.py b/fabric/utils.py index b5aeaeb8e8..f6c737fa13 100644 --- a/fabric/utils.py +++ b/fabric/utils.py @@ -21,7 +21,7 @@ def abort(msg): """ from fabric.state import output if output.aborts: - logger.error( "FATAL: %s" % msg ) + logger.error( "Fatal error: %s" % msg ) logger.error( "Aborting." ) sys.exit(1) @@ -37,7 +37,7 @@ def warn(msg): """ from fabric.state import output if output.warnings: - logger.warn( "%s" % msg ) + logger.warn( "Warning: %s" % msg ) def indent(text, spaces=4, strip=False): @@ -68,7 +68,7 @@ def indent(text, spaces=4, strip=False): return output -def puts(text, show_prefix=True, end="\n", flush=False): +def puts(text, show_prefix=True, end="", flush=False): """ An alias for ``print`` whose output is managed by Fabric's output controls.