Merge pull request #1441 from Tarrasch/dont_nest_stacktraces_for_run_…

…failures

Don't print nested stack traces for run() failures

luigi/worker.py

@@ -102,11 +102,14 @@ def __init__(self, task, worker_id, result_queue, random_seed=False, worker_time
         self.timeout_time = time.time() + worker_timeout if worker_timeout else None
 
     def _run_get_new_deps(self):
+        run_again = False
         try:
             task_gen = self.task.run(tracking_url_callback=self.tracking_url_callback)
         except TypeError as ex:
             if 'unexpected keyword argument' not in getattr(ex, 'message', ex.args[0]):
                 raise
+            run_again = True
+        if run_again:
             task_gen = self.task.run()
         if not isinstance(task_gen, types.GeneratorType):
             return None

test/cmdline_test.py

@@ -84,6 +84,11 @@ class FooSubClass(FooBaseClass):
     pass
 
 
+class ATaskThatFails(luigi.Task):
+    def run(self):
+        raise ValueError()
+
+
 class CmdlineTest(unittest.TestCase):
 
     def setUp(self):
@@ -270,6 +275,20 @@ def test_bin_mentions_misspelled_task(self):
         self.assertTrue(stderr.find(b'FooBaseClass') != -1)
         self.assertTrue(stderr.find(b'--x') != 0)
 
+    def test_stack_trace_has_no_inner(self):
+        """
+        Test that the stack trace for failing tasks are short
+
+        The stack trace shouldn't contain unreasonably much implementation
+        details of luigi In particular it should say that the task is
+        misspelled and not that the local parameters do not exist.
+        """
+        returncode, stdout, stderr = self._run_cmdline(['./bin/luigi', '--module', 'cmdline_test', 'ATaskThatFails', '--local-scheduler', '--no-lock'])
+        print(stdout)
+
+        self.assertFalse(stdout.find(b"run() got an unexpected keyword argument 'tracking_url_callback'") != -1)
+        self.assertFalse(stdout.find(b'During handling of the above exception, another exception occurred') != -1)
+
 
 if __name__ == '__main__':
     # Needed for one of the tests