From 320f1ba0958446036b42d4f2148174c9edf803de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tibor=20Cserv=C3=A1k?= Date: Fri, 29 Nov 2024 16:23:00 +0100 Subject: [PATCH] [fix] Enable and disable checkers This PR fixes the enabling and disabling checker options, focusing on resolving ambiguities in option processing. Key updates: Ambiguity handling: When ambiguous checker options are provided with `-e` or `-d` flags, an error is now raised. The user receives suggestions for specifying the option by using namespaces (checker, prefix, guideline, etc.). If a checker name matches multiple checkers as a prefix, it triggers an ambiguity error. Suggestions are provided to help users choose between the checker name or the prefix. Additional namespaces: A new `checker` namespace is created. All label types can be a namespace as well, specified with a `:` separator before the checker option. Default profile settings improvements: Only those checkers enabled that has profile:default label in the label files. Prefix-based configuration is no longer applied for the default profile, ensuring precise and predictable behavior. Getting rid off `W` prefix: It is no longer available to enable or disable a warrning with W. --- .../analyzers/clangtidy/analyzer.py | 2 + .../analyzers/clangtidy/config_handler.py | 10 - .../analyzers/config_handler.py | 116 +++++++----- analyzer/codechecker_analyzer/checkers.py | 7 +- .../tests/functional/analyze/test_analyze.py | 2 +- .../compiler_warning_wno_group.output | 4 +- .../compiler_warning_wno_simple2.output | 4 +- .../compiler_warning_wunused.output | 4 +- analyzer/tests/unit/test_checker_handling.py | 171 ++++++++++++++++-- config/labels/analyzers/clang-tidy.json | 5 +- .../functional/report_viewer_api/__init__.py | 4 +- 11 files changed, 235 insertions(+), 94 deletions(-) diff --git a/analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py b/analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py index cbc803edfc..b7f21d8eac 100644 --- a/analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py +++ b/analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py @@ -288,6 +288,8 @@ def get_analyzer_checkers(cls): ("clang-diagnostic-" + warning, "") for warning in get_warnings()) + checker_description.append(("clang-diagnostic-error", "")) + cls.__analyzer_checkers = checker_description return checker_description diff --git a/analyzer/codechecker_analyzer/analyzers/clangtidy/config_handler.py b/analyzer/codechecker_analyzer/analyzers/clangtidy/config_handler.py index 68625e126a..31a2a8ca0f 100644 --- a/analyzer/codechecker_analyzer/analyzers/clangtidy/config_handler.py +++ b/analyzer/codechecker_analyzer/analyzers/clangtidy/config_handler.py @@ -41,13 +41,3 @@ def add_checker(self, checker_name, description='', return super().add_checker(checker_name, description, state) - - def set_checker_enabled(self, checker_name, enabled=True): - """ - Enable checker, keep description if already set. - """ - if checker_name.startswith('W') or \ - checker_name.startswith('clang-diagnostic'): - self.add_checker(checker_name) - - super().set_checker_enabled(checker_name, enabled) diff --git a/analyzer/codechecker_analyzer/analyzers/config_handler.py b/analyzer/codechecker_analyzer/analyzers/config_handler.py index e701bd96af..25c47c5914 100644 --- a/analyzer/codechecker_analyzer/analyzers/config_handler.py +++ b/analyzer/codechecker_analyzer/analyzers/config_handler.py @@ -12,7 +12,6 @@ from abc import ABCMeta from enum import Enum -from string import Template import collections import platform import sys @@ -86,7 +85,7 @@ def add_checker(self, checker_name, description='', """ self.__available_checkers[checker_name] = (state, description) - def set_checker_enabled(self, checker_name, enabled=True): + def set_checker_enabled(self, checker_name, enabled=True, is_strict=False): """ Explicitly handle checker state, keep description if already set. """ @@ -94,7 +93,8 @@ def set_checker_enabled(self, checker_name, enabled=True): regex = "^" + re.escape(str(checker_name)) + "\\b.*$" for ch_name, values in self.__available_checkers.items(): - if re.match(regex, ch_name): + if (is_strict and ch_name == checker_name) \ + or (not is_strict and re.match(regex, ch_name)): _, description = values state = CheckerState.ENABLED if enabled \ else CheckerState.DISABLED @@ -118,7 +118,7 @@ def checks(self): """ return self.__available_checkers - def __gen_name_variations(self): + def __gen_name_variations(self, only_prefix=False, unique=False): """ Generate all applicable name variations from the given checker list. """ @@ -134,9 +134,9 @@ def __gen_name_variations(self): # ['misc', 'misc-dangling', 'misc-dangling-handle'] # from 'misc-dangling-handle'. v = [delim.join(parts[:(i + 1)]) for i in range(len(parts))] - reserved_names += v + reserved_names += v[:-1] if only_prefix else v - return reserved_names + return list(set(reserved_names)) if unique else reserved_names def initialize_checkers(self, checkers, @@ -184,7 +184,7 @@ def initialize_checkers(self, else: # Turn default checkers on. for checker in default_profile_checkers: - self.set_checker_enabled(checker) + self.set_checker_enabled(checker, is_strict=True) self.enable_all = enable_all # If enable_all is given, almost all checkers should be enabled. @@ -207,48 +207,68 @@ def initialize_checkers(self, self.set_checker_enabled(checker_name) # Set user defined enabled or disabled checkers from the command line. - - # Construct a list of reserved checker names. - # (It is used to check if a profile name is valid.) - reserved_names = self.__gen_name_variations() - profiles = checker_labels.get_description('profile') - guidelines = checker_labels.occurring_values('guideline') - - templ = Template("The ${entity} name '${identifier}' conflicts with a " - "checker name prefix '${identifier}'. Please use -e " - "${entity}:${identifier} to enable checkers of the " - "${identifier} ${entity} or use -e " - "prefix:${identifier} to select checkers which have " - "a name starting with '${identifier}'.") - for identifier, enabled in cmdline_enable: - if "prefix:" in identifier: - identifier = identifier.replace("prefix:", "") - self.set_checker_enabled(identifier, enabled) - - elif ':' in identifier: - for checker in checker_labels.checkers_by_labels([identifier]): - self.set_checker_enabled(checker, enabled) - - elif identifier in profiles: - if identifier in reserved_names: - LOG.error(templ.substitute(entity="profile", - identifier=identifier)) - sys.exit(1) - else: - for checker in checker_labels.checkers_by_labels( - [f'profile:{identifier}']): - self.set_checker_enabled(checker, enabled) - - elif identifier in guidelines: - if identifier in reserved_names: - LOG.error(templ.substitute(entity="guideline", - identifier=identifier)) + labels = checker_labels.labels() \ + if callable(getattr(checker_labels, 'labels', None)) \ + else ["guideline", "profile", "severity", "sei-cert"] + + if ":" in identifier: + identifier_namespace = identifier.split(":")[0] + all_namespaces = ["checker", "prefix"] + labels + + if identifier_namespace not in all_namespaces: + LOG.error("The %s namespace is not known. Please select" + "one of these existing namespace options: %s", + identifier_namespace, ", ".join(all_namespaces)) sys.exit(1) - else: - for checker in checker_labels.checkers_by_labels( - [f'guideline:{identifier}']): - self.set_checker_enabled(checker, enabled) + + identifier = identifier.split(":", 1)[1] + self.initialize_checkers_by_namespace( + identifier_namespace, identifier, enabled) else: - self.set_checker_enabled(identifier, enabled) + all_options = dict(zip(labels, map( + lambda label: checker_labels.get_description(label).keys() + if checker_labels.get_description(label) + else checker_labels.occurring_values(label), labels))) + + all_options["prefix"] = list(set(self.__gen_name_variations( + only_prefix=True, unique=True))) + + all_options["checker"] = self.__available_checkers + + possible_options = {} + for label, options in all_options.items(): + if identifier in options: + possible_options[label] = identifier + + if len(possible_options) == 1: + self.initialize_checkers_by_namespace( + *list(possible_options.items())[0], enabled) + elif len(possible_options) > 1: + error_options = ", ".join(f"{label}:{option}" + for label, option + in possible_options.items()) + + LOG.error("The %s is ambigous. Please select one of these" + " options to clarify the checker list: %s.", + identifier, error_options) + sys.exit(1) + else: + # The identifier is not known but we just pass it + # and handle it in a different section. + continue + + def initialize_checkers_by_namespace(self, + identifier_namespace, + identifier, + enabled): + if identifier_namespace == "checker": + self.set_checker_enabled(identifier, enabled, is_strict=True) + elif identifier_namespace == "prefix": + self.set_checker_enabled(identifier, enabled) + else: + checker_labels = analyzer_context.get_context().checker_labels + for checker in checker_labels.checkers_by_labels( + [f"{identifier_namespace}:{identifier}"]): + self.set_checker_enabled(checker, enabled, is_strict=True) diff --git a/analyzer/codechecker_analyzer/checkers.py b/analyzer/codechecker_analyzer/checkers.py index 23f0c401c3..8e150f05d3 100644 --- a/analyzer/codechecker_analyzer/checkers.py +++ b/analyzer/codechecker_analyzer/checkers.py @@ -19,12 +19,7 @@ def available(ordered_checkers, available_checkers): """ missing_checkers = set() for checker_name, _ in ordered_checkers: - # TODO: This label list shouldn't be hard-coded here. - if checker_name.startswith('profile:') or \ - checker_name.startswith('guideline:') or \ - checker_name.startswith('severity:') or \ - checker_name.startswith('sei-cert:') or \ - checker_name.startswith('prefix:'): + if ":" in checker_name: continue name_match = False diff --git a/analyzer/tests/functional/analyze/test_analyze.py b/analyzer/tests/functional/analyze/test_analyze.py index be90f4cbd2..8e21b7d5c5 100644 --- a/analyzer/tests/functional/analyze/test_analyze.py +++ b/analyzer/tests/functional/analyze/test_analyze.py @@ -886,7 +886,7 @@ def test_disable_all_warnings(self): analyze_cmd = [self._codechecker_cmd, "check", "-l", build_json, "--analyzers", "clang-tidy", "-d", "clang-diagnostic", - "-e", "clang-diagnostic-unused"] + "-e", "prefix:clang-diagnostic-unused"] source_file = os.path.join(self.test_dir, "compiler_warning.c") build_log = [{"directory": self.test_workspace, diff --git a/analyzer/tests/functional/analyze_and_parse/test_files/compiler_warning_wno_group.output b/analyzer/tests/functional/analyze_and_parse/test_files/compiler_warning_wno_group.output index a559c8c8da..f08ce4d96e 100644 --- a/analyzer/tests/functional/analyze_and_parse/test_files/compiler_warning_wno_group.output +++ b/analyzer/tests/functional/analyze_and_parse/test_files/compiler_warning_wno_group.output @@ -1,7 +1,7 @@ NORMAL#CodeChecker log --output $LOGFILE$ --build "make compiler_warning_wno_group" --quiet -NORMAL#CodeChecker analyze $LOGFILE$ --output $OUTPUT$ --analyzers clang-tidy -e clang-diagnostic-unused +NORMAL#CodeChecker analyze $LOGFILE$ --output $OUTPUT$ --analyzers clang-tidy -e prefix:clang-diagnostic-unused NORMAL#CodeChecker parse $OUTPUT$ -CHECK#CodeChecker check --build "make compiler_warning_wno_group" --output $OUTPUT$ --quiet --analyzers clang-tidy -e clang-diagnostic-unused +CHECK#CodeChecker check --build "make compiler_warning_wno_group" --output $OUTPUT$ --quiet --analyzers clang-tidy -e prefix:clang-diagnostic-unused -------------------------------------------------------------------------------- [] - Starting build... [] - Using CodeChecker ld-logger. diff --git a/analyzer/tests/functional/analyze_and_parse/test_files/compiler_warning_wno_simple2.output b/analyzer/tests/functional/analyze_and_parse/test_files/compiler_warning_wno_simple2.output index 1704a5155a..d659feafbe 100644 --- a/analyzer/tests/functional/analyze_and_parse/test_files/compiler_warning_wno_simple2.output +++ b/analyzer/tests/functional/analyze_and_parse/test_files/compiler_warning_wno_simple2.output @@ -1,7 +1,7 @@ NORMAL#CodeChecker log --output $LOGFILE$ --build "make compiler_warning_unused" --quiet -NORMAL#CodeChecker analyze $LOGFILE$ --output $OUTPUT$ --analyzers clang-tidy -d clang-diagnostic-unused +NORMAL#CodeChecker analyze $LOGFILE$ --output $OUTPUT$ --analyzers clang-tidy -d prefix:clang-diagnostic-unused NORMAL#CodeChecker parse $OUTPUT$ -CHECK#CodeChecker check --build "make compiler_warning_unused" --output $OUTPUT$ --quiet --analyzers clang-tidy -d clang-diagnostic-unused +CHECK#CodeChecker check --build "make compiler_warning_unused" --output $OUTPUT$ --quiet --analyzers clang-tidy -d prefix:clang-diagnostic-unused -------------------------------------------------------------------------------- [] - Starting build... [] - Using CodeChecker ld-logger. diff --git a/analyzer/tests/functional/analyze_and_parse/test_files/compiler_warning_wunused.output b/analyzer/tests/functional/analyze_and_parse/test_files/compiler_warning_wunused.output index 1704a5155a..d659feafbe 100644 --- a/analyzer/tests/functional/analyze_and_parse/test_files/compiler_warning_wunused.output +++ b/analyzer/tests/functional/analyze_and_parse/test_files/compiler_warning_wunused.output @@ -1,7 +1,7 @@ NORMAL#CodeChecker log --output $LOGFILE$ --build "make compiler_warning_unused" --quiet -NORMAL#CodeChecker analyze $LOGFILE$ --output $OUTPUT$ --analyzers clang-tidy -d clang-diagnostic-unused +NORMAL#CodeChecker analyze $LOGFILE$ --output $OUTPUT$ --analyzers clang-tidy -d prefix:clang-diagnostic-unused NORMAL#CodeChecker parse $OUTPUT$ -CHECK#CodeChecker check --build "make compiler_warning_unused" --output $OUTPUT$ --quiet --analyzers clang-tidy -d clang-diagnostic-unused +CHECK#CodeChecker check --build "make compiler_warning_unused" --output $OUTPUT$ --quiet --analyzers clang-tidy -d prefix:clang-diagnostic-unused -------------------------------------------------------------------------------- [] - Starting build... [] - Using CodeChecker ld-logger. diff --git a/analyzer/tests/unit/test_checker_handling.py b/analyzer/tests/unit/test_checker_handling.py index 81e36f09ef..657ae7a401 100644 --- a/analyzer/tests/unit/test_checker_handling.py +++ b/analyzer/tests/unit/test_checker_handling.py @@ -23,7 +23,7 @@ from codechecker_analyzer.analyzers.cppcheck.analyzer import Cppcheck from codechecker_analyzer.analyzers.config_handler import CheckerState from codechecker_analyzer.analyzers.clangtidy.config_handler \ - import is_compiler_warning + import is_compiler_warning, ClangTidyConfigHandler from codechecker_analyzer.arg import AnalyzerConfig, CheckerConfig from codechecker_analyzer.cmd.analyze import \ is_analyzer_config_valid, is_checker_config_valid @@ -37,14 +37,15 @@ class MockClangsaCheckerLabels: def checkers_by_labels(self, labels): if labels[0] == 'profile:default': - return ['core', 'deadcode', 'security.FloatLoopCounter'] + return ['deadcode.DeadStores', 'security.FloatLoopCounter'] if labels[0] == 'prefix:security': return ['security.insecureAPI.bzero', 'security.insecureAPI.getpw'] if labels[0] == 'profile:security': - return ['alpha.security'] + return ['alpha.security.ArrayBound', + 'alpha.security.MallocOverflow'] if labels[0] == 'profile:sensitive': return ['alpha.core.BoolAssignment', @@ -60,9 +61,14 @@ def checkers_by_labels(self, labels): def get_description(self, label): if label == 'profile': - return ['default', 'sensitive', 'security', 'portability', - 'extreme'] - return [] + return { + "default": "", + "sensitive": "", + "security": "", + "portability": "", + "extreme": "" + } + return {} def occurring_values(self, label): if label == 'guideline': @@ -141,7 +147,7 @@ def test_no_disabled_checks(self): any(arg.startswith('-analyzer-disable-checker') for arg in self.__class__.cmd)) - def test_checker_initializer(self): + def test_clangsa_checker_initializer(self): """ Test initialize_checkers() function. """ @@ -384,16 +390,23 @@ def checkers_by_labels(self, labels): return [ 'bugprone-assert-side-effect', 'bugprone-dangling-handle', - 'bugprone-inaccurate-erase'] + 'bugprone-inaccurate-erase', + 'clang-diagnostic-format', + 'clang-diagnostic-format-nonliteral', + 'clang-diagnostic-format-security'] return [] def get_description(self, label): if label == 'profile': - return ['default', 'sensitive', 'security', 'portability', - 'extreme'] - - return [] + return { + "default": "", + "sensitive": "", + "security": "", + "portability": "", + "extreme": "" + } + return {} def occurring_values(self, label): if label == 'guideline': @@ -479,6 +492,121 @@ def _is_disabled(self, checker, analyzer_cmd): return enable < disable + def test_clangtidy_checker_initializer(self): + """ + Test initialize_checkers() function. + """ + def all_with_status(status): + def f(checks, checkers): + result = set(check for check, data in checks.items() + if data[0] == status) + return set(checkers) <= result + return f + + checkers = ClangTidy.get_analyzer_checkers() + + format_prefix = "clang-diagnostic-format" + + format_matched_default_checkers = [ + "clang-diagnostic-format-nonliteral", + "clang-diagnostic-format-security" + ] + + format_matched_not_default_checkers = [ + "clang-diagnostic-format-non-iso", + "clang-diagnostic-format-pedantic", + ] + + cfg_handler = ClangTidyConfigHandler() + + # Check the ambigous option handling. + with self.assertLogs(level='ERROR') as log: + with self.assertRaises(SystemExit) as e: + cfg_handler.initialize_checkers(checkers, + [("clang-diagnostic-format", + True)]) + + err_ambigous_checker = re.compile(r"ERROR:.*?is ambigous\. Please " + r"select one of these options to " + r"clarify the checker list:.*$") + + match = err_ambigous_checker.search(log.output[0]) + + self.assertIsNotNone(match) + self.assertEqual(e.exception.code, 1) + + # Check if the specified checker and the default checkers are enabled + # when the clang-diagnostic-format is enabled by 'checker:' namespace. + cfg_handler.initialize_checkers(checkers, + [(f"checker:{format_prefix}", True)]) + self.assertIn(format_prefix, cfg_handler.checks()) + self.assertTrue(all_with_status(CheckerState.ENABLED) + (cfg_handler.checks(), [format_prefix])) + self.assertTrue(all_with_status(CheckerState.ENABLED) + (cfg_handler.checks(), + format_matched_default_checkers)) + self.assertTrue(all_with_status(CheckerState.DISABLED) + (cfg_handler.checks(), + format_matched_not_default_checkers)) + + # Check if the specified checker is the only one that enabled when the + # clang-diagnostic-format is enabled by 'checker:' namespace and the + # default profile is disabled. + cfg_handler.initialize_checkers(checkers, + [("default", False), + (f"checker:{format_prefix}", True)]) + self.assertIn(format_prefix, cfg_handler.checks()) + self.assertTrue(all_with_status(CheckerState.ENABLED) + (cfg_handler.checks(), [format_prefix])) + self.assertTrue(all_with_status(CheckerState.DISABLED) + (cfg_handler.checks(), + format_matched_default_checkers)) + self.assertTrue(all_with_status(CheckerState.DISABLED) + (cfg_handler.checks(), + format_matched_not_default_checkers)) + + # Check if the specified checker is disabled by 'checker:' namespace + # but the default profile is enabled. + cfg_handler.initialize_checkers(checkers, + [(f"checker:{format_prefix}", False)]) + self.assertIn(format_prefix, cfg_handler.checks()) + self.assertTrue(all_with_status(CheckerState.DISABLED) + (cfg_handler.checks(), [format_prefix])) + self.assertTrue(all_with_status(CheckerState.ENABLED) + (cfg_handler.checks(), + format_matched_default_checkers)) + self.assertTrue(all_with_status(CheckerState.DISABLED) + (cfg_handler.checks(), + format_matched_not_default_checkers)) + + # Check the prefix matched chackers when the 'prefix:' namespace + # enables them. + cfg_handler.initialize_checkers(checkers, + [(f"prefix:{format_prefix}", True)]) + self.assertIn(format_prefix, cfg_handler.checks()) + self.assertTrue(all_with_status(CheckerState.ENABLED) + (cfg_handler.checks(), [format_prefix])) + self.assertTrue(all_with_status(CheckerState.ENABLED) + (cfg_handler.checks(), + format_matched_default_checkers)) + self.assertTrue(all_with_status(CheckerState.ENABLED) + (cfg_handler.checks(), + format_matched_not_default_checkers)) + + # Check the prefix matched chackers when the 'prefix:' namespace + # disables them. + cfg_handler.initialize_checkers(checkers, + [(f"prefix:{format_prefix}", False)]) + self.assertIn(format_prefix, cfg_handler.checks()) + self.assertTrue(all_with_status(CheckerState.DISABLED) + (cfg_handler.checks(), [format_prefix])) + self.assertTrue(all_with_status(CheckerState.DISABLED) + (cfg_handler.checks(), + format_matched_default_checkers)) + self.assertTrue(all_with_status(CheckerState.DISABLED) + (cfg_handler.checks(), + format_matched_not_default_checkers)) + def test_disable_clangsa_checkers(self): """ Test that checker config still disables clang-analyzer-*. @@ -510,9 +638,8 @@ def test_disable_clangsa_checkers(self): for arg in analyzer.construct_analyzer_cmd(result_handler): self.assertFalse(arg.startswith('-checks')) - self.assertEqual( - analyzer.config_handler.checks()['Wreserved-id-macro'][0], - CheckerState.ENABLED) + self.assertNotIn("Wreserved-id-macro", + analyzer.config_handler.checks().keys()) def test_analyze_wrong_parameters(self): """ @@ -603,7 +730,7 @@ def test_clang_diags_as_compiler_warnings(self): analyzer = create_analyzer_tidy([ # This should enable -Wvla and -Wvla-extension. - '--enable', 'clang-diagnostic-vla', + '--enable', 'prefix:clang-diagnostic-vla', '--disable', 'clang-diagnostic-unused-value']) result_handler = create_result_handler(analyzer) @@ -645,10 +772,14 @@ def checkers_by_labels(self, labels): def get_description(self, label): if label == 'profile': - return ['default', 'sensitive', 'security', 'portability', - 'extreme'] - - return [] + return { + "default": "", + "sensitive": "", + "security": "", + "portability": "", + "extreme": "" + } + return {} def occurring_values(self, label): if label == 'guideline': diff --git a/config/labels/analyzers/clang-tidy.json b/config/labels/analyzers/clang-tidy.json index bb52fd319f..077c267a3f 100644 --- a/config/labels/analyzers/clang-tidy.json +++ b/config/labels/analyzers/clang-tidy.json @@ -2345,7 +2345,10 @@ "severity:MEDIUM" ], "clang-diagnostic-error": [ - "severity:CRITICAL" + "severity:CRITICAL", + "profile:default", + "profile:extreme", + "profile:sensitive" ], "clang-diagnostic-exceptions": [ "doc_url:https://clang.llvm.org/docs/DiagnosticsReference.html#wexceptions", diff --git a/web/tests/functional/report_viewer_api/__init__.py b/web/tests/functional/report_viewer_api/__init__.py index 8d8e6ce41d..873f6a0c50 100644 --- a/web/tests/functional/report_viewer_api/__init__.py +++ b/web/tests/functional/report_viewer_api/__init__.py @@ -85,7 +85,7 @@ def setup_class_common(workspace_name): 'workspace': TEST_WORKSPACE, 'checkers': ['-d', 'clang-diagnostic', '-e', 'clang-diagnostic-division-by-zero', - '-e', 'clang-diagnostic-return-type'], + '-e', 'checker:clang-diagnostic-return-type'], 'tag': tag, 'analyzers': ['clangsa', 'clang-tidy'] } @@ -191,7 +191,7 @@ def setup_class_common(workspace_name): '-d', 'unix.Malloc', '-d', 'clang-diagnostic', '-e', 'clang-diagnostic-division-by-zero', - '-e', 'clang-diagnostic-return-type'] + '-e', 'checker:clang-diagnostic-return-type'] ret = codechecker.check_and_store(codechecker_cfg, test_project_name_third, project.path(test_project))