reimplementation of heldout-test evaluation structure

src/darjeeling/candidate.py

@@ -1,13 +1,13 @@
 # -*- coding: utf-8 -*-
-__all__ = ('Candidate',)
+__all__ = ('Candidate', 'DiffCandidate')
 
-from typing import Dict, List, Tuple
+from typing import Dict, List, Tuple, Optional
 import typing
 
 from bugzoo.core.patch import Patch
 import attr
 
-from .core import Replacement, FileLine
+from .core import (Replacement, FileLine)
 from .transformation import Transformation
 from .util import tuple_from_iterable
 
@@ -19,13 +19,13 @@
 class Candidate:
     """Represents a repair as a set of atomic program transformations."""
     problem: 'Problem' = attr.ib(hash=False, eq=False)
-    transformations: Tuple[Transformation, ...] = \
+    transformations: Optional[Tuple[Transformation, ...]] = \
         attr.ib(converter=tuple_from_iterable)
 
     def to_diff(self) -> Patch:
         """Transforms this candidate patch into a concrete, unified diff."""
         replacements = \
-            map(lambda t: t.to_replacement(), self.transformations)
+            map(lambda t: t.to_replacement(), self.transformations) if self.transformations else {}
         replacements_by_file: Dict[str, List[Replacement]] = {}
         for rep in replacements:
             fn = rep.location.filename
@@ -40,7 +40,10 @@ def lines_changed(self) -> List[FileLine]:
         Returns a list of source lines that are changed by this candidate
         patch.
         """
-        return [t.line for t in self.transformations]
+        if self.transformations:
+            return [t.line for t in self.transformations]
+        else:
+            return []
 
     @property
     def id(self) -> str:
@@ -50,3 +53,71 @@ def id(self) -> str:
 
     def __repr__(self) -> str:
         return "Candidate<#{}>".format(self.id)
+
+
+@attr.s(frozen=True, repr=False, slots=True, auto_attribs=True)
+class DiffPatch:
+    _file: str = attr.ib()
+    _patch: Patch = attr.ib(factory=Patch)
+
+    def to_diff(self) -> Patch:
+        return self._patch
+
+    @property
+    def files(self) -> List[str]:
+        return self._patch.files
+
+    @property
+    def patch(self) -> Patch:
+        return self._patch
+
+    @property
+    def file_name(self) -> str:
+        return self._file
+
+    def __repr__(self) -> str:
+        return "DiffPatch<{}>".format(self.file_name)
+
+
+@attr.s(frozen=True, repr=False, slots=True, auto_attribs=True)
+class DiffCandidate(Candidate):
+    """Represents a repair as a set of atomic program transformations."""
+    _diffpatch: DiffPatch = attr.ib()
+
+    def lines_changed(self) -> List[FileLine]:
+        locs: List[FileLine] = []
+        # no accessibility to bugzoo.core.patch subcontent
+        # lines = [(f.old_fn,l) for f in self.get_file_patches()\
+        #          for h in f.__hunks \
+        #          for l in range(h.__old_start_at,h.__old_start_at+len(h.__lines))\
+        #         ]
+        # for f,l in lines:
+        #     locs.append(FileLine(f,l))
+        return locs
+
+    def to_diff(self) -> Patch:
+        return self._diffpatch.to_diff()
+
+    def get_file_patches(self):
+        return self._diffpatch._patch.__file_patches
+
+    @property
+    def diffpatch(self) -> DiffPatch:
+        return self._diffpatch
+
+    @property
+    def patch(self) -> Patch:
+        return self._diffpatch.patch
+
+    @property
+    def file(self) -> str:
+        return self._diffpatch.file_name
+
+    @property
+    def id(self) -> str:
+        """An eight-character hexadecimal identifier for this candidate."""
+        hex_hash = hex(abs(hash(self)))
+        return hex_hash[2:10]
+
+    def __repr__(self) -> str:
+        return "DiffCandidate<{}#{}>".format(self.file, self.id)

src/darjeeling/cli/__init__.py

@@ -16,11 +16,11 @@
 from ..environment import Environment
 from ..problem import Problem
 from ..version import __version__ as VERSION
-from ..config import Config
+from ..config import Config, EvaluateConfig
 from ..events import CsvEventLogger, WebSocketEventHandler
 from ..plugins import LOADED_PLUGINS
 from ..resources import ResourceUsageTracker
-from ..session import Session
+from ..session import Session, EvaluateSession
 from ..exceptions import BadConfigurationException
 from ..util import duration_str
 
@@ -96,6 +96,18 @@ def _default_log_filename(self) -> str:
         num = max(used_numbers) + 1
         return os.path.join(os.getcwd(), 'darjeeling.log.{}'.format(num))
 
+    @property
+    def _default_eval_log_filename(self) -> str:
+        # find all log file numbers that have been used in this directory
+        used_numbers = [int(s.rpartition('.')[-1])
+                        for s in glob.glob('evaluation.log.[0-9]*')]
+
+        if not used_numbers:
+            return os.path.join(os.getcwd(), 'evaluation.log.0')
+
+        num = max(used_numbers) + 1
+        return os.path.join(os.getcwd(), 'evaluation.log.{}'.format(num))
+
     @cement.ex(
         help='generates a test suite coverage report for a given problem',
         arguments=[
@@ -241,7 +253,7 @@ def repair(self) -> bool:
         # setup logging to file
         if should_log_to_file:
             if not log_to_filename:
-                log_to_filename = self._default_log_filename
+                log_to_filename = self._default_eval_log_filename
             logger.info(f'logging to file: {log_to_filename}')
             logger.add(log_to_filename, level='TRACE')
 
@@ -301,6 +313,63 @@ def repair(self) -> bool:
             else:
                 sys.exit(1)
 
+    @cement.ex(
+        help='evaluate a repair specified by patch using additional criteria',
+        arguments=[
+            (['filename'],
+             {'help': ('a Darjeeling configuration file describing a faulty '
+                       'program and how it should be repaired.')}),
+            (['--patch-dir'],
+             {'help': 'path containing patches to restore and evaluate.',
+              'dest': 'dir_patches',
+              'type': str}),
+            (['--log-to-file'],
+             {'help': 'path to store the log file.',
+              'type': str}),
+            (['--threads'],
+             {'dest': 'threads',
+              'type': int,
+              'help': ('number of threads over which the repair workload '
+                       'should be distributed')})
+        ]
+    )
+    def evaluate(self) -> None:
+        """Evaluates a given program."""
+        # load the configuration file
+        filename = self.app.pargs.filename
+        filename = os.path.abspath(filename)
+        cfg_dir = os.path.dirname(filename)
+        dir_patches: Optional[str] = self.app.pargs.dir_patches
+        threads: Optional[int] = self.app.pargs.threads
+        log_to_filename: Optional[str] = self.app.pargs.log_to_file
+
+        logger.remove()
+        logger.enable('darjeeling')
+        for plugin_name in LOADED_PLUGINS:
+            logger.enable(plugin_name)
+
+        with open(filename, 'r') as f:
+            yml = yaml.safe_load(f)
+
+        if not log_to_filename:
+            log_to_filename = self._default_log_filename
+        logger.info(f'logging to file: {log_to_filename}')
+        logger.add(log_to_filename, level='TRACE')
+        cfg = EvaluateConfig.from_yml(yml=yml,
+                                      dir_=cfg_dir,
+                                      dir_patches=dir_patches,
+                                      threads=threads)
+
+        with Environment() as environment:
+            try:
+                session = EvaluateSession.from_config(environment, cfg)
+            except BadConfigurationException:
+                print("ERROR: bad configuration file")
+                sys.exit(1)
+
+            session.run()
+            session.close()
+
 
 class CLI(cement.App):
     class Meta:

src/darjeeling/config.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 __all__ = ('Config', 'OptimizationsConfig', 'CoverageConfig',
-           'LocalizationConfig')
+           'LocalizationConfig', 'EvaluateConfig')
 
 from typing import Any, Collection, Dict, List, NoReturn, Optional, Set
 import datetime
@@ -138,10 +138,10 @@ class Config:
     """
     dir_patches: str = attr.ib()
     program: ProgramDescriptionConfig
-    transformations: ProgramTransformationsConfig
-    localization: LocalizationConfig
+    transformations: Optional[ProgramTransformationsConfig]
+    localization: Optional[LocalizationConfig]
     search: SearcherConfig
-    coverage: CoverageConfig
+    coverage: Optional[CoverageConfig]
     resource_limits: ResourceLimits
     seed: int = attr.ib(default=0)
     optimizations: OptimizationsConfig = attr.ib(factory=OptimizationsConfig)
@@ -240,7 +240,7 @@ def err(m: str) -> NoReturn:
         # coverage config
         if 'coverage' in yml:
             if plus:
-                yml['coverage']['method']['type']='plus'
+                yml['coverage']['method']['type'] = 'plus'
             coverage = CoverageConfig.from_dict(yml['coverage'], dir_)
         else:
             m = "'coverage' section is expected"
@@ -279,3 +279,93 @@ def err(m: str) -> NoReturn:
                       search=search,
                       optimizations=opts,
                       dir_patches=dir_patches)
+
+
+@attr.s(frozen=True, auto_attribs=True)
+class EvaluateConfig(Config):
+    """A configuration for Darjeeling to evaluate patches with additional content.
+
+    Attributes
+    ----------
+    dir_patches: str
+        The absolute path to the directory to which patches are saved.
+    seed: int
+        The seed that should be used by the random number generator.
+    threads: int
+        The number of threads over which the search should be distributed.
+    program: ProgramDescriptionConfig
+        A description of the program under transformation.
+    resource_limits: ResourceLimits
+        Limits on the resources that may be consumed during the search.
+
+    """
+    # search: SearcherConfig
+    # program: ProgramDescriptionConfig
+    # resource_limits: ResourceLimits
+    # dir_patches: str = attr.ib()
+    # threads: int = attr.ib(default=1)
+
+    @staticmethod
+    def from_yml(yml: Dict[str, Any],
+                 dir_: Optional[str] = None,
+                 *,
+                 terminate_early: bool = True,
+                 plus: bool = False,
+                 seed: Optional[int] = None,
+                 threads: Optional[int] = None,
+                 run_redundant_tests: bool = False,
+                 limit_candidates: Optional[int] = None,
+                 limit_time_minutes: Optional[int] = None,
+                 dir_patches: Optional[str] = None
+                 ) -> 'EvaluateConfig':
+        """Loads a configuration from a YAML dictionary.
+
+        Raises
+        ------
+        BadConfigurationException
+            If an illegal configuration is provided.
+        """
+        def err(m: str) -> NoReturn:
+            raise BadConfigurationException(m)
+
+        if dir_patches is None and 'save-patches-to' in yml:
+            dir_patches = yml['save-patches-to']
+            if not isinstance(dir_patches, str):
+                err("'save-patches-to' property should be a string")
+            if not os.path.isabs(dir_patches):
+                if not dir_:
+                    err("'save-patches-to' must be absolute for non-file-based configurations")
+                dir_patches = os.path.join(dir_, dir_patches)
+        elif dir_patches is None:
+            if not dir_:
+                err("'save-patches-to' must be specified for non-file-based configurations")
+            dir_patches = os.path.join(dir_, 'patches')
+
+        if threads is None and 'threads' in yml:
+            if not isinstance(yml['threads'], int):
+                err("'threads' property should be an int")
+            threads = yml['threads']
+        elif threads is None:
+            threads = 1
+
+        # resource limits
+        yml.setdefault('resource-limits', {})
+
+        resource_limits = \
+            ResourceLimits.from_dict(yml['resource-limits'], dir_)
+
+        if 'program' not in yml:
+            err("'program' section is missing")
+        program = ProgramDescriptionConfig.from_dict(dict_=yml['program'], dir_=dir_, heldout=True)
+
+        search = SearcherConfig.from_dict({'type': 'reviewer'}, dir_)
+
+        return EvaluateConfig(threads=threads,
+                              program=program,
+                              search=search,
+                              dir_patches=dir_patches,
+                              resource_limits=resource_limits,
+                              transformations=None,
+                              localization=None,
+                              coverage=None
+                              )

src/darjeeling/coverage/config.py

@@ -112,7 +112,7 @@ def build(self,
         # exclude yacc and lex files
         def is_yacc_or_lex_file(filename: str) -> bool:
             return filename.endswith(".y") or filename.endswith(".l")
-        
+
         covered_files = set(filename for test_coverage in coverage.values() for filename in test_coverage.lines.files)
         restrict_to_files = set(filename for filename in covered_files if not is_yacc_or_lex_file(filename))
         coverage = coverage.restrict_to_files(restrict_to_files)