Merge pull request #54 from BerkeleyLearnVerify/dfremont-fixes

Fix ScenicSampler with more than 10 objects (again)

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "verifai"
-version = "2.1.0"
+version = "2.1.1"
 description = "A toolkit for the formal design and analysis of systems that include artificial intelligence (AI) and machine learning (ML) components."
 authors = [
 	{ name = "Tommaso Dreossi" },

src/verifai/features/features.py

@@ -721,7 +721,21 @@ def __repr__(self):
         return f'ScalarArray({self.domain}, {self.shape})'
 
 class Struct(Domain):
-    """A domain consisting of named sub-domains."""
+    """A domain consisting of named sub-domains.
+
+    The order of the sub-domains is arbitrary: two Structs are considered equal
+    if they have the same named sub-domains, regardless of order. As the order
+    is an implementation detail, accessing the values of sub-domains in points
+    sampled from a Struct should be done by name:
+
+    >>> struct = Struct({'a': Box((0, 1)), 'b': Box((2, 3))})
+    >>> point = struct.uniformPoint()
+    >>> point.b
+    (2.20215292046797,)
+
+    Within a given version of VerifAI, the sub-domain order is consistent, so
+    that the order of columns in error tables is also consistent.
+    """
 
     def __init__(self, domains):
         self.namedDomains = tuple(sorted(domains.items(), key=lambda i: i[0]))

src/verifai/samplers/scenic_sampler.py

@@ -194,7 +194,10 @@ def spaceForScenario(scenario, ignoredProperties):
     assert scenario.egoObject is scenario.objects[0]
     doms = (domainForObject(obj, ignoredProperties)
             for obj in scenario.objects)
-    objects = Struct({ f'object{i}': dom for i, dom in enumerate(doms) })
+    objects = Struct({
+        ScenicSampler.nameForObject(i): dom
+        for i, dom in enumerate(doms)
+    })
 
     # create domains for global parameters
     paramDoms = {}
@@ -279,15 +282,30 @@ def nextSample(self, feedback=None):
         return self.pointForScene(self.lastScene)
 
     def pointForScene(self, scene):
-        """Convert a sampled Scenic :obj:`Scene` to a point in our feature space."""
+        """Convert a sampled Scenic :obj:`~scenic.core.scenarios.Scene` to a point in our feature space.
+
+        The `FeatureSpace` used by this sampler consists of 2 features:
+
+            * ``objects``, which is a `Struct` consisting of attributes ``object0``,
+              ``object1``, etc. with the properties of the corresponding objects
+              in the Scenic program. The names of these attributes may change in a
+              future version of VerifAI: use the `nameForObject` function to
+              generate them.
+            * ``params``, which is a `Struct` storing the values of the
+              :term:`global parameters` of the Scenic program (use
+              `paramDictForSample` to extract them).
+        """
         lengths, dom = self.space.domains
         assert lengths is None
         assert scene.egoObject is scene.objects[0]
         objDomain = dom.domainNamed['objects']
         assert len(objDomain.domains) == len(scene.objects)
-        objects = (pointForObject(objDomain.domainNamed[f'object{i}'], obj)
-                   for i, obj in enumerate(scene.objects))
-        objPoint = objDomain.makePoint(*objects)
+        objects = {
+            self.nameForObject(i):
+            pointForObject(objDomain.domainNamed[self.nameForObject(i)], obj)
+            for i, obj in enumerate(scene.objects)
+        }
+        objPoint = objDomain.makePoint(**objects)
 
         paramDomain = dom.domainNamed['params']
         params = {}
@@ -298,8 +316,17 @@ def pointForScene(self, scene):
 
         return self.space.makePoint(objects=objPoint, params=paramPoint)
 
+    @staticmethod
+    def nameForObject(i):
+        """Name used in the `FeatureSpace` for the Scenic object with index i.
+
+        That is, if ``scene`` is a :obj:`~scenic.core.scenarios.Scene`, the object
+        ``scene.objects[i]``.
+        """
+        return f'object{i}'
+
     def paramDictForSample(self, sample):
-        """Recover the dict of global parameters from a `ScenicSampler` sample."""
+        """Recover the dict of :term:`global parameters` from a `ScenicSampler` sample."""
         params = sample.params._asdict()
         corrected = {}
         for newName, quotedParam in self.quotedParams.items():

tests/scenic/test_scenic.py

@@ -71,9 +71,15 @@ def test_object_order(new_Object):
     sample = sampler.nextSample()
     objects = sample.objects
     assert len(objects) == 11
-    for i, obj in enumerate(objects):
+    for i in range(len(objects)):
+        name = ScenicSampler.nameForObject(i)
+        obj = getattr(objects, name)
         assert obj.position[:2] == pytest.approx((2*i, 0))
 
+    flat = sampler.space.flatten(sample)
+    unflat = sampler.space.unflatten(flat)
+    assert unflat == sample
+
 ## Active sampling
 
 def test_active_sampling(new_Object):