Merge pull request #30 from MiXaiLL76/dev_tests

v1.5.5

faster_coco_eval/core/coco.py

@@ -105,8 +105,11 @@ def __init__(self, annotation_file=None):
     def createIndex(self):
         # create index
         logger.debug("creating index...")
-        anns, cats, imgs = {}, {}, {}
+        anns, cats, imgs, annToImgs = {}, {}, {}, {}
         imgToAnns, catToImgs = defaultdict(list), defaultdict(list)
+        imgCatToAnnsIdx = defaultdict(dict)
+        imgToAnnsIdx = defaultdict(dict)
+
         annsImgIds_dict = {}
         if "images" in self.dataset:
             for img in self.dataset["images"]:
@@ -120,6 +123,15 @@ def createIndex(self):
                 if annsImgIds_dict.get(ann["image_id"]):
                     imgToAnns[ann["image_id"]].append(ann)
                     anns[ann["id"]] = ann
+                    annToImgs[ann["id"]] = ann["image_id"]
+                    imgCatToAnnsIdx[(ann["image_id"], ann["category_id"])][
+                        ann["id"]
+                    ] = len(
+                        imgCatToAnnsIdx[(ann["image_id"], ann["category_id"])]
+                    )
+                    imgToAnnsIdx[ann["image_id"]][ann["id"]] = len(
+                        imgToAnnsIdx[ann["image_id"]]
+                    )
 
         if "categories" in self.dataset:
             for cat in self.dataset["categories"]:
@@ -135,6 +147,9 @@ def createIndex(self):
         self.anns = anns
         self.imgToAnns = imgToAnns
         self.catToImgs = catToImgs
+        self.annToImgs = annToImgs
+        self.imgCatToAnnsIdx = imgCatToAnnsIdx
+        self.imgToAnnsIdx = imgToAnnsIdx
         self.imgs = imgs
         self.cats = cats
 
@@ -453,21 +468,64 @@ def annToMask(self, ann):
         return m
 
     def get_ann_ids(self, img_ids=[], cat_ids=[], area_rng=[], iscrowd=None):
+        """Get ann ids that satisfy given filter conditions.
+
+        :param img_ids (int array) : get anns for given imgs
+        :param cat_ids (int array) : get anns for given cats
+        :param area_rng (float array) : get anns for given area range
+            (e.g. [0 inf])
+        :return: ids (int array)  : integer array of ann ids
+
+        """
         return self.getAnnIds(img_ids, cat_ids, area_rng, iscrowd)
 
     def get_cat_ids(self, cat_names=[], sup_names=[], cat_ids=[]):
+        """Get cat ids that satisfy given filter conditions.
+
+        :param cat_names (str array) : get cats for given cat names
+        :param sup_names (str array) : get cats for given supercategory
+            names
+        :param cat_ids (int array) : get cats for given cat ids
+        :return: ids (int array)  : integer array of cat ids
+
+        """
         return self.getCatIds(cat_names, sup_names, cat_ids)
 
     def get_img_ids(self, img_ids=[], cat_ids=[]):
+        """Get img ids that satisfy given filter conditions.
+
+        :param img_ids (int array) : get imgs for given ids
+        :param cat_ids (int array) : get imgs with all given cats
+        :return: ids (int array)  : integer array of img ids
+
+        """
         return self.getImgIds(img_ids, cat_ids)
 
     def load_anns(self, ids):
+        """Load anns with the specified ids.
+
+        :param ids (int array)       : integer ids specifying anns
+        :return: anns (object array) : loaded ann objects
+
+        """
         return self.loadAnns(ids)
 
     def load_cats(self, ids):
+        """Load cats with the specified ids.
+
+        :param ids (int array)       : integer ids specifying cats
+        :return: cats (object array) : loaded cat objects
+
+        """
         return self.loadCats(ids)
 
     def load_imgs(self, ids):
+        """Load anns with the specified ids.
+
+        :param ids (int array)       : integer ids specifying img
+        :return: imgs (object array) : loaded img objects
+
+        """
         return self.loadImgs(ids)
 
     @property
@@ -477,3 +535,15 @@ def img_ann_map(self):
     @property
     def cat_img_map(self):
         return self.catToImgs
+
+    @property
+    def ann_img_map(self):
+        return self.annToImgs
+
+    @property
+    def img_ann_idx_map(self):
+        return self.imgToAnnsIdx
+
+    @property
+    def img_cat_ann_idx_map(self):
+        return self.imgCatToAnnsIdx

faster_coco_eval/core/cocoeval.py

@@ -592,6 +592,13 @@ def setKpParams(self):
         )
 
     def __init__(self, iouType="segm", kpt_sigmas=None):
+        """Params for coco evaluation api.
+
+        IouType: the type of iou to use for evaluation, can be 'segm', 'bbox',
+            or 'keypoints'
+        kpt_sigmas: list of keypoint sigma values.
+
+        """
         if iouType == "segm" or iouType == "bbox":
             self.setDetParams()
         elif iouType == "keypoints":

faster_coco_eval/core/faster_eval_api.py

@@ -9,7 +9,6 @@
 
 import faster_coco_eval.faster_eval_api_cpp as _C
 
-from . import mask as maskUtils
 from .cocoeval import COCOeval
 
 logger = logging.getLogger(__name__)
@@ -157,16 +156,21 @@ def accumulate(self):
         self.matched = False
         try:
             if self.extra_calc:
+
+                num_iou_thresholds, _, _, num_area_ranges, _ = self.eval[
+                    "counts"
+                ]
+
                 self.detection_matches = np.vstack(
                     np.array(self.eval["detection_matches"]).reshape(
-                        self.eval["counts"][0], self.eval["counts"][3], -1
+                        num_iou_thresholds, num_area_ranges, -1
                     )
                 )
                 assert self.detection_matches.shape[1] <= len(self.cocoDt.anns)
 
                 self.ground_truth_matches = np.vstack(
                     np.array(self.eval["ground_truth_matches"]).reshape(
-                        self.eval["counts"][0], self.eval["counts"][3], -1
+                        num_iou_thresholds, num_area_ranges, -1
                     )
                 )
                 assert self.ground_truth_matches.shape[1] <= len(
@@ -175,7 +179,7 @@ def accumulate(self):
 
                 self.ground_truth_orig_id = np.vstack(
                     np.array(self.eval["ground_truth_orig_id"]).reshape(
-                        self.eval["counts"][0], self.eval["counts"][3], -1
+                        num_iou_thresholds, num_area_ranges, -1
                     )
                 )
                 assert self.ground_truth_orig_id.shape[1] <= len(
@@ -192,6 +196,8 @@ def accumulate(self):
         self.print_function("DONE (t={:0.2f}s).".format(toc - tic))
 
     def math_matches(self):
+        """For each ground truth, find the best matching detection and set the
+        detection as matched."""
         for gidx, ground_truth_matches in enumerate(self.ground_truth_matches):
             gt_ids = self.ground_truth_orig_id[gidx]
 
@@ -203,24 +209,29 @@ def math_matches(self):
                 if gt_id <= -1:
                     continue
 
-                _gt_ann = self.cocoGt.anns.get(gt_id)
-                if _gt_ann is None:
-                    continue
+                _gt_ann = self.cocoGt.anns[gt_id]
+                _dt_ann = self.cocoDt.anns[dt_id]
+                _img_id = self.cocoGt.ann_img_map[gt_id]
+                _catId = _gt_ann["category_id"] if self.params.useCats else -1
 
-                _dt_ann = self.cocoDt.anns.get(dt_id)
-                if _dt_ann is None:
-                    continue
+                if self.params.useCats:
+                    _catId = _gt_ann["category_id"]
+                    _map_gt_dict = self.cocoGt.img_cat_ann_idx_map
+                    _map_dt_dict = self.cocoDt.img_cat_ann_idx_map
+                    _map_id = (_img_id, _catId)
+                else:
+                    _catId = -1
+                    _map_gt_dict = self.cocoGt.img_ann_idx_map
+                    _map_dt_dict = self.cocoDt.img_ann_idx_map
+                    _map_id = _img_id
 
-                if int(_gt_ann["image_id"]) != int(_dt_ann["image_id"]):
-                    continue
+                iou_gt_id = _map_gt_dict[_map_id].get(gt_id)
+                iou_dt_id = _map_dt_dict[_map_id].get(dt_id)
 
-                if self.params.useCats == 1:
-                    if int(_gt_ann["category_id"]) != int(
-                        _dt_ann["category_id"]
-                    ):
-                        continue
+                if iou_gt_id is None or iou_dt_id is None:
+                    continue
 
-                iou = self.computeAnnIoU(_gt_ann, _dt_ann)
+                iou = self.ious[(_img_id, _catId)][iou_dt_id, iou_gt_id]
 
                 if not _gt_ann.get("matched", False):
                     _dt_ann["tp"] = True
@@ -251,63 +262,16 @@ def math_matches(self):
             if self.cocoGt.anns[gt_id].get("matched") is None:
                 self.cocoGt.anns[gt_id]["fn"] = True
 
-    def computeAnnIoU(self, gt_ann, dt_ann):
-        g = []
-        d = []
-
-        if self.params.iouType == "segm":
-            g.append(gt_ann["rle"])
-            d.append(dt_ann["rle"])
-        elif self.params.iouType == "bbox":
-            g.append(gt_ann["bbox"])
-            d.append(dt_ann["bbox"])
-
-        iscrowd = [0 for _ in g]
-
-        _iou = maskUtils.iou(d, g, iscrowd)
-
-        if len(_iou) == 0:
-            return 0
-        else:
-            return _iou.max()
-
-    def compute_mIoU(self, categories=None, raw=False):
-        if self.params.iouType == "keypoints":
-            return np.array(
-                [val.max() for val in self.ious.values() if len(val)]
-            ).mean()
-
-        g = []
-        d = []
-        s = []
-
+    def compute_mIoU(self):
+        """Compute the mIoU metric."""
+        ious = []
         for _, dt_ann in self.cocoDt.anns.items():
-            if dt_ann.get("tp", False):
-                gt_ann = self.cocoGt.anns[dt_ann["gt_id"]]
-                if categories is None or gt_ann["category_id"] in categories:
-                    s.append(dt_ann.get("score", 1))
-                    if self.params.iouType == "segm":
-                        g.append(gt_ann["rle"])
-                        d.append(dt_ann["rle"])
-                    elif self.params.iouType == "bbox":
-                        g.append(gt_ann["bbox"])
-                        d.append(dt_ann["bbox"])
-                    else:
-                        raise Exception("unknown iouType for iou computation")
-
-        iscrowd = [0 for _ in g]
-
-        ious = maskUtils.iou(d, g, iscrowd)
-        if raw:
-            return ious
-
-        if len(ious) == 0:
-            return 0
-        else:
-            ious = ious.diagonal()
-            return ious.mean()
+            if dt_ann.get("iou", False):
+                ious.append(dt_ann["iou"])
+        return sum(ious) / len(ious)
 
     def compute_mAUC(self):
+        """Compute the mAUC metric."""
         aucs = []
 
         for K in range(self.eval["counts"][2]):
@@ -376,6 +340,11 @@ def stats_as_dict(self):
 
     @staticmethod
     def calc_auc(recall_list, precision_list):
+        """
+        Calculate area under precision recall curve
+        recall_list: list of recall values
+        precision_list: list of precision values
+        """
         # https://towardsdatascience.com/how-to-efficiently-implement-area-under-precision-recall-curve-pr-auc-a85872fd7f14
         # mrec = np.concatenate(([0.], recall_list, [1.]))
         # mpre = np.concatenate(([0.], precision_list, [0.]))