Algorithm-Repo/SourceCode/MachineLearningUtils/TestTools/AIModelMetricsUtils/object_detection_metric.py

"""
检测：
[{'Label': 1, 'Confidence': 1.0, 'BoundingBox'：[2, 3, 4, 5]}]
[{'Label': 1, 'Confidence': 1.0, 'BoundingBox'：[2, 3, 4, 5]},{'Label': 2, 'Confidence': 1.0, 'BoundingBox'：[2, 3, 4, 5]}]
[{'Label': 0, 'Confidence': 1.0, 'BoundingBox'：[0, 0, 0, 0]}]
[]

"""
import json
import sys
import numpy as np

# 检测：
from enum import Enum


class MethodAveragePrecision(Enum):
    """
    Class representing if the coordinates are relative to the
    image size or are absolute values.
    """
    EveryPointInterpolation = 1
    ElevenPointInterpolation = 2


# https://github.com/rafaelpadilla/Object-Detection-Metrics

class Evaluator_object_detection(object):
    def __init__(self, iou_thres, method=MethodAveragePrecision.EveryPointInterpolation):
        self.method = method
        self.iou_thres = iou_thres
        self.wrong_file = {}
        self.background_images_all_indexs = []
        self.background_images_results = {}
        self.background_images_results_count = {}
        self.all_image_dict = {}
        self.all_no_background_images_pos_results = {}
        self.all_no_background_images_fp_results = {}
        self.all_no_background_images_tp_results = {}
        self.pred_results = []
        self.gt_results = []
        self.classes = []
        self.wrong_file['gt_wrong'] = []
        self.wrong_file['pred_wrong'] = []
        self.background_images_results_count['background_images_all_nums'] = 0
        self.all_image_dict['images_all_nums'] = 0

    def generate_metrics(self):
        classes = sorted(self.classes)
        # Precision x Recall is obtained individually by each class
        # Loop through by classes
        ret = []  # list containing metrics (precision, recall, average precision) of each class
        for c in classes:
            if c == 0:
                continue
            # Get only detection of class c
            dects = []
            [dects.append(d) for d in self.pred_results if d[1] == c]
            # Get only ground truths of class c, use filename as key
            gts = {}
            npos = 0
            for g in self.gt_results:
                if g[1] == c:
                    npos += 1
                    gts[g[0]] = gts.get(g[0], []) + [g]

                    if str(c) not in self.all_no_background_images_pos_results.keys():
                        self.all_no_background_images_pos_results[str(c)] = [g[0]]
                    else:
                        self.all_no_background_images_pos_results[str(c)].append(g[0])

            # sort detections by decreasing confidence
            dects = sorted(dects, key=lambda conf: conf[2], reverse=True)
            TP = np.zeros(len(dects))
            FP = np.zeros(len(dects))
            # create dictionary with amount of gts for each image
            det = {key: np.zeros(len(gts[key])) for key in gts}

            # print("Evaluating class: %s (%d detections)" % (str(c), len(dects)))
            # Loop through detections
            for d in range(len(dects)):
                # print('dect %s => %s' % (dects[d][0], dects[d][3],))
                # Find ground truth image
                gt = gts[dects[d][0]] if dects[d][0] in gts else []
                iouMax = sys.float_info.min
                for j in range(len(gt)):
                    # print('Ground truth gt => %s' % (gt[j][3],))
                    iou = Evaluator_object_detection.iou(dects[d][3], gt[j][3])
                    if iou > iouMax:
                        iouMax = iou
                        jmax = j
                # Assign detection as true positive/don't care/false positive
                if iouMax >= self.iou_thres:
                    if det[dects[d][0]][jmax] == 0:
                        TP[d] = 1  # count as true positive
                        det[dects[d][0]][jmax] = 1  # flag as already 'seen'
                        # print("TP")

                        if str(c) not in self.all_no_background_images_tp_results.keys():
                            self.all_no_background_images_tp_results[str(c)] = [dects[d][0]]
                        else:
                            self.all_no_background_images_tp_results[str(c)].append(dects[d][0])

                    else:
                        FP[d] = 1  # count as false positive
                        if str(c) not in self.all_no_background_images_fp_results.keys():
                            self.all_no_background_images_fp_results[str(c)] = [dects[d][0]]
                        else:
                            self.all_no_background_images_fp_results[str(c)].append(dects[d][0])
                        # print("FP")
                # - A detected "cat" is overlaped with a GT "cat" with IOU >= IOUThreshold.
                else:
                    FP[d] = 1  # count as false positive
                    if str(c) not in self.all_no_background_images_fp_results.keys():
                        self.all_no_background_images_fp_results[str(c)] = [dects[d][0]]
                    else:
                        self.all_no_background_images_fp_results[str(c)].append(dects[d][0])
                    # print("FP")
            # compute precision, recall and average precision
            acc_FP = np.cumsum(FP)
            acc_TP = np.cumsum(TP)
            rec = acc_TP / npos
            prec = np.divide(acc_TP, (acc_FP + acc_TP))
            # Depending on the method, call the right implementation
            if self.method == MethodAveragePrecision.EveryPointInterpolation:
                [ap, mpre, mrec, ii] = Evaluator_object_detection.CalculateAveragePrecision(rec, prec)
            else:
                [ap, mpre, mrec, _] = Evaluator_object_detection.ElevenPointInterpolatedAP(rec, prec)
            # add class result in the dictionary to be returned
            r = {
                'class': c,
                'precision': prec,
                'recall': rec,
                'AP': ap,
                'interpolated precision': mpre,
                'interpolated recall': mrec,
                'total positives': npos,
                'total TP': np.sum(TP),
                'total FP': np.sum(FP)
            }
            ret.append(r)

        return ret

    def add_batch(self, gt_file, pred_file, image_index):
        if gt_file == []:
            self.wrong_file['gt_wrong'].append(image_index)
        elif pred_file == []:
            self.wrong_file['pred_wrong'].append(image_index)
        elif gt_file != [] and pred_file != []:
            # 判断gt为背景的图像：
            self.all_image_dict['images_all_nums'] += 1
            if len(gt_file) == 1 and gt_file[0]['Label'] == 0 and gt_file[0]['BoundingBox'] == [0, 0, 0, 0]:
                self.background_images_results_count['background_images_all_nums'] += 1
                if len(pred_file) == 1 and pred_file[0]['Label'] == 0 and gt_file[0]['BoundingBox'] == [0, 0, 0, 0]:
                    # if 'gtlabel_0_predlabel_{}'.format(0) not in self.background_images_results.keys():
                    #     self.background_images_results['gtlabel_0_predlabel_{}'.format(0)] = [image_index]
                    # else:
                    #     self.background_images_results['gtlabel_0_predlabel_{}'.format(0)].append(image_index)

                    if 'gtlabel_0_predlabel_{}'.format(0) not in self.background_images_results_count.keys():
                        self.background_images_results_count['gtlabel_0_predlabel_{}'.format(0)] = 1
                    else:
                        self.background_images_results_count['gtlabel_0_predlabel_{}'.format(0)] += 1
                else:
                    for i in range(len(pred_file)):
                        pred_label = pred_file[i]['Label']

                        if 'gtlabel_0_predlabel_{}'.format(pred_label) not in self.background_images_results.keys():
                            self.background_images_results['gtlabel_0_predlabel_{}'.format(pred_label)] = [image_index]
                        else:
                            self.background_images_results['gtlabel_0_predlabel_{}'.format(pred_label)].append(
                                image_index)

                        if 'gtlabel_0_predlabel_{}'.format(
                                pred_label) not in self.background_images_results_count.keys():
                            self.background_images_results_count['gtlabel_0_predlabel_{}'.format(pred_label)] = 1
                        else:
                            self.background_images_results_count['gtlabel_0_predlabel_{}'.format(pred_label)] += 1
            else:
                gt_len = len(gt_file)
                pred_len = len(pred_file)
                for i in range(gt_len):
                    each_gt_label = gt_file[i]['Label']
                    each_gt_bb_left = gt_file[i]['BoundingBox'][0]
                    each_gt_bb_top = gt_file[i]['BoundingBox'][1]
                    each_gt_bb_right = gt_file[i]['BoundingBox'][2]
                    each_gt_bb_bottom = gt_file[i]['BoundingBox'][3]
                    self.gt_results.append([image_index, each_gt_label, 1.0,
                                            (each_gt_bb_left, each_gt_bb_top, each_gt_bb_right, each_gt_bb_bottom)])
                    if each_gt_label not in self.classes:
                        self.classes.append(each_gt_label)

                for i in range(pred_len):
                    each_pred_label = pred_file[i]['Label']
                    each_pred_confidence = pred_file[i]['Confidence']
                    each_pred_bb_left = pred_file[i]['BoundingBox'][0]
                    each_pred_bb_top = pred_file[i]['BoundingBox'][1]
                    each_pred_bb_right = pred_file[i]['BoundingBox'][2]
                    each_pred_bb_bottom = pred_file[i]['BoundingBox'][3]
                    self.pred_results.append([image_index, each_pred_label, each_pred_confidence, (
                    each_pred_bb_left, each_pred_bb_top, each_pred_bb_right, each_pred_bb_bottom)])
                    if each_pred_label not in self.classes:
                        self.classes.append(each_pred_label)

    @staticmethod
    def iou(boxA, boxB):
        # if boxes dont intersect
        if Evaluator_object_detection._boxesIntersect(boxA, boxB) is False:
            return 0
        interArea = Evaluator_object_detection._getIntersectionArea(boxA, boxB)
        union = Evaluator_object_detection._getUnionAreas(boxA, boxB, interArea=interArea)
        # intersection over union
        iou = interArea / union
        assert iou >= 0
        return iou

    # boxA = (Ax1,Ay1,Ax2,Ay2)
    # boxB = (Bx1,By1,Bx2,By2)
    @staticmethod
    def _boxesIntersect(boxA, boxB):
        if boxA[0] > boxB[2]:
            return False  # boxA is right of boxB
        if boxB[0] > boxA[2]:
            return False  # boxA is left of boxB
        if boxA[3] < boxB[1]:
            return False  # boxA is above boxB
        if boxA[1] > boxB[3]:
            return False  # boxA is below boxB
        return True

    @staticmethod
    def _getIntersectionArea(boxA, boxB):
        xA = max(boxA[0], boxB[0])
        yA = max(boxA[1], boxB[1])
        xB = min(boxA[2], boxB[2])
        yB = min(boxA[3], boxB[3])
        # intersection area
        return (xB - xA + 1) * (yB - yA + 1)

    @staticmethod
    def _getUnionAreas(boxA, boxB, interArea=None):
        area_A = Evaluator_object_detection._getArea(boxA)
        area_B = Evaluator_object_detection._getArea(boxB)
        if interArea is None:
            interArea = Evaluator_object_detection._getIntersectionArea(boxA, boxB)
        return float(area_A + area_B - interArea)

    @staticmethod
    def _getArea(box):
        return (box[2] - box[0] + 1) * (box[3] - box[1] + 1)

    @staticmethod
    def CalculateAveragePrecision(rec, prec):
        mrec = []
        mrec.append(0)
        [mrec.append(e) for e in rec]
        mrec.append(1)
        mpre = []
        mpre.append(0)
        [mpre.append(e) for e in prec]
        mpre.append(0)
        for i in range(len(mpre) - 1, 0, -1):
            mpre[i - 1] = max(mpre[i - 1], mpre[i])
        ii = []
        for i in range(len(mrec) - 1):
            if mrec[1 + i] != mrec[i]:
                ii.append(i + 1)
        ap = 0
        for i in ii:
            ap = ap + np.sum((mrec[i] - mrec[i - 1]) * mpre[i])
        # return [ap, mpre[1:len(mpre)-1], mrec[1:len(mpre)-1], ii]
        return [ap, mpre[0:len(mpre) - 1], mrec[0:len(mpre) - 1], ii]

    @staticmethod
    # 11-point interpolated average precision
    def ElevenPointInterpolatedAP(rec, prec):
        # def CalculateAveragePrecision2(rec, prec):
        mrec = []
        # mrec.append(0)
        [mrec.append(e) for e in rec]
        # mrec.append(1)
        mpre = []
        # mpre.append(0)
        [mpre.append(e) for e in prec]
        # mpre.append(0)
        recallValues = np.linspace(0, 1, 11)
        recallValues = list(recallValues[::-1])
        rhoInterp = []
        recallValid = []
        # For each recallValues (0, 0.1, 0.2, ... , 1)
        for r in recallValues:
            # Obtain all recall values higher or equal than r
            argGreaterRecalls = np.argwhere(mrec[:] >= r)
            pmax = 0
            # If there are recalls above r
            if argGreaterRecalls.size != 0:
                pmax = max(mpre[argGreaterRecalls.min():])
            recallValid.append(r)
            rhoInterp.append(pmax)
        # By definition AP = sum(max(precision whose recall is above r))/11
        ap = sum(rhoInterp) / 11
        # Generating values for the plot
        rvals = []
        rvals.append(recallValid[0])
        [rvals.append(e) for e in recallValid]
        rvals.append(0)
        pvals = []
        pvals.append(0)
        [pvals.append(e) for e in rhoInterp]
        pvals.append(0)
        # rhoInterp = rhoInterp[::-1]
        cc = []
        for i in range(len(rvals)):
            p = (rvals[i], pvals[i - 1])
            if p not in cc:
                cc.append(p)
            p = (rvals[i], pvals[i])
            if p not in cc:
                cc.append(p)
        recallValues = [i[0] for i in cc]
        rhoInterp = [i[1] for i in cc]
        return [ap, rhoInterp, recallValues, None]


if __name__ == "__main__":

    from object_detection_test.test_files_generate import test_pred_files_get, test_gt_files_get
    import os

    currentPath = os.path.dirname(os.path.abspath(__file__))
    path_pred = os.path.join(currentPath, 'object_detection_test\\all_files\\pred')
    pred_files = test_pred_files_get(path_pred)

    path_gt = os.path.join(currentPath, 'object_detection_test\\all_files\\gt')
    gt_files = test_gt_files_get(path_gt)

    evaluator = Evaluator_object_detection(iou_thres=0.3, method=MethodAveragePrecision.EveryPointInterpolation)
    assert len(gt_files) == len(pred_files)
    for i in range(len(gt_files)):
        image_index = str(i)
        evaluator.add_batch(json.loads(gt_files[i]), json.loads(pred_files[i]), image_index)

    wrong_file = evaluator.wrong_file
    print(wrong_file)
    background_images_results = evaluator.background_images_results
    print(background_images_results)
    all_image_dict = evaluator.all_image_dict
    print(all_image_dict)

    metricsPerClass = evaluator.generate_metrics()
    for mc in metricsPerClass:
        c = mc['class']
        precision = mc['precision']
        recall = mc['recall']
        average_precision = mc['AP']
        ipre = mc['interpolated precision']
        irec = mc['interpolated recall']
        # Print AP per class
        print('%s: %f' % (c, average_precision))

        precision_all1 = precision[-1]
        precision_all = mc['total TP'] / (mc['total TP'] + mc['total FP'])
        recall_all1 = recall[-1]
        recall_all = mc['total TP'] / mc['total positives']
        print()