###########################################################################################
#                                                                                         #
# Evaluator class: Implements the most popular metrics for object detection               #
#                                                                                         #
# Developed by: Rafael Padilla (rafael.padilla@smt.ufrj.br)                               #
#        SMT - Signal Multimedia and Telecommunications Lab                               #
#        COPPE - Universidade Federal do Rio de Janeiro                                   #
#        Last modification: Oct 9th 2018                                                 #
###########################################################################################

import imp
import shutil
import json
from pprint import pprint as pp
from utils import *
from BoundingBoxes import *
from BoundingBox import *
import numpy as np
import os
import sys
from collections import Counter
import matplotlib.pyplot as plt
plt.switch_backend('agg')


# os.environ['debug_mode'] = 'true'


def debug_print(*args):
    if 'debug_mode' in os.environ:
        print(*args)


class Evaluator:
    def GetPascalVOCMetrics(
            self,
            boundingboxes,
            IOUThreshold=0.5,
            method=MethodAveragePrecision.EveryPointInterpolation, scoreThreshold=0.5, results_path='/results'):
        """Get the metrics used by the VOC Pascal 2012 challenge.
        Get
        Args:
            boundingboxes: Object of the class BoundingBoxes representing ground truth and detected
            bounding boxes;
            IOUThreshold: IOU threshold indicating which detections will be considered TP or FP
            (default value = 0.5);
            method (default = EveryPointInterpolation): It can be calculated as the implementation
            in the official PASCAL VOC toolkit (EveryPointInterpolation), or applying the 11-point
            interpolatio as described in the paper "The PASCAL Visual Object Classes(VOC) Challenge"
            or EveryPointInterpolation"  (ElevenPointInterpolation);
        Returns:
            A list of dictionaries. Each dictionary contains information and metrics of each class.
            The keys of each dictionary are:
            dict['class']: class representing the current dictionary;
            dict['precision']: array with the precision values;
            dict['recall']: array with the recall values;
            dict['AP']: average precision;
            dict['interpolated precision']: interpolated precision values;
            dict['interpolated recall']: interpolated recall values;
            dict['total positives']: total number of ground truth positives;
            dict['total TP']: total number of True Positive detections;
            dict['total FP']: total number of False Positive detections;
        """
        right_path = os.path.join(results_path, 'right')
        wrong_path = os.path.join(results_path, 'wrong')
        os.makedirs(right_path, exist_ok=True)
        os.makedirs(wrong_path, exist_ok=True)

        ret = [
        ]  # list containing metrics (precision, recall, average precision) of each class
        # List with all ground truths (Ex: [imageName,class,confidence=1, (bb coordinates XYX2Y2)])
        groundTruths = []
        # List with all detections (Ex: [imageName,class,confidence,(bb coordinates XYX2Y2)])
        detections = []
        # Get all classes
        classes = []

        gt_dict = {}

        # 统计每张图片中gt框的个数
        # k:img_nm
        # v: {"gt": 0, "detected": 0}
        img_dict = {}

        # Loop through all bounding boxes and separate them into GTs and detections
        for bb in boundingboxes.getBoundingBoxes():
            # [imageName, class, confidence, (bb coordinates XYX2Y2)]
            if bb.getBBType() == BBType.GroundTruth:
                groundTruths.append([
                    bb.getImageName(),
                    bb.getClassId(), 1,
                    bb.getAbsoluteBoundingBox(BBFormat.XYX2Y2),
                    bb.getIndex()
                ])
                k = tuple((bb.getImageName(), bb.getClassId(),
                           bb.getAbsoluteBoundingBox(BBFormat.XYX2Y2),
                           bb.getIndex()))
                # 0:default(位置漏检), 1:与至少一个检测框IOU>= IOUThreshold（类别漏检）, 2:正确检出（IOU和类别均正确）（正确检出）
                gt_dict[k] = 0
                k = bb.getImageName()
                if k not in img_dict:
                    img_dict[k] = {
                        "gt": 0,
                        "detected": 0,
                        "wrong_tag": 0,
                        "info": {
                            "FP_view": {
                                "right": [],
                                "wrong_location": [],
                                "wrong_category": [],
                                "wrong_duplicate": []
                            },
                            "FN_view": {
                                "right": [],
                                "wrong_location": [],
                                "wrong_category": []
                            }
                        }
                    }
                img_dict[k]["gt"] += 1
            else:
                detections.append([
                    bb.getImageName(),
                    bb.getClassId(),
                    bb.getConfidence(),
                    bb.getAbsoluteBoundingBox(BBFormat.XYX2Y2),
                    bb.getIndex()
                ])
            # get class
            if bb.getClassId() not in classes:
                classes.append(bb.getClassId())
        classes = sorted(classes)
        # Precision x Recall is obtained individually by each class
        # Loop through by classes
        for c in classes:
            debug_print('[!!!]CURRENT CLASS IS [', c, ']')
            # Get only detection of class c
            dects = []
            [dects.append(d) for d in detections if d[1] == c]
            # [dects.append(d) for d in detections if d[1] == c and d[2] >= scoreThreshold)]
            # Get only ground truths of class c
            gts = []
            [gts.append(g) for g in groundTruths if g[1] == c]
            npos = len(gts)
            # sort detections by decreasing confidence
            dects = sorted(dects, key=lambda conf: conf[2], reverse=True)
            TP = np.zeros(len(dects))
            FP = np.zeros(len(dects))
            FP_duplicate = np.zeros(len(dects))
            FP_category = np.zeros(len(dects))
            FP_location = np.zeros(len(dects))

            # create dictionary with amount of gts for each image
            det = Counter([cc[0] for cc in gts])
            for key, val in det.items():
                det[key] = np.zeros(val)
            # print("Evaluating class: %s (%d detections)" % (str(c), len(dects)))
            # Loop through detections
            for d in range(len(dects)):
                # print('dect %s => %s' % (dects[d][0], dects[d][3],))
                # Find ground truth image
                debug_print('[!!!]current det_id', d)
                debug_print('     det_boundingbox', dects[d][3])
                dt = dects[d]
                dt_index = int(dt[4])

                gt = [gt for gt in gts if gt[0] == dects[d][0]]
                iouMax = sys.float_info.min

                detected_gt = []
                jmax = -1
                for j in range(len(gt)):
                    # print('Ground truth gt => %s' % (gt[j][3],))
                    iou = Evaluator.iou(dects[d][3], gt[j][3])
                    if iou > iouMax:
                        iouMax = iou
                        jmax = j
                        detected_gt = gt[j]
                if jmax != -1:
                    debug_print('     gt_boundingbox', gt[jmax][3])
                else:
                    pass
                debug_print('     iouMax', iouMax)
                debug_print('     IOUThreshold', IOUThreshold)

                img_nm = dt[0]
                if iouMax >= IOUThreshold:
                    k = tuple((detected_gt[0], detected_gt[1], detected_gt[3],
                               detected_gt[4]))
                    # Assign detection as true positive/don't care/false positive
                    if img_nm not in img_dict:
                        img_dict[img_nm] = {
                            "gt": 0,
                            "detected": 0,
                            "wrong_tag": 0,
                            "info": {
                                "FP_view": {
                                    "right": [],
                                    "wrong_location": [],
                                    "wrong_category": [],
                                    "wrong_duplicate": []
                                },
                                "FN_view": {
                                    "right": [],
                                    "wrong_location": [],
                                    "wrong_category": []
                                }
                            }
                        }
                    debug_print('     reason: iouMax >= IOUThreshold')
                    if dt[2] >= scoreThreshold:
                        debug_print('     reason: conf >= scoreThreshold')
                        if det[dects[d][0]][jmax] == 0:
                            TP[d] = 1  # count as true positive
                            # flag as already 'seen'
                            det[dects[d][0]][jmax] = 1
                            debug_print('     ret:', 'TP')
                            # print("TP")
                            # 2:正确检出（IOU和类别均正确）
                            gt_dict[k] = 2
                            # print("TP")
                            img_dict[img_nm]["detected"] += 1
                            img_dict[img_nm]["info"]["FP_view"][
                                "right"].append(dt_index)
                        else:
                            debug_print(
                                '     reason: but this gt_box already been assigned to a det_box')
                            debug_print('     ret:', 'FP(wrong_duplicate)')
                            FP[d] = 1  # count as false positive
                            FP_duplicate[d] = 1
                            img_dict[img_nm]["info"]["FP_view"][
                                "wrong_duplicate"].append(dt_index)
                            # print('fp in img, classify as wrong result!')
                            # img_dict.pop(img_nm, None)
                            img_dict[img_nm]["wrong_tag"] = 1
                            # print("FP")
                    else:
                        debug_print('     reason: but conf < scoreThreshold')
                        debug_print('     ret:', 'FP(wrong_category)')
                        FP[d] = 1  # count as false positive
                        FP_category[d] = 1
                        img_dict[img_nm]["info"]["FP_view"][
                            "wrong_category"].append(dt_index)
                        # img_dict.pop(img_nm, None)
                        img_dict[img_nm]["wrong_tag"] = 1
                        # 没有正确检出
                        if gt_dict[k] == 0:
                            gt_dict[k] = 1
                # - A detected "cat" is overlaped with a GT "cat" with IOU >= IOUThreshold.
                else:
                    debug_print('     reason: iouMax < IOUThreshold')
                    debug_print('     ret:', 'FP(wrong_location)')
                    FP[d] = 1  # count as false positive
                    FP_location[d] = 1
                    # vzyl:0305
                    if img_nm not in img_dict:
                        img_dict[img_nm] = {
                            "gt": 0,
                            "detected": 0,
                            "wrong_tag": 0,
                            "info": {
                                "FP_view": {
                                    "right": [],
                                    "wrong_location": [],
                                    "wrong_category": [],
                                    "wrong_duplicate": []
                                },
                                "FN_view": {
                                    "right": [],
                                    "wrong_location": [],
                                    "wrong_category": []
                                }
                            }
                        }
                    img_dict[img_nm]["info"]["FP_view"][
                        "wrong_location"].append(dt_index)
                    # img_dict.pop(img_nm, None)
                    img_dict[img_nm]["wrong_tag"] = 1
                    # print("FP")

            # compute precision, recall and average precision
            acc_FP = np.cumsum(FP)
            acc_TP = np.cumsum(TP)
            acc_FP_duplicate = np.sum(FP_duplicate)
            acc_FP_category = np.sum(FP_category)
            acc_FP_location = np.sum(FP_location)

            assert (np.sum(TP) + np.sum(FP)) == len(dects)
            assert (np.sum(FP) == (acc_FP_duplicate + acc_FP_category +
                                   acc_FP_location))

            rec = acc_TP / npos if npos > 0 else np.zeros_like(acc_TP)
            # rec = acc_TP / npos
            prec = np.divide(acc_TP, (acc_FP + acc_TP))
            # cal pre and recall
            confs = [x[2] for x in dects]

            debug_print('[!!!]precision and recall calculate detail:')
            debug_print('     all det_box confs(decrease):', confs)
            debug_print('     all det_box precsions_related(decrease):', prec)
            debug_print('     all det_box recalls_related(increase):', rec)
            # ret_c = 0
            ret_prec = 0
            ret_rec = 0
            flag = 0

            if confs:
                debug_print('     reason: scoreThreshold', scoreThreshold)
                if confs[0] < scoreThreshold:
                    debug_print('     reason: confs[0]', confs[0])
                    debug_print('     reason: confs[0] < scoreThreshold')
                    ret_prec = 0
                    ret_rec = 0
                else:
                    for i, conf in enumerate(confs):
                        if conf < scoreThreshold:
                            # ret_c = confs[i-1]
                            ret_prec = prec[i-1]
                            ret_rec = rec[i-1]
                            flag = 1
                            debug_print(
                                '     reason: confs[{}] ='.format(i-1), confs[i-1])
                            debug_print(
                                '     reason: confs[{}] >= scoreThreshold'.format(i-1))
                            debug_print(
                                '     reason: confs[{}] ='.format(i), confs[i])
                            debug_print(
                                '     reason: confs[{}] < scoreThreshold'.format(i))
                            debug_print(
                                '     ret: use confs[{}]\'s conf({})'.format(i-1, confs[i-1]))
                            debug_print(
                                '     ret: get its precision and recall')
                            break
                    if not flag:
                        debug_print('     reason: no conf < scoreThreshold')
                        debug_print(
                            '     ret: use final conf({})'.format(confs[-1]))
                        debug_print('     ret: get its precision and recall')
                        ret_prec = prec[-1]
                        ret_rec = rec[-1]
            else:
                debug_print('     reason: no det_box')

            ret_f1score = 2 * ret_prec * ret_rec / \
                (ret_prec + ret_rec) if (ret_prec + ret_rec) > 0 else 0

            debug_print('     ret: precision=', ret_prec)
            debug_print('     ret: recall=', ret_rec)
            debug_print('     ret: f1-score=', ret_f1score)

            # rec: 0-1
            debug_print('[!!!]precision_for_recall0.85 claculate detail:')
            debug_print('     all det_box recalls_related(increase):', rec)
            ret_rec85 = 0
            ret_prec85 = 0
            flag = 0
            for i, rc in enumerate(rec):
                if rc >= 0.85:
                    debug_print('     reason: found a recall >= 0.85')
                    debug_print('     reason: recall[{}]={}'.format(i, rc))
                    debug_print('     ret: use its related precision')
                    ret_rec85 = rc
                    ret_prec85 = prec[i]
                    # print('ret_rec85', f'{ret_rec85}')
                    # print('ret_prec85', f'{ret_prec85}')
                    flag = 1
                    break
            if not flag:
                debug_print('    reason: all recall < 0.85')
            debug_print('     ret: precision_for_recall0.85=', ret_prec85)
            # Depending on the method, call the right implementation
            debug_print('[!!!]AP claculate detail:')
            if method == MethodAveragePrecision.EveryPointInterpolation:
                [ap, mpre, mrec,
                 ii] = Evaluator.CalculateAveragePrecision(rec, prec)
            elif method == MethodAveragePrecision.ElevenPointInterpolation:
                [ap, mpre, mrec,
                 _] = Evaluator.ElevenPointInterpolatedAP(rec, prec)
            else:
                [ap, mpre, mrec,
                 _] = Evaluator.ElevenPointInterpolatedAP(rec, prec)
            debug_print('    after interpolation AP curve:')
            debug_print('    interpolated_precision', mpre)
            debug_print('    interpolated_recall', mrec)
            debug_print('    calculate AP(area under curve)', ap)
            debug_print('    ret: AP=', ap)
            # add class result in the dictionary to be returned
            r = {
                'IOUThreshold': IOUThreshold,
                'class': c,
                'precision': prec,
                'recall': rec,
                'AP': ap,
                'interpolated precision': mpre,
                'interpolated recall': mrec,
                'total TP': np.sum(TP),
                'total FP': np.sum(FP),
                'total FP_duplicate': acc_FP_duplicate,
                'total FP_category': acc_FP_category,
                'total FP_location': acc_FP_location,
                'total positives': npos,
                'ret_prec85': ret_prec85,
                'ret_prec': ret_prec,
                'ret_rec': ret_rec,
                'ret_f1': ret_f1score
            }
            debug_print('\n'*2)
            ret.append(r)
        # cal FN
        detected_gt = 0
        FN_category = 0
        FN_location = 0
        for k, v in gt_dict.items():
            img_nm = k[0]
            index = int(k[3])
            if v == 0:
                FN_location += 1
                img_dict[img_nm]["info"]["FN_view"]["wrong_location"].append(
                    index)
            elif v == 1:
                FN_category += 1
                img_dict[img_nm]["info"]["FN_view"]["wrong_category"].append(
                    index)
            else:
                detected_gt += 1
                img_dict[img_nm]["info"]["FN_view"]["right"].append(index)

        assert (detected_gt + FN_category +
                FN_location) == len(groundTruths)
        gt_ret = {
            'detected_gt': detected_gt,
            'FN_category': FN_category,
            'FN_location': FN_location
        }
        pp(ret)
        pp(gt_ret)

        import copy
        # sort index list
        for img_nm in img_dict.keys():
            for view in img_dict[img_nm]["info"].keys():
                for cate, index_list in img_dict[img_nm]["info"][view].items():
                    img_dict[img_nm]["info"][view][cate] = copy.deepcopy(
                        sorted(index_list))
        # print(img_dict)

        # fit hikmars3
        if not(os.getenv('hikmars_version') and os.getenv('hikmars_version') == '3'):
            results_path = right_path

        # move img
        for k, v in img_dict.items():
            img_nm = k
            # modify json
            json_path = os.path.join(results_path, '{}.json'.format(img_nm))
            json_info = {}
            with open(json_path, 'r+') as f:
                json_info = json.load(f)
                json_info["info"] = img_dict[img_nm]["info"]

            with open(json_path, "w") as f:
                json.dump(json_info, f)
            try:
                if v["gt"] > 0 and v["gt"] == v["detected"] and v["wrong_tag"] != 1:
                    if results_path != right_path:
                        shutil.move(
                            '{}/{}.json'.format(results_path, img_nm), right_path)
                else:
                    shutil.move(
                        '{}/{}.json'.format(results_path, img_nm), wrong_path)
            except:
                debug_print('no such file {}, skipped.'.format(json_path))
        return ret, gt_ret

    def PlotPrecisionRecallCurve(
            self,
            boundingBoxes,
            IOUThreshold=0.5,
            scoreThreshold=0.5,
            method=MethodAveragePrecision.EveryPointInterpolation,
            showAP=False,
            showInterpolatedPrecision=False,
            results_path=None,
            savePath=None,
            showGraphic=True):
        """PlotPrecisionRecallCurve
        Plot the Precision x Recall curve for a given class.
        Args:
            boundingBoxes: Object of the class BoundingBoxes representing ground truth and detected
            bounding boxes;
            IOUThreshold (optional): IOU threshold indicating which detections will be considered
            TP or FP (default value = 0.5);
            method (default = EveryPointInterpolation): It can be calculated as the implementation
            in the official PASCAL VOC toolkit (EveryPointInterpolation), or applying the 11-point
            interpolatio as described in the paper "The PASCAL Visual Object Classes(VOC) Challenge"
            or EveryPointInterpolation"  (ElevenPointInterpolation).
            showAP (optional): if True, the average precision value will be shown in the title of
            the graph (default = False);
            showInterpolatedPrecision (optional): if True, it will show in the plot the interpolated
             precision (default = False);
            savePath (optional): if informed, the plot will be saved as an image in this path
            (ex: /home/mywork/ap.png) (default = None);
            showGraphic (optional): if True, the plot will be shown (default = True)
        Returns:
            A list of dictionaries. Each dictionary contains information and metrics of each class.
            The keys of each dictionary are:
            dict['class']: class representing the current dictionary;
            dict['precision']: array with the precision values;
            dict['recall']: array with the recall values;
            dict['AP']: average precision;
            dict['interpolated precision']: interpolated precision values;
            dict['interpolated recall']: interpolated recall values;
            dict['total positives']: total number of ground truth positives;
            dict['total TP']: total number of True Positive detections;
            dict['total FP']: total number of False Negative detections;
        """
        if not results_path:
            results_path = '/results'

        results, gt_ret = self.GetPascalVOCMetrics(
            boundingBoxes, IOUThreshold, method, scoreThreshold, results_path)
        result = None
        # Each resut represents a class
        for result in results:
            if result is None:
                raise IOError('Error: Class %d could not be found.' % classId)

            classId = result['class']
            precision = result['precision']
            recall = result['recall']
            average_precision = result['AP']
            mpre = result['interpolated precision']
            mrec = result['interpolated recall']
            npos = result['total positives']
            total_tp = result['total TP']
            total_fp = result['total FP']

            plt.close()
            # if showInterpolatedPrecision:
            if method == MethodAveragePrecision.EveryPointInterpolation:
                plt.plot(mrec,
                         mpre,
                         '--r',
                         label='Interpolated precision (every point)')
            elif method == MethodAveragePrecision.ElevenPointInterpolation:
                # Uncomment the line below if you want to plot the area
                # plt.plot(mrec, mpre, 'or', label='11-point interpolated precision')
                # Remove duplicates, getting only the highest precision of each recall value
                nrec = []
                nprec = []
                for idx in range(len(mrec)):
                    r = mrec[idx]
                    if r not in nrec:
                        idxEq = np.argwhere(mrec == r)
                        nrec.append(r)
                        nprec.append(max([mpre[int(id)] for id in idxEq]))
                plt.plot(nrec,
                         nprec,
                         'or',
                         label='11-point interpolated precision')
                plt.plot(nrec, nprec, label='11-point interpolated precision')

            plt.plot(recall, precision, label='Precision')
            plt.xlabel('recall')
            plt.ylabel('precision')

            if showAP:
                ap_str = "{0:.2f}%".format(average_precision * 100)
                # ap_str = "{0:.4f}%".format(average_precision * 100)
                plt.title('Precision x Recall curve \nClass: %s, AP: %s' %
                          (str(classId), ap_str))
            else:
                plt.title('Precision x Recall curve \nClass: %s' %
                          str(classId))
            plt.legend(shadow=True)
            plt.grid()
            # plt.xlim(0, 1)
            # plt.ylim(0, 1)
            ############################################################
            # Uncomment the following block to create plot with points #
            ############################################################
            # plt.plot(recall, precision, 'bo')
            # labels = ['R', 'Y', 'J', 'A', 'U', 'C', 'M', 'F', 'D', 'B', 'H', 'P', 'E', 'X', 'N', 'T',
            # 'K', 'Q', 'V', 'I', 'L', 'S', 'G', 'O']
            # dicPosition = {}
            # dicPosition['left_zero'] = (-30,0)
            # dicPosition['left_zero_slight'] = (-30,-10)
            # dicPosition['right_zero'] = (30,0)
            # dicPosition['left_up'] = (-30,20)
            # dicPosition['left_down'] = (-30,-25)
            # dicPosition['right_up'] = (20,20)
            # dicPosition['right_down'] = (20,-20)
            # dicPosition['up_zero'] = (0,30)
            # dicPosition['up_right'] = (0,30)
            # dicPosition['left_zero_long'] = (-60,-2)
            # dicPosition['down_zero'] = (-2,-30)
            # vecPositions = [
            #     dicPosition['left_down'],
            #     dicPosition['left_zero'],
            #     dicPosition['right_zero'],
            #     dicPosition['right_zero'],  #'R', 'Y', 'J', 'A',
            #     dicPosition['left_up'],
            #     dicPosition['left_up'],
            #     dicPosition['right_up'],
            #     dicPosition['left_up'],  # 'U', 'C', 'M', 'F',
            #     dicPosition['left_zero'],
            #     dicPosition['right_up'],
            #     dicPosition['right_down'],
            #     dicPosition['down_zero'],  #'D', 'B', 'H', 'P'
            #     dicPosition['left_up'],
            #     dicPosition['up_zero'],
            #     dicPosition['right_up'],
            #     dicPosition['left_up'],  # 'E', 'X', 'N', 'T',
            #     dicPosition['left_zero'],
            #     dicPosition['right_zero'],
            #     dicPosition['left_zero_long'],
            #     dicPosition['left_zero_slight'],  # 'K', 'Q', 'V', 'I',
            #     dicPosition['right_down'],
            #     dicPosition['left_down'],
            #     dicPosition['right_up'],
            #     dicPosition['down_zero']
            # ]  # 'L', 'S', 'G', 'O'
            # for idx in range(len(labels)):
            #     box = dict(boxstyle='round,pad=.5',facecolor='yellow',alpha=0.5)
            #     plt.annotate(labels[idx],
            #                 xy=(recall[idx],precision[idx]), xycoords='data',
            #                 xytext=vecPositions[idx], textcoords='offset points',
            #                 arrowprops=dict(arrowstyle="->", connectionstyle="arc3"),
            #                 bbox=box)
            if savePath is not None:
                plt.savefig(os.path.join(savePath, str(classId) + '.png'))
            if showGraphic is True:
                plt.show()
                # plt.waitforbuttonpress()
                plt.pause(0.05)
        return results, gt_ret

    @staticmethod
    def CalculateAveragePrecision(rec, prec):
        mrec = []
        mrec.append(0)
        [mrec.append(e) for e in rec]
        mrec.append(1)
        mpre = []
        mpre.append(0)
        [mpre.append(e) for e in prec]
        mpre.append(0)
        for i in range(len(mpre) - 1, 0, -1):
            mpre[i - 1] = max(mpre[i - 1], mpre[i])
        ii = []
        for i in range(len(mrec) - 1):
            if mrec[1:][i] != mrec[0:-1][i]:
                ii.append(i + 1)
        ap = 0
        for i in ii:
            ap = ap + np.sum((mrec[i] - mrec[i - 1]) * mpre[i])
        # return [ap, mpre[1:len(mpre)-1], mrec[1:len(mpre)-1], ii]
        return [ap, mpre[0:len(mpre) - 1], mrec[0:len(mpre) - 1], ii]

    @staticmethod
    # 11-point interpolated average precision
    def ElevenPointInterpolatedAP(rec, prec):
        # def CalculateAveragePrecision2(rec, prec):
        mrec = []
        # mrec.append(0)
        [mrec.append(e) for e in rec]
        # mrec.append(1)
        mpre = []
        # mpre.append(0)
        [mpre.append(e) for e in prec]
        # mpre.append(0)
        recallValues = np.linspace(0, 1, 11)
        recallValues = list(recallValues[::-1])
        rhoInterp = []
        recallValid = []
        # For each recallValues (0, 0.1, 0.2, ... , 1)
        for r in recallValues:
            # Obtain all recall values higher or equal than r
            argGreaterRecalls = np.argwhere(mrec[:] >= r)
            pmax = 0
            # If there are recalls above r
            if argGreaterRecalls.size != 0:
                pmax = max(mpre[argGreaterRecalls.min():])
            recallValid.append(r)
            rhoInterp.append(pmax)
        # By definition AP = sum(max(precision whose recall is above r))/11
        ap = sum(rhoInterp) / 11
        # Generating values for the plot
        rvals = []
        rvals.append(recallValid[0])
        [rvals.append(e) for e in recallValid]
        rvals.append(0)
        pvals = []
        pvals.append(0)
        [pvals.append(e) for e in rhoInterp]
        pvals.append(0)
        # rhoInterp = rhoInterp[::-1]
        cc = []
        for i in range(len(rvals)):
            p = (rvals[i], pvals[i - 1])
            if p not in cc:
                cc.append(p)
            p = (rvals[i], pvals[i])
            if p not in cc:
                cc.append(p)
        recallValues = [i[0] for i in cc]
        rhoInterp = [i[1] for i in cc]
        return [ap, rhoInterp, recallValues, None]

    # For each detections, calculate IOU with reference
    @staticmethod
    def _getAllIOUs(reference, detections):
        ret = []
        bbReference = reference.getAbsoluteBoundingBox(BBFormat.XYX2Y2)
        # img = np.zeros((200,200,3), np.uint8)
        for d in detections:
            bb = d.getAbsoluteBoundingBox(BBFormat.XYX2Y2)
            iou = Evaluator.iou(bbReference, bb)
            # Show blank image with the bounding boxes
            # img = add_bb_into_image(img, d, color=(255,0,0), thickness=2, label=None)
            # img = add_bb_into_image(img, reference, color=(0,255,0), thickness=2, label=None)
            ret.append((iou, reference, d))  # iou, reference, detection
        # cv2.imshow("comparing",img)
        # cv2.waitKey(0)
        # cv2.destroyWindow("comparing")
        # sort by iou (from highest to lowest)
        return sorted(ret, key=lambda i: i[0], reverse=True)

    @staticmethod
    def iou(boxA, boxB):
        # if boxes dont intersect
        if Evaluator._boxesIntersect(boxA, boxB) is False:
            return 0
        interArea = Evaluator._getIntersectionArea(boxA, boxB)
        union = Evaluator._getUnionAreas(boxA, boxB, interArea=interArea)
        # intersection over union
        iou = interArea / union
        assert iou >= 0
        return iou

    # boxA = (Ax1,Ay1,Ax2,Ay2)
    # boxB = (Bx1,By1,Bx2,By2)
    @staticmethod
    def _boxesIntersect(boxA, boxB):
        if boxA[0] > boxB[2]:
            return False  # boxA is right of boxB
        if boxB[0] > boxA[2]:
            return False  # boxA is left of boxB
        if boxA[3] < boxB[1]:
            return False  # boxA is above boxB
        if boxA[1] > boxB[3]:
            return False  # boxA is below boxB
        return True

    @staticmethod
    def _getIntersectionArea(boxA, boxB):
        xA = max(boxA[0], boxB[0])
        yA = max(boxA[1], boxB[1])
        xB = min(boxA[2], boxB[2])
        yB = min(boxA[3], boxB[3])
        # intersection area
        return (xB - xA + 1) * (yB - yA + 1)

    @staticmethod
    def _getUnionAreas(boxA, boxB, interArea=None):
        area_A = Evaluator._getArea(boxA)
        area_B = Evaluator._getArea(boxB)
        if interArea is None:
            interArea = Evaluator._getIntersectionArea(boxA, boxB)
        return float(area_A + area_B - interArea)

    @staticmethod
    def _getArea(box):
        return (box[2] - box[0] + 1) * (box[3] - box[1] + 1)
