yolov8ToEZB/py/metrics.py

import numpy as np
from sklearn.preprocessing import normalize


def classification_acc_metric(names, x, *args):
    """ Classification Acc Metric """
    # get probability
    output_name = args[-1]
    label = args[0]
    prob = x[output_name]  # (N, K)

    # soft-max
    prob -= np.max(prob, axis=-1, keepdims=True)  # avoid numerical issues
    prob = np.exp(prob)
    prob /= np.sum(prob, axis=-1, keepdims=True)

    # acc
    target = prob.argmax(axis=-1)  # top 1
    acc = np.mean(target == label)
    return acc


def classification_entropy_metric(names, x, *args):
    """ Classification Entropy Loss Metric """
    # get probability
    output_name, label = args[-1], args[0]
    prob = x[output_name]

    # log-soft-max
    prob -= np.max(prob, axis=-1, keepdims=True)
    prob = np.exp(prob)
    prob /= np.sum(prob, axis=-1, keepdims=True)
    prob = np.log(prob)
    prob = np.nan_to_num(prob)  # for numerical stability

    # cross-entropy loss value. Remember, larger means better, so we do not multiply -1 here.
    return np.mean(prob[range(prob.shape[0]), label])


def recognition_pairs_metric(names, x, *args):
    """ Recognition Pairs Metric """
    # get register embedding and test embedding
    output_name, label = args[-1], args[0]
    embedding = x[output_name]
    register, test = embedding[::2], embedding[1::2]

    # cosine similarity & acc
    register, test = normalize(register), normalize(test)
    cosine_similarity = np.diag(np.matmul(register, test.T))
    best_acc, best_threshold = 0.0, 0.0
    for threshold in np.linspace(0, 1, 20):
        pred = cosine_similarity > threshold
        tp = np.sum(np.logical_and(pred, label))
        tn = np.sum(np.logical_and(np.logical_not(pred), np.logical_not(label)))
        acc = 1.0 * (tp + tn) / len(label)
        if acc > best_acc:
            best_acc, best_threshold = acc, threshold
    return best_acc


def recognition_before_after_metric(names, x, *args):
    """ Compare Output Before And After Quantization"""

    # get data
    output_name = args[-1]
    output_before_quantization = args[0]
    before_embedding = output_before_quantization[output_name]
    after_embedding = x[output_name]
    if before_embedding.ndim > 2:
        before_embedding = before_embedding.reshape(before_embedding.shape[0], -1)
    if after_embedding.ndim > 2:
        after_embedding = after_embedding.reshape(after_embedding.shape[0], -1)

    # A face's similarity must be close to 1.0 before and after quantization
    before_embedding, after_embedding = normalize(before_embedding), normalize(after_embedding)
    cosine_similarity = np.diag(np.matmul(before_embedding, after_embedding.T))
    thresholds = [0.85, 0.875, 0.90, 0.925, 0.95, 0.975]
    ave_acc = 0.0
    for threshold in thresholds:
        ave_acc += 1.0 * np.sum(cosine_similarity > threshold) / len(cosine_similarity)
    ave_acc /= len(thresholds)
    return ave_acc


def classification_top_1_acc(names, x, *args):
    """ classification top 1 acc """
    # prob
    output_name = args[-1]
    label = args[0]
    prob = x[output_name].squeeze()

    if prob.ndim == 1:
        prob = prob[None, :]

    # soft-max
    prob -= np.max(prob, axis=-1, keepdims=True)  # avoid numerical issues
    prob = np.exp(prob)
    prob /= np.sum(prob, axis=-1, keepdims=True)

    # top 1 acc
    target = prob.argmax(axis=-1)
    acc = np.mean(target == label)
    return acc


def classification_top_5_acc(names, x, *args):
    """ classification top 5 acc """
    # prob
    output_name = args[-1]
    label = args[0]
    prob = x[output_name].squeeze()

    if prob.ndim == 1:
        prob = prob[None, :]

    # soft-max
    prob -= np.max(prob, axis=-1, keepdims=True)  # avoid numerical issues
    prob = np.exp(prob)
    prob /= np.sum(prob, axis=-1, keepdims=True)

    # top 5 acc
    index = np.argsort(prob, axis=-1)[:, -5:]
    tile_label = np.tile(label, (5, 1)).T
    acc = 1.0 * np.sum(index == tile_label) / len(label)
    return acc


def cos_similarity_metric(names, x, *args):
    output_name = args[-1]
    before_emb = args[0][output_name]
    after_emb = x[output_name]

    if np.isnan(np.min(after_emb)) or np.isinf(np.min(after_emb)) or np.isinf(np.max(after_emb)):
        return 0.0

    if before_emb.ndim == 1:
        before_emb = before_emb[None, ...]
    if after_emb.ndim == 1:
        after_emb = after_emb[None, ...]
    if before_emb.ndim > 2:
        before_emb = before_emb.reshape(before_emb.shape[0], -1)
    if after_emb.ndim > 2:
        after_emb = after_emb.reshape(after_emb.shape[0], -1)

    before_emb, after_emb = normalize(before_emb), normalize(after_emb)
    sim = (before_emb * after_emb).sum(axis=1).mean()
    return sim


last_sim_mean, last_sim_std = None, None


def cos_similarity_mean_std_metric(names, x, *args):
    global last_sim_mean, last_sim_std
    output_name = args[-1]
    before_emb = args[0][output_name]
    after_emb = x[output_name]
    if before_emb.ndim == 1:
        before_emb = before_emb[None, ...]
    if after_emb.ndim == 1:
        after_emb = after_emb[None, ...]
    if before_emb.ndim > 2:
        before_emb = before_emb.reshape(before_emb.shape[0], -1)
    if after_emb.ndim > 2:
        after_emb = after_emb.reshape(after_emb.shape[0], -1)
    before_emb, after_emb = normalize(before_emb), normalize(after_emb)
    sim = (before_emb * after_emb).sum(axis=1)
    sim_mean, sim_std = sim.mean(), sim.std()
    if last_sim_mean is None and last_sim_std is None:
        last_sim_mean, last_sim_std = sim_mean, sim_std
    mean_loss, std_loss = sim_mean - last_sim_mean, last_sim_std - sim_std
    loss = mean_loss + std_loss
    return loss


def mse_metric(names, x, *args):
    output_name = args[-1]
    before_output = args[0][output_name]
    after_output = x[output_name]
    mse = ((before_output - after_output) ** 2).mean()
    return -1 * mse


def yolo_metric(names, x, *args):
    num_outputs = 3  # YOLO has 3 outputs
    output_names = args[-num_outputs:]

    sim = 0.0
    for name in output_names:
        src_data = args[0][name]
        dst_data = x[name]
        if np.isnan(np.min(dst_data)) or np.isinf(np.min(dst_data)) or np.isinf(np.max(dst_data)):
            return 0.0

        n = src_data.shape[0]

        src_xyxy = src_data[..., :4].reshape(n, -1)  # [x, y, x, y, obj, cls]
        dst_xyxy = dst_data[..., :4].reshape(n, -1)

        src_obj = src_data[..., 4].reshape(n, -1)
        dst_obj = dst_data[..., 4].reshape(n, -1)

        src_cls = src_data[..., 5:].reshape(n, -1)
        dst_cls = dst_data[..., 5:].reshape(n, -1)

        sim_xyxy = (normalize(src_xyxy) * normalize(dst_xyxy)).sum(axis=1).mean()
        sim_obj = (normalize(src_obj) * normalize(dst_obj)).sum(axis=1).mean()
        sim_cls = (normalize(src_cls) * normalize(dst_cls)).sum(axis=1).mean()
        sim += 0.3 * sim_xyxy + 0.3 * sim_obj + 0.4 * sim_cls
        print(name, sim_xyxy, sim_obj, sim_cls)
    sim /= 3
    return sim


"""
You Can Add Your Custom Metric Here. 
"""

def yolo_metric_hjj(names, x, *args):
    num_outputs = 3  # YOLO has 3 outputs
    output_names = args[-num_outputs:]

    sim = 0.0
    for name in output_names:
        src_data = args[0][name]
        dst_data = x[name]
        # print(src_data.shape, dst_data.shape)
        
        if np.isnan(np.min(dst_data)) or np.isinf(np.min(dst_data)) or np.isinf(np.max(dst_data)):
            return 0.0

        n, _, h, w = src_data.shape
        # src_data = np.transpose(src_data, axes=[0, 2, 1])
        # dst_data = np.transpose(dst_data, axes=[0, 2, 1])
        
        src_xyxy = src_data[:, :4].reshape(n, -1)  # [x, y, x, y, obj, cls]
        dst_xyxy = dst_data[:, :4].reshape(n, -1)

        src_cls = src_data[:, 4:].reshape(n, -1)
        dst_cls = dst_data[:, 4:].reshape(n, -1)

        # sim_xyxy = (normalize(src_xyxy) * normalize(dst_xyxy)).sum(axis=1).mean()
        # sim_cls = (normalize(src_cls) * normalize(dst_cls)).sum(axis=1).mean()
        
        sim_xyxy = -np.abs(normalize(src_xyxy) - normalize(dst_xyxy)).sum(axis=1).mean()
        sim_cls = -np.abs(normalize(src_cls) - normalize(dst_cls)).sum(axis=1).mean()
        
        sim += 0.6 * sim_xyxy + 0.4 * sim_cls
        # print(name, sim_xyxy, sim_obj, sim_cls)
    return sim / num_outputs