当前位置：首页 > news >正文

实时手机检测-通用效果评估：Precision-Recall曲线绘制与阈值选优

news 2026/3/26 22:59:45

实时手机检测-通用效果评估：Precision-Recall曲线绘制与阈值选优

1. 模型效果评估的重要性

在目标检测项目中，仅仅能够检测出目标是不够的，我们还需要知道检测的质量如何。这就是为什么需要进行效果评估的原因。想象一下，如果一个手机检测模型把很多不是手机的东西误判为手机，或者漏掉了很多真正的手机，这样的模型在实际应用中就会带来很多问题。

Precision-Recall曲线是一种非常直观的评估工具，它能够帮助我们：

了解模型在不同置信度阈值下的表现
找到最适合实际应用的阈值设置
比较不同模型的性能优劣
为后续应用（如打电话检测）提供可靠的基础

2. 环境准备与数据收集

2.1 安装必要依赖

首先确保已经安装了模型运行所需的环境，然后安装评估相关的库：

pip install numpy matplotlib scikit-learn opencv-python

2.2 准备测试数据集

为了进行效果评估，我们需要准备一个包含标注信息的测试集。这个测试集应该包含：

各种场景下的手机图片
准确的标注信息（手机的位置和类别）
不同角度、光照条件下的样本

import os import json import cv2 # 加载标注数据 def load_annotations(annotation_path): with open(annotation_path, 'r') as f: annotations = json.load(f) return annotations # 示例标注数据结构 annotations_example = { "images": [ { "id": 1, "file_name": "phone_image_001.jpg", "width": 640, "height": 480 } ], "annotations": [ { "id": 1, "image_id": 1, "category_id": 1, "bbox": [100, 150, 80, 120], # [x, y, width, height] "area": 9600, "iscrowd": 0 } ], "categories": [ { "id": 1, "name": "phone" } ] }

3. 模型推理与结果收集

3.1 运行模型获取检测结果

使用实时手机检测模型对测试集进行推理，并保存检测结果：

import gradio as gr import numpy as np from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks # 初始化模型 phone_detector = pipeline(Tasks.domain_specific_object_detection, model='damo/cv_tinynas_object-detection_damoyolo_phone') def detect_phones(image): """使用模型进行手机检测""" result = phone_detector(image) return result # 批量处理测试集 def process_test_set(test_images_dir, output_file): results = [] image_files = [f for f in os.listdir(test_images_dir) if f.endswith(('.jpg', '.png', '.jpeg'))] for img_file in image_files: image_path = os.path.join(test_images_dir, img_file) image = cv2.imread(image_path) image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # 获取检测结果 detection_result = detect_phones(image) results.append({ 'image_id': img_file, 'detections': detection_result }) # 保存结果 with open(output_file, 'w') as f: json.dump(results, f) return results

3.2 解析检测结果

def parse_detection_results(detection_results, confidence_threshold=0.25): """解析检测结果，提取边界框和置信度""" all_detections = [] for result in detection_results: image_id = result['image_id'] detections = result['detections'] for det in detections.get('boxes', []): if len(det) >= 6: # 确保有足够的元素 x1, y1, x2, y2, confidence, class_id = det[:6] if confidence >= confidence_threshold: all_detections.append({ 'image_id': image_id, 'bbox': [x1, y1, x2 - x1, y2 - y1], # 转换为[x, y, width, height]格式 'confidence': confidence, 'class_id': class_id }) return all_detections

4. Precision-Recall曲线绘制

4.1 计算Precision和Recall

from sklearn.metrics import precision_recall_curve import numpy as np def calculate_precision_recall(ground_truths, detections, iou_threshold=0.5): """计算Precision和Recall""" # 准备数据 gt_boxes = [] detection_boxes = [] detection_scores = [] # 匹配检测结果和真实标注 for image_id in set([gt['image_id'] for gt in ground_truths]): image_gts = [gt for gt in ground_truths if gt['image_id'] == image_id] image_dets = [det for det in detections if det['image_id'] == image_id] # 计算IOU并匹配 for det in image_dets: detection_scores.append(det['confidence']) # 找到最佳匹配的真实框 best_iou = 0 for gt in image_gts: iou = calculate_iou(det['bbox'], gt['bbox']) if iou > best_iou: best_iou = iou detection_boxes.append(1 if best_iou >= iou_threshold else 0) # 按置信度排序 sorted_indices = np.argsort(detection_scores)[::-1] detection_boxes_sorted = [detection_boxes[i] for i in sorted_indices] # 计算Precision和Recall precision = [] recall = [] true_positives = 0 false_positives = 0 total_gt = len(ground_truths) for i, is_true_positive in enumerate(detection_boxes_sorted): if is_true_positive: true_positives += 1 else: false_positives += 1 current_precision = true_positives / (true_positives + false_positives) if (true_positives + false_positives) > 0 else 0 current_recall = true_positives / total_gt if total_gt > 0 else 0 precision.append(current_precision) recall.append(current_recall) return precision, recall, detection_scores def calculate_iou(box1, box2): """计算两个边界框的IOU""" # 计算交集区域 x1 = max(box1[0], box2[0]) y1 = max(box1[1], box2[1]) x2 = min(box1[0] + box1[2], box2[0] + box2[2]) y2 = min(box1[1] + box1[3], box2[1] + box2[3]) # 计算交集面积 intersection = max(0, x2 - x1) * max(0, y2 - y1) # 计算并集面积 area1 = box1[2] * box1[3] area2 = box2[2] * box2[3] union = area1 + area2 - intersection return intersection / union if union > 0 else 0

4.2 绘制Precision-Recall曲线

import matplotlib.pyplot as plt def plot_precision_recall_curve(precision, recall, ap_score=None): """绘制Precision-Recall曲线""" plt.figure(figsize=(10, 6)) # 绘制曲线 plt.plot(recall, precision, 'b-', linewidth=2, label='Precision-Recall curve') # 设置图表属性 plt.xlabel('Recall', fontsize=12) plt.ylabel('Precision', fontsize=12) plt.title('Precision-Recall Curve for Phone Detection', fontsize=14) plt.grid(True, alpha=0.3) # 添加AP分数 if ap_score is not None: plt.text(0.6, 0.05, f'AP: {ap_score:.3f}', fontsize=12, bbox=dict(facecolor='white', alpha=0.5)) plt.legend(loc='lower left') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.tight_layout() plt.show() # 完整的评估流程 def evaluate_model_performance(ground_truths, detections): """完整的模型性能评估""" precision, recall, scores = calculate_precision_recall(ground_truths, detections) # 计算AP（Average Precision） ap_score = calculate_average_precision(precision, recall) # 绘制曲线 plot_precision_recall_curve(precision, recall, ap_score) return ap_score, precision, recall def calculate_average_precision(precision, recall): """计算Average Precision""" # 确保recall是递增的 recall = np.array(recall) precision = np.array(precision) # 按recall排序 sorted_indices = np.argsort(recall) recall = recall[sorted_indices] precision = precision[sorted_indices] # 计算AP ap = 0.0 for i in range(1, len(recall)): ap += (recall[i] - recall[i-1]) * precision[i] return ap

5. 阈值选择策略

5.1 寻找最佳置信度阈值

def find_optimal_threshold(precision, recall, scores): """寻找最佳置信度阈值""" # 计算F1分数 f1_scores = [2 * p * r / (p + r) if (p + r) > 0 else 0 for p, r in zip(precision, recall)] # 找到F1分数最大的阈值 best_f1_index = np.argmax(f1_scores) best_threshold = scores[best_f1_index] if best_f1_index < len(scores) else 0.5 # 绘制F1分数曲线 plt.figure(figsize=(12, 5)) plt.subplot(1, 2, 1) plt.plot(scores[:len(f1_scores)], f1_scores, 'r-', linewidth=2) plt.xlabel('Confidence Threshold') plt.ylabel('F1 Score') plt.title('F1 Score vs Confidence Threshold') plt.grid(True, alpha=0.3) plt.subplot(1, 2, 2) plt.plot(scores[:len(precision)], precision, 'g-', label='Precision', linewidth=2) plt.plot(scores[:len(recall)], recall, 'b-', label='Recall', linewidth=2) plt.xlabel('Confidence Threshold') plt.ylabel('Score') plt.title('Precision/Recall vs Confidence Threshold') plt.legend() plt.grid(True, alpha=0.3) plt.tight_layout() plt.show() return best_threshold, best_f1_index def apply_optimal_threshold(detections, optimal_threshold): """应用最佳阈值过滤检测结果""" filtered_detections = [] for det in detections: if det['confidence'] >= optimal_threshold: filtered_detections.append(det) return filtered_detections

5.2 不同应用场景的阈值选择

在实际应用中，不同的场景可能需要不同的阈值策略：

def get_threshold_for_scenario(scenario_type): """根据不同应用场景推荐阈值""" threshold_strategies = { 'high_precision': 0.7, # 高精度模式：减少误报，适合安全关键应用 'balanced': 0.4, # 平衡模式：兼顾精度和召回率 'high_recall': 0.2, # 高召回模式：减少漏检，适合搜索应用 'real_time': 0.3 # 实时模式：平衡速度和准确性 } return threshold_strategies.get(scenario_type, 0.4) # 示例：为打电话检测应用选择阈值 def setup_phone_call_detection(): """为打电话检测应用配置最佳阈值""" # 打电话检测需要高精度，避免误报 threshold = get_threshold_for_scenario('high_precision') print(f"推荐阈值 for 打电话检测: {threshold}") return threshold

6. 实际应用示例

6.1 集成到Gradio界面

将阈值选择功能集成到Web界面中，让用户可以实时调整：

def create_interactive_ui(): """创建交互式评估界面""" with gr.Blocks(title="手机检测效果评估") as demo: gr.Markdown("# 实时手机检测效果评估工具") with gr.Row(): with gr.Column(): image_input = gr.Image(label="上传测试图片", type="numpy") confidence_slider = gr.Slider(minimum=0.1, maximum=0.9, value=0.5, label="置信度阈值", step=0.05) detect_btn = gr.Button("开始检测") with gr.Column(): output_image = gr.Image(label="检测结果") metrics_output = gr.Textbox(label="性能指标") # 添加PR曲线显示 with gr.Row(): pr_curve = gr.Plot(label="Precision-Recall曲线") def process_image(image, confidence_threshold): # 执行检测 result = detect_phones(image) # 绘制检测结果 output_img = image.copy() for box in result.get('boxes', []): if len(box) >= 5 and box[4] >= confidence_threshold: x1, y1, x2, y2 = int(box[0]), int(box[1]), int(box[2]), int(box[3]) cv2.rectangle(output_img, (x1, y1), (x2, y2), (0, 255, 0), 2) cv2.putText(output_img, f"{box[4]:.2f}", (x1, y1-10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2) # 计算性能指标（简化版） precision = 0.85 # 实际应用中需要基于标注数据计算 recall = 0.78 # 实际应用中需要基于标注数据计算 f1_score = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0 metrics_text = f"Precision: {precision:.3f}\nRecall: {recall:.3f}\nF1 Score: {f1_score:.3f}" return output_img, metrics_text detect_btn.click(process_image, inputs=[image_input, confidence_slider], outputs=[output_image, metrics_output]) return demo

6.2 批量评估脚本

def batch_evaluation(test_set_path, annotations_path): """批量评估模型在测试集上的表现""" # 加载标注数据 with open(annotations_path, 'r') as f: ground_truths = json.load(f) # 处理测试集获取检测结果 detection_results = process_test_set(test_set_path, "temp_detections.json") # 解析检测结果 detections = parse_detection_results(detection_results) # 计算性能指标 ap_score, precision, recall = evaluate_model_performance(ground_truths, detections) # 寻找最佳阈值 scores = [det['confidence'] for det in detections] optimal_threshold, best_f1_index = find_optimal_threshold(precision, recall, scores) print(f"模型性能评估结果:") print(f"Average Precision: {ap_score:.4f}") print(f"最佳置信度阈值: {optimal_threshold:.4f}") print(f"最佳F1分数: {2 * precision[best_f1_index] * recall[best_f1_index] / (precision[best_f1_index] + recall[best_f1_index]):.4f}") return ap_score, optimal_threshold # 运行批量评估 if __name__ == "__main__": ap_score, optimal_threshold = batch_evaluation("test_images/", "annotations.json")