""" 评估器模块：实现模型评估流程 """ import numpy as np import tensorflow as tf import time import os from typing import List, Dict, Tuple, Optional, Any, Union, Callable import pandas as pd import matplotlib.pyplot as plt import json from config.system_config import SAVED_MODELS_DIR from models.base_model import TextClassificationModel from evaluation.metrics import ClassificationMetrics from utils.logger import get_logger from utils.file_utils import ensure_dir, save_json logger = get_logger("Evaluator") class ModelEvaluator: """模型评估器，负责评估模型性能""" def __init__(self, model: TextClassificationModel, class_names: Optional[List[str]] = None, output_dir: Optional[str] = None, batch_size: Optional[int] = None): """ 初始化模型评估器 Args: model: 要评估的模型 class_names: 类别名称列表 output_dir: 输出目录，用于保存评估结果 batch_size: 批大小，如果为None则使用模型默认值 """ self.model = model self.class_names = class_names self.batch_size = batch_size or model.batch_size # 设置输出目录 if output_dir is None: self.output_dir = os.path.join(SAVED_MODELS_DIR, 'evaluation', model.model_name) else: self.output_dir = output_dir ensure_dir(self.output_dir) # 创建评估指标计算器 self.metrics = ClassificationMetrics(class_names) # 评估结果 self.evaluation_results = None logger.info(f"初始化模型评估器，模型: {model.model_name}") def evaluate(self, x_test: Union[np.ndarray, tf.data.Dataset], y_test: Optional[np.ndarray] = None, batch_size: Optional[int] = None, verbose: int = 1) -> Dict[str, float]: """ 评估模型 Args: x_test: 测试数据特征 y_test: 测试数据标签 batch_size: 批大小 verbose: 详细程度 Returns: 评估结果 """ batch_size = batch_size or self.batch_size logger.info(f"开始评估模型: {self.model.model_name}") start_time = time.time() # 使用模型评估 model_metrics = self.model.evaluate(x_test, y_test, verbose=verbose) # 获取预测结果 y_prob = self.model.predict(x_test, batch_size=batch_size, verbose=0) y_pred = np.argmax(y_prob, axis=1) # 处理y_test，确保y_test是一维数组 if isinstance(x_test, tf.data.Dataset): # 如果是TensorFlow Dataset，需要从中提取y_test y_test_extracted = np.concatenate([y for _, y in x_test], axis=0) y_test = y_test_extracted if y_test is not None and len(y_test.shape) > 1 and y_test.shape[1] > 1: y_test = np.argmax(y_test, axis=1) # 计算所有指标 all_metrics = self.metrics.calculate_all_metrics(y_test, y_pred, y_prob) # 合并模型内置指标和自定义指标 metrics_names = self.model.model.metrics_names model_metrics_dict = {name: float(value) for name, value in zip(metrics_names, model_metrics)} all_metrics.update(model_metrics_dict) # 记录评估时间 evaluation_time = time.time() - start_time all_metrics['evaluation_time'] = evaluation_time # 保存评估结果 self.evaluation_results = { 'metrics': all_metrics, 'confusion_matrix': self.metrics.confusion_matrix(y_test, y_pred).tolist(), 'classification_report': self.metrics.classification_report(y_test, y_pred, output_dict=True) } logger.info(f"模型评估完成，用时: {evaluation_time:.2f} 秒") logger.info(f"主要评估指标: accuracy={all_metrics.get('accuracy', 'N/A'):.4f}, " f"f1_macro={all_metrics.get('f1_macro', 'N/A'):.4f}") return all_metrics def save_evaluation_results(self, save_plots: bool = True) -> str: """ 保存评估结果 Args: save_plots: 是否保存可视化图表 Returns: 结果保存路径 """ if self.evaluation_results is None: raise ValueError("请先调用evaluate方法进行评估") # 保存评估结果为JSON results_path = os.path.join(self.output_dir, 'evaluation_results.json') with open(results_path, 'w', encoding='utf-8') as f: json.dump(self.evaluation_results, f, ensure_ascii=False, indent=4) # 保存评估指标为CSV metrics_df = pd.DataFrame( self.evaluation_results['metrics'].items(), columns=['Metric', 'Value'] ).set_index('Metric') metrics_path = os.path.join(self.output_dir, 'metrics.csv') metrics_df.to_csv(metrics_path) # 保存可视化图表 if save_plots: self._save_plots() logger.info(f"评估结果已保存到: {self.output_dir}") return self.output_dir def _save_plots(self) -> None: """保存评估结果可视化图表""" if self.evaluation_results is None: raise ValueError("请先调用evaluate方法进行评估") # 创建可视化目录 plots_dir = os.path.join(self.output_dir, 'plots') ensure_dir(plots_dir) # 混淆矩阵图 cm_path = os.path.join(plots_dir, 'confusion_matrix.png') cm = np.array(self.evaluation_results['confusion_matrix']) # 将混淆矩阵转换为NumPy数组 if isinstance(cm, list): cm = np.array(cm) # 绘制混淆矩阵 self.metrics.plot_confusion_matrix( np.arange(cm.shape[0]), # 假设标签 np.arange(cm.shape[1]), # 假设预测 normalize='true', save_path=cm_path ) # 保存评估指标条形图 metrics_path = os.path.join(plots_dir, 'metrics_bar.png') metrics = self.evaluation_results['metrics'] # 选择要展示的主要指标 main_metrics = { 'accuracy': metrics.get('accuracy', 0), 'precision_macro': metrics.get('precision_macro', 0), 'recall_macro': metrics.get('recall_macro', 0), 'f1_macro': metrics.get('f1_macro', 0) } # 绘制条形图 plt.figure(figsize=(10, 6)) plt.bar(main_metrics.keys(), main_metrics.values()) plt.title('Main Evaluation Metrics') plt.ylabel('Score') plt.ylim(0, 1) plt.xticks(rotation=45, ha='right') plt.tight_layout() plt.savefig(metrics_path) plt.close() # 如果有类别级别的指标，绘制每个类别的指标 if 'classification_report' in self.evaluation_results: report = self.evaluation_results['classification_report'] # 提取每个类别的精确率、召回率和F1值 class_metrics = {} for key, value in report.items(): if key not in ['accuracy', 'macro avg', 'weighted avg', 'samples avg']: if isinstance(value, dict): class_metrics[key] = value if class_metrics: # 绘制每个类别的F1分数 class_f1_path = os.path.join(plots_dir, 'class_f1_scores.png') classes = list(class_metrics.keys()) f1_scores = [metrics['f1-score'] for metrics in class_metrics.values()] plt.figure(figsize=(12, 6)) bars = plt.bar(classes, f1_scores) # 在柱状图上方显示数值 for bar in bars: height = bar.get_height() plt.text(bar.get_x() + bar.get_width() / 2., height + 0.01, f'{height:.2f}', ha='center', va='bottom', rotation=0) plt.title('F1 Score by Class') plt.ylabel('F1 Score') plt.ylim(0, 1.1) plt.xticks(rotation=45, ha='right') plt.tight_layout() plt.savefig(class_f1_path) plt.close() # 绘制每个类别的精确率和召回率 class_prec_rec_path = os.path.join(plots_dir, 'class_precision_recall.png') precisions = [metrics['precision'] for metrics in class_metrics.values()] recalls = [metrics['recall'] for metrics in class_metrics.values()] plt.figure(figsize=(12, 6)) x = np.arange(len(classes)) width = 0.35 plt.bar(x - width / 2, precisions, width, label='Precision') plt.bar(x + width / 2, recalls, width, label='Recall') plt.ylabel('Score') plt.title('Precision and Recall by Class') plt.xticks(x, classes, rotation=45, ha='right') plt.legend() plt.ylim(0, 1.1) plt.tight_layout() plt.savefig(class_prec_rec_path) plt.close() logger.info(f"评估可视化图表已保存到: {plots_dir}") def compare_models(self, other_evaluators: List['ModelEvaluator'], metrics: Optional[List[str]] = None, save_path: Optional[str] = None) -> pd.DataFrame: """ 比较多个模型的评估结果 Args: other_evaluators: 其他模型评估器列表 metrics: 要比较的指标列表，默认为['accuracy', 'precision_macro', 'recall_macro', 'f1_macro'] save_path: 比较结果的保存路径 Returns: 比较结果DataFrame """ if self.evaluation_results is None: raise ValueError("请先调用evaluate方法进行评估") # 默认比较指标 if metrics is None: metrics = ['accuracy', 'precision_macro', 'recall_macro', 'f1_macro'] # 收集所有模型的评估指标 models_metrics = {} # 当前模型 models_metrics[self.model.model_name] = { metric: self.evaluation_results['metrics'].get(metric, 'N/A') for metric in metrics } # 其他模型 for evaluator in other_evaluators: if evaluator.evaluation_results is None: logger.warning(f"模型 {evaluator.model.model_name} 尚未评估，跳过") continue models_metrics[evaluator.model.model_name] = { metric: evaluator.evaluation_results['metrics'].get(metric, 'N/A') for metric in metrics } # 创建比较DataFrame comparison_df = pd.DataFrame(models_metrics).T # 保存比较结果 if save_path: comparison_df.to_csv(save_path) logger.info(f"模型比较结果已保存到: {save_path}") # 绘制比较条形图 plt.figure(figsize=(12, 6)) comparison_df.plot(kind='bar', figsize=(12, 6)) plt.title('Model Comparison') plt.ylabel('Score') plt.ylim(0, 1) plt.legend(loc='lower right') plt.tight_layout() # 如果save_path是CSV文件，将其替换为PNG文件 if save_path.endswith('.csv'): plot_path = save_path.replace('.csv', '.png') else: plot_path = save_path + '.png' plt.savefig(plot_path) plt.close() logger.info(f"模型比较图表已保存到: {plot_path}") return comparison_df def evaluate_class_performance(self, x_test: Union[np.ndarray, tf.data.Dataset], y_test: Optional[np.ndarray] = None, batch_size: Optional[int] = None, verbose: int = 0) -> pd.DataFrame: """ 评估模型在各个类别上的性能 Args: x_test: 测试数据特征 y_test: 测试数据标签 batch_size: 批大小 verbose: 详细程度 Returns: 各类别性能指标DataFrame """ batch_size = batch_size or self.batch_size # 获取预测结果 y_prob = self.model.predict(x_test, batch_size=batch_size, verbose=verbose) y_pred = np.argmax(y_prob, axis=1) # 处理y_test，确保y_test是一维数组 if isinstance(x_test, tf.data.Dataset): # 如果是TensorFlow Dataset，需要从中提取y_test y_test_extracted = np.concatenate([y for _, y in x_test], axis=0) y_test = y_test_extracted if y_test is not None and len(y_test.shape) > 1 and y_test.shape[1] > 1: y_test = np.argmax(y_test, axis=1) # 获取分类报告 report = self.metrics.classification_report(y_test, y_pred, output_dict=True) # 提取各类别指标 class_metrics = {} for key, value in report.items(): if key not in ['accuracy', 'macro avg', 'weighted avg', 'samples avg']: if isinstance(value, dict): class_metrics[key] = value # 转换为DataFrame class_performance_df = pd.DataFrame(class_metrics).T # 添加支持度（样本数量） class_counts = np.bincount(y_test) for idx, count in enumerate(class_counts): if str(idx) in class_performance_df.index: class_performance_df.loc[str(idx), 'support'] = count # 添加类别名称 if self.class_names: class_performance_df['class_name'] = [ self.class_names[int(idx)] if int(idx) < len(self.class_names) else idx for idx in class_performance_df.index ] # 保存类别性能指标 performance_path = os.path.join(self.output_dir, 'class_performance.csv') class_performance_df.to_csv(performance_path) logger.info(f"各类别性能指标已保存到: {performance_path}") return class_performance_df def plot_error_analysis(self, x_test: Union[np.ndarray, tf.data.Dataset], y_test: Optional[np.ndarray] = None, batch_size: Optional[int] = None, num_samples: int = 10, save_path: Optional[str] = None) -> None: """ 绘制误分类样本分析 Args: x_test: 测试数据特征 y_test: 测试数据标签 batch_size: 批大小 num_samples: 要展示的误分类样本数量 save_path: 保存路径 """ # 仅适用于文本数据的分析，需要原始文本 logger.info("误分类样本分析需要原始文本数据，此方法可能需要根据实际数据类型进行修改") # 在实际应用中，这里应该根据实际数据类型进行修改 # 例如，对于序列化的文本，可能需要反序列化，或者使用词汇表将索引转换回文本 # 此处仅展示一个基本框架 batch_size = batch_size or self.batch_size # 获取预测结果 y_prob = self.model.predict(x_test, batch_size=batch_size, verbose=0) y_pred = np.argmax(y_prob, axis=1) # 处理y_test，确保y_test是一维数组 if isinstance(x_test, tf.data.Dataset): # 如果是TensorFlow Dataset，需要从中提取y_test和x_test dataset_iterator = iter(x_test) x_test_extracted = [] y_test_extracted = [] try: while True: x, y = next(dataset_iterator) x_test_extracted.append(x) y_test_extracted.append(y) except StopIteration: pass x_test = np.concatenate(x_test_extracted, axis=0) y_test = np.concatenate(y_test_extracted, axis=0) if y_test is not None and len(y_test.shape) > 1 and y_test.shape[1] > 1: y_test = np.argmax(y_test, axis=1) # 找出误分类样本 misclassified_indices = np.where(y_pred != y_test)[0] # 如果没有误分类样本，返回 if len(misclassified_indices) == 0: logger.info("没有误分类样本") return # 随机选择一些误分类样本 if len(misclassified_indices) > num_samples: misclassified_indices = np.random.choice(misclassified_indices, num_samples, replace=False) # 保存误分类样本分析结果 misclassified_data = [] for idx in misclassified_indices: true_label = y_test[idx] pred_label = y_pred[idx] true_class = self.class_names[true_label] if self.class_names else str(true_label) pred_class = self.class_names[pred_label] if self.class_names else str(pred_label) # 对于序列化的文本，此处需要进行反序列化 # 这里仅作示例，实际应用中需要根据具体数据类型修改 sample_text = f"Sample {idx}" misclassified_data.append({ 'sample_id': idx, 'true_label': true_label, 'predicted_label': pred_label, 'true_class': true_class, 'predicted_class': pred_class, 'confidence': float(y_prob[idx, pred_label]), 'sample_text': sample_text }) # 创建DataFrame misclassified_df = pd.DataFrame(misclassified_data) # 保存结果 if save_path: misclassified_df.to_csv(save_path) logger.info(f"误分类样本分析已保存到: {save_path}") return misclassified_df