diff --git a/autoware_ml/detection3d/datasets/transforms/__init__.py b/autoware_ml/detection3d/datasets/transforms/__init__.py index 6bc932f1a..a63ff1eea 100644 --- a/autoware_ml/detection3d/datasets/transforms/__init__.py +++ b/autoware_ml/detection3d/datasets/transforms/__init__.py @@ -1,3 +1,4 @@ +from .local_3d_bbox import Local3DBBoxExpand from .object_min_points_filter import ObjectMinPointsFilter -__all__ = ["ObjectMinPointsFilter"] +__all__ = ["ObjectMinPointsFilter", "Local3DBBoxExpand"] diff --git a/autoware_ml/detection3d/datasets/transforms/local_3d_bbox.py b/autoware_ml/detection3d/datasets/transforms/local_3d_bbox.py new file mode 100644 index 000000000..cd340cbe1 --- /dev/null +++ b/autoware_ml/detection3d/datasets/transforms/local_3d_bbox.py @@ -0,0 +1,77 @@ +from typing import List, Optional + +import numpy as np +from mmcv.transforms import BaseTransform +from mmdet3d.structures.ops import box_np_ops +from mmengine.registry import TRANSFORMS + + +@TRANSFORMS.register_module() +class Local3DBBoxExpand(BaseTransform): + """Locally expand the 3D bounding boxes by scaling the width, which it doesn't scale the points. + + Args: + expand_widths: (List[float]): Uniformly sampled expand width. + width_dim: (int): The dimension of the width. Default is 4, which is the width dimension of the 3D + bounding box. Since 3D Bbox is in the format of [x, y, z, dx, dy, dz, heading], the width dimension is the + 4th dimension. + label_ids: (List[int]): The label IDs to expand. If None, all label IDs will be expanded. + """ + + def __init__( + self, + expand_widths: List[float], + expand_lengths: Optional[List[float]] = None, + length_dim: int = 3, + width_dim: int = 4, + label_ids: List[int] = None, + ) -> None: + + super().__init__() + assert isinstance(expand_widths, list) + assert len(expand_widths) == 2 + assert expand_widths[0] < expand_widths[1] + if expand_lengths is not None: + assert isinstance(expand_lengths, list) + assert len(expand_lengths) == 2 + assert expand_lengths[0] < expand_lengths[1] + self.expand_lengths = expand_lengths + self.length_dim = length_dim + self.expand_widths = expand_widths + self.width_dim = width_dim + self.label_ids = label_ids + + def transform(self, input_dict: dict) -> dict: + """Call function to locally augment the 3D bounding boxes by scaling the width. + + Args: + input_dict (dict): Result dict from loading pipeline. + + Returns: + dict: Results after locally augmenting the 3D bounding boxes by scaling the width, 'gt_bboxes_3d' \ + key is updated in the result dict. + """ + # Label mask + if self.label_ids is not None: + label_masks = [True if label in self.label_ids else False for label in input_dict["gt_labels_3d"]] + else: + label_masks = np.ones(len(input_dict["gt_labels_3d"]), dtype=bool) + + for i in range(len(input_dict["gt_bboxes_3d"])): + if not label_masks[i]: + continue + + expand_width = np.random.uniform(self.expand_widths[0], self.expand_widths[1]) + input_dict["gt_bboxes_3d"].tensor[i, self.width_dim] += expand_width + if self.expand_lengths is not None: + expand_length = np.random.uniform(self.expand_lengths[0], self.expand_lengths[1]) + input_dict["gt_bboxes_3d"].tensor[i, self.length_dim] += expand_length + + return input_dict + + def __repr__(self) -> str: + """str: Return a string that describes the module.""" + repr_str = self.__class__.__name__ + repr_str += f"(expand_widths={self.expand_widths}, expand_lengths={self.expand_lengths}, \ + length_dim={self.length_dim}, width_dim={self.width_dim}, label_ids={self.label_ids})" + return repr_str diff --git a/autoware_ml/detection3d/evaluation/t4metric/t4metric_v2.py b/autoware_ml/detection3d/evaluation/t4metric/t4metric_v2.py index c7865320f..f7f1af021 100644 --- a/autoware_ml/detection3d/evaluation/t4metric/t4metric_v2.py +++ b/autoware_ml/detection3d/evaluation/t4metric/t4metric_v2.py @@ -985,11 +985,27 @@ def _aggregate_metrics_data( # Create precision_interpolate and recall_interpolate keys iterable_metrics[ - f"T4MetricV2_label_detection/{label_name}_precisions_{matching_mode}_{threshold}" + f"T4MetricV2_label_detection/{label_name}_interp-precisions_{matching_mode}_{threshold}" ] = ap.precision_interp.tolist() iterable_metrics[ - f"T4MetricV2_label_detection/{label_name}_recalls_{matching_mode}_{threshold}" + f"T4MetricV2_label_detection/{label_name}_interp-recalls_{matching_mode}_{threshold}" ] = ap.recall_interp.tolist() + iterable_metrics[ + f"T4MetricV2_label_detection/{label_name}_interp-confs_{matching_mode}_{threshold}" + ] = ap.conf_interp.tolist() + + # TP error metrics (e.g. ATE, AOE, ASE, AVE, AAE) + if ap.tp_error_metrics is not None: + for tp_error_metric in ap.tp_error_metrics: + mode = tp_error_metric.mode + average_mode = tp_error_metric.average_mode + + iterable_metrics[ + f"T4MetricV2_label_detection/{label_name}_{mode}_values_{matching_mode}_{threshold}" + ] = tp_error_metric.values.tolist() + iterable_metrics[ + f"T4MetricV2_label_detection/{label_name}_{mode}_interp-values_{matching_mode}_{threshold}" + ] = tp_error_metric.interpolated_values.tolist() return iterable_metrics @@ -1044,6 +1060,40 @@ def _process_metrics_for_aggregation(self, metrics_score: MetricsScore, evaluato ap.optimal_precision ) + # Number of prediction matches (TPs) and matches at the optimal confidence threshold + metric_dict[f"T4MetricV2_label/{label_name}_num-match_{matching_mode}_{threshold}"] = ap.num_tp + metric_dict[f"T4MetricV2_label/{label_name}_min-recall-num-match_{matching_mode}_{threshold}"] = ( + ap.num_tp_at_min_recall_conf + ) + metric_dict[ + f"T4MetricV2_label/{label_name}_medium-recall-num-match_{matching_mode}_{threshold}" + ] = ap.num_tp_at_medium_recall_conf + metric_dict[f"T4MetricV2_label/{label_name}_optimal-num-match_{matching_mode}_{threshold}"] = ( + ap.num_tp_at_optimal_conf + ) + + # TP error metrics (e.g. ATE, AOE, ASE, AVE, AAE) + if ap.tp_error_metrics is not None: + for tp_error_metric in ap.tp_error_metrics: + mode = tp_error_metric.mode + average_mode = tp_error_metric.average_mode + + metric_dict[ + f"T4MetricV2_label/{label_name}_tp-error_{average_mode}_{matching_mode}_{threshold}" + ] = tp_error_metric.avg_metric + metric_dict[ + f"T4MetricV2_label/{label_name}_tp-error-min-recall-conf_{average_mode}_{matching_mode}_{threshold}" + ] = tp_error_metric.min_recall_conf + metric_dict[ + f"T4MetricV2_label/{label_name}_tp-error-optimal-{average_mode}_{matching_mode}_{threshold}" + ] = tp_error_metric.optimal_avg_metric + metric_dict[ + f"T4MetricV2_label/{label_name}_tp-error-medium-{average_mode}_{matching_mode}_{threshold}" + ] = tp_error_metric.medium_avg_metric + metric_dict[ + f"T4MetricV2_label/{label_name}_tp-error-medium-recall-conf-{average_mode}_{matching_mode}_{threshold}" + ] = tp_error_metric.medium_recall_conf + # Label metadata key metric_dict[f"metadata_label/test_{label_name}_num_predictions"] = label_num_preds metric_dict[f"metadata_label/test_{label_name}_num_ground_truths"] = label_num_gts @@ -1054,6 +1104,41 @@ def _process_metrics_for_aggregation(self, metrics_score: MetricsScore, evaluato metric_dict[map_key] = map_instance.map metric_dict[maph_key] = map_instance.maph + # Add mean TP errors (e.g. mATE, mAOE, mASE, mAVE, mAAE) + if map_instance.mean_tp_errors is not None: + for mean_tp_error_name, mean_tp_error_value in map_instance.mean_tp_errors.items(): + metric_dict[f"T4MetricV2/mean-tp-error_{mean_tp_error_name}_{matching_mode}"] = mean_tp_error_value + + optimal_mean_tp_errors = map_instance.optimal_mean_tp_errors.get(mean_tp_error_name, None) + if optimal_mean_tp_errors is not None: + metric_dict[f"T4MetricV2/mean-tp-error-optimal-{mean_tp_error_name}_{matching_mode}"] = ( + optimal_mean_tp_errors + ) + + medium_mean_tp_errors = map_instance.medium_mean_tp_errors.get(mean_tp_error_name, None) + if medium_mean_tp_errors is not None: + metric_dict[f"T4MetricV2/mean-tp-error-medium-{mean_tp_error_name}_{matching_mode}"] = ( + medium_mean_tp_errors + ) + + # Add NuScenes Detection Score (NDS) based on mAP and mAPH + if map_instance.map_based_nds is not None: + metric_dict[f"T4MetricV2/{map_instance.map_based_nds.metric_prefix_name}_nds_{matching_mode}"] = ( + map_instance.map_based_nds.nds + ) + if map_instance.medium_map_based_nds is not None: + metric_dict[ + f"T4MetricV2/{map_instance.medium_map_based_nds.metric_prefix_name}_nds_{matching_mode}" + ] = map_instance.medium_map_based_nds.nds + if map_instance.mapH_based_nds is not None: + metric_dict[f"T4MetricV2/{map_instance.mapH_based_nds.metric_prefix_name}_nds_{matching_mode}"] = ( + map_instance.mapH_based_nds.nds + ) + if map_instance.medium_mapH_based_nds is not None: + metric_dict[ + f"T4MetricV2/{map_instance.medium_mapH_based_nds.metric_prefix_name}_nds_{matching_mode}" + ] = map_instance.medium_mapH_based_nds.nds + total_num_preds = num_preds # Selected evaluator @@ -1109,7 +1194,10 @@ def _write_aggregated_metrics( aggregated_metrics[evaluator_name]["metadata_label"][label_name] = {} aggregated_metrics[evaluator_name]["metadata_label"][label_name][key] = value - elif key.startswith("T4MetricV2/mAP_") or key.startswith("T4MetricV2/mAPH_"): + elif key.startswith("T4MetricV2/tp-mean-error"): + # These are TP error metrics, put them in the metrics section + aggregated_metrics[evaluator_name]["tp_mean_errors"][key] = value + elif key.startswith("T4MetricV2/mAP_") or key.startswith("T4MetricV2/mAPH_") or "nds" in key: # These are overall metrics, put them in the metrics section aggregated_metrics[evaluator_name]["metrics"][key] = value else: