From 95846b24b7e99b9e906bf44a8f615421bad50893 Mon Sep 17 00:00:00 2001 From: Hayato Mizushima Date: Fri, 26 Jul 2024 12:49:34 +0900 Subject: [PATCH] feat: add scenario migration script --- .../annotationless_perception_migration.py | 72 +++++++++++++++++++ .../scenario.ja.yaml | 27 ++----- .../annotationless_perception/scenario.yaml | 2 - 3 files changed, 78 insertions(+), 23 deletions(-) create mode 100644 log_evaluator_migration_tool/annotationless_perception_migration.py diff --git a/log_evaluator_migration_tool/annotationless_perception_migration.py b/log_evaluator_migration_tool/annotationless_perception_migration.py new file mode 100644 index 00000000..736ffcfd --- /dev/null +++ b/log_evaluator_migration_tool/annotationless_perception_migration.py @@ -0,0 +1,72 @@ +import argparse +from pathlib import Path + +import yaml + + +def convert_scenario(scenario_path: Path) -> None: + scenario_file = scenario_path.open("r+") + yaml_obj = yaml.safe_load(scenario_file) + + if yaml_obj["Evaluation"]["UseCaseName"] != "annotationless_perception": + print(f"{scenario_path} does not require conversion") # noqa + scenario_file.close() + return + + if ( + yaml_obj["ScenarioFormatVersion"] != "3.0.0" + or yaml_obj["Evaluation"]["UseCaseFormatVersion"] != "1.0.0" + ): + print(f"{scenario_path} does not match format version") # noqa + scenario_file.close() + return + + yaml_obj["Evaluation"]["UseCaseFormatVersion"] = "2.0.0" + + class_cond = yaml_obj["Evaluation"]["Conditions"]["ClassConditions"] + + for k, v in class_cond.items(): + pass_range_dict = expand_pass_range(v["PassRange"]) + for name, value in v["Threshold"].items(): + threshold_values = {} + for m_k, m_v in value.items(): + threshold_values[m_k] = { + "lower": m_v * pass_range_dict[m_k][0], + "upper": m_v * pass_range_dict[m_k][1], + } + class_cond[k] |= {name: threshold_values} + class_cond[k].pop("Threshold") + class_cond[k].pop("PassRange") + + # 既存の内容を消す + scenario_file.seek(0) + scenario_file.truncate() + + # 更新済みの内容を書き込む + yaml.safe_dump(yaml_obj, scenario_file, sort_keys=False) + scenario_file.close() + + +def expand_pass_range(pass_range_dict: dict) -> dict: + rtn_dict = {} + for k, v in pass_range_dict.items(): + s_lower, s_upper = v.split("-") + rtn_dict[k] = [float(s_lower), float(s_upper)] + return rtn_dict + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "data_directory", + help="data_directory of log_evaluator", + ) + args = parser.parse_args() + scenario_paths = Path(args.data_directory).resolve().glob("**/scenario*.y*ml") # yaml or yml + for scenario_path in sorted(scenario_paths): + print(f"convert {scenario_path}") # noqa + convert_scenario(scenario_path) + + +if __name__ == "__main__": + main() diff --git a/sample/annotationless_perception/scenario.ja.yaml b/sample/annotationless_perception/scenario.ja.yaml index 3d6d848e..0e2d81b8 100644 --- a/sample/annotationless_perception/scenario.ja.yaml +++ b/sample/annotationless_perception/scenario.ja.yaml @@ -10,29 +10,14 @@ Evaluation: ClassConditions: # クラス毎の条件を記述する。条件を設定がないクラスが出力された場合はメトリクスだけ計算される。評価には影響しない # サンプルデータではTRUCKのclassも出力されるが条件を記述してないので、TRUCKは必ずSuccessになる - # result.jsonlから条件指定を行う場合、ここに記述があるキーのみ更新される。 - # result.jsonlにTRUCKのメトリクスが出ていても、この例ではTRUCKのキーを指定していないので評価条件には追加されない。 CAR: # classification key - Threshold: - # 記述のないキーについては評価されない（必ず成功になる） - lateral_deviation: { max: 0.4, mean: 0.019 } - yaw_deviation: { max: 3.1411, mean: 0.05 } - predicted_path_deviation_5.00: { max: 16.464, mean: 1.8 } - total_objects_count_r60.00_h10.00: { metric_value: 10 } - PassRange: - min: 0.0-2.0 # lower[<=1.0]-upper[>=1.0] - max: 0.0-2.0 # lower[<=1.0]-upper[>=1.0] - mean: 0.5-2.0 # lower[<=1.0]-upper[>=1.0] - metric_value: 0.9-1.1 + lateral_deviation: { mean: { lower: 0.01, upper: 0.03 } } + yaw_deviation: { mean: { lower: 0.05, upper: 0.09 } } + predicted_path_deviation_5.00: { mean: { lower: 1.5, upper: 3.0 } } + yaw_rate: { mean: { lower: 0.01, upper: 0.03 } } BUS: # classification key - Threshold: - # Only lateral_deviation is evaluated. - yaw_rate: { max: 0.05 } # Only max is evaluated. - PassRange: - min: 0.0-2.0 # lower[<=1.0]-upper[>=1.0] - max: 0.0-2.0 # lower[<=1.0]-upper[>=1.0] - mean: 0.5-2.0 # lower[<=1.0]-upper[>=1.0] - metric_value: 0.9-1.1 + yaw_rate: { mean: { lower: 0, upper: 0.01 } } Datasets: - sample_dataset: VehicleId: default + diff --git a/sample/annotationless_perception/scenario.yaml b/sample/annotationless_perception/scenario.yaml index c9b7b70c..7355aa84 100644 --- a/sample/annotationless_perception/scenario.yaml +++ b/sample/annotationless_perception/scenario.yaml @@ -10,8 +10,6 @@ Evaluation: ClassConditions: # Describe the conditions for each class. If a class with no conditions is output, only the metrics are calculated. It does not affect the evaluation. # In the sample data, the class of TRUCK is also output, but the condition is not described, so TRUCK is always Success. - # When specifying conditions from result.jsonl, only keys described here will be updated. - # Even though TRUCK metrics appear in result.jsonl, they are not added to the evaluation condition because the TRUCK key is not specified in this example. CAR: # classification key lateral_deviation: { mean: { lower: 0.01, upper: 0.03 } } yaw_deviation: { mean: { lower: 0.05, upper: 0.09 } }