feat: add scenario migration script

log_evaluator_migration_tool/annotationless_perception_migration.py

@@ -0,0 +1,72 @@
+import argparse
+from pathlib import Path
+
+import yaml
+
+
+def convert_scenario(scenario_path: Path) -> None:
+    scenario_file = scenario_path.open("r+")
+    yaml_obj = yaml.safe_load(scenario_file)
+
+    if yaml_obj["Evaluation"]["UseCaseName"] != "annotationless_perception":
+        print(f"{scenario_path} does not require conversion")  # noqa
+        scenario_file.close()
+        return
+
+    if (
+        yaml_obj["ScenarioFormatVersion"] != "3.0.0"
+        or yaml_obj["Evaluation"]["UseCaseFormatVersion"] != "1.0.0"
+    ):
+        print(f"{scenario_path} does not match format version")  # noqa
+        scenario_file.close()
+        return
+
+    yaml_obj["Evaluation"]["UseCaseFormatVersion"] = "2.0.0"
+
+    class_cond = yaml_obj["Evaluation"]["Conditions"]["ClassConditions"]
+
+    for k, v in class_cond.items():
+        pass_range_dict = expand_pass_range(v["PassRange"])
+        for name, value in v["Threshold"].items():
+            threshold_values = {}
+            for m_k, m_v in value.items():
+                threshold_values[m_k] = {
+                    "lower": m_v * pass_range_dict[m_k][0],
+                    "upper": m_v * pass_range_dict[m_k][1],
+                }
+            class_cond[k] |= {name: threshold_values}
+        class_cond[k].pop("Threshold")
+        class_cond[k].pop("PassRange")
+
+    # 既存の内容を消す
+    scenario_file.seek(0)
+    scenario_file.truncate()
+
+    # 更新済みの内容を書き込む
+    yaml.safe_dump(yaml_obj, scenario_file, sort_keys=False)
+    scenario_file.close()
+
+
+def expand_pass_range(pass_range_dict: dict) -> dict:
+    rtn_dict = {}
+    for k, v in pass_range_dict.items():
+        s_lower, s_upper = v.split("-")
+        rtn_dict[k] = [float(s_lower), float(s_upper)]
+    return rtn_dict
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "data_directory",
+        help="data_directory of log_evaluator",
+    )
+    args = parser.parse_args()
+    scenario_paths = Path(args.data_directory).resolve().glob("**/scenario*.y*ml")  # yaml or yml
+    for scenario_path in sorted(scenario_paths):
+        print(f"convert {scenario_path}")  # noqa
+        convert_scenario(scenario_path)
+
+
+if __name__ == "__main__":
+    main()

sample/annotationless_perception/scenario.ja.yaml

@@ -10,29 +10,14 @@ Evaluation:
     ClassConditions:
       # クラス毎の条件を記述する。条件を設定がないクラスが出力された場合はメトリクスだけ計算される。評価には影響しない
       # サンプルデータではTRUCKのclassも出力されるが条件を記述してないので、TRUCKは必ずSuccessになる
-      # result.jsonlから条件指定を行う場合、ここに記述があるキーのみ更新される。
-      # result.jsonlにTRUCKのメトリクスが出ていても、この例ではTRUCKのキーを指定していないので評価条件には追加されない。
       CAR: # classification key
-        Threshold:
-          # 記述のないキーについては評価されない（必ず成功になる）
-          lateral_deviation: { max: 0.4, mean: 0.019 }
-          yaw_deviation: { max: 3.1411, mean: 0.05 }
-          predicted_path_deviation_5.00: { max: 16.464, mean: 1.8 }
-          total_objects_count_r60.00_h10.00: { metric_value: 10 }
-        PassRange:
-          min: 0.0-2.0 # lower[<=1.0]-upper[>=1.0]
-          max: 0.0-2.0 # lower[<=1.0]-upper[>=1.0]
-          mean: 0.5-2.0 # lower[<=1.0]-upper[>=1.0]
-          metric_value: 0.9-1.1
+        lateral_deviation: { mean: { lower: 0.01, upper: 0.03 } }
+        yaw_deviation: { mean: { lower: 0.05, upper: 0.09 } }
+        predicted_path_deviation_5.00: { mean: { lower: 1.5, upper: 3.0 } }
+        yaw_rate: { mean: { lower: 0.01, upper: 0.03 } }
       BUS: # classification key
-        Threshold:
-          # Only lateral_deviation is evaluated.
-          yaw_rate: { max: 0.05 } # Only max is evaluated.
-        PassRange:
-          min: 0.0-2.0 # lower[<=1.0]-upper[>=1.0]
-          max: 0.0-2.0 # lower[<=1.0]-upper[>=1.0]
-          mean: 0.5-2.0 # lower[<=1.0]-upper[>=1.0]
-          metric_value: 0.9-1.1
+        yaw_rate: { mean: { lower: 0, upper: 0.01 } }
   Datasets:
     - sample_dataset:
         VehicleId: default
+

sample/annotationless_perception/scenario.yaml

@@ -10,8 +10,6 @@ Evaluation:
     ClassConditions:
       # Describe the conditions for each class. If a class with no conditions is output, only the metrics are calculated. It does not affect the evaluation.
       # In the sample data, the class of TRUCK is also output, but the condition is not described, so TRUCK is always Success.
-      # When specifying conditions from result.jsonl, only keys described here will be updated.
-      # Even though TRUCK metrics appear in result.jsonl, they are not added to the evaluation condition because the TRUCK key is not specified in this example.
       CAR: # classification key
         lateral_deviation: { mean: { lower: 0.01, upper: 0.03 } }
         yaw_deviation: { mean: { lower: 0.05, upper: 0.09 } }