Update evaluation scripts (#37)

Update evaluation scripts to most recent. --------- Co-authored-by: am <am> Co-authored-by: Holger Roth <[email protected]>
Project-MONAI · Oct 29, 2024 · 3f65e36 · 3f65e36
1 parent 31ec9eb
commit 3f65e36
Show file tree

Hide file tree

Showing 29 changed files with 3,873 additions and 405 deletions.
diff --git a/m3/demo/gradio_m3.py b/m3/demo/gradio_m3.py
@@ -676,6 +676,7 @@ def update_sys_message(sys_message, sv):
     sv.sys_msg = sys_message
     return sv
 
+
 def update_modality_prompt(modality_prompt, sv):
     """Update the modality prompt"""
     logger.debug(f"Updating the modality prompt")
@@ -785,7 +786,16 @@ def create_demo(source, model_path, conv_mode, server_port):
         clear_btn.click(
             fn=clear_all_convs,
             inputs=[sv],
-            outputs=[sv, prompt_edit, chat_history, history_text, history_text_full, sys_prompt_text, model_cards_text, modality_prompt_dropdown],
+            outputs=[
+                sv,
+                prompt_edit,
+                chat_history,
+                history_text,
+                history_text_full,
+                sys_prompt_text,
+                model_cards_text,
+                modality_prompt_dropdown,
+            ],
         )
 
         # States

diff --git a/m3/eval/eval_all.sh b/m3/eval/eval_all.sh
@@ -27,7 +27,8 @@ export CONV_MODE=$3
 : ${CONTAINER:?"CONTAINER env var is not set!"}
 : ${DATASETS:?"DATASETS env var is not set!"}
 : ${CODE:?"CODE env var is not set!"}
-
+: ${ACCOUNT:?"ACCOUNT env var is not set!"}
+
 
 sbatch eval_radvqa.slurm $MODEL_PATH $OUTPUT_FOLDER_NAME $CONV_MODE 
 sbatch eval_slakevqa.slurm $MODEL_PATH $OUTPUT_FOLDER_NAME $CONV_MODE 
@@ -36,6 +37,7 @@ sbatch eval_pathvqa.slurm $MODEL_PATH $OUTPUT_FOLDER_NAME $CONV_MODE
 sbatch eval_mimicvqa.slurm $MODEL_PATH $OUTPUT_FOLDER_NAME $CONV_MODE 
 
 sbatch eval_report_mimiccxr.slurm $MODEL_PATH $OUTPUT_FOLDER_NAME $CONV_MODE 
+sbatch eval_report_mimiccxr_expert.slurm $MODEL_PATH $OUTPUT_FOLDER_NAME $CONV_MODE 
 
 sbatch eval_chestxray14_class.slurm $MODEL_PATH $OUTPUT_FOLDER_NAME $CONV_MODE 
 sbatch eval_chexpert_class.slurm $MODEL_PATH $OUTPUT_FOLDER_NAME $CONV_MODE 

diff --git a/m3/eval/eval_chestxray14_class.slurm b/m3/eval/eval_chestxray14_class.slurm
@@ -3,11 +3,10 @@
 #SBATCH --job-name=chestxray14_class_eval
 #SBATCH --nodes=1
 #SBATCH --gres=gpu:8
-#SBATCH --time=2:00:00
+#SBATCH --time=4:00:00
 #SBATCH --partition=interactive,interactive_singlenode,grizzly,polar,polar2,polar3,polar4
 #SBATCH --exclusive
 #SBATCH --dependency=singleton
-#SBATCH -A healthcareeng_monai
 
 # Copyright (c) MONAI Consortium
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -36,8 +35,9 @@ export CONV_MODE=$3
 : ${CONTAINER:?"CONTAINER env var is not set!"}
 : ${DATASETS:?"DATASETS env var is not set!"}
 : ${CODE:?"CODE env var is not set!"}
+: ${ACCOUNT:?"ACCOUNT env var is not set!"}
 
-srun -o job_chestxray14_class_${OUTPUT_FOLDER_NAME}_eval.log --container-image=$CONTAINER \
+srun -o job_chestxray14_class_${OUTPUT_FOLDER_NAME}_eval.log --container-image=$CONTAINER --acount=$ACCOUNT \
      --container-mounts=$CODE:/data/code,${DATASETS}:/data/datasets,$MODEL_PATH:/data/model_path \
      --no-container-mount-home \
      --container-remap-root \

diff --git a/m3/eval/eval_chestxray14_expert_class.slurm b/m3/eval/eval_chestxray14_expert_class.slurm
@@ -3,10 +3,9 @@
 #SBATCH --job-name=chestxray14_expert_class_eval
 #SBATCH --nodes=1
 #SBATCH --gres=gpu:8
-#SBATCH --time=2:00:00
+#SBATCH --time=4:00:00
 #SBATCH --partition=interactive,interactive_singlenode,grizzly,polar,polar2,polar3,polar4
 #SBATCH --exclusive
-#SBATCH -A healthcareeng_monai
 
 # Copyright (c) MONAI Consortium
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -35,12 +34,10 @@ export CONV_MODE=$3
 : ${CONTAINER:?"CONTAINER env var is not set!"}
 : ${DATASETS:?"DATASETS env var is not set!"}
 : ${CODE:?"CODE env var is not set!"}
+: ${ACCOUNT:?"ACCOUNT env var is not set!"}
 
-# DATASETS='/lustre/fsw/portfolios/healthcareeng/projects/healthcareeng_monai/VLM/datasets'
-# TODO standardize dataset location
 
-
-srun -o job_chestxray14_expert_class_${OUTPUT_FOLDER_NAME}_eval.log --container-image=$CONTAINER \
+srun -o job_chestxray14_expert_class_${OUTPUT_FOLDER_NAME}_eval.log --container-image=$CONTAINER --acount=$ACCOUNT \
      --container-mounts=$CODE:/data/code,${DATASETS}:/data/datasets,$MODEL_PATH:/data/model_path \
      --no-container-mount-home \
      --container-remap-root \

diff --git a/m3/eval/eval_chexpert_class.slurm b/m3/eval/eval_chexpert_class.slurm
@@ -7,7 +7,6 @@
 #SBATCH --partition=interactive,interactive_singlenode,grizzly,polar,polar2,polar3,polar4
 #SBATCH --exclusive
 #SBATCH --dependency=singleton
-#SBATCH -A healthcareeng_monai
 
 # Copyright (c) MONAI Consortium
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -36,8 +35,9 @@ export CONV_MODE=$3
 : ${CONTAINER:?"CONTAINER env var is not set!"}
 : ${DATASETS:?"DATASETS env var is not set!"}
 : ${CODE:?"CODE env var is not set!"}
+: ${ACCOUNT:?"ACCOUNT env var is not set!"}
 
-srun -o job_chexpert_class_${OUTPUT_FOLDER_NAME}_eval.log --container-image=$CONTAINER \
+srun -o job_chexpert_class_${OUTPUT_FOLDER_NAME}_eval.log --container-image=$CONTAINER --acount=$ACCOUNT \
      --container-mounts=$CODE:/data/code,${DATASETS}:/data/datasets,$MODEL_PATH:/data/model_path \
      --no-container-mount-home \
      --container-remap-root \

diff --git a/m3/eval/eval_chexpert_expert_class.slurm b/m3/eval/eval_chexpert_expert_class.slurm
@@ -6,7 +6,6 @@
 #SBATCH --time=2:00:00
 #SBATCH --partition=interactive,interactive_singlenode,grizzly,polar,polar2,polar3,polar4
 #SBATCH --exclusive
-#SBATCH -A healthcareeng_monai
 
 # Copyright (c) MONAI Consortium
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -35,9 +34,10 @@ export CONV_MODE=$3
 : ${CONTAINER:?"CONTAINER env var is not set!"}
 : ${DATASETS:?"DATASETS env var is not set!"}
 : ${CODE:?"CODE env var is not set!"}
+: ${ACCOUNT:?"ACCOUNT env var is not set!"}
 
 
-srun -o job_chexpert_expert_class_${OUTPUT_FOLDER_NAME}_eval.log --container-image=$CONTAINER \
+srun -o job_chexpert_expert_class_${OUTPUT_FOLDER_NAME}_eval.log --container-image=$CONTAINER --acount=$ACCOUNT \
      --container-mounts=$CODE:/data/code,${DATASETS}:/data/datasets,$MODEL_PATH:/data/model_path \
      --no-container-mount-home \
      --container-remap-root \

diff --git a/m3/eval/eval_mimicvqa.slurm b/m3/eval/eval_mimicvqa.slurm
@@ -7,7 +7,6 @@
 #SBATCH --time=2:00:00
 #SBATCH --partition=interactive,interactive_singlenode,grizzly,polar,polar2,polar3,polar4
 #SBATCH --dependency=singleton
-#SBATCH -A healthcareeng_monai
 
 # Copyright (c) MONAI Consortium
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -36,6 +35,7 @@ export CONV_MODE=$3
 : ${CONTAINER:?"CONTAINER env var is not set!"}
 : ${DATASETS:?"DATASETS env var is not set!"}
 : ${CODE:?"CODE env var is not set!"}
+: ${ACCOUNT:?"ACCOUNT env var is not set!"}
 
 # common
 export GENERATION_CONFIG='{"max_new_tokens":1024,"do_sample":false,"temperature":0}'
@@ -44,7 +44,7 @@ export GENERATION_CONFIG='{"max_new_tokens":1024,"do_sample":false,"temperature"
 
 
 srun -o job_mimicvqa_${OUTPUT_FOLDER_NAME}_eval.log \
-    --container-image=$CONTAINER \
+    --container-image=$CONTAINER --acount=$ACCOUNT \
     --container-mounts=$CODE:/data/code,$DATASETS:/data/datasets,$MODEL_PATH:/data/model_path \
     --no-container-mount-home \
     --container-remap-root \

diff --git a/m3/eval/eval_pathvqa.slurm b/m3/eval/eval_pathvqa.slurm
@@ -6,7 +6,6 @@
 #SBATCH --time=2:00:00
 #SBATCH --partition=interactive,interactive_singlenode,grizzly,polar,polar2,polar3,polar4
 #SBATCH --dependency=singleton
-#SBATCH -A healthcareeng_monai
 
 # Copyright (c) MONAI Consortium
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -35,12 +34,13 @@ export CONV_MODE=$3
 : ${CONTAINER:?"CONTAINER env var is not set!"}
 : ${DATASETS:?"DATASETS env var is not set!"}
 : ${CODE:?"CODE env var is not set!"}
+: ${ACCOUNT:?"ACCOUNT env var is not set!"}
 
 # common
 export GENERATION_CONFIG='{"max_new_tokens":1024,"do_sample":false,"temperature":0}'
 
 
-srun -o job_pathvqa_${OUTPUT_FOLDER_NAME}_eval.log --container-image=$CONTAINER \
+srun -o job_pathvqa_${OUTPUT_FOLDER_NAME}_eval.log --container-image=$CONTAINER --acount=$ACCOUNT \
      --container-mounts=$CODE:/data/code,$DATASETS:/data/datasets,$MODEL_PATH:/data/model_path \
      --no-container-mount-home \
      --container-remap-root \

diff --git a/m3/eval/eval_radvqa.slurm b/m3/eval/eval_radvqa.slurm
@@ -6,7 +6,6 @@
 #SBATCH --time=1:00:00
 #SBATCH --partition=interactive,interactive_singlenode,grizzly,polar,polar2,polar3,polar4
 #SBATCH --dependency=singleton
-#SBATCH -A healthcareeng_monai
 
 # Copyright (c) MONAI Consortium
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -35,12 +34,13 @@ export CONV_MODE=$3
 : ${CONTAINER:?"CONTAINER env var is not set!"}
 : ${DATASETS:?"DATASETS env var is not set!"}
 : ${CODE:?"CODE env var is not set!"}
+: ${ACCOUNT:?"ACCOUNT env var is not set!"}
 
 # common
 export GENERATION_CONFIG='{"max_new_tokens":1024,"do_sample":false,"temperature":0}'
 
 
-srun -o job_radvqa_${OUTPUT_FOLDER_NAME}_eval.log --container-image=$CONTAINER \
+srun -o job_radvqa_${OUTPUT_FOLDER_NAME}_eval.log --container-image=$CONTAINER --acount=$ACCOUNT \
      --container-mounts=$CODE:/data/code,$DATASETS:/data/datasets,$MODEL_PATH:/data/model_path \
      --no-container-mount-home \
      --container-remap-root \