Skip to content

Commit

Permalink
WIP combined scores
Browse files Browse the repository at this point in the history
  • Loading branch information
@beveradb
beveradb committed Dec 17, 2024
1 parent 3017ae5 commit 829dabd
Show file tree
Hide file tree
Showing 2 changed files with 317 additions and 27 deletions.
266 changes: 266 additions & 0 deletions audio_separator/models-scores.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,266 @@
{
"1_HP-UVR.pth": {
"model_name": "VR Arch Single Model v5: 1_HP-UVR",
"track_scores": [
{
"track_name": "Moosmusic - Big Dummy Shake",
"scores": {
"vocals": {
"SDR": 9.86522,
"SIR": 23.3587,
"SAR": 10.1785,
"ISR": 15.0739
},
"instrumental": {
"SDR": 15.7431,
"SIR": 20.9606,
"SAR": 17.3032,
"ISR": 29.0623
}
}
}
]
},
"2_HP-UVR.pth": {
"model_name": "VR Arch Single Model v5: 2_HP-UVR",
"track_scores": [
{
"track_name": "Moosmusic - Big Dummy Shake",
"scores": {
"vocals": {
"SDR": 9.86277,
"SIR": 23.2713,
"SAR": 10.0964,
"ISR": 14.7435
},
"instrumental": {
"SDR": 15.4816,
"SIR": 20.1623,
"SAR": 17.3278,
"ISR": 28.9186
}
}
}
]
},
"3_HP-Vocal-UVR.pth": {
"model_name": "VR Arch Single Model v5: 3_HP-Vocal-UVR",
"track_scores": [
{
"track_name": "Moosmusic - Big Dummy Shake",
"scores": {
"vocals": {
"SDR": 10.2015,
"SIR": 25.0552,
"SAR": 10.3513,
"ISR": 15.9666
},
"instrumental": {
"SDR": 16.2483,
"SIR": 21.8777,
"SAR": 17.4968,
"ISR": 30.7329
}
}
}
]
},
"4_HP-Vocal-UVR.pth": {
"model_name": "VR Arch Single Model v5: 4_HP-Vocal-UVR",
"track_scores": [
{
"track_name": "Moosmusic - Big Dummy Shake",
"scores": {
"vocals": {
"SDR": 10.1621,
"SIR": 23.2506,
"SAR": 10.2838,
"ISR": 16.6347
},
"instrumental": {
"SDR": 15.8716,
"SIR": 22.3494,
"SAR": 17.176,
"ISR": 28.9295
}
}
}
]
},
"5_HP-Karaoke-UVR.pth": {
"model_name": "VR Arch Single Model v5: 5_HP-Karaoke-UVR",
"track_scores": [
{
"track_name": "Moosmusic - Big Dummy Shake",
"scores": {
"vocals": {
"SDR": 9.3986,
"SIR": 23.8072,
"SAR": 9.52931,
"ISR": 13.2465
},
"instrumental": {
"SDR": 15.4412,
"SIR": 18.7661,
"SAR": 17.0021,
"ISR": 29.6517
}
}
}
]
},
"6_HP-Karaoke-UVR.pth": {
"model_name": "VR Arch Single Model v5: 6_HP-Karaoke-UVR",
"track_scores": [
{
"track_name": "Moosmusic - Big Dummy Shake",
"scores": {
"vocals": {
"SDR": 9.62845,
"SIR": 24.0414,
"SAR": 9.55387,
"ISR": 13.5417
},
"instrumental": {
"SDR": 15.6841,
"SIR": 19.0452,
"SAR": 17.1219,
"ISR": 31.5764
}
}
}
]
},
"7_HP2-UVR.pth": {
"model_name": "VR Arch Single Model v5: 7_HP2-UVR",
"track_scores": [
{
"track_name": "Moosmusic - Big Dummy Shake",
"scores": {
"vocals": {
"SDR": 9.9246,
"SIR": 23.9273,
"SAR": 10.3074,
"ISR": 14.8333
},
"instrumental": {
"SDR": 15.7343,
"SIR": 20.1956,
"SAR": 17.6699,
"ISR": 31.2578
}
}
}
]
},
"8_HP2-UVR.pth": {
"model_name": "VR Arch Single Model v5: 8_HP2-UVR",
"track_scores": [
{
"track_name": "Moosmusic - Big Dummy Shake",
"scores": {
"vocals": {
"SDR": 9.94006,
"SIR": 23.715,
"SAR": 10.3701,
"ISR": 14.653
},
"instrumental": {
"SDR": 15.5981,
"SIR": 20.4122,
"SAR": 17.3973,
"ISR": 29.4197
}
}
}
]
},
"9_HP2-UVR.pth": {
"model_name": "VR Arch Single Model v5: 9_HP2-UVR",
"track_scores": [
{
"track_name": "Moosmusic - Big Dummy Shake",
"scores": {
"vocals": {
"SDR": 9.89634,
"SIR": 23.8526,
"SAR": 10.3504,
"ISR": 14.5682
},
"instrumental": {
"SDR": 15.7111,
"SIR": 20.3581,
"SAR": 17.6392,
"ISR": 29.5016
}
}
}
]
},
"10_SP-UVR-2B-32000-1.pth": {
"model_name": "VR Arch Single Model v5: 10_SP-UVR-2B-32000-1",
"track_scores": [
{
"track_name": "Moosmusic - Big Dummy Shake",
"scores": {
"vocals": {
"SDR": 9.54673,
"SIR": 23.3362,
"SAR": 9.91098,
"ISR": 13.8572
},
"instrumental": {
"SDR": 15.0055,
"SIR": 19.9132,
"SAR": 16.537,
"ISR": 23.2056
}
}
}
]
},
"11_SP-UVR-2B-32000-2.pth": {
"model_name": "VR Arch Single Model v5: 11_SP-UVR-2B-32000-2",
"track_scores": [
{
"track_name": "Moosmusic - Big Dummy Shake",
"scores": {
"vocals": {
"SDR": 9.52838,
"SIR": 23.6181,
"SAR": 9.90405,
"ISR": 13.8069
},
"instrumental": {
"SDR": 15.0352,
"SIR": 19.8583,
"SAR": 16.5648,
"ISR": 23.329
}
}
}
]
},
"12_SP-UVR-3B-44100.pth": {
"model_name": "VR Arch Single Model v5: 12_SP-UVR-3B-44100",
"track_scores": [
{
"track_name": "Moosmusic - Big Dummy Shake",
"scores": {
"vocals": {
"SDR": 9.70757,
"SIR": 23.4678,
"SAR": 10.1294,
"ISR": 14.3904
},
"instrumental": {
"SDR": 15.2341,
"SIR": 20.0053,
"SAR": 17.2023,
"ISR": 28.5095
}
}
}
]
}
}
78 changes: 51 additions & 27 deletions tests/model-metrics/test-all-models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,17 @@ def evaluate_track(track_name, track_path, test_model, mus_db):
agg_scores.loc[(test_model, "accompaniment", "ISR")],
)

# Return the aggregate scores in a structured format with 6 significant figures
model_results = {
"track_name": track_name,
"scores": {
"vocals": {metric: float(f"{agg_scores.loc[(test_model, 'vocals', metric)]:.6g}") for metric in ["SDR", "SIR", "SAR", "ISR"]},
"instrumental": {metric: float(f"{agg_scores.loc[(test_model, 'accompaniment', metric)]:.6g}") for metric in ["SDR", "SIR", "SAR", "ISR"]},
},
}

return scores, model_results


def convert_decimal_to_float(obj):
"""Recursively converts Decimal objects to floats in a nested structure."""
Expand All @@ -109,8 +120,6 @@ def convert_decimal_to_float(obj):

def main():
logger.info("Starting model evaluation script...")

# Create results directory
os.makedirs(RESULTS_PATH, exist_ok=True)

# Initialize MUSDB once at the start
Expand All @@ -122,47 +131,62 @@ def main():
separator = Separator()
models_by_type = separator.list_supported_model_files()

# Just get the first model for testing
test_model = None
# Get first track from MUSDB18
logger.info("Looking for first track in MUSDB18 dataset...")
first_track = next(Path(MUSDB_PATH).glob("**/mixture.wav"))
track_path = first_track.parent
track_name = track_path.name
logger.info(f"Found track: {track_name} at path: {track_path}")

# Load existing results if available
combined_results_path = "audio_separator/models-scores.json"
combined_results = {}
if os.path.exists(combined_results_path):
logger.info("Loading existing combined results...")
with open(combined_results_path) as f:
combined_results = json.load(f)

# Process all models
for model_type, models in models_by_type.items():
for model_name, model_info in models.items():
test_model = None
if isinstance(model_info, str):
test_model = model_info
break
elif isinstance(model_info, dict):
for file_name in model_info.keys():
if file_name.endswith((".onnx", ".pth", ".ckpt")):
test_model = file_name
break
if test_model:
break

logger.info(f"Selected model for testing: {test_model}")
if test_model:
# Initialize model entry if it doesn't exist
if test_model not in combined_results:
combined_results[test_model] = {"model_name": model_name, "track_scores": []}

# Get first track from MUSDB18
logger.info("Looking for first track in MUSDB18 dataset...")
first_track = next(Path(MUSDB_PATH).glob("**/mixture.wav"))
track_path = first_track.parent
track_name = track_path.name
logger.info(f"Found track: {track_name} at path: {track_path}")
# Skip if track already evaluated for this model
track_already_evaluated = any(track_score["track_name"] == track_name for track_score in combined_results[test_model]["track_scores"])
if track_already_evaluated:
logger.info(f"Skipping already evaluated track for model: {test_model}")
continue

try:
logger.info("Starting track evaluation...")
evaluate_track(track_name, track_path, test_model, mus)
logger.info("Completed evaluation")
logger.info(f"Processing model: {test_model}")
try:
_, model_results = evaluate_track(track_name, track_path, test_model, mus)
combined_results[test_model]["track_scores"].append(model_results)

except Exception as e:
logger.error(f"Error during evaluation: {str(e)}")
logger.error("Error details:", exc_info=True)
return 1
# Save results after each successful evaluation
os.makedirs(os.path.dirname(combined_results_path), exist_ok=True)
with open(combined_results_path, "w") as f:
json.dump(combined_results, f, indent=2)

# Clean up any test directories that might have been created
test_dir = os.path.join(RESULTS_PATH, test_model, track_name, "test")
if os.path.exists(test_dir):
import shutil
logger.info(f"Updated combined results file with {test_model} - {track_name}")

shutil.rmtree(test_dir)
except Exception as e:
logger.error(f"Error evaluating model {test_model}: {str(e)}")
logger.error("Error details:", exc_info=True)
continue

logger.info("Evaluation complete")
return 0


Expand Down

0 comments on commit 829dabd

Please sign in to comment.