Skip to content

Commit

Permalink
test_examples: split test baseline and test config
Browse files Browse the repository at this point in the history 
...and rename "gaudi" to gaudi1" so we can use the
new baseline fixture.

Signed-off-by: U. Artie Eoff <[email protected]>
  • Loading branch information
@uartie
uartie committed Feb 24, 2025
1 parent d4521ac commit bae2585
Show file tree
Hide file tree
Showing 37 changed files with 655 additions and 407 deletions.
440 changes: 440 additions & 0 deletions tests/baselines/fixture/tests/test_examples.json
View file

Large diffs are not rendered by default.

6 changes: 2 additions & 4 deletions .../baselines/CodeLlama_13b_Instruct_hf.json → ...s/examples/CodeLlama_13b_Instruct_hf.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@
"deepspeed": {
"learning_rate": 5e-5,
"train_batch_size": 48,
"train_runtime": 542.2985,
"train_samples_per_second": 18.789,
"perplexity": 6.877496628184696,
"metrics": ["perplexity", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--dataset_config_name wikitext-2-raw-v1",
"--gradient_checkpointing",
Expand All @@ -20,4 +18,4 @@
}
}
}
}
}
6 changes: 2 additions & 4 deletions tests/baselines/LlamaGuard_7b.json → tests/configs/examples/LlamaGuard_7b.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@
"deepspeed": {
"learning_rate": 3e-5,
"train_batch_size": 32,
"eval_f1": 0.8873483535528596,
"train_runtime": 62.4539,
"train_samples_per_second": 342.169,
"metrics": ["eval_f1", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--max_seq_length 128",
"--add_pad_token True",
Expand All @@ -20,4 +18,4 @@
}
}
}
}
}
6 changes: 2 additions & 4 deletions tests/baselines/Llama_3_1_8B.json → tests/configs/examples/Llama_3_1_8B.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@
"single_card": {
"learning_rate": 3e-4,
"train_batch_size": 10,
"perplexity": 2.7317,
"train_runtime": 1435.24322,
"train_samples_per_second": 13.3044,
"metrics": ["perplexity", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--bf16",
"--gradient_checkpointing",
Expand All @@ -32,4 +30,4 @@
}
}
}
}
}
4 changes: 1 addition & 3 deletions ...elines/Llama_3_2_11B_Vision_Instruct.json → ...amples/Llama_3_2_11B_Vision_Instruct.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@
"multi_card": {
"learning_rate": 5e-5,
"train_batch_size": 2,
"train_runtime": 350,
"train_samples_per_second": 20.48,
"eval_accuracy": 0.6,
"metrics": ["eval_accuracy", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--bf16",
"--gradient_accumulation_steps 8",
Expand Down
6 changes: 2 additions & 4 deletions tests/baselines/Qwen2_72B.json → tests/configs/examples/Qwen2_72B.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@
"deepspeed": {
"learning_rate": 3e-4,
"train_batch_size": 8,
"perplexity": 3.7020898897918824,
"train_runtime": 918.8018,
"train_samples_per_second": 7.554,
"metrics": ["perplexity", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--bf16 True",
"--subset None",
Expand Down Expand Up @@ -44,4 +42,4 @@
}
}
}
}
}
8 changes: 3 additions & 5 deletions tests/baselines/Qwen2_7B.json → tests/configs/examples/Qwen2_7B.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@
"multi_card": {
"learning_rate": 3e-4,
"train_batch_size": 32,
"train_runtime": 410,
"train_samples_per_second": 120,
"metrics": ["train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--bf16 True",
"--subset ''",
Expand Down Expand Up @@ -44,8 +43,7 @@
"multi_card": {
"learning_rate": 3e-4,
"train_batch_size": 2,
"train_runtime": 423.995,
"train_samples_per_second": 7.342,
"metrics": ["train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--bf16 True",
"--subset ''",
Expand All @@ -71,4 +69,4 @@
}
}
}
}
}
20 changes: 6 additions & 14 deletions tests/baselines/albert_large_v2.json → tests/configs/examples/albert_large_v2.json
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
{
"gaudi": {
"gaudi1": {
"squad": {
"num_train_epochs": 2,
"eval_batch_size": 4,
"distribution": {
"single_card": {
"learning_rate": 6e-5,
"train_batch_size": 32,
"eval_f1": 91.8679,
"train_runtime": 2900.5518,
"train_samples_per_second": 62.298,
"metrics": ["eval_f1", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--max_seq_length 384",
"--use_hpu_graphs_for_inference"
Expand All @@ -18,9 +16,7 @@
"multi_card": {
"learning_rate": 6e-5,
"train_batch_size": 32,
"eval_f1": 92.7647,
"train_runtime": 464.9893,
"train_samples_per_second": 494.936,
"metrics": ["eval_f1", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--max_seq_length 384",
"--use_hpu_graphs_for_inference"
Expand All @@ -37,9 +33,7 @@
"single_card": {
"learning_rate": 6e-5,
"train_batch_size": 128,
"eval_f1": 92.4235,
"train_runtime": 571.138,
"train_samples_per_second": 321.635,
"metrics": ["eval_f1", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--max_seq_length 384",
"--use_hpu_graphs_for_inference"
Expand All @@ -48,9 +42,7 @@
"multi_card": {
"learning_rate": 7e-5,
"train_batch_size": 128,
"eval_f1": 92.2111,
"train_runtime": 115.15,
"train_samples_per_second": 2464.403,
"metrics": ["eval_f1", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--max_seq_length 384",
"--use_hpu_graphs_for_inference"
Expand All @@ -59,4 +51,4 @@
}
}
}
}
}
20 changes: 6 additions & 14 deletions tests/baselines/albert_xxlarge_v1.json → ...s/configs/examples/albert_xxlarge_v1.json
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
{
"gaudi": {
"gaudi1": {
"squad": {
"num_train_epochs": 1,
"eval_batch_size": 2,
"distribution": {
"single_card": {
"learning_rate": 1e-5,
"train_batch_size": 12,
"eval_f1": 95.1334,
"train_runtime": 9474.1784,
"train_samples_per_second": 9.464,
"metrics": ["eval_f1", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--max_seq_length 384",
"--use_hpu_graphs_for_inference"
Expand All @@ -18,9 +16,7 @@
"multi_card": {
"learning_rate": 5e-5,
"train_batch_size": 12,
"eval_f1": 95.1145,
"train_runtime": 1347.7824,
"train_samples_per_second": 71.285,
"metrics": ["eval_f1", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--max_seq_length 384",
"--use_hpu_graphs_for_inference"
Expand All @@ -37,9 +33,7 @@
"single_card": {
"learning_rate": 2e-5,
"train_batch_size": 16,
"eval_f1": 95.1484,
"train_runtime": 1523.3401,
"train_samples_per_second": 58.697,
"metrics": ["eval_f1", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--max_seq_length 384",
"--use_hpu_graphs_for_inference"
Expand All @@ -48,9 +42,7 @@
"multi_card": {
"learning_rate": 7e-5,
"train_batch_size": 16,
"eval_f1": 95.1898,
"train_runtime": 243.0459,
"train_samples_per_second": 416.256,
"metrics": ["eval_f1", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--max_seq_length 384",
"--use_hpu_graphs_for_inference"
Expand All @@ -59,4 +51,4 @@
}
}
}
}
}
5 changes: 1 addition & 4 deletions ...nes/ast_finetuned_speech_commands_v2.json → ...les/ast_finetuned_speech_commands_v2.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,7 @@
"multi_card": {
"learning_rate": 5e-4,
"train_batch_size": 32,
"eval_accuracy": 0.1871,
"train_runtime": 139.9477,
"train_samples_per_second": 1955.74,
"eval_samples_per_second": 2301.088,
"metrics": ["eval_accuracy", "train_runtime", "train_samples_per_second", "eval_samples_per_second"],
"extra_arguments": [
"--audio_column_name audio",
"--label_column_name language",
Expand Down
18 changes: 5 additions & 13 deletions tests/baselines/bert_base_uncased.json → ...s/configs/examples/bert_base_uncased.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,7 @@
"single_card": {
"learning_rate": 5e-5,
"train_batch_size": 24,
"eval_f1": 87.3749,
"train_runtime": 568.832,
"train_samples_per_second": 158.687,
"metrics": ["eval_f1", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--max_seq_length 384",
"--use_hpu_graphs_for_inference"
Expand All @@ -17,9 +15,7 @@
"multi_card": {
"learning_rate": 2e-4,
"train_batch_size": 24,
"eval_f1": 87.6017,
"train_runtime": 97.7157,
"train_samples_per_second": 1240.638,
"metrics": ["eval_f1", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--max_seq_length 384",
"--use_hpu_graphs_for_inference"
Expand All @@ -34,9 +30,7 @@
"single_card": {
"learning_rate": 6e-5,
"train_batch_size": 64,
"eval_f1": 0.8998,
"train_runtime": 31.044,
"train_samples_per_second": 558.201,
"metrics": ["eval_f1", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--max_seq_length 128",
"--use_hpu_graphs_for_inference"
Expand All @@ -45,14 +39,12 @@
"multi_card": {
"learning_rate": 5e-4,
"train_batch_size": 64,
"eval_f1": 0.8765,
"train_runtime": 28.3865,
"train_samples_per_second": 3643.715,
"metrics": ["eval_f1", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--max_seq_length 128",
"--use_hpu_graphs_for_inference"
]
}
}
}
}
}
34 changes: 9 additions & 25 deletions ...ert_large_uncased_whole_word_masking.json → ...ert_large_uncased_whole_word_masking.json
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
{
"gaudi": {
"gaudi1": {
"squad": {
"num_train_epochs": 1,
"eval_batch_size": 8,
"distribution": {
"single_card": {
"learning_rate": 3e-5,
"train_batch_size": 24,
"eval_f1": 93.1962,
"train_runtime": 1678.3456,
"train_samples_per_second": 54.101,
"metrics": ["eval_f1", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--max_seq_length 384",
"--use_hpu_graphs_for_inference"
Expand All @@ -18,9 +16,7 @@
"multi_card": {
"learning_rate": 7e-5,
"train_batch_size": 24,
"eval_f1": 93.1869,
"train_runtime": 309.9553,
"train_samples_per_second": 398.459,
"metrics": ["eval_f1", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--max_seq_length 384",
"--use_hpu_graphs_for_inference"
Expand All @@ -35,9 +31,7 @@
"single_card": {
"learning_rate": 3e-5,
"train_batch_size": 32,
"eval_f1": 0.9022,
"train_runtime": 90.3943,
"train_samples_per_second": 172.792,
"metrics": ["eval_f1", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--max_seq_length 128",
"--use_hpu_graphs_for_inference"
Expand All @@ -46,9 +40,7 @@
"multi_card": {
"learning_rate": 3e-5,
"train_batch_size": 16,
"eval_f1": 0.8897,
"train_runtime": 65.644,
"train_samples_per_second": 919.623,
"metrics": ["eval_f1", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--max_seq_length 128",
"--use_hpu_graphs_for_inference"
Expand All @@ -65,9 +57,7 @@
"single_card": {
"learning_rate": 3e-5,
"train_batch_size": 32,
"eval_f1": 93.2753,
"train_runtime": 342.1722,
"train_samples_per_second": 286.435,
"metrics": ["eval_f1", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--max_seq_length 384",
"--use_hpu_graphs_for_inference"
Expand All @@ -76,9 +66,7 @@
"multi_card": {
"learning_rate": 3e-5,
"train_batch_size": 32,
"eval_f1": 91.71,
"train_runtime": 80.307,
"train_samples_per_second": 2150.333,
"metrics": ["eval_f1", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--max_seq_length 384",
"--use_hpu_graphs_for_inference"
Expand All @@ -93,9 +81,7 @@
"single_card": {
"learning_rate": 3e-5,
"train_batch_size": 256,
"eval_f1": 0.867,
"train_runtime": 33.2909,
"train_samples_per_second": 1100.598,
"metrics": ["eval_f1", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--max_seq_length 128",
"--use_hpu_graphs_for_inference"
Expand All @@ -104,9 +90,7 @@
"multi_card": {
"learning_rate": 3e-5,
"train_batch_size": 40,
"eval_f1": 0.8452579034941764,
"train_runtime": 31.445,
"train_samples_per_second": 2845.068,
"metrics": ["eval_f1", "train_runtime", "train_samples_per_second"],
"extra_arguments": [
"--max_seq_length 128",
"--use_hpu_graphs_for_inference"
Expand Down
Loading

0 comments on commit bae2585

Please sign in to comment.