From b9b73afeb3d00de7411b17bba4ac25f890ef5e4b Mon Sep 17 00:00:00 2001 From: "U. Artie Eoff" Date: Thu, 20 Feb 2025 11:55:42 -0500 Subject: [PATCH] test_feature_extraction: use baseline fixture Signed-off-by: U. Artie Eoff --- .../tests/test_feature_extraction.json | 10 ++++++++++ tests/test_feature_extraction.py | 19 ++++++++++++------- 2 files changed, 22 insertions(+), 7 deletions(-) create mode 100644 tests/baselines/fixture/tests/test_feature_extraction.json diff --git a/tests/baselines/fixture/tests/test_feature_extraction.json b/tests/baselines/fixture/tests/test_feature_extraction.json new file mode 100644 index 0000000000..bf336f6c17 --- /dev/null +++ b/tests/baselines/fixture/tests/test_feature_extraction.json @@ -0,0 +1,10 @@ +{ + "tests/test_feature_extraction.py::GaudiFeatureExtractionTester::test_latency_graph_bf16": { + "gaudi1": { + "time_per_iter": 0.7987 + }, + "gaudi2": { + "time_per_iter": 0.6812 + } + } +} \ No newline at end of file diff --git a/tests/test_feature_extraction.py b/tests/test_feature_extraction.py index 85f25354b5..a8cff668a4 100644 --- a/tests/test_feature_extraction.py +++ b/tests/test_feature_extraction.py @@ -29,12 +29,6 @@ adapt_transformers_to_gaudi() -if OH_DEVICE_CONTEXT in ["gaudi2"]: - # Gaudi2 CI baselines - LATENCY_GTE_SMALL_BF16_GRAPH_BASELINE = 0.6812 -else: - # Gaudi1 CI baselines - LATENCY_GTE_SMALL_BF16_GRAPH_BASELINE = 0.7987 MODEL_NAME = "Supabase/gte-small" INPUT_TEXTS = [ @@ -94,6 +88,13 @@ class GaudiFeatureExtractionTester(TestCase): Tests for Supabase/gte-small feature extraction on Gaudi """ + @pytest.fixture(autouse=True) + def _use_(self, baseline): + """ + https://docs.pytest.org/en/stable/how-to/unittest.html#using-autouse-fixtures-and-accessing-other-fixtures + """ + self.baseline = baseline + def test_inference_default(self): """ Tests for equivalent CPU and HPU outputs @@ -135,4 +136,8 @@ def test_latency_graph_bf16(self): torch.hpu.synchronize() end_time = time.time() time_per_iter = (end_time - start_time) * 1000 / test_iters # time in ms - self.assertLess(time_per_iter, 1.05 * LATENCY_GTE_SMALL_BF16_GRAPH_BASELINE) + self.baseline.assertRef( + compare=lambda actual, ref: actual < (1.05 * ref), + context=[OH_DEVICE_CONTEXT], + time_per_iter=time_per_iter, + )