show stats in tests

.github/workflows/playwright.yml

@@ -19,9 +19,3 @@ jobs:
       run: npx playwright install --with-deps
     - name: Run Playwright tests
       run: npx playwright test
-    - uses: actions/upload-artifact@v4
-      if: ${{ !cancelled() }}
-      with:
-        name: playwright-report
-        path: playwright-report/
-        retention-days: 30

playwright.config.ts

@@ -6,7 +6,7 @@ export default defineConfig({
   forbidOnly: !!process.env.CI,
   retries: process.env.CI ? 2 : 0,
   workers: process.env.CI ? 1 : undefined,
-  reporter: 'html',
+  reporter: [['dot'], ['./tests/stats-reporter.ts']],
   timeout: 120 * 1000,
   use: {
     baseURL: 'http://localhost:5173',

tests/json2pbf.spec.ts

@@ -1,28 +1,30 @@
 import { test, expect } from '@playwright/test';
 import { codecs } from '../demo/config';
 
-type Configuration = { arraySize: number, iterations: number, thresolds: { packTime: number, unpackTime: number, sendTime: number, receiveTime: number } };  
+type Configuration = { arraySize: number, iterations: number };  
 
 const configurations: Configuration[] = [
-  { arraySize: 10**4, iterations: 1000, thresolds: { packTime: 3000, unpackTime: 3000, sendTime: 100, receiveTime: 100 } },
-  { arraySize: 10**5, iterations: 100, thresolds: { packTime: 3000, unpackTime: 3000, sendTime: 100, receiveTime: 100 } },
-  { arraySize: 10**6, iterations: 10, thresolds: { packTime: 6000, unpackTime: 6000, sendTime: 100, receiveTime: 100 } },
-  { arraySize: 10**7, iterations: 1, thresolds: { packTime: 6000, unpackTime: 6000, sendTime: 100, receiveTime: 100 } },
+  { arraySize: 10**4, iterations: 1000 },
+  { arraySize: 10**5, iterations: 100 },
+  { arraySize: 10**6, iterations: 10},
+  { arraySize: 10**7, iterations: 1 },
 ]
 
 for (let i = 0; i < codecs.length; i++) {
   test.describe(codecs[i].name, () => {
-    for (const { arraySize, iterations, thresolds } of configurations) {
-      test(`size: ${arraySize}, iterations: ${iterations}`, async ({ page }) => {
+    for (const { arraySize, iterations } of configurations) {
+      test(`size: ${arraySize}, iterations: ${iterations}`, async ({ page }, testInfo) => {
         await page.goto('/?test');
         const result = await page.evaluate(
           ([codec, arraySize, iterations]) => (window as any).performTest(codec, arraySize, iterations),
           [i, arraySize, iterations]
         );
         expect(result.equal).toBe(true);
-        for (const name in thresolds) {
-          expect(result[name], name).toBeLessThan(thresolds[name]);
-        }
+
+        testInfo.attach('stats', {
+          body: JSON.stringify({ codec: codecs[i].name, case: `10**${Math.log10(arraySize)} x ${iterations}`, ...result }), 
+          contentType: 'application/json'}
+        )
       });    
     }  
   })

tests/stats-reporter.ts

@@ -0,0 +1,42 @@
+import type {
+    Reporter, TestCase, TestResult
+} from '@playwright/test/reporter';
+
+class StatsReporter implements Reporter {
+    private results: Record<string, Record<string, string>> = {};
+
+    onBegin() {
+        this.results = {}
+    }
+
+    onTestEnd(_test: TestCase, result: TestResult) {
+        if (!result.attachments[0] || !result.attachments[0].body) {
+            return;
+        }
+        const body = result.attachments[0].body as any;
+        const stats = JSON.parse(body);
+        if (!this.results[stats.codec]) {
+            this.results[stats.codec] = {};
+        }
+
+        const { packTime, unpackTime, receiveTime, sendTime } = stats;
+        const total = packTime + unpackTime + receiveTime + sendTime;
+
+        const fmt = (n: number) => ('      '+Math.trunc(n)).slice(-6);
+
+        this.results[stats.codec][stats.case] = `${fmt(total)} (pack: ${fmt(packTime)}, unpack: ${fmt(unpackTime)}, receive: ${fmt(receiveTime)}, send: ${fmt(sendTime)})`;
+    }
+
+    onEnd() {
+        for (const codec in this.results) {
+            console.log(`${codec}:`)
+            const codecStats = this.results[codec];
+            for (const case_ in codecStats) {
+                const caseStats = codecStats[case_];
+                console.log(`\t${case_}:\t${caseStats}`);
+            }
+        }
+    }
+}
+
+export default StatsReporter;