test: Improve contract e2e flakyness FE-1232 (#1765)

The goal of FE-1232 was to understand why Chrome Beta Contract tests were failing. Turns out there were a lot of issues with the contract tests, like action timing out and the reports saying that the tests passed. The two other main issues were the fuelWalletTestHelper failing to find the Popup page, and the aria label checks straight up not working. While working on figuring out why the tests were failing and if they were a Chrome Beta breaking change, I took the opportunity to improve some things. Changes: - Make **Contract Tests** shard for improved speed - await for `checkAriaLabelsContainsText` to avoid race condition issues - Import `expect` in `elements.ts`. - `checkAriaLabelsContainsText` errors fixed as we now wait for the button to trigger the Popup - Add pollling to `waitSuccessTransaction` - Improve logs - `getWalletPopupPage` increased timeout and clearer logs.
FuelLabs · Jan 14, 2025 · ab1ed94 · ab1ed94
1 parent 28628cb
commit ab1ed94
Show file tree

Hide file tree

Showing 13 changed files with 144 additions and 47 deletions.
diff --git a/.github/actions/e2e-tests-contracts/action.yaml b/.github/actions/e2e-tests-contracts/action.yaml
@@ -14,6 +14,10 @@ inputs:
     description: Browser to run tests on (chromium or chrome-beta)
     required: false
     default: chromium
+  shard:
+    description: Shard configuration for test parallelization (e.g. 1/4)
+    required: false
+    default: ''
 
 runs:
   using: 'composite'
@@ -35,7 +39,7 @@ runs:
 
     - name: Run E2E Contract Tests
       shell: bash
-      run: xvfb-run --auto-servernum -- pnpm test:e2e:contracts --project=${{ inputs.browser }}
+      run: xvfb-run --auto-servernum -- pnpm test:e2e:contracts --project=${{ inputs.browser }} ${{ inputs.shard != '' && format('--shard {0}', inputs.shard) || '' }}
       env:
         PORT: 5173
         VITE_FUEL_PROVIDER_URL: ${{ inputs.providerUrl }}

diff --git a/.github/workflows/pr-tests-e2e-contracts.yml b/.github/workflows/pr-tests-e2e-contracts.yml
@@ -11,9 +11,13 @@ concurrency:
 
 jobs:
   tests-e2e-contracts:
-    name: Test (Chrome Stable)
-    runs-on: buildjet-8vcpu-ubuntu-2204
-    timeout-minutes: 15
+    name: Test (Chrome Stable) [Shard ${{ matrix.shard }}]
+    runs-on: buildjet-8vcpu-ubuntu-2204	
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3, 4]
     steps:
       - uses: actions/checkout@v4
       - uses: FuelLabs/github-actions/setups/node@master
@@ -40,27 +44,31 @@ jobs:
       - name: Build & Deploy Contracts
         run: pnpm deploy:contracts
         working-directory: ./packages/e2e-contract-tests
-
       - name: Run E2E Contract Tests
         uses: ./.github/actions/e2e-tests-contracts
         with:
           providerUrl: "http://localhost:4000/v1/graphql"
           masterMnemonic: ${{ secrets.VITE_MASTER_WALLET_MNEMONIC }}
           genesisSecret: "0xa449b1ffee0e2205fa924c6740cc48b3b473aa28587df6dab12abc245d1f5298"
           browser: 'chromium'
+          shard: ${{ matrix.shard }}/4
 
       - uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: playwright-e2e-contract-tests-report
+          name: playwright-e2e-contract-tests-report-shard-${{ matrix.shard }}
           path: packages/e2e-contract-tests/playwright-results
           retention-days: 30
 
   tests-e2e-contracts-beta:
-    name: Test (Chrome Beta)
-    runs-on: buildjet-8vcpu-ubuntu-2204
-    timeout-minutes: 15
+    name: Test (Chrome Beta) [Shard ${{ matrix.shard }}]
+    runs-on: buildjet-8vcpu-ubuntu-2204	
+    timeout-minutes: 20
     continue-on-error: true
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3, 4]
     steps:
       - uses: actions/checkout@v4
       - uses: FuelLabs/github-actions/setups/node@master
@@ -88,17 +96,19 @@ jobs:
       - name: Build & Deploy Contracts
         run: pnpm deploy:contracts
         working-directory: ./packages/e2e-contract-tests
+
       - uses: ./.github/actions/e2e-tests-contracts
         with:
           providerUrl: "http://localhost:4000/v1/graphql"
           masterMnemonic: ${{ secrets.VITE_MASTER_WALLET_MNEMONIC }}
           genesisSecret: "0xa449b1ffee0e2205fa924c6740cc48b3b473aa28587df6dab12abc245d1f5298"
           browser: 'chrome-beta'
+          shard: ${{ matrix.shard }}/4
 
       - uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: playwright-e2e-contract-tests-report-beta
+          name: playwright-e2e-contract-tests-report-beta-shard-${{ matrix.shard }}
           path: packages/e2e-contract-tests/playwright-results
           retention-days: 30
 

diff --git a/packages/e2e-contract-tests/playwright/e2e/DepositHalfEth.test.ts b/packages/e2e-contract-tests/playwright/e2e/DepositHalfEth.test.ts
@@ -57,19 +57,25 @@ test.describe('Deposit Half ETH', () => {
 
     const depositHalfButton = getButtonByText(page, 'Deposit Half ETH', true);
     await expectButtonToBeEnabled(depositHalfButton);
+    await page.waitForTimeout(1000); // Wait for slow VM
     await depositHalfButton.click();
+    await page.waitForTimeout(1000); // Wait for slow VM
 
     const walletNotificationPage =
       await fuelWalletTestHelper.getWalletPopupPage();
 
     // Test if asset name is defined (not unknown)
-    checkAriaLabelsContainsText(
+    await checkAriaLabelsContainsText(
       walletNotificationPage,
       'Asset Name',
       'Ethereum'
     );
     // Test if sender name is defined (not unknown)
-    checkAriaLabelsContainsText(walletNotificationPage, 'Sender Name', '');
+    await checkAriaLabelsContainsText(
+      walletNotificationPage,
+      'Sender Name',
+      ''
+    );
 
     // test forward asset name is shown
     await hasText(walletNotificationPage, 'Ethereum');

diff --git a/packages/e2e-contract-tests/playwright/e2e/ForwardAndMintMulticall.test.ts b/packages/e2e-contract-tests/playwright/e2e/ForwardAndMintMulticall.test.ts
@@ -66,19 +66,25 @@ test.describe('Forward and Mint Multicall', () => {
       'Deposit And Mint Multicall'
     );
     await expectButtonToBeEnabled(forwardHalfAndMintButton);
+    await page.waitForTimeout(1000); // Wait for slow VM
     await forwardHalfAndMintButton.click();
+    await page.waitForTimeout(1000); // Wait for slow VM
 
     const walletNotificationPage =
       await fuelWalletTestHelper.getWalletPopupPage();
 
     // Test if asset name is defined (not unknown)
-    checkAriaLabelsContainsText(
+    await checkAriaLabelsContainsText(
       walletNotificationPage,
       'Asset Name',
       'Ethereum'
     );
     // Test if sender name is defined (not unknown)
-    checkAriaLabelsContainsText(walletNotificationPage, 'Sender Name', '');
+    await checkAriaLabelsContainsText(
+      walletNotificationPage,
+      'Sender Name',
+      ''
+    );
 
     // test forward asset name is shown
     await hasText(walletNotificationPage, 'Ethereum');

diff --git a/packages/e2e-contract-tests/playwright/e2e/ForwardCustomAsset.test.ts b/packages/e2e-contract-tests/playwright/e2e/ForwardCustomAsset.test.ts
@@ -78,19 +78,25 @@ test.describe('Forward Custom Asset', () => {
       'Forward Custom Asset'
     );
     await expectButtonToBeEnabled(forwardCustomAssetButton);
+    await page.waitForTimeout(1000); // Wait for slow VM
     await forwardCustomAssetButton.click();
+    await page.waitForTimeout(1000); // Wait for slow VM
 
     const walletNotificationPage =
       await fuelWalletTestHelper.getWalletPopupPage();
 
     // Test if asset name is defined (not unknown)
-    checkAriaLabelsContainsText(
+    await checkAriaLabelsContainsText(
       walletNotificationPage,
       'Asset Name',
       'Ethereum'
     );
     // Test if sender name is defined (not unknown)
-    checkAriaLabelsContainsText(walletNotificationPage, 'Sender Name', '');
+    await checkAriaLabelsContainsText(
+      walletNotificationPage,
+      'Sender Name',
+      ''
+    );
 
     // test the asset name is shown
     await hasText(walletNotificationPage, 'Unknown', 0, 5000, true);

diff --git a/packages/e2e-contract-tests/playwright/e2e/ForwardEth.test.ts b/packages/e2e-contract-tests/playwright/e2e/ForwardEth.test.ts
@@ -56,30 +56,32 @@ test.describe('Forward Eth', () => {
 
     const forwardEthButton = getButtonByText(page, 'Forward ETH');
     await expectButtonToBeEnabled(forwardEthButton);
+    await page.waitForTimeout(1000); // Wait for slow VM
     await forwardEthButton.click();
+    await page.waitForTimeout(1000); // Wait for slow VM
 
     const walletNotificationPage =
       await fuelWalletTestHelper.getWalletPopupPage();
 
-    // Test if asset name is defined (not unknown)
-    checkAriaLabelsContainsText(
+    await checkAriaLabelsContainsText(
       walletNotificationPage,
       'Asset Name',
       'Ethereum'
     );
     // Test if sender name is defined (not unknown)
-    checkAriaLabelsContainsText(walletNotificationPage, 'Sender Name', '');
-
-    // test the asset name is shown
+    await checkAriaLabelsContainsText(
+      walletNotificationPage,
+      'Sender Name',
+      ''
+    );
     await hasText(walletNotificationPage, 'Ethereum');
 
     // test asset id is correct
-    await hasText(walletNotificationPage, shortAddress(await getBaseAssetId()));
+    const baseAssetId = await getBaseAssetId();
+    await hasText(walletNotificationPage, shortAddress(baseAssetId));
 
     // test forward eth amount is correct
     await hasText(walletNotificationPage, `${forwardEthAmount} ETH`);
-
-    // test gas fee is correct
     await hasText(walletNotificationPage, 'Fee (network)');
 
     // test to and from addresses
@@ -94,10 +96,13 @@ test.describe('Forward Eth', () => {
     await fuelWalletTestHelper.walletApprove();
     await waitSuccessTransaction(page);
     const postDepositBalanceEth = await fuelWallet.getBalance();
-    expect(
-      Number.parseFloat(
-        preDepositBalanceEth.sub(postDepositBalanceEth).format({ precision: 4 })
-      )
-    ).toBe(Number.parseFloat(forwardEthAmount));
+
+    const difference = preDepositBalanceEth
+      .sub(postDepositBalanceEth)
+      .format({ precision: 4 });
+
+    expect(Number.parseFloat(difference)).toBe(
+      Number.parseFloat(forwardEthAmount)
+    );
   });
 });
diff --git a/packages/e2e-contract-tests/playwright/e2e/ForwardHalfAndExternalMint.test.ts b/packages/e2e-contract-tests/playwright/e2e/ForwardHalfAndExternalMint.test.ts
@@ -70,19 +70,25 @@ test.describe('Forward Half ETH and Mint External Custom Asset', () => {
       'Forward Half And External Mint'
     );
     await expectButtonToBeEnabled(forwardHalfAndMintButton);
+    await page.waitForTimeout(1000); // Wait for slow VM
     await forwardHalfAndMintButton.click();
+    await page.waitForTimeout(1000); // Wait for slow VM
 
     const walletNotificationPage =
       await fuelWalletTestHelper.getWalletPopupPage();
 
     // Test if asset name is defined (not unknown)
-    checkAriaLabelsContainsText(
+    await checkAriaLabelsContainsText(
       walletNotificationPage,
       'Asset Name',
       'Ethereum'
     );
     // Test if sender name is defined (not unknown)
-    checkAriaLabelsContainsText(walletNotificationPage, 'Sender Name', '');
+    await checkAriaLabelsContainsText(
+      walletNotificationPage,
+      'Sender Name',
+      ''
+    );
 
     // test forward asset name is shown
     await hasText(walletNotificationPage, 'Ethereum');

diff --git a/packages/e2e-contract-tests/playwright/e2e/ForwardHalfAndMint.test.ts b/packages/e2e-contract-tests/playwright/e2e/ForwardHalfAndMint.test.ts
@@ -67,19 +67,25 @@ test.describe('Forward Half ETH and Mint Custom Asset', () => {
       'Forward Half And Mint'
     );
     await expectButtonToBeEnabled(forwardHalfAndMintButton);
+    await page.waitForTimeout(1000); // Wait for slow VM
     await forwardHalfAndMintButton.click();
+    await page.waitForTimeout(1000); // Wait for slow VM
 
     const walletNotificationPage =
       await fuelWalletTestHelper.getWalletPopupPage();
 
     // Test if asset name is defined (not unknown)
-    checkAriaLabelsContainsText(
+    await checkAriaLabelsContainsText(
       walletNotificationPage,
       'Asset Name',
       'Ethereum'
     );
     // Test if sender name is defined (not unknown)
-    checkAriaLabelsContainsText(walletNotificationPage, 'Sender Name', '');
+    await checkAriaLabelsContainsText(
+      walletNotificationPage,
+      'Sender Name',
+      ''
+    );
 
     // test forward asset name is shown
     await hasText(walletNotificationPage, 'Ethereum');

diff --git a/packages/e2e-contract-tests/playwright/e2e/ForwardHalfCustomAsset.test.ts b/packages/e2e-contract-tests/playwright/e2e/ForwardHalfCustomAsset.test.ts
@@ -81,19 +81,25 @@ test.describe('Forward Half Custom Asset', () => {
       'Forward Half Custom Asset'
     );
     await expectButtonToBeEnabled(forwardHalfCustomAssetButton);
+    await page.waitForTimeout(1000); // Wait for slow VM
     await forwardHalfCustomAssetButton.click();
+    await page.waitForTimeout(1000); // Wait for slow VM
 
     const walletNotificationPage =
       await fuelWalletTestHelper.getWalletPopupPage();
 
     // Test if asset name is defined (not unknown)
-    checkAriaLabelsContainsText(
+    await checkAriaLabelsContainsText(
       walletNotificationPage,
       'Asset Name',
       'Ethereum'
     );
     // Test if sender name is defined (not unknown)
-    checkAriaLabelsContainsText(walletNotificationPage, 'Sender Name', '');
+    await checkAriaLabelsContainsText(
+      walletNotificationPage,
+      'Sender Name',
+      ''
+    );
     // test the forward asset name is shown
     await hasText(walletNotificationPage, 'Unknown', 0, 5000, true);
     // test forward asset id is correct

diff --git a/packages/e2e-contract-tests/playwright/e2e/MintAsset.test.ts b/packages/e2e-contract-tests/playwright/e2e/MintAsset.test.ts
@@ -62,6 +62,8 @@ test.describe('Mint Assets', () => {
 
     // test asset is correct
     const assetId = calculateAssetId(MAIN_CONTRACT_ID, await getBaseAssetId());
+    await page.waitForTimeout(10000); // Wait for slow VM
+
     const walletNotificationPage =
       await fuelWalletTestHelper.getWalletPopupPage();
     // short address function copied from app package
@@ -126,20 +128,25 @@ test.describe('Mint Assets', () => {
     const mintButton = getButtonByText(page, 'Mint Asset configuration');
 
     await expectButtonToBeEnabled(mintButton);
+    await page.waitForTimeout(1000); // Wait for slow VM
     await mintButton.click();
-
+    await page.waitForTimeout(1000); // Wait for slow VM
     // test asset is correct
     const walletNotificationPage =
       await fuelWalletTestHelper.getWalletPopupPage();
 
     // Test if asset name is defined (not unknown)
-    checkAriaLabelsContainsText(
+    await checkAriaLabelsContainsText(
       walletNotificationPage,
       'Asset Name',
       'Ethereum'
     );
     // Test if sender name is defined (not unknown)
-    checkAriaLabelsContainsText(walletNotificationPage, 'Sender Name', '');
+    await checkAriaLabelsContainsText(
+      walletNotificationPage,
+      'Sender Name',
+      ''
+    );
 
     // scroll to bottom of page to ensure all text is visible
     await walletNotificationPage.evaluate(() =>

diff --git a/packages/e2e-contract-tests/playwright/e2e/utils/elements.ts b/packages/e2e-contract-tests/playwright/e2e/utils/elements.ts
@@ -1,4 +1,4 @@
-import type { Page } from '@playwright/test';
+import { type Page, expect } from '@playwright/test';
 
 export async function checkAriaLabelsContainsText(
   walletNotificationPage: Page,
@@ -9,10 +9,11 @@ export async function checkAriaLabelsContainsText(
   const count = await locator.count();
 
   for (let i = 0; i < count; i++) {
+    const innerHTML = await locator.nth(i).innerHTML();
     if (text === '') {
-      expect(locator.nth(i).innerHTML()).not.toBe('');
+      await expect(innerHTML).not.toBe('');
     } else {
-      expect((await locator.nth(i).innerHTML()).includes(text)).toBeTruthy();
+      await expect(innerHTML.includes(text)).toBeTruthy();
     }
   }
 }