Replace IntegrationTests with Benchmarks (#34)

.github/workflows/benchmarks.yaml

@@ -0,0 +1,94 @@
+name: Benchmark PR vs main
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+    paths:
+    - '**.swift'
+    - '**.yml'
+
+jobs:
+  benchmark-delta:
+
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 15
+    continue-on-error: true
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Homebrew Mac
+        if: ${{ runner.os == 'Macos' }}
+        run: |
+          echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH
+          brew install jemalloc
+
+      - name: Ubuntu deps
+        if: ${{ runner.os == 'Linux' }}
+        run: |
+          sudo apt-get install -y libjemalloc-dev
+
+      - name: Git URL token override and misc
+        run: |
+          #git config --global url."https://ordo-ci:${{ secrets.CI_MACHINE_PAT }}@github.com".insteadOf "https://github.com"
+          #/usr/bin/ordo-performance
+          [ -d Benchmarks ] && echo "hasBenchmark=1" >> $GITHUB_ENV
+          echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH
+      - name: Run benchmarks for PR branch
+        if: ${{ env.hasBenchmark == '1' }}
+        run: |
+          cd Benchmarks
+          swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update pull_request
+      - name: Switch to branch 'main'
+        if: ${{ env.hasBenchmark == '1' }}
+        run: |
+          git stash
+          git checkout main
+      - name: Run benchmarks for branch 'main'
+        if: ${{ env.hasBenchmark == '1' }}
+        run: |
+          cd Benchmarks
+          swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update main
+      - name: Compare PR and main
+        if: ${{ env.hasBenchmark == '1' }}
+        id: benchmark
+        run: |
+          echo $(date) >> $GITHUB_STEP_SUMMARY
+          echo "exitStatus=1" >> $GITHUB_ENV
+          cd Benchmarks
+          swift package benchmark baseline check main pull_request --format markdown >> $GITHUB_STEP_SUMMARY
+          echo "exitStatus=0" >> $GITHUB_ENV
+        continue-on-error: true
+      - if: ${{ env.exitStatus == '0' }}
+        name: Pull request comment text success
+        id: prtestsuccess
+        run: |
+          echo 'PRTEST<<EOF' >> $GITHUB_ENV
+          echo "[Pull request benchmark comparison [${{ matrix.os }}] with 'main' run at $(date -Iseconds)](https://github.com/swift-extras/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }})" >> $GITHUB_ENV
+          echo 'EOF' >> $GITHUB_ENV
+      - if: ${{ env.exitStatus == '1' }}
+        name: Pull request comment text failure
+        id: prtestfailure
+        run: |
+          echo 'PRTEST<<EOF' >> $GITHUB_ENV
+          echo "[Pull request benchmark comparison [${{ matrix.os }}] with 'main' run at $(date -Iseconds)](https://github.com/swift-extras/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }})" >> $GITHUB_ENV
+          echo "_Pull request had performance regressions_" >> $GITHUB_ENV
+          echo 'EOF' >> $GITHUB_ENV
+      - name: Comment PR
+        if: ${{ env.hasBenchmark == '1' }}
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          message: ${{ env.PRTEST }}
+          comment_includes: "Pull request benchmark comparison [${{ matrix.os }}] with"
+      - name: Exit with correct status
+        run: |
+          exit ${{ env.exitStatus }}

.github/workflows/ci.yaml

@@ -58,27 +58,6 @@ jobs:
     - name: Build & run
       run: swift run -c release
 
-  "tuxOS-Integration-Tests":
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        images:
-        - swift:5.10
-    container:
-      image: ${{ matrix.images }}
-    env:
-      MAX_ALLOCS_ALLOWED_base64_decoding: 1000
-      MAX_ALLOCS_ALLOWED_base64_encoding: 1000
-      MAX_ALLOCS_ALLOWED_base32_decoding: 1000
-      MAX_ALLOCS_ALLOWED_base32_encoding: 1000
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-    - name: Test
-      run: ./run-tests.sh
-      working-directory: ./IntegrationTests
-
   "macOS-Tests":
     runs-on: macOS-14
     strategy:

Benchmarks/.gitignore

@@ -0,0 +1,2 @@
+.benchmarkBaselines
+Package.resolved

Benchmarks/Benchmarks/BaseN/BaseN.swift

@@ -0,0 +1,56 @@
+import Benchmark
+import ExtrasBase64
+
+let benchmarks = {
+    Benchmark.defaultConfiguration = .init(
+        metrics: [
+            .cpuTotal,
+            .throughput,
+            .mallocCountTotal,
+        ],
+        warmupIterations: 10,
+        scalingFactor: .kilo
+    )
+
+    Benchmark("Base32.encode") { benchmark in
+        let bytes = Array(UInt8(0) ... UInt8(255))
+
+        benchmark.startMeasurement()
+
+        for _ in benchmark.scaledIterations {
+            blackHole(Base32.encodeToString(bytes: bytes))
+        }
+    }
+
+    Benchmark("Base32.decode") { benchmark in
+        let bytes = Array(UInt8(0) ... UInt8(255))
+        let base32 = Base32.encodeToString(bytes: bytes)
+
+        benchmark.startMeasurement()
+
+        for _ in benchmark.scaledIterations {
+            try blackHole(Base32.decode(string: base32))
+        }
+    }
+
+    Benchmark("Base64.encode") { benchmark in
+        let bytes = Array(UInt8(0) ... UInt8(255))
+
+        benchmark.startMeasurement()
+
+        for _ in benchmark.scaledIterations {
+            blackHole(Base64.encodeToString(bytes: bytes))
+        }
+    }
+
+    Benchmark("Base64.decode") { benchmark in
+        let bytes = Array(UInt8(0) ... UInt8(255))
+        let base64 = Base64.encodeToString(bytes: bytes)
+
+        benchmark.startMeasurement()
+
+        for _ in benchmark.scaledIterations {
+            try blackHole(Base64.decode(string: base64))
+        }
+    }
+}

Benchmarks/Package.swift

@@ -0,0 +1,26 @@
+// swift-tools-version: 5.9
+// The swift-tools-version declares the minimum version of Swift required to build this package.
+
+import PackageDescription
+
+let package = Package(
+    name: "Benchmarks",
+    platforms: [.macOS(.v14)],
+    dependencies: [
+        .package(url: "https://github.com/ordo-one/package-benchmark.git", from: "1.0.0"),
+        .package(name: "swift-extras-base64", path: ".."),
+    ],
+    targets: [
+        .executableTarget(
+            name: "BaseN",
+            dependencies: [
+                .product(name: "ExtrasBase64", package: "swift-extras-base64"),
+                .product(name: "Benchmark", package: "package-benchmark"),
+            ],
+            path: "Benchmarks/BaseN",
+            plugins: [
+                .plugin(name: "BenchmarkPlugin", package: "package-benchmark"),
+            ]
+        ),
+    ]
+)