Skip to content

Commit 4ce194f

Browse files
adam-fowleradam-fowler
authored
Replace IntegrationTests with Benchmarks (#34)
1 parent 1e8b1cc commit 4ce194f

File tree

34 files changed

+178
-1569
lines changed

34 files changed

+178
-1569
lines changed

‎.github/workflows/benchmarks.yaml

+94
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
name: Benchmark PR vs main
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
branches:
7+
- main
8+
paths:
9+
- '**.swift'
10+
- '**.yml'
11+
12+
jobs:
13+
benchmark-delta:
14+
15+
runs-on: ${{ matrix.os }}
16+
timeout-minutes: 15
17+
continue-on-error: true
18+
19+
strategy:
20+
matrix:
21+
os: [ubuntu-latest]
22+
23+
steps:
24+
- uses: actions/checkout@v4
25+
with:
26+
fetch-depth: 0
27+
28+
- name: Homebrew Mac
29+
if: ${{ runner.os == 'Macos' }}
30+
run: |
31+
echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH
32+
brew install jemalloc
33+
34+
- name: Ubuntu deps
35+
if: ${{ runner.os == 'Linux' }}
36+
run: |
37+
sudo apt-get install -y libjemalloc-dev
38+
39+
- name: Git URL token override and misc
40+
run: |
41+
#git config --global url."https://ordo-ci:${{ secrets.CI_MACHINE_PAT }}@github.com".insteadOf "https://github.com"
42+
#/usr/bin/ordo-performance
43+
[ -d Benchmarks ] && echo "hasBenchmark=1" >> $GITHUB_ENV
44+
echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH
45+
- name: Run benchmarks for PR branch
46+
if: ${{ env.hasBenchmark == '1' }}
47+
run: |
48+
cd Benchmarks
49+
swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update pull_request
50+
- name: Switch to branch 'main'
51+
if: ${{ env.hasBenchmark == '1' }}
52+
run: |
53+
git stash
54+
git checkout main
55+
- name: Run benchmarks for branch 'main'
56+
if: ${{ env.hasBenchmark == '1' }}
57+
run: |
58+
cd Benchmarks
59+
swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update main
60+
- name: Compare PR and main
61+
if: ${{ env.hasBenchmark == '1' }}
62+
id: benchmark
63+
run: |
64+
echo $(date) >> $GITHUB_STEP_SUMMARY
65+
echo "exitStatus=1" >> $GITHUB_ENV
66+
cd Benchmarks
67+
swift package benchmark baseline check main pull_request --format markdown >> $GITHUB_STEP_SUMMARY
68+
echo "exitStatus=0" >> $GITHUB_ENV
69+
continue-on-error: true
70+
- if: ${{ env.exitStatus == '0' }}
71+
name: Pull request comment text success
72+
id: prtestsuccess
73+
run: |
74+
echo 'PRTEST<<EOF' >> $GITHUB_ENV
75+
echo "[Pull request benchmark comparison [${{ matrix.os }}] with 'main' run at $(date -Iseconds)](https://github.com/swift-extras/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }})" >> $GITHUB_ENV
76+
echo 'EOF' >> $GITHUB_ENV
77+
- if: ${{ env.exitStatus == '1' }}
78+
name: Pull request comment text failure
79+
id: prtestfailure
80+
run: |
81+
echo 'PRTEST<<EOF' >> $GITHUB_ENV
82+
echo "[Pull request benchmark comparison [${{ matrix.os }}] with 'main' run at $(date -Iseconds)](https://github.com/swift-extras/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }})" >> $GITHUB_ENV
83+
echo "_Pull request had performance regressions_" >> $GITHUB_ENV
84+
echo 'EOF' >> $GITHUB_ENV
85+
- name: Comment PR
86+
if: ${{ env.hasBenchmark == '1' }}
87+
uses: thollander/actions-comment-pull-request@v2
88+
with:
89+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
90+
message: ${{ env.PRTEST }}
91+
comment_includes: "Pull request benchmark comparison [${{ matrix.os }}] with"
92+
- name: Exit with correct status
93+
run: |
94+
exit ${{ env.exitStatus }}

‎.github/workflows/ci.yaml

-21
Original file line numberDiff line numberDiff line change
@@ -58,27 +58,6 @@ jobs:
5858
- name: Build & run
5959
run: swift run -c release
6060

61-
"tuxOS-Integration-Tests":
62-
runs-on: ubuntu-latest
63-
strategy:
64-
fail-fast: false
65-
matrix:
66-
images:
67-
- swift:5.10
68-
container:
69-
image: ${{ matrix.images }}
70-
env:
71-
MAX_ALLOCS_ALLOWED_base64_decoding: 1000
72-
MAX_ALLOCS_ALLOWED_base64_encoding: 1000
73-
MAX_ALLOCS_ALLOWED_base32_decoding: 1000
74-
MAX_ALLOCS_ALLOWED_base32_encoding: 1000
75-
steps:
76-
- name: Checkout
77-
uses: actions/checkout@v4
78-
- name: Test
79-
run: ./run-tests.sh
80-
working-directory: ./IntegrationTests
81-
8261
"macOS-Tests":
8362
runs-on: macOS-14
8463
strategy:

‎Benchmarks/.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
.benchmarkBaselines
2+
Package.resolved

‎Benchmarks/Benchmarks/BaseN/BaseN.swift

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
import Benchmark
2+
import ExtrasBase64
3+
4+
let benchmarks = {
5+
Benchmark.defaultConfiguration = .init(
6+
metrics: [
7+
.cpuTotal,
8+
.throughput,
9+
.mallocCountTotal,
10+
],
11+
warmupIterations: 10,
12+
scalingFactor: .kilo
13+
)
14+
15+
Benchmark("Base32.encode") { benchmark in
16+
let bytes = Array(UInt8(0) ... UInt8(255))
17+
18+
benchmark.startMeasurement()
19+
20+
for _ in benchmark.scaledIterations {
21+
blackHole(Base32.encodeToString(bytes: bytes))
22+
}
23+
}
24+
25+
Benchmark("Base32.decode") { benchmark in
26+
let bytes = Array(UInt8(0) ... UInt8(255))
27+
let base32 = Base32.encodeToString(bytes: bytes)
28+
29+
benchmark.startMeasurement()
30+
31+
for _ in benchmark.scaledIterations {
32+
try blackHole(Base32.decode(string: base32))
33+
}
34+
}
35+
36+
Benchmark("Base64.encode") { benchmark in
37+
let bytes = Array(UInt8(0) ... UInt8(255))
38+
39+
benchmark.startMeasurement()
40+
41+
for _ in benchmark.scaledIterations {
42+
blackHole(Base64.encodeToString(bytes: bytes))
43+
}
44+
}
45+
46+
Benchmark("Base64.decode") { benchmark in
47+
let bytes = Array(UInt8(0) ... UInt8(255))
48+
let base64 = Base64.encodeToString(bytes: bytes)
49+
50+
benchmark.startMeasurement()
51+
52+
for _ in benchmark.scaledIterations {
53+
try blackHole(Base64.decode(string: base64))
54+
}
55+
}
56+
}

‎Benchmarks/Package.swift

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
// swift-tools-version: 5.9
2+
// The swift-tools-version declares the minimum version of Swift required to build this package.
3+
4+
import PackageDescription
5+
6+
let package = Package(
7+
name: "Benchmarks",
8+
platforms: [.macOS(.v14)],
9+
dependencies: [
10+
.package(url: "https://github.com/ordo-one/package-benchmark.git", from: "1.0.0"),
11+
.package(name: "swift-extras-base64", path: ".."),
12+
],
13+
targets: [
14+
.executableTarget(
15+
name: "BaseN",
16+
dependencies: [
17+
.product(name: "ExtrasBase64", package: "swift-extras-base64"),
18+
.product(name: "Benchmark", package: "package-benchmark"),
19+
],
20+
path: "Benchmarks/BaseN",
21+
plugins: [
22+
.plugin(name: "BenchmarkPlugin", package: "package-benchmark"),
23+
]
24+
),
25+
]
26+
)

0 commit comments

Comments
 (0)