Skip to content

Commit eeb7e0a

Browse files
ahoppenahoppen
committed
WIP: Performance testing
1 parent 8b55263 commit eeb7e0a

File tree

5 files changed

+196
-11
lines changed

5 files changed

+196
-11
lines changed

‎.github/workflows/pull_request.yml

Lines changed: 82 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,85 @@ concurrency:
99
cancel-in-progress: true
1010

1111
jobs:
12-
tests:
13-
name: Test
14-
uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main
15-
with:
16-
linux_exclude_swift_versions: "[{\"swift_version\": \"5.8\"}]"
17-
soundness:
18-
name: Soundness
19-
uses: swiftlang/github-workflows/.github/workflows/soundness.yml@main
20-
with:
21-
license_header_check_project_name: "Swift.org"
22-
api_breakage_check_allowlist_path: "api-breakages.txt"
12+
# tests:
13+
# name: Test
14+
# uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main
15+
# with:
16+
# linux_exclude_swift_versions: "[{\"swift_version\": \"5.8\"}]"
17+
# soundness:
18+
# name: Soundness
19+
# uses: swiftlang/github-workflows/.github/workflows/soundness.yml@main
20+
# with:
21+
# license_header_check_project_name: "Swift.org"
22+
# api_breakage_check_allowlist_path: "api-breakages.txt"
23+
performance_test:
24+
name: Performance test
25+
runs-on: ubuntu-latest
26+
container:
27+
image: swift:latest
28+
permissions:
29+
pull-requests: write
30+
steps:
31+
- name: Install libjemalloc-dev
32+
run: apt-get update && apt-get install -y libjemalloc-dev
33+
- name: Checkout repository
34+
uses: actions/checkout@v4
35+
with:
36+
fetch-depth: 0
37+
- name: Mark the workspace as safe
38+
# https://github.com/actions/checkout/issues/766
39+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
40+
- name: Measure PR performance
41+
run: |
42+
swift package --package-path Benchmarks/ --allow-writing-to-directory Benchmarks/.benchmarkBaselines/ benchmark baseline update "${{ github.head_ref }}"
43+
- name: Measure base branch performance
44+
run: |
45+
git checkout ${{ github.base_ref }}
46+
swift package --package-path Benchmarks/ --allow-writing-to-directory Benchmarks/.benchmarkBaselines/ benchmark baseline update "${{ github.base_ref }}"
47+
- name: Compare performance measurements
48+
id: compare_performance
49+
run: |
50+
find Benchmarks
51+
if ! swift package --package-path Benchmarks benchmark baseline check "${{ github.base_ref }}" "${{ github.head_ref }}" --format markdown > /tmp/comparison.md 2>/tmp/comparison-stderr.txt; then
52+
echo "has_significant_changes=true" >> "$GITHUB_OUTPUT"
53+
else
54+
echo "has_significant_changes=false" >> "$GITHUB_OUTPUT"
55+
fi
56+
- name: Install gh
57+
if: ${{ steps.compare_performance.outputs.has_significant_changes == 'true' }}
58+
# Installation instructions from https://github.com/cli/cli/blob/trunk/docs/install_linux.md#debian-ubuntu-linux-raspberry-pi-os-apt
59+
run: |
60+
(type -p wget >/dev/null || (apt update && apt-get install wget -y)) \
61+
&& mkdir -p -m 755 /etc/apt/keyrings \
62+
&& out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \
63+
&& cat $out | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
64+
&& chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
65+
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
66+
&& apt update \
67+
&& apt install gh -y
68+
- name: Post comment
69+
if: ${{ steps.compare_performance.outputs.has_significant_changes == 'true' }}
70+
env:
71+
GH_TOKEN: ${{ github.token }}
72+
run: |
73+
if grep benchmarkThresholdRegression /tmp/comparison-stderr.txt > /dev/null; then
74+
PERFORMANCE_CHANGE_MESSAGE="This PR has regressed performance characteristics. Please review whether the changes reported below are expected or if you can do something to improve them."
75+
END_OF_COMMENT
76+
elif grep benchmarkThresholdImprovement /tmp/comparison-stderr.txt > /dev/null; then
77+
PERFORMANCE_CHANGE_MESSAGE="This PR has improved performance characteristics. Thank you 🚀"
78+
else
79+
PERFORMANCE_CHANGE_MESSAGE="This PR has changed performance characteristics. Please review that the measurements reported below are expected or if you can do something to improve them."
80+
fi
81+
82+
cat > /tmp/performance_change_header.md <<EOF
83+
$PERFORMANCE_CHANGE_MESSAGE
84+
85+
<details><summary><b>Performance report</b></summary>
86+
87+
EOF
88+
89+
cat "</details>" > /tmp/performance_change_footer.md
90+
91+
COMMENT="$(cat /tmp/performance_change_header.md /tmp/comparison.md /tmp/performance_change_footer.md)"
92+
gh pr comment ${{ github.event.number }} --body "$COMMENT"
93+

‎Benchmarks/.gitignore

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
.DS_Store
2+
/.build
3+
/Packages
4+
xcuserdata/
5+
DerivedData/
6+
.swiftpm/configuration/registries.json
7+
.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
8+
.netrc
9+
.benchmarkBaselines

‎Benchmarks/.vscode/launch.json

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
{
2+
"configurations": [
3+
{
4+
"type": "swift",
5+
"request": "launch",
6+
"args": [],
7+
"cwd": "${workspaceFolder:Benchmarks}",
8+
"name": "Debug Benchmarks",
9+
"program": "${workspaceFolder:Benchmarks}/.build/debug/Benchmarks",
10+
"preLaunchTask": "swift: Build Debug Benchmarks"
11+
},
12+
{
13+
"type": "swift",
14+
"request": "launch",
15+
"args": [],
16+
"cwd": "${workspaceFolder:Benchmarks}",
17+
"name": "Release Benchmarks",
18+
"program": "${workspaceFolder:Benchmarks}/.build/release/Benchmarks",
19+
"preLaunchTask": "swift: Build Release Benchmarks"
20+
},
21+
{
22+
"type": "swift",
23+
"request": "launch",
24+
"args": [],
25+
"cwd": "${workspaceFolder:Benchmarks}",
26+
"name": "Debug MyBenchmark",
27+
"program": "${workspaceFolder:Benchmarks}/.build/debug/MyBenchmark",
28+
"preLaunchTask": "swift: Build Debug MyBenchmark"
29+
},
30+
{
31+
"type": "swift",
32+
"request": "launch",
33+
"args": [],
34+
"cwd": "${workspaceFolder:Benchmarks}",
35+
"name": "Release MyBenchmark",
36+
"program": "${workspaceFolder:Benchmarks}/.build/release/MyBenchmark",
37+
"preLaunchTask": "swift: Build Release MyBenchmark"
38+
}
39+
]
40+
}

‎Benchmarks/Benchmarks/MyBenchmark/Benchmark.swift

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import Benchmark
2+
import Foundation
3+
4+
let benchmarks: @Sendable () -> Void = {
5+
let configuration = Benchmark.Configuration(
6+
metrics: [.cpuUser],
7+
// thresholds: [.instructions: BenchmarkThresholds(relative: [.p25: 0.5, .p50: 0.5, .p75: 0.5, .p90: 0.5])]
8+
)
9+
Benchmark(
10+
"Fibonacci computation",
11+
configuration: configuration
12+
) { benchmark in
13+
blackHole(fibonacci(30))
14+
}
15+
}
16+
17+
func fibonacci(_ x: Int) -> Int {
18+
if x <= 0 {
19+
return 0
20+
}
21+
if x == 1 {
22+
return 1
23+
}
24+
return fibonacci(x - 2) + fibonacci(x - 1)
25+
}

‎Benchmarks/Package.swift

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// swift-tools-version: 5.8
2+
//===----------------------------------------------------------------------===//
3+
//
4+
// This source file is part of the SwiftCertificates open source project
5+
//
6+
// Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors
7+
// Licensed under Apache License v2.0
8+
//
9+
// See LICENSE.txt for license information
10+
// See CONTRIBUTORS.txt for the list of SwiftCertificates project authors
11+
//
12+
// SPDX-License-Identifier: Apache-2.0
13+
//
14+
//===----------------------------------------------------------------------===//
15+
16+
import PackageDescription
17+
18+
let package = Package(
19+
name: "benchmarks",
20+
platforms: [
21+
.macOS(.v13)
22+
],
23+
dependencies: [
24+
// .package(name: "swift-format", path: "../"),
25+
.package(url: "https://github.com/ordo-one/package-benchmark.git", from: "1.11.1"),
26+
],
27+
targets: [
28+
.executableTarget(
29+
name: "MyBenchmark",
30+
dependencies: [
31+
.product(name: "Benchmark", package: "package-benchmark"),
32+
// .product(name: "SwiftFormat", package: "swift-format"),
33+
],
34+
path: "Benchmarks/MyBenchmark",
35+
plugins: [
36+
.plugin(name: "BenchmarkPlugin", package: "package-benchmark")
37+
]
38+
)
39+
]
40+
)

0 commit comments

Comments
 (0)