Skip to content

Commit 256b5c7

Browse files
efiopefiop
committed
osxpkg: fix building process
Turns out the keychain auto-locks in 300 seconds, and depending on how long our previous stages take (e.g. pip install), it might be already locked when we try to use it.
1 parent d215056 commit 256b5c7

8 files changed

+420
-24
lines changed

‎.github/workflows/build.yaml

+146-24
Original file line numberDiff line numberDiff line change
@@ -1,72 +1,194 @@
11
name: Build package
22
on:
3+
pull_request:
34
push:
45
branches:
56
- main
67
workflow_dispatch:
78
permissions:
89
contents: read
910
id-token: write
11+
1012
jobs:
11-
macos:
13+
binary:
1214
runs-on: macos-11
13-
environment: aws
1415
steps:
1516
- uses: actions/checkout@v3
17+
1618
- name: Set up Python 3.10
1719
uses: actions/setup-python@v4
1820
with:
1921
python-version: "3.10"
20-
- name: Set up Ruby 2.6
21-
uses: actions/setup-ruby@v1
22-
with:
23-
ruby-version: 2.6
24-
- name: Set up certificates
22+
23+
- name: Install app certificates
2524
env:
2625
ITERATIVE_APPLICATION_CERT: ${{ secrets.ITERATIVE_APPLICATION_CERT }}
2726
ITERATIVE_APPLICATION_CERT_PASS: ${{ secrets.ITERATIVE_APPLICATION_CERT_PASS }}
28-
ITERATIVE_INSTALLER_CERT: ${{ secrets.ITERATIVE_INSTALLER_CERT }}
29-
ITERATIVE_INSTALLER_CERT_PASS: ${{ secrets.ITERATIVE_INSTALLER_CERT_PASS }}
3027
run: |
3128
echo $ITERATIVE_APPLICATION_CERT | base64 -d > application_cert.p12
32-
echo $ITERATIVE_INSTALLER_CERT | base64 -d > installer_cert.p12
3329
security create-keychain -p 123456 build.keychain
30+
# NOTE: signing and steps before it might take longer than default 300 seconds, so we extend the
31+
# timeout until the keychain automatically locks again.
32+
security set-keychain-settings -lut 3600 build.keychain
3433
security default-keychain -s build.keychain
3534
security unlock-keychain -p 123456 build.keychain
3635
security import application_cert.p12 -k build.keychain -P $ITERATIVE_APPLICATION_CERT_PASS -T /usr/bin/codesign
37-
security import installer_cert.p12 -k build.keychain -P $ITERATIVE_INSTALLER_CERT_PASS -T /usr/bin/productsign
3836
security set-key-partition-list -S 'apple-tool:,apple:,codesign:,productsign:' -s -k 123456 build.keychain
3937
security find-identity -v
40-
- name: Check signing
41-
run: |
42-
which codesign
43-
which productsign
44-
codesign -f -s 62687162A75C39558A9EA17B57E8C65306BABE92 upload.py
38+
4539
- name: Install requirements
4640
run: |
4741
pip install wheel
4842
pip install -r requirements.txt
49-
brew install mitchellh/gon/gon
43+
5044
- name: Download dvc pkg
5145
run: python download.py
46+
5247
- name: Install dvc requirements
5348
run: |
5449
# available lxml (webdav dependency) wheels are built with an old
5550
# SDK, which breaks osxpkg notarization. Building from scratch
5651
# instead.
5752
pip install ./dvc[all] --no-binary lxml
5853
pip install -r dvc/scripts/build-requirements.txt
54+
55+
- name: Build binary
56+
run: |
57+
echo 'PKG = "osxpkg"' > dvc/dvc/utils/build.py
58+
python dvc/scripts/pyinstaller/build.py
59+
cp -a dvc/scripts/pyinstaller/dist dist
60+
61+
- name: Sign binary
62+
run: python sign_bin.py --application-id 62687162A75C39558A9EA17B57E8C65306BABE92 --keychain build.keychain
63+
64+
- name: Create tar with binary files
65+
run: tar cvf dist.tar.gz dist
66+
67+
- name: Upload binary to artifacts
68+
uses: actions/upload-artifact@v2
69+
with:
70+
name: binary
71+
path: dist.tar.gz
72+
retention-days: 1
73+
74+
installer:
75+
needs: binary
76+
runs-on: macos-11
77+
steps:
78+
- uses: actions/checkout@v3
79+
80+
- name: Set up Python 3.10
81+
uses: actions/setup-python@v4
82+
with:
83+
python-version: "3.10"
84+
85+
- name: Set up Ruby 2.6
86+
uses: actions/setup-ruby@v1
87+
with:
88+
ruby-version: 2.6
89+
90+
- name: Install installer certificates
91+
env:
92+
ITERATIVE_INSTALLER_CERT: ${{ secrets.ITERATIVE_INSTALLER_CERT }}
93+
ITERATIVE_INSTALLER_CERT_PASS: ${{ secrets.ITERATIVE_INSTALLER_CERT_PASS }}
94+
run: |
95+
echo $ITERATIVE_INSTALLER_CERT | base64 -d > installer_cert.p12
96+
security create-keychain -p 123456 build.keychain
97+
# NOTE: signing and steps before it might take longer than default 300 seconds, so we extend the
98+
# timeout until the keychain automatically locks again.
99+
security set-keychain-settings -lut 3600 build.keychain
100+
security default-keychain -s build.keychain
101+
security unlock-keychain -p 123456 build.keychain
102+
security import installer_cert.p12 -k build.keychain -P $ITERATIVE_INSTALLER_CERT_PASS -T /usr/bin/productsign
103+
security set-key-partition-list -S 'apple-tool:,apple:,codesign:,productsign:' -s -k 123456 build.keychain
104+
105+
- name: Check certificates
106+
run: security find-identity -v
107+
108+
- name: Install requirements
109+
run: |
110+
pip install wheel
111+
pip install -r requirements.txt
59112
gem install --no-document fpm
60-
- name: Build and sign
61-
timeout-minutes: 120
113+
114+
- name: Download signed binary
115+
uses: actions/download-artifact@v3
116+
with:
117+
name: binary
118+
119+
- name: Unpack tar with binary files
120+
run: tar -xvf dist.tar.gz
121+
122+
- name: Build installer
123+
run: python build_installer.py
124+
125+
- name: Sign installer
126+
run: python sign_installer.py --installer-id 374191642324A98B1D9835CCD256EDEE2C710051 --keychain build.keychain
127+
128+
- name: Upload installer to artifacts
129+
uses: actions/upload-artifact@v2
130+
with:
131+
name: installer
132+
path: dvc*.pkg
133+
retention-days: 1
134+
135+
notarize:
136+
needs: installer
137+
runs-on: macos-11
138+
steps:
139+
- uses: actions/checkout@v3
140+
141+
- name: Set up Python 3.10
142+
uses: actions/setup-python@v4
143+
with:
144+
python-version: "3.10"
145+
146+
- name: Install requirements
147+
run: |
148+
brew install mitchellh/gon/gon
149+
150+
- name: Download signed installer
151+
uses: actions/download-artifact@v3
152+
with:
153+
name: installer
154+
155+
- name: Notarize and staple installer
62156
env:
63157
APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
64-
run: |
65-
sed -i "" "s/timeout=180/timeout=7200/g" dvc/scripts/pyinstaller/sign.py
66-
python dvc/scripts/build.py osxpkg --sign-application --application-id 62687162A75C39558A9EA17B57E8C65306BABE92 --sign-installer --installer-id 374191642324A98B1D9835CCD256EDEE2C710051 --notarize --apple-id-username [email protected] --apple-id-password $APPLE_ID_PASSWORD
158+
run: python notarize_installer.py --apple-id-username [email protected] --apple-id-password $APPLE_ID_PASSWORD
159+
160+
- name: Upload stapled installer to artifacts
161+
uses: actions/upload-artifact@v2
162+
with:
163+
name: stapled_installer
164+
path: dvc*.pkg
165+
retention-days: 1
166+
167+
upload:
168+
needs: notarize
169+
runs-on: ubuntu-latest
170+
environment: aws
171+
steps:
172+
- uses: actions/checkout@v3
173+
174+
- name: Set up Python 3.10
175+
uses: actions/setup-python@v4
176+
with:
177+
python-version: "3.10"
178+
179+
- name: Install requirements
180+
run: pip install awscli
181+
182+
- name: Download signed, notarized and stapled installer
183+
uses: actions/download-artifact@v3
184+
with:
185+
name: stapled_installer
186+
67187
- uses: aws-actions/configure-aws-credentials@v1
68188
with:
69189
aws-region: us-east-2
70190
role-to-assume: arn:aws:iam::260760892802:role/dvc-public-osxpkg-deployer
71-
- name: Upload
72-
run: python upload.py dvc/scripts/fpm/dvc-*.pkg
191+
192+
- name: Upload to aws
193+
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
194+
run: python upload.py dvc-*.pkg

‎after-install.sh

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
#!/bin/sh
2+
ln -sf /usr/local/lib/dvc/dvc /usr/local/bin/dvc

‎after-remove.sh

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
#!/bin/sh
2+
rm -rf /usr/local/bin/dvc

‎build_installer.py

+75
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
import os
2+
import pathlib
3+
import shutil
4+
from subprocess import STDOUT, check_call, check_output
5+
6+
path = pathlib.Path(__file__).parent.absolute()
7+
build = path / "build"
8+
install = build / "usr"
9+
10+
flags = [
11+
"--description",
12+
'"Data Version Control | Git for Data & Models"',
13+
"-n",
14+
"dvc",
15+
"-s",
16+
"dir",
17+
"-f",
18+
"--license",
19+
'"Apache License 2.0"',
20+
]
21+
22+
install /= "local"
23+
bash_dir = install / "etc" / "bash_completion.d"
24+
dirs = ["usr"]
25+
flags.extend(
26+
[
27+
"--osxpkg-identifier-prefix",
28+
"com.iterative",
29+
"--after-install",
30+
path / "after-install.sh",
31+
"--after-remove",
32+
path / "after-remove.sh",
33+
]
34+
)
35+
36+
try:
37+
shutil.rmtree(build)
38+
except FileNotFoundError:
39+
pass
40+
41+
lib = install / "lib"
42+
lib.mkdir(parents=True)
43+
shutil.copytree(path / "dist" / "dvc", lib / "dvc")
44+
45+
bash_dir.mkdir(parents=True)
46+
bash_completion = check_output(
47+
[lib / "dvc" / "dvc", "completion", "-s", "bash"], text=True
48+
)
49+
(bash_dir / "dvc").write_text(bash_completion)
50+
51+
zsh_dir = install / "share" / "zsh" / "site-functions"
52+
zsh_dir.mkdir(parents=True)
53+
zsh_completion = check_output(
54+
[lib / "dvc" / "dvc", "completion", "-s", "zsh"], text=True
55+
)
56+
(zsh_dir / "_dvc").write_text(zsh_completion)
57+
58+
version = check_output([lib / "dvc" / "dvc", "--version"], text=True).strip()
59+
60+
check_call(
61+
[
62+
"fpm",
63+
"--verbose",
64+
"-t",
65+
"osxpkg",
66+
*flags,
67+
"-v",
68+
version,
69+
"-C",
70+
build,
71+
*dirs,
72+
],
73+
cwd=path,
74+
stderr=STDOUT,
75+
)

‎entitlements.plist

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
3+
<plist version="1.0">
4+
<dict>
5+
<!-- These are required for binaries built by PyInstaller -->
6+
<key>com.apple.security.cs.allow-jit</key>
7+
<true/>
8+
<key>com.apple.security.cs.allow-unsigned-executable-memory</key>
9+
<true/>
10+
</dict>
11+
</plist>

‎notarize_installer.py

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
import argparse
2+
import json
3+
import os
4+
import pathlib
5+
import sys
6+
from subprocess import STDOUT, check_call
7+
8+
if sys.platform != "darwin":
9+
raise NotImplementedError
10+
11+
parser = argparse.ArgumentParser()
12+
parser.add_argument(
13+
"path",
14+
nargs="?",
15+
help="Path to the osxpkg to notarize. If not specified - try to find one.",
16+
)
17+
parser.add_argument("--apple-id-username", required=True, help="Apple ID username.")
18+
parser.add_argument(
19+
"--apple-id-password",
20+
required=True,
21+
help=(
22+
"Apple ID app-specific password. Note that this is not a regular "
23+
"Apple ID password, so you need to generate one at "
24+
"https://appleid.apple.com/account/manage"
25+
),
26+
)
27+
args = parser.parse_args()
28+
29+
path = pathlib.Path(__file__).parent.absolute()
30+
31+
if args.path:
32+
pkg = pathlib.Path(args.path)
33+
else:
34+
pkgs = list(path.glob("*.pkg"))
35+
if not pkgs:
36+
print("No pkgs found")
37+
sys.exit(1)
38+
39+
if len(pkgs) > 1:
40+
print("Too many packages")
41+
sys.exit(1)
42+
43+
(pkg,) = pkgs
44+
45+
46+
config = {
47+
"notarize": {
48+
"path": os.fspath(pkg),
49+
"bundle_id": "com.iterative.dvc",
50+
"staple": True,
51+
},
52+
"apple_id": {
53+
"username": args.apple_id_username,
54+
"password": args.apple_id_password,
55+
},
56+
}
57+
58+
(path / "config.json").write_text(json.dumps(config))
59+
60+
check_call(
61+
["gon", "config.json"],
62+
cwd=path,
63+
stderr=STDOUT,
64+
)

0 commit comments

Comments
 (0)