Skip to content

Commit 66ae65b

Browse files
terem42terem42
committed
squashed history
0 parents  commit 66ae65b

File tree

140 files changed

+47867
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

140 files changed

+47867
-0
lines changed

‎.github/workflows/debian-packages-build.yml

+184
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
name: Build test and deploy ZFS packages for Debian 10, Ubuntu 18.04 and later
2+
on:
3+
push:
4+
branches:
5+
- master
6+
paths-ignore:
7+
- 'README.md'
8+
jobs:
9+
build:
10+
name: Building packages
11+
runs-on: ubuntu-18.04
12+
steps:
13+
- name: clone zfs upstream repo
14+
shell: bash
15+
run: |
16+
mkdir ~/zfs && cd ~/zfs
17+
git init
18+
git fetch https://github.com/openzfs/zfs.git refs/heads/master
19+
git checkout FETCH_HEAD
20+
- name: clone our zfs ubuntu repo
21+
shell: bash
22+
run: |
23+
echo "ref triggered = ${{ github.ref }}"
24+
mkdir ~/zfs/debian && cd ~/zfs/debian
25+
git init
26+
git fetch https://github.com/${{ github.repository }}.git ${{ github.ref }} --depth=1
27+
git checkout FETCH_HEAD
28+
- name: retrieve build packages
29+
shell: bash
30+
run: |
31+
sudo DEBIAN_FRONTEND=noninteractive apt-get -yq update
32+
sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install dpkg-dev alien autoconf automake build-essential dkms fakeroot gawk gdebi-core libacl1-dev libaio-dev libattr1-dev libblkid-dev libdevmapper-dev libelf-dev libselinux-dev libssl-dev libtool libudev-dev nfs-kernel-server python3 python3-dev python3-cffi python3-setuptools uuid-dev zlib1g-dev linux-headers-$(uname -r) python3-all-dev python3-sphinx
33+
- name: build zfs packages
34+
shell: bash
35+
run: |
36+
echo "current folder"
37+
pwd
38+
cd ~/zfs
39+
ls ./debian/changelog
40+
dpkg-buildpackage -b -rfakeroot -us -uc --no-sign
41+
mkdir ./zfs-deb-packages
42+
cp ../*.deb ./zfs-deb-packages
43+
ls ./zfs-deb-packages -al
44+
- name: cache created packages
45+
uses: actions/cache@v1
46+
with:
47+
path: ~/zfs/zfs-deb-packages
48+
key: ${{ runner.OS }}-zfs-${{ github.sha }}
49+
test:
50+
needs: build
51+
name: Testing packages
52+
runs-on: ubuntu-18.04
53+
strategy:
54+
fail-fast: true
55+
matrix:
56+
testbatch:
57+
- 'alloc_class,limits,arc,channel_program,cp_files,deadman,btree,cache,history,hkdf,inuse,zfs_property,zfs_receive,zfs_reservation,zfs_send,zfs_set,zfs_share,zfs_snapshot,zfs_unload-key,zfs_unmount,zfs_unshare,zfs_upgrade,zpool,zpool_destroy,zpool_detach'
58+
- 'rootpool,scrub_mirror,pool_checkpoint'
59+
- 'acl,atime,bootfs,cachefile,casenorm,chattr,checksum,clean_mirror,compression,ctime,delegate,devices,events,exec,fault,features,grow_pool'
60+
- 'grow_replicas,mv_files,cli_user,zfs_mount,zfs_promote,zfs_rollback,zpool_events,zpool_expand,zpool_export,zpool_add,zpool_attach,zpool_clear,zpool_create'
61+
- 'zpool_get,zpool_history,zpool_import,zpool_labelclear,zpool_offline,zpool_online'
62+
- 'zpool_reopen,zpool_remove,zpool_replace,zpool_scrub,zpool_set,zpool_status,zpool_sync,zpool_upgrade,redacted_send,redundancy'
63+
- 'zfs_destroy,large_files,largest_pool,link_count,migration,inheritance,refquota,refreserv,rename_dirs,replacement,log_spacemap,persist_l2arc,reservation'
64+
- 'zfs,zfs_bookmark,zfs_change-key,zfs_clone,zfs_copies,zfs_create,zfs_diff,zfs_get,zfs_inherit,zfs_load-key,zfs_rename,mmap,mmp,mount,nestedfs,no_space,nopwrite,online_offline,pool_names,poolversion'
65+
- 'zdb,mmp'
66+
- 'privilege,quota,raidz,redundancy,rsend'
67+
- 'slog,snapshot,snapused,sparse,threadsappend,tmpfile,truncate,upgrade,userquota,vdev_zaps,write_dirs,xattr,zvol,libzfs'
68+
steps:
69+
- name: Restore cached packages
70+
uses: actions/cache@v1
71+
with:
72+
path: ./zfs-deb-packages
73+
key: ${{ runner.OS }}-zfs-${{ github.sha }}
74+
- name: setup local repo and put earlier-built ZFS packages there
75+
shell: bash
76+
run: |
77+
cd ./zfs-deb-packages
78+
ls ./ -alt
79+
pwd
80+
sudo bash -c "dpkg-scanpackages -m . > Packages"
81+
sudo bash -c "echo 'deb [trusted=yes] file:$(pwd) ./' >> /etc/apt/sources.list"
82+
sudo bash -c "echo -e \"Package: *\nPin: origin \\\"\\\"\nPin-Priority: 1001\" > /etc/apt/preferences.d/90_zfs"
83+
sudo DEBIAN_FRONTEND=noninteractive apt-get -yq update
84+
- name: install earlier-built ZFS packages along with packages required for conducting tests
85+
shell: bash
86+
run: |
87+
sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install linux-modules-extra-$(uname -r)
88+
sudo bash -c "echo \"zfs-dkms zfs-dkms/note-incompatible-licenses note true\" | debconf-set-selections"
89+
sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install zfs-dkms zfsutils-linux zfs-zed zfs-test parted lsscsi ksh attr acl fio quota nfs-kernel-server samba-common-bin linux-tools-generic linux-tools-$(uname -r) linux-cloud-tools-$(uname -r)
90+
zfs version
91+
- name: run tests
92+
shell: bash
93+
run: |
94+
sudo rm -rf /etc/hostid
95+
/usr/share/zfs/zfs-tests.sh -v -T ${{ matrix.testbatch }}
96+
- name: show test output on failure
97+
if: failure()
98+
shell: bash
99+
run: |
100+
find /var/tmp/test_results/current/log -type f -name '*' -printf "%f\n" -exec cut -c -800 {} \;
101+
deploy:
102+
needs: test
103+
name: Deploying artefacts to debian repository
104+
runs-on: ubuntu-18.04
105+
steps:
106+
- name: Restore cached packages
107+
uses: actions/cache@v1
108+
with:
109+
path: ./zfs-deb-packages
110+
key: ${{ runner.OS }}-zfs-${{ github.sha }}
111+
- name: adding required packages for repo publishing
112+
shell: bash
113+
run: |
114+
sudo bash -c "echo 'deb http://repo.aptly.info/ squeeze main' >> /etc/apt/sources.list"
115+
wget -qO - https://www.aptly.info/pubkey.txt | sudo apt-key add -
116+
sudo DEBIAN_FRONTEND=noninteractive apt-get -yq update
117+
sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install gnupg aptly
118+
- name: create repo and add packages
119+
shell: bash
120+
run: |
121+
aptly repo create -distribution="zfs-debian-experimental" -component="main" zfs-debian-experimental
122+
sed -i 's/\"architectures\": \[\]/\"architectures\": \[\"amd64\"\]/g' ~/.aptly.conf
123+
echo -n "${{ secrets.GPG_KEY }}" > ./apt-private-key.asc
124+
echo ${{ secrets.GPG_PASS }} | gpg --batch --import ./apt-private-key.asc
125+
rm -rf ./apt-private-key.asc
126+
ls ./zfs-deb-packages/*.deb -1 | xargs aptly repo add zfs-debian-experimental
127+
aptly publish repo --gpg-key="$(gpg --list-public-keys --keyid-format 0xLONG | sed -n -E 's/^pub\s+\w{1,30}[0-9]+\/0x([0-9A-Z]+)\s+.+$/\1/p')" --batch=true --passphrase="${{ secrets.GPG_PASS }}" zfs-debian-experimental
128+
rm -rf ./zfs-deb-packages
129+
- uses: actions/checkout@v2
130+
with:
131+
ref: gh-pages
132+
- name: push updates
133+
shell: bash
134+
run: |2
135+
git config --global user.email "$(git log -n 1 --format=format:%ae)"
136+
git config --global user.name "$(git log -n 1 --format=format:%an)"
137+
rm ./public -rf
138+
cp -r /home/runner/.aptly/public ./
139+
export PAGE_FOOTER_DATE_TAG=$(date -Is)
140+
sed -i -E "s/(<span class=\"datetag\">)[^<]*(<\/span>)/\1${PAGE_FOOTER_DATE_TAG}\2/" ./index.html
141+
python - <<END
142+
import os
143+
indexTextStart = """<!DOCTYPE html>
144+
<html>
145+
<head><title>Index of {folderPath}</title></head>
146+
<body>
147+
<h2>Index of {folderPath}</h2>
148+
<hr>
149+
<ul>
150+
<li>
151+
<a href='../'>../</a>
152+
</li>
153+
"""
154+
indexTextEnd = """
155+
</ul>
156+
<hr/>
157+
<p>Page generated: ${PAGE_FOOTER_DATE_TAG}</p>
158+
</body>
159+
</html>
160+
"""
161+
def index_folder(folderPath):
162+
print("Indexing: " + folderPath +'/')
163+
files = os.listdir(folderPath)
164+
root = folderPath
165+
if folderPath == '.':
166+
root = 'Root'
167+
indexText = indexTextStart.format(folderPath=root)
168+
for file in files:
169+
if file != 'index.html':
170+
indexText += "\t\t<li>\n\t\t\t<a href='" + file + "'>" + file + "</a>\n\t\t</li>\n"
171+
if os.path.isdir(folderPath+'/'+file):
172+
index_folder(folderPath + '/' + file)
173+
indexText += indexTextEnd
174+
index = open(folderPath+'/index.html', "w")
175+
index.write(indexText)
176+
index_folder('./public')
177+
END
178+
rm -rf ./.git
179+
git init
180+
git checkout --orphan "gh-pages"
181+
git remote add origin "https://x-access-token:${{ secrets.GITHUB_PERSONAL_TOKEN }}@github.com/${{ github.repository }}.git"
182+
git add --all
183+
git commit -am "automatic pages update from ${{ github.ref }} commit ${{ github.sha }}"
184+
git push --force --set-upstream origin gh-pages

‎NEWS

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
zfs-linux (0.8.0-1) experimental; urgency=medium
2+
3+
ZFS 0.8.0 introduced the following NEW features:
4+
5+
* Native encryption
6+
* Raw encrypted 'zfs send/receive'
7+
* Device removal
8+
* Pool checkpoints
9+
* Pool TRIM
10+
* Pool initialization
11+
* Project accounting and quota
12+
* Channel programs
13+
* Pyzfs
14+
* Python 3 compatibility
15+
* Direct IO
16+
17+
And the following performance improvements:
18+
19+
* Sequential scrub and resilver
20+
* Allocation classes
21+
* Administrative commands
22+
* Parallel allocation
23+
* Deferred resilvers
24+
* ZFS Intent Log (ZIL)
25+
* Volumes
26+
* QAT
27+
28+
Details about the features and performance improvements can be found at:
29+
30+
https://github.com/zfsonlinux/zfs/releases/tag/zfs-0.8.0
31+
32+
NEWS on the Debian packaging side:
33+
34+
* ZFS doesn't depend on the spl-dkms package anymore, because
35+
upstream had merged SPL source into ZFS source tree, and that
36+
module is now included in the zfs-dkms package.
37+
38+
-- Mo Zhou <[email protected]> Fri, 24 May 2019 03:27:43 +0000

‎README.md

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
![img](http://zfsonlinux.org/images/zfs-linux.png)
2+
3+
This repository is intended to help ZFS enthusiasts to build, test and deploy Debian packages for ZFS on Linux,
4+
using latest master branch from [official repo](https://github.com/openzfs/zfs) and patches to it, listed in patches/series folder. Official distributions are often lag behing dev progress, so, those wanting to use latest "bleeding edge" features, can leverage this possibility.
5+
6+
The repository is based on excellent work of [Debian ZFS team](https://salsa.debian.org/zfsonlinux-team/zfs.git) with a few customizations, allowed to build run smoontly on Github workflow runners.
7+
8+
Attached to this repository exist a Github actions workflow located [here](./.github/workflows/debian-packages-build.yml), which upon each master commit, checks out [ZFS upstream repo](https://github.com/openzfs/zfs)
9+
then applies patches, builds, tests and upon tests success, deploys built and tested debian binary packages into tiny, yet fully functional [APT repository](https://terem42.github.io/zfs-debian/), created using Github Pages.
10+
11+
You can later reference this repository in your deployments or fork this repo (insructions below) and build your custom version of packages, having all the beeding edge changes you want. This approach gives you complete freedom over the building
12+
version and applied patches over the source code, while still being guided by master branch from official repo.
13+
If you want to change upstream repo source or branch from offcial repo to your forked repo, just edit .github/workflow/ubuntu-packages-build.yml workflow file.
14+
15+
# Official Resources
16+
17+
* [Site](http://zfsonlinux.org)
18+
* [Wiki](https://github.com/zfsonlinux/zfs/wiki)
19+
* [Mailing lists](https://github.com/zfsonlinux/zfs/wiki/Mailing-Lists)
20+
* [OpenZFS site](http://open-zfs.org/)
21+
22+
# Installation
23+
24+
Add the signing key for this repo:
25+
26+
````
27+
sudo bash -c "wget -O - https://terem42.github.io/zfs-debian/apt_pub.gpg | apt-key add -"
28+
````
29+
30+
Next, install packages:
31+
32+
````
33+
sudo add-apt-repository 'deb https://terem42.github.io/zfs-debian/public zfs-debian-experimental main'
34+
sudo bash -c "echo \"zfs-dkms zfs-dkms/note-incompatible-licenses note true\" | debconf-set-selections"
35+
sudo apt install -t zfs-debian-experimental --yes zfs-dkms zfsutils-linux
36+
````
37+
# List of tests skipped at Github runners
38+
I've added patch, adding several tests to the "known to skip or fail" list, see it [here](./patches/terem-0002-added-known-to-skip-tests-on-github-workflow-vm.patch). Check it for complete list of these tests. If you want to use self-hosted runners, do not forget to exclude this patch from ./patches/series file to have complete test coverage.
39+
40+
# Forking the repo
41+
Fork and setup this repo for personal ZFS packages tailored to your needs in 5 easy steps:
42+
* create github secret GITHUB_PERSONAL_TOKEN containing github personal acess token, it is nesessary for pushing changes into gh-pages branch after succesfull build and test (for security reasons, github.token used by Gihtub Actions workflow does not trigger properly Github Pages site rebuild after push to gh-pages branch).
43+
* either import your own or generate GPG key (it is used by github workflow to sign packages, read [here](https://help.github.com/en/github/authenticating-to-github/generating-a-new-gpg-key) on how to create gpg key, afterwards export public key into apt_pub.gpg file an commit changes into gh-pages branch
44+
* export as ASCII-armored your private GPG key into file, then create another Github secret GPG_KEY with the file contents
45+
* create github secret GPG_PASS holding your password for the private key
46+
* edit ./index.html file within gh-pages branch to match your repo and info you want, it will hold your repo entry page, explaining details on how to use this repo

0 commit comments

Comments
 (0)