FreeBSD port

Buildkit code is mostly generic enough to support FreeBSD, however
there are some quirks / infrastructural pieces that need to be
addressed for full support, to name some

-    contenthash.NewFromStat attempts to set Devmajor / Devminor for
    regular files, assuming that RDev is zero for regular
    files. Unlike on Linux, it's not the case for FreeBSD.

-    containerdexecutor.Run uses bind mounts for rootfs. Bind mounts
    are not supported in FreeBSD and we should use nullfs instead

-    There is no CI job to run tests on FreeBSD

-    Some dependencies weren't ported

This change ports buildkit to FreeBSD

Signed-off-by: Artem Khramov <[email protected]>
Co-authored-by: Akihiro Suda <[email protected]>

.github/workflows/test-os.yml

@@ -85,3 +85,28 @@ jobs:
         with:
           name: test-reports
           path: ./bin/testreports
+
+  test-freebsd-amd64:
+    runs-on: macos-12
+    env:
+      VAGRANT_VAGRANTFILE: hack/Vagrantfile.freebsd13
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Cache Vagrant boxes
+        uses: actions/cache@v2
+        with:
+          path: ~/.vagrant.d/boxes
+          key: ${{ runner.os }}-vagrant-${{ hashFiles('hack/Vagrantfile.freebsd13') }}
+          restore-keys: |
+            ${{ runner.os }}-vagrant-
+      -
+        name: Set up vagrant
+        run: vagrant up
+      -
+        name: Integration smoke test
+        run: |
+          vagrant ssh -- "cp /vagrant/hack/dockerfiles/freebsd-smoke.Dockerfile /vagrant/cmd/buildctl/Dockerfile"
+          vagrant ssh -- "cd /vagrant/cmd/buildctl; go run . build --frontend dockerfile.v0 --local context=. --local dockerfile=."

Dockerfile

@@ -116,6 +116,10 @@ COPY --link --from=buildctl /usr/bin/buildctl /
 FROM scratch AS binaries-windows
 COPY --link --from=buildctl /usr/bin/buildctl /buildctl.exe
 
+FROM scratch AS binaries-freebsd
+COPY --link --from=buildkitd /usr/bin/buildkitd /
+COPY --link --from=buildctl /usr/bin/buildctl /
+
 FROM binaries-$TARGETOS AS binaries
 # enable scanning for this stage
 ARG BUILDKIT_SBOM_SCAN_STAGE=true
@@ -215,6 +219,9 @@ ENTRYPOINT ["buildkitd"]
 
 FROM binaries AS buildkit-darwin
 
+FROM binaries AS buildkit-freebsd
+ENTRYPOINT ["/buildkitd"]
+
 FROM binaries AS buildkit-windows
 # this is not in binaries-windows because it is not intended for release yet, just CI
 COPY --link --from=buildkitd /usr/bin/buildkitd /buildkitd.exe

cache/contenthash/filehash_unix.go

@@ -20,8 +20,8 @@ func setUnixOpt(path string, fi os.FileInfo, stat *fstypes.Stat) error {
 	stat.Gid = s.Gid
 
 	if !fi.IsDir() {
-		if s.Mode&syscall.S_IFBLK != 0 ||
-			s.Mode&syscall.S_IFCHR != 0 {
+		if s.Mode&syscall.S_IFLNK == 0 && (s.Mode&syscall.S_IFBLK != 0 ||
+			s.Mode&syscall.S_IFCHR != 0) {
 			stat.Devmajor = int64(unix.Major(uint64(s.Rdev)))
 			stat.Devminor = int64(unix.Minor(uint64(s.Rdev)))
 		}

cache/manager_test.go

@@ -2022,8 +2022,8 @@ func checkDescriptor(ctx context.Context, t *testing.T, cs content.Store, desc o
 }
 
 func TestMergeOp(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip("Depends on unimplemented merge-op support on Windows")
+	if runtime.GOOS == "windows" || runtime.GOOS == "freebsd" {
+		t.Skipf("Depends on unimplemented merge-op support on %s", runtime.GOOS)
 	}
 
 	// This just tests the basic Merge method and some of the logic with releasing merge refs.
@@ -2140,8 +2140,8 @@ func TestMergeOp(t *testing.T) {
 }
 
 func TestDiffOp(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip("Depends on unimplemented diff-op support on Windows")
+	if runtime.GOOS == "windows" || runtime.GOOS == "freebsd" {
+		t.Skipf("Depends on unimplemented diff-op support on %s", runtime.GOOS)
 	}
 
 	// This just tests the basic Diff method and some of the logic with releasing diff refs.

cmd/buildkitd/main_containerd_worker.go

@@ -1,5 +1,5 @@
-//go:build linux || windows
-// +build linux windows
+//go:build linux || windows || freebsd
+// +build linux windows freebsd
 
 package main
 

docs/freebsd.md

@@ -0,0 +1,16 @@
+# Experimental FreeBSD support
+
+Buildkit also has experimental FreeBSD support. The container infrastructure on FreeBSD is still at an early stage, so problems may occur.
+
+Dependencies:
+
+- [runj](https://github.com/samuelkarp/runj)
+- [containerd](https://github.com/containerd/containerd)
+
+These dependencies can be installed from the ports tree, or using the `pkg` command:
+
+```csh
+% pkg install containerd runj
+```
+
+For BuildKit build instructions see [`..github/CONTRIBUTING.md`](../.github/CONTRIBUTING.md).

executor/containerdexecutor/executor.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"runtime"
 	"sync"
 	"syscall"
 	"time"
@@ -209,11 +210,20 @@ func (w *containerdExecutor) Run(ctx context.Context, id string, root executor.M
 		cioOpts = append(cioOpts, cio.WithTerminal)
 	}
 
-	task, err := container.NewTask(ctx, cio.NewCreator(cioOpts...), containerd.WithRootFS([]mount.Mount{{
+	rootfs := containerd.WithRootFS([]mount.Mount{{
 		Source:  rootfsPath,
 		Type:    "bind",
 		Options: []string{"rbind"},
-	}}))
+	}})
+	if runtime.GOOS == "freebsd" {
+		rootfs = containerd.WithRootFS([]mount.Mount{{
+			Source:  rootfsPath,
+			Type:    "nullfs",
+			Options: []string{},
+		}})
+	}
+
+	task, err := container.NewTask(ctx, cio.NewCreator(cioOpts...), rootfs)
 	if err != nil {
 		return nil, err
 	}

executor/oci/spec.go

@@ -197,7 +197,9 @@ func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mou
 	}
 
 	if tracingSocket != "" {
-		s.Mounts = append(s.Mounts, getTracingSocketMount(tracingSocket))
+		if mount := getTracingSocketMount(tracingSocket); mount != nil {
+			s.Mounts = append(s.Mounts, *mount)
+		}
 	}
 
 	s.Mounts = dedupMounts(s.Mounts)

executor/oci/spec_freebsd.go

@@ -0,0 +1,57 @@
+package oci
+
+import (
+	"github.com/containerd/containerd/oci"
+	"github.com/docker/docker/pkg/idtools"
+	"github.com/moby/buildkit/solver/pb"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+)
+
+func generateMountOpts(resolvConf, hostsFile string) ([]oci.SpecOpts, error) {
+	return nil, nil
+}
+
+// generateSecurityOpts may affect mounts, so must be called after generateMountOpts
+func generateSecurityOpts(mode pb.SecurityMode, apparmorProfile string, selinuxB bool) ([]oci.SpecOpts, error) {
+	if mode == pb.SecurityMode_INSECURE {
+		return nil, errors.New("no support for running in insecure mode on FreeBSD")
+	}
+	return nil, nil
+}
+
+// generateProcessModeOpts may affect mounts, so must be called after generateMountOpts
+func generateProcessModeOpts(mode ProcessMode) ([]oci.SpecOpts, error) {
+	if mode == NoProcessSandbox {
+		return nil, errors.New("no support for NoProcessSandbox on FreeBSD")
+	}
+	return nil, nil
+}
+
+func generateIDmapOpts(idmap *idtools.IdentityMapping) ([]oci.SpecOpts, error) {
+	if idmap == nil {
+		return nil, nil
+	}
+	return nil, errors.New("no support for IdentityMapping on FreeBSD")
+}
+
+func generateRlimitOpts(ulimits []*pb.Ulimit) ([]oci.SpecOpts, error) {
+	if len(ulimits) == 0 {
+		return nil, nil
+	}
+	return nil, errors.New("no support for POSIXRlimit on FreeBSD")
+}
+
+// tracing is not implemented on FreeBSD
+func getTracingSocketMount(socket string) *specs.Mount {
+	return nil
+}
+
+// tracing is not implemented on FreeBSD
+func getTracingSocket() string {
+	return ""
+}
+
+func cgroupNamespaceSupported() bool {
+	return false
+}

executor/oci/spec_unix.go → executor/oci/spec_linux.go

@@ -1,6 +1,3 @@
-//go:build !windows
-// +build !windows
-
 package oci
 
 import (
@@ -134,8 +131,8 @@ func withDefaultProfile() oci.SpecOpts {
 	}
 }
 
-func getTracingSocketMount(socket string) specs.Mount {
-	return specs.Mount{
+func getTracingSocketMount(socket string) *specs.Mount {
+	return &specs.Mount{
 		Destination: tracingSocketPath,
 		Type:        "bind",
 		Source:      socket,

executor/oci/spec_windows.go

@@ -52,8 +52,8 @@ func generateRlimitOpts(ulimits []*pb.Ulimit) ([]oci.SpecOpts, error) {
 	return nil, errors.New("no support for POSIXRlimit on Windows")
 }
 
-func getTracingSocketMount(socket string) specs.Mount {
-	return specs.Mount{
+func getTracingSocketMount(socket string) *specs.Mount {
+	return &specs.Mount{
 		Destination: filepath.FromSlash(tracingSocketPath),
 		Source:      socket,
 		Options:     []string{"ro"},

hack/Vagrantfile.freebsd13

@@ -0,0 +1,46 @@
+# This code is taken from the Vagrantfile from libjail-rs
+# https://github.com/fubarnetes/libjail-rs/blob/727353bd6565c5e7a9be2664258d0197a1c8bb35/Vagrantfile
+# licensed under BSD-3 Clause License:
+# BSD 3-Clause License
+
+# Copyright (c) 2018, Fabian Freyer <[email protected]> All rights reserved.
+
+# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+# * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+# * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+# * Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Vagrant.configure("2") do |config|
+  # Stable version
+  #
+  config.vm.define "fbsd_13_2" do |fbsd_13_2|
+    fbsd_13_2.vm.box = "freebsd/FreeBSD-13.2-RELEASE"
+  end
+
+  config.vm.synced_folder ".", "/vagrant", type: "rsync", rsync__auto: true
+
+  config.vm.provision "shell", inline: <<-SHELL
+    kldload nullfs
+    echo 'FreeBSD: { url: "pkg+http://pkg.FreeBSD.org/${ABI}/latest" }' > /usr/local/etc/pkg/repos/FreeBSD.conf
+
+    pkg bootstrap
+    pkg install -y go git containerd runj wait_on
+
+    mkdir -p /vagrant/coverage
+    daemon -o /var/log/containerd.log containerd
+
+    mkdir -p /run/buildkit
+    cd /vagrant/cmd/buildkitd
+    go build -buildvcs=false .
+    echo "launching buildkitd..."
+    mkdir -p /run/buildkit
+    daemon ./buildkitd
+    wait_on -t 5 /run/buildkit
+    echo "launched buildkitd"
+  SHELL
+end

hack/dockerfiles/freebsd-smoke.Dockerfile

@@ -0,0 +1,3 @@
+FROM dougrabson/freebsd-minimal:13.1
+
+RUN echo "Hello, buildkit!"

snapshot/diffapply_freebsd.go

@@ -0,0 +1,17 @@
+package snapshot
+
+import (
+	"context"
+
+	"github.com/containerd/containerd/leases"
+	"github.com/containerd/containerd/snapshots"
+	"github.com/pkg/errors"
+)
+
+func (sn *mergeSnapshotter) diffApply(ctx context.Context, dest Mountable, diffs ...Diff) (_ snapshots.Usage, rerr error) {
+	return snapshots.Usage{}, errors.New("diffApply not yet supported on FreeBSD")
+}
+
+func needsUserXAttr(ctx context.Context, sn Snapshotter, lm leases.Manager) (bool, error) {
+	return false, errors.New("needs userxattr not supported on FreeBSD")
+}

snapshot/diffapply_unix.go

@@ -1,5 +1,5 @@
-//go:build !windows
-// +build !windows
+//go:build !windows && !freebsd
+// +build !windows,!freebsd
 
 package snapshot
 

snapshot/localmounter_freebsd.go

@@ -0,0 +1,66 @@
+package snapshot
+
+import (
+	"os"
+
+	"github.com/containerd/containerd/mount"
+	"github.com/pkg/errors"
+)
+
+func (lm *localMounter) Mount() (string, error) {
+	lm.mu.Lock()
+	defer lm.mu.Unlock()
+
+	if lm.mounts == nil && lm.mountable != nil {
+		mounts, release, err := lm.mountable.Mount()
+		if err != nil {
+			return "", err
+		}
+		lm.mounts = mounts
+		lm.release = release
+	}
+
+	if len(lm.mounts) == 1 && lm.mounts[0].Type == "nullfs" {
+		ro := false
+		for _, opt := range lm.mounts[0].Options {
+			if opt == "ro" {
+				ro = true
+				break
+			}
+		}
+		if !ro {
+			return lm.mounts[0].Source, nil
+		}
+	}
+
+	dir, err := os.MkdirTemp("", "buildkit-mount")
+	if err != nil {
+		return "", errors.Wrap(err, "failed to create temp dir")
+	}
+
+	if err := mount.All(lm.mounts, dir); err != nil {
+		os.RemoveAll(dir)
+		return "", errors.Wrapf(err, "failed to mount %s: %+v", dir, lm.mounts)
+	}
+	lm.target = dir
+	return dir, nil
+}
+
+func (lm *localMounter) Unmount() error {
+	lm.mu.Lock()
+	defer lm.mu.Unlock()
+
+	if lm.target != "" {
+		if err := mount.Unmount(lm.target, 0); err != nil {
+			return err
+		}
+		os.RemoveAll(lm.target)
+		lm.target = ""
+	}
+
+	if lm.release != nil {
+		return lm.release()
+	}
+
+	return nil
+}

snapshot/localmounter_unix.go → snapshot/localmounter_linux.go

@@ -1,6 +1,3 @@
-//go:build !windows
-// +build !windows
-
 package snapshot
 
 import (