Skip to content

Commit

Permalink
Add distro test for go modules
Browse files Browse the repository at this point in the history
Makes sure distros are tested with go modules.
Also fixes the Windows implementation.

Signed-off-by: Brian Goff <[email protected]>
  • Loading branch information
cpuguy83 committed May 6, 2024
1 parent e0c11b5 commit 3d9466d
Show file tree
Hide file tree
Showing 5 changed files with 178 additions and 43 deletions.
20 changes: 6 additions & 14 deletions frontend/windows/handle_zip.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,7 @@ const (
workerImgRef = "mcr.microsoft.com/mirror/docker/library/ubuntu:jammy"
outputDir = "/tmp/output"
buildScriptName = "_build.sh"
)

var (
varCacheAptMount = dalec.WithMountedAptCache("/var/cache/apt", "dalec-windows-var-cache-apt")
varLibAptMount = dalec.WithMountedAptCache("/var/lib/apt", "dalec-windows-var-lib-apt")
aptCachePrefix = "jammy-windowscross"
)

func handleZip(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) {
Expand Down Expand Up @@ -53,9 +49,6 @@ func handleZip(ctx context.Context, client gwclient.Client) (*gwclient.Result, e
}

st := getZipLLB(worker, spec.Name, bin)
if err != nil {
return nil, nil, err
}

def, err := st.Marshal(ctx)
if err != nil {
Expand Down Expand Up @@ -116,8 +109,7 @@ func installBuildDeps(deps []string) llb.StateOption {

return s.Run(
shArgs("apt-get update && apt-get install -y "+strings.Join(sorted, " ")),
varCacheAptMount,
varLibAptMount,
dalec.WithMountedAptCache(aptCachePrefix),
).Root()
}
}
Expand Down Expand Up @@ -152,6 +144,8 @@ func withSourcesMounted(dst string, states map[string]llb.State, sources map[str
}

func buildBinaries(spec *dalec.Spec, worker llb.State, sOpt dalec.SourceOpts, targetKey string) (llb.State, error) {
worker = worker.With(installBuildDeps(spec.GetBuildDeps(targetKey)))

sources, err := specToSourcesLLB(worker, spec, sOpt)
if err != nil {
return llb.Scratch(), err
Expand All @@ -161,9 +155,8 @@ func buildBinaries(spec *dalec.Spec, worker llb.State, sOpt dalec.SourceOpts, ta
buildScript := createBuildScript(spec)
binaries := maps.Keys(spec.Artifacts.Binaries)
script := generateInvocationScript(binaries)
work := worker.With(installBuildDeps(spec.GetBuildDeps(targetKey)))

artifacts := work.Run(
artifacts := worker.Run(
shArgs(script.String()),
llb.Dir("/build"),
withSourcesMounted("/build", patched, spec.Sources),
Expand Down Expand Up @@ -200,8 +193,7 @@ func workerImg(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) llb.State {
return llb.Image(workerImgRef, llb.WithMetaResolver(sOpt.Resolver), dalec.WithConstraints(opts...)).
Run(
shArgs("apt-get update && apt-get install -y build-essential binutils-mingw-w64 g++-mingw-w64-x86-64 gcc git make pkg-config quilt zip"),
varCacheAptMount,
varLibAptMount,
dalec.WithMountedAptCache(aptCachePrefix),
).Root()
}

Expand Down
41 changes: 28 additions & 13 deletions helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,17 +54,32 @@ func (f runOptionFunc) SetRunOption(i *llb.ExecInfo) {
f(i)
}

func WithRemovedDockerCleanFile() llb.RunOption {
// WithMountedAptCache gives an [llb.RunOption] that mounts the apt cache directories.
// It uses the given namePrefix as the prefix for the cache keys.
// namePrefix should be distinct per distro version.
func WithMountedAptCache(namePrefix string) llb.RunOption {
return runOptionFunc(func(ei *llb.ExecInfo) {
llb.AddMount("/etc/apt/apt.conf.d/docker-clean", llb.Scratch())
})
}

func WithMountedAptCache(dst, cacheName string) llb.RunOption {
return runOptionFunc(func(ei *llb.ExecInfo) {
WithRemovedDockerCleanFile().SetRunOption(ei)
llb.AddMount(dst, llb.Scratch(), llb.AsPersistentCacheDir(cacheName, llb.CacheMountLocked)).
SetRunOption(ei)
// This is in the "official" docker image for ubuntu/debian.
// This file prevents us from actually caching anything.
// To resolve that we delete the file.
ei.State = ei.State.File(
llb.Rm("/etc/apt/apt.conf.d/docker-clean", llb.WithAllowNotFound(true)),
constraintsOptFunc(func(c *llb.Constraints) {
*c = ei.Constraints
}),
)

llb.AddMount(
"/var/cache/apt",
llb.Scratch(),
llb.AsPersistentCacheDir(namePrefix+"dalec-var-cache-apt", llb.CacheMountLocked),
).SetRunOption(ei)

llb.AddMount(
"/var/lib/apt",
llb.Scratch(),
llb.AsPersistentCacheDir(namePrefix+"dalec-var-lib-apt", llb.CacheMountLocked),
).SetRunOption(ei)
})
}

Expand Down Expand Up @@ -215,16 +230,16 @@ func CacheDirsToRunOpt(mounts map[string]CacheDirConfig, distroKey, archKey stri
opts = append(opts, llb.AddMount(p, llb.Scratch(), llb.AsPersistentCacheDir(key, mode)))
}

return runOptFunc(func(ei *llb.ExecInfo) {
return RunOptFunc(func(ei *llb.ExecInfo) {
for _, opt := range opts {
opt.SetRunOption(ei)
}
})
}

type runOptFunc func(*llb.ExecInfo)
type RunOptFunc func(*llb.ExecInfo)

func (f runOptFunc) SetRunOption(ei *llb.ExecInfo) {
func (f RunOptFunc) SetRunOption(ei *llb.ExecInfo) {
f(ei)
}

Expand Down
74 changes: 73 additions & 1 deletion test/mariner2_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -157,20 +157,36 @@ index 0000000..5260cb1

Dependencies: &dalec.PackageDependencies{
Runtime: map[string][]string{
"bash": {},
"bash": {},
"coreutils": {},
},
},

Build: dalec.ArtifactBuild{
Steps: []dalec.BuildStep{
// These are "build" steps where we aren't really building things just verifying
// that sources are in the right place and have the right permissions and content
{
// file added by patch
Command: "test -f ./src1",
},
{
Command: "test -x ./src1",
},
{
Command: "test ! -d ./src1",
},
{
Command: "./src1 | grep 'hello world'",
},
{
// file added by patch
Command: "ls -lh ./src2/file2",
},
{
// file added by patch
Command: "test -f ./src2/file2",
},
{
// file added by patch
Command: "test -x ./src2/file2",
Expand Down Expand Up @@ -316,4 +332,60 @@ echo "$BAR" > bar.txt

runTest(t, distroSigningTest(t, &spec, signTarget))
})

t.Run("go module", func(t *testing.T) {
t.Parallel()
ctx := startTestSpan(baseCtx, t)

spec := &dalec.Spec{
Name: "test-build-with-gomod",
Version: "0.0.1",
Revision: "1",
License: "MIT",
Website: "https://github.com/azure/dalec",
Vendor: "Dalec",
Packager: "Dalec",
Description: "Testing container target",
Sources: map[string]dalec.Source{
"src": {
Generate: []*dalec.SourceGenerator{
{
Gomod: &dalec.GeneratorGomod{},
},
},
Inline: &dalec.SourceInline{
Dir: &dalec.SourceInlineDir{
Files: map[string]*dalec.SourceInlineFile{
"main.go": {Contents: gomodFixtureMain},
"go.mod": {Contents: gomodFixtureMod},
"go.sum": {Contents: gomodFixtureSum},
},
},
},
},
},
Dependencies: &dalec.PackageDependencies{
Build: map[string][]string{
// TODO: This works at least for now, but is distro specific and
// could break on new distros (though that is still unlikely).
"golang": {},
},
},
Build: dalec.ArtifactBuild{
Steps: []dalec.BuildStep{
{Command: "[ -d \"${GOMODCACHE}/github.com/cpuguy83/[email protected]\" ]"},
{Command: "[ -d ./src ]"},
{Command: "[ -f ./src/main.go ]"},
{Command: "[ -f ./src/go.mod ]"},
{Command: "[ -f ./src/go.sum ]"},
{Command: "cd ./src && go build"},
},
},
}

testEnv.RunTest(ctx, t, func(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) {
req := newSolveRequest(withBuildTarget(buildTarget), withSpec(ctx, t, spec))
return client.Solve(ctx, req)
})
})
}
30 changes: 15 additions & 15 deletions test/source_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -224,12 +224,9 @@ func TestSourceHTTP(t *testing.T) {
})
}

func TestSourceWithGomod(t *testing.T) {
t.Parallel()

// Create a very simple fake module with a limited depdenency tree just to
// keep the test as fast/reliable as possible.
const mainGo = `package main
// Create a very simple fake module with a limited depdenency tree just to
// keep the test as fast/reliable as possible.
const gomodFixtureMain = `package main
import (
"fmt"
Expand All @@ -243,26 +240,29 @@ func main() {
}
`

const gomod = `module testgomodsource
const gomodFixtureMod = `module testgomodsource
go 1.20.0
go 1.20
require github.com/cpuguy83/tar2go v0.3.1
`

const gosum = `
const gomodFixtureSum = `
github.com/cpuguy83/tar2go v0.3.1 h1:DMWlaIyoh9FBWR4hyfZSOEDA7z8rmCiGF1IJIzlTlR8=
github.com/cpuguy83/tar2go v0.3.1/go.mod h1:2Ys2/Hu+iPHQRa4DjIVJ7UAaKnDhAhNACeK3A0Rr5rM=
`

func TestSourceWithGomod(t *testing.T) {
t.Parallel()

const downgradePatch = `diff --git a/go.mod b/go.mod
index 0c18614..8a3a0ee 100644
--- a/go.mod
+++ b/go.mod
@@ -2,4 +2,4 @@ module testgomodsource
go 1.20.0
go 1.20
-require github.com/cpuguy83/tar2go v0.3.1
+require github.com/cpuguy83/tar2go v0.3.0
diff --git a/go.sum b/go.sum
Expand Down Expand Up @@ -313,9 +313,9 @@ index ea874f5..ba38f84 100644
Inline: &dalec.SourceInline{
Dir: &dalec.SourceInlineDir{
Files: map[string]*dalec.SourceInlineFile{
"main.go": {Contents: mainGo},
"go.mod": {Contents: gomod},
"go.sum": {Contents: gosum},
"main.go": {Contents: gomodFixtureMain},
"go.mod": {Contents: gomodFixtureMod},
"go.sum": {Contents: gomodFixtureSum},
},
},
},
Expand Down
56 changes: 56 additions & 0 deletions test/windows_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -336,6 +336,62 @@ echo "$BAR" > bar.txt
return gwclient.NewResult(), nil
})
})

t.Run("go module", func(t *testing.T) {
t.Parallel()
ctx := startTestSpan(baseCtx, t)

spec := &dalec.Spec{
Name: "test-build-with-gomod",
Version: "0.0.1",
Revision: "1",
License: "MIT",
Website: "https://github.com/azure/dalec",
Vendor: "Dalec",
Packager: "Dalec",
Description: "Testing container target",
Sources: map[string]dalec.Source{
"src": {
Generate: []*dalec.SourceGenerator{
{
Gomod: &dalec.GeneratorGomod{},
},
},
Inline: &dalec.SourceInline{
Dir: &dalec.SourceInlineDir{
Files: map[string]*dalec.SourceInlineFile{
"main.go": {Contents: gomodFixtureMain},
"go.mod": {Contents: gomodFixtureMod},
"go.sum": {Contents: gomodFixtureSum},
},
},
},
},
},
Dependencies: &dalec.PackageDependencies{
Build: map[string][]string{
// TODO: This works at least for now, but is distro specific and
// could break on new distros (though that is still unlikely).
"golang": {},
},
},
Build: dalec.ArtifactBuild{
Steps: []dalec.BuildStep{
{Command: "[ -d \"${GOMODCACHE}/github.com/cpuguy83/[email protected]\" ]"},
{Command: "[ -d ./src ]"},
{Command: "[ -f ./src/main.go ]"},
{Command: "[ -f ./src/go.mod ]"},
{Command: "[ -f ./src/go.sum ]"},
{Command: "cd ./src && go build"},
},
},
}

testEnv.RunTest(ctx, t, func(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) {
req := newSolveRequest(withBuildTarget(buildTarget), withSpec(ctx, t, spec))
return client.Solve(ctx, req)
})
})
}

func reqToState(ctx context.Context, gwc gwclient.Client, sr gwclient.SolveRequest, t *testing.T) llb.State {
Expand Down

0 comments on commit 3d9466d

Please sign in to comment.