Skip to content

Commit a37d19e

Browse files
Honny1Honny1
committed
Add --parents option for COPY in Dockerfiles
It also includes an implementation of the --parents flag for the buildah copy command. Fixes: https://issues.redhat.com/browse/RUN-2193 Fixes: #5557 Signed-off-by: Jan Rodák <[email protected]>
1 parent 5a57a86 commit a37d19e

File tree

19 files changed

+307
-50
lines changed

19 files changed

+307
-50
lines changed

‎add.go

+47-23
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,13 @@ type AddAndCopyOptions struct {
9494
// RetryDelay is how long to wait before retrying attempts to retrieve
9595
// remote contents.
9696
RetryDelay time.Duration
97+
// ParentsPatterns are patterns to preserve parent directories of source content
98+
// Key: source path (same as argument sources of Add function)
99+
// Value: path(pattern) that specifies leading directories in the paths of items being copied,
100+
// relative to either the top of the build context, or to the
101+
// "pivot point", a location in the source path marked by a path component named "."
102+
// (i.e., where "/./" occurs in the path).
103+
ParentsPatterns map[string]string
97104
}
98105

99106
// gitURLFragmentSuffix matches fragments to use as Git reference and build
@@ -263,6 +270,18 @@ func globbedToGlobbable(glob string) string {
263270
return result
264271
}
265272

273+
// getParentsPrefixToRemove gets from the pattern the prefix before the "pivot point",
274+
// the location in the source path marked by the path component named "."
275+
// (i.e. where "/./" occurs in the path).
276+
// In case "/./" is not present is returned "/".
277+
func getParentsPrefixToRemove(pattern string) string {
278+
prefix, _, found := strings.Cut(pattern, "/./")
279+
if !found {
280+
return string(filepath.Separator)
281+
}
282+
return filepath.Clean(prefix)
283+
}
284+
266285
// Add copies the contents of the specified sources into the container's root
267286
// filesystem, optionally extracting contents of local files that look like
268287
// non-empty archives.
@@ -476,9 +495,12 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
476495
if err := copier.Mkdir(mountPoint, extractDirectory, mkdirOptions); err != nil {
477496
return fmt.Errorf("ensuring target directory exists: %w", err)
478497
}
479-
480498
// Copy each source in turn.
481499
for _, src := range sources {
500+
parentsPrefixToRemove := ""
501+
if pattern, ok := options.ParentsPatterns[src]; ok {
502+
parentsPrefixToRemove = getParentsPrefixToRemove(pattern)
503+
}
482504
var multiErr *multierror.Error
483505
var getErr, closeErr, renameErr, putErr error
484506
var wg sync.WaitGroup
@@ -498,17 +520,18 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
498520
var cloneDir, subdir string
499521
cloneDir, subdir, getErr = define.TempDirForURL(tmpdir.GetTempDir(), "", src)
500522
getOptions := copier.GetOptions{
501-
UIDMap: srcUIDMap,
502-
GIDMap: srcGIDMap,
503-
Excludes: options.Excludes,
504-
ExpandArchives: extract,
505-
ChownDirs: chownDirs,
506-
ChmodDirs: chmodDirsFiles,
507-
ChownFiles: chownFiles,
508-
ChmodFiles: chmodDirsFiles,
509-
StripSetuidBit: options.StripSetuidBit,
510-
StripSetgidBit: options.StripSetgidBit,
511-
StripStickyBit: options.StripStickyBit,
523+
UIDMap: srcUIDMap,
524+
GIDMap: srcGIDMap,
525+
Excludes: options.Excludes,
526+
ExpandArchives: extract,
527+
ParentsPrefixToRemove: parentsPrefixToRemove,
528+
ChownDirs: chownDirs,
529+
ChmodDirs: chmodDirsFiles,
530+
ChownFiles: chownFiles,
531+
ChmodFiles: chmodDirsFiles,
532+
StripSetuidBit: options.StripSetuidBit,
533+
StripSetgidBit: options.StripSetgidBit,
534+
StripStickyBit: options.StripStickyBit,
512535
}
513536
writer := io.WriteCloser(pipeWriter)
514537
repositoryDir := filepath.Join(cloneDir, subdir)
@@ -645,17 +668,18 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
645668
return false, false, nil
646669
})
647670
getOptions := copier.GetOptions{
648-
UIDMap: srcUIDMap,
649-
GIDMap: srcGIDMap,
650-
Excludes: options.Excludes,
651-
ExpandArchives: extract,
652-
ChownDirs: chownDirs,
653-
ChmodDirs: chmodDirsFiles,
654-
ChownFiles: chownFiles,
655-
ChmodFiles: chmodDirsFiles,
656-
StripSetuidBit: options.StripSetuidBit,
657-
StripSetgidBit: options.StripSetgidBit,
658-
StripStickyBit: options.StripStickyBit,
671+
UIDMap: srcUIDMap,
672+
GIDMap: srcGIDMap,
673+
Excludes: options.Excludes,
674+
ExpandArchives: extract,
675+
ChownDirs: chownDirs,
676+
ChmodDirs: chmodDirsFiles,
677+
ChownFiles: chownFiles,
678+
ChmodFiles: chmodDirsFiles,
679+
ParentsPrefixToRemove: parentsPrefixToRemove,
680+
StripSetuidBit: options.StripSetuidBit,
681+
StripSetgidBit: options.StripSetgidBit,
682+
StripStickyBit: options.StripStickyBit,
659683
}
660684
getErr = copier.Get(contextDir, contextDir, getOptions, []string{globbedToGlobbable(globbed)}, writer)
661685
closeErr = writer.Close()

‎cmd/buildah/addcopy.go

+8
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ type addCopyResults struct {
3838
retry int
3939
retryDelay string
4040
excludes []string
41+
parents bool
4142
}
4243

4344
func createCommand(addCopy string, desc string, short string, opts *addCopyResults) *cobra.Command {
@@ -116,6 +117,7 @@ func init() {
116117

117118
copyFlags := copyCommand.Flags()
118119
applyFlagVars(copyFlags, &copyOpts)
120+
copyFlags.BoolVar(&copyOpts.parents, "parents", false, "preserve leading directories in the paths of items being copied.")
119121

120122
rootCmd.AddCommand(addCommand)
121123
rootCmd.AddCommand(copyCommand)
@@ -247,6 +249,12 @@ func addAndCopyCmd(c *cobra.Command, args []string, verb string, iopts addCopyRe
247249
InsecureSkipTLSVerify: systemContext.DockerInsecureSkipTLSVerify,
248250
MaxRetries: iopts.retry,
249251
}
252+
if iopts.parents {
253+
options.ParentsPatterns = map[string]string{}
254+
for _, pattern := range args {
255+
options.ParentsPatterns[pattern] = pattern
256+
}
257+
}
250258
if iopts.contextdir != "" {
251259
var excludes []string
252260

‎copier/copier.go

+75-22
Original file line numberDiff line numberDiff line change
@@ -343,22 +343,26 @@ func Stat(root string, directory string, options StatOptions, globs []string) ([
343343

344344
// GetOptions controls parts of Get()'s behavior.
345345
type GetOptions struct {
346-
UIDMap, GIDMap []idtools.IDMap // map from hostIDs to containerIDs in the output archive
347-
Excludes []string // contents to pretend don't exist, using the OS-specific path separator
348-
ExpandArchives bool // extract the contents of named items that are archives
349-
ChownDirs *idtools.IDPair // set ownership on directories. no effect on archives being extracted
350-
ChmodDirs *os.FileMode // set permissions on directories. no effect on archives being extracted
351-
ChownFiles *idtools.IDPair // set ownership of files. no effect on archives being extracted
352-
ChmodFiles *os.FileMode // set permissions on files. no effect on archives being extracted
353-
StripSetuidBit bool // strip the setuid bit off of items being copied. no effect on archives being extracted
354-
StripSetgidBit bool // strip the setgid bit off of items being copied. no effect on archives being extracted
355-
StripStickyBit bool // strip the sticky bit off of items being copied. no effect on archives being extracted
356-
StripXattrs bool // don't record extended attributes of items being copied. no effect on archives being extracted
357-
KeepDirectoryNames bool // don't strip the top directory's basename from the paths of items in subdirectories
358-
Rename map[string]string // rename items with the specified names, or under the specified names
359-
NoDerefSymlinks bool // don't follow symlinks when globs match them
360-
IgnoreUnreadable bool // ignore errors reading items, instead of returning an error
361-
NoCrossDevice bool // if a subdirectory is a mountpoint with a different device number, include it but skip its contents
346+
UIDMap, GIDMap []idtools.IDMap // map from hostIDs to containerIDs in the output archive
347+
Excludes []string // contents to pretend don't exist, using the OS-specific path separator
348+
ExpandArchives bool // extract the contents of named items that are archives
349+
ChownDirs *idtools.IDPair // set ownership on directories. no effect on archives being extracted
350+
ChmodDirs *os.FileMode // set permissions on directories. no effect on archives being extracted
351+
ChownFiles *idtools.IDPair // set ownership of files. no effect on archives being extracted
352+
ChmodFiles *os.FileMode // set permissions on files. no effect on archives being extracted
353+
// ParentsPrefixToRemove is removed from source path and what is left is used as name to maintain
354+
// the sources parent directory in the destination if is empty is used only file name as name in tar.
355+
// Note: Parent directories are copied to the tarstream
356+
ParentsPrefixToRemove string
357+
StripSetuidBit bool // strip the setuid bit off of items being copied. no effect on archives being extracted
358+
StripSetgidBit bool // strip the setgid bit off of items being copied. no effect on archives being extracted
359+
StripStickyBit bool // strip the sticky bit off of items being copied. no effect on archives being extracted
360+
StripXattrs bool // don't record extended attributes of items being copied. no effect on archives being extracted
361+
KeepDirectoryNames bool // don't strip the top directory's basename from the paths of items in subdirectories
362+
Rename map[string]string // rename items with the specified names, or under the specified names
363+
NoDerefSymlinks bool // don't follow symlinks when globs match them
364+
IgnoreUnreadable bool // ignore errors reading items, instead of returning an error
365+
NoCrossDevice bool // if a subdirectory is a mountpoint with a different device number, include it but skip its contents
362366
}
363367

364368
// Get produces an archive containing items that match the specified glob
@@ -1215,6 +1219,7 @@ func copierHandlerGet(bulkWriter io.Writer, req request, pm *fileutils.PatternMa
12151219
return errorResponse("copier: get: error reading info about directory %q: %v", req.Directory, err)
12161220
}
12171221
cb := func() error {
1222+
alreadyCopied := map[string]struct{}{}
12181223
tw := tar.NewWriter(bulkWriter)
12191224
defer tw.Close()
12201225
hardlinkChecker := new(hardlinkChecker)
@@ -1354,6 +1359,9 @@ func copierHandlerGet(bulkWriter io.Writer, req request, pm *fileutils.PatternMa
13541359
ok = filepath.SkipDir
13551360
}
13561361
}
1362+
if len(req.GetOptions.ParentsPrefixToRemove) > 0 {
1363+
rel = filepath.Clean(strings.TrimPrefix(path, filepath.Join(req.Directory, req.GetOptions.ParentsPrefixToRemove)))
1364+
}
13571365
// add the item to the outgoing tar stream
13581366
if err := copierHandlerGetOne(info, symlinkTarget, rel, path, options, tw, hardlinkChecker, idMappings); err != nil {
13591367
if req.GetOptions.IgnoreUnreadable && errorIsPermission(err) {
@@ -1379,17 +1387,37 @@ func copierHandlerGet(bulkWriter io.Writer, req request, pm *fileutils.PatternMa
13791387
if skip {
13801388
continue
13811389
}
1382-
// add the item to the outgoing tar stream. in
1383-
// cases where this was a symlink that we
1384-
// dereferenced, be sure to use the name of the
1385-
// link.
1386-
if err := copierHandlerGetOne(info, "", filepath.Base(queue[i]), item, req.GetOptions, tw, hardlinkChecker, idMappings); err != nil {
1390+
1391+
copyFunc := func(name string, path string, fileInfo os.FileInfo) error {
1392+
// add the item to the outgoing tar stream. in
1393+
// cases where this was a symlink that we
1394+
// dereferenced, be sure to use the name of the
1395+
// link.
1396+
if err := copierHandlerGetOne(fileInfo, "", name, path, req.GetOptions, tw, hardlinkChecker, idMappings); err != nil {
1397+
return err
1398+
}
1399+
itemsCopied++
1400+
return nil
1401+
}
1402+
1403+
name := filepath.Base(queue[i])
1404+
if len(req.GetOptions.ParentsPrefixToRemove) > 0 {
1405+
name = filepath.Clean(strings.TrimPrefix(item, filepath.Join(req.Directory, req.GetOptions.ParentsPrefixToRemove)))
1406+
alreadyCopied, err = copyParentsDirs(name, item, alreadyCopied, copyFunc)
1407+
if err != nil {
1408+
if req.GetOptions.IgnoreUnreadable && errorIsPermission(err) {
1409+
continue
1410+
}
1411+
return fmt.Errorf("copier: get: %q: %w", queue[i], err)
1412+
}
1413+
}
1414+
1415+
if err := copyFunc(name, item, info); err != nil {
13871416
if req.GetOptions.IgnoreUnreadable && errorIsPermission(err) {
13881417
continue
13891418
}
13901419
return fmt.Errorf("copier: get: %q: %w", queue[i], err)
13911420
}
1392-
itemsCopied++
13931421
}
13941422
}
13951423
if itemsCopied == 0 {
@@ -1400,6 +1428,31 @@ func copierHandlerGet(bulkWriter io.Writer, req request, pm *fileutils.PatternMa
14001428
return &response{Stat: statResponse.Stat, Get: getResponse{}}, cb, nil
14011429
}
14021430

1431+
func copyParentsDirs(name string, path string, alreadyCopied map[string]struct{}, copy func(string, string, os.FileInfo) error) (map[string]struct{}, error) {
1432+
parentName := filepath.Dir(name)
1433+
parentPath := filepath.Dir(path)
1434+
for parentName != "/" && parentName != "." {
1435+
if _, ok := alreadyCopied[parentPath]; ok {
1436+
parentName = filepath.Dir(parentName)
1437+
parentPath = filepath.Dir(parentPath)
1438+
continue
1439+
}
1440+
1441+
parentInfo, err := os.Lstat(parentPath)
1442+
if err != nil {
1443+
return alreadyCopied, fmt.Errorf("copier: get: lstat %q: %w", parentPath, err)
1444+
}
1445+
if err := copy(strings.TrimPrefix(parentName, "/"), parentPath, parentInfo); err != nil {
1446+
return alreadyCopied, err
1447+
}
1448+
1449+
alreadyCopied[parentPath] = struct{}{}
1450+
parentName = filepath.Dir(parentName)
1451+
parentPath = filepath.Dir(parentPath)
1452+
}
1453+
return alreadyCopied, nil
1454+
}
1455+
14031456
func handleRename(rename map[string]string, name string) string {
14041457
if rename == nil {
14051458
return name

‎docs/buildah-copy.1.md

+8
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,12 @@ is preserved.
6565

6666
Path to an alternative .containerignore (.dockerignore) file. Requires \-\-contextdir be specified.
6767

68+
**--parents**
69+
70+
Preserve leading directories in the paths of items being copied, relative to either the
71+
top of the build context, or to the "pivot point", a location in the source path marked
72+
by a path component named "." (i.e., where "/./" occurs in the path).
73+
6874
**--quiet**, **-q**
6975

7076
Refrain from printing a digest of the copied content.
@@ -93,6 +99,8 @@ buildah copy containerID '/myapp/app.conf' '/myapp/app.conf'
9399

94100
buildah copy --exclude=**/*.md docs containerID 'docs' '/docs'
95101

102+
buildah copy --parents containerID './x/a.txt' './y/a.txt' '/parents'
103+
96104
buildah copy --chown myuser:mygroup containerID '/myapp/app.conf' '/myapp/app.conf'
97105

98106
buildah copy --chmod 660 containerID '/myapp/app.conf' '/myapp/app.conf'

‎imagebuildah/stage_executor.go

+7-4
Original file line numberDiff line numberDiff line change
@@ -368,9 +368,6 @@ func (s *StageExecutor) Copy(excludes []string, copies ...imagebuilder.Copy) err
368368
if cp.Link {
369369
return errors.New("COPY --link is not supported")
370370
}
371-
if cp.Parents {
372-
return errors.New("COPY --parents is not supported")
373-
}
374371
if len(cp.Excludes) > 0 {
375372
excludes = append(slices.Clone(excludes), cp.Excludes...)
376373
}
@@ -550,6 +547,7 @@ func (s *StageExecutor) performCopy(excludes []string, copies ...imagebuilder.Co
550547
} else {
551548
logrus.Debugf("COPY %#v, %#v", excludes, copy)
552549
}
550+
parentsPattern := map[string]string{}
553551
for _, src := range copy.Src {
554552
if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
555553
// Source is a URL, allowed for ADD but not COPY.
@@ -560,7 +558,9 @@ func (s *StageExecutor) performCopy(excludes []string, copies ...imagebuilder.Co
560558
return fmt.Errorf("source can't be a URL for COPY")
561559
}
562560
} else {
563-
sources = append(sources, filepath.Join(contextDir, src))
561+
source := filepath.Join(contextDir, src)
562+
parentsPattern[source] = src
563+
sources = append(sources, source)
564564
}
565565
}
566566
options := buildah.AddAndCopyOptions{
@@ -582,6 +582,9 @@ func (s *StageExecutor) performCopy(excludes []string, copies ...imagebuilder.Co
582582
MaxRetries: s.executor.maxPullPushRetries,
583583
RetryDelay: s.executor.retryPullPushDelay,
584584
}
585+
if copy.Parents {
586+
options.ParentsPatterns = parentsPattern
587+
}
585588
if len(copy.Files) > 0 {
586589
// If we are copying heredoc files, we need to temporary place
587590
// them in the context dir and then move to container via copier

‎tests/bud.bats

+8-1
Original file line numberDiff line numberDiff line change
@@ -394,7 +394,7 @@ _EOF
394394
assert "$output" !~ "First symlink content"
395395

396396
# Modify the symlink
397-
ln -sf samplefile2 $contextdir/tomount
397+
ln -sf samplefile2 $contextdir/tomount
398398

399399
# on third run since we have changed symlink so cache must burst.
400400
run_buildah build $WITH_POLICY_JSON --layers -t source -f $contextdir/Containerfile $contextdir
@@ -6225,6 +6225,13 @@ _EOF
62256225
assert "$output" !~ "test2.txt"
62266226
}
62276227

6228+
@test "bud with copy --parents" {
6229+
run_buildah build -t test $WITH_POLICY_JSON $BUDFILES/copy-parents
6230+
assert "$output" !~ "./no_parents/a.txt
6231+
./parents/x/a.txt
6232+
./parents/y/a.txt"
6233+
}
6234+
62286235
@test "bud with containerfile secret" {
62296236
_prefetch alpine
62306237
mytmpdir=${TEST_SCRATCH_DIR}/my-dir1

‎tests/bud/copy-parents/Containerfile

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
FROM alpine
2+
COPY ./x/a.txt ./y/a.txt /no_parents/
3+
COPY --parents ./x/a.txt ./y/a.txt /parents/
4+
run find /no_parents
5+
run find /parents

‎tests/bud/copy-parents/x/a.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
A-FILE

‎tests/bud/copy-parents/x/y/a.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
A-FILE

‎tests/bud/copy-parents/x/y/b.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Hello

‎tests/bud/copy-parents/y/a.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
A-FILE

‎tests/bud/copy-parents/y/b.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Hello

0 commit comments

Comments
 (0)