Add --parents option for COPY in Dockerfiles

It also includes an implementation of the --parents flag for the buildah copy command.

Fixes: https://issues.redhat.com/browse/RUN-2193
Fixes: #5557

Signed-off-by: Jan Rodák <[email protected]>

Author: Honny1

add.go

@@ -94,6 +94,8 @@ type AddAndCopyOptions struct {
 	// RetryDelay is how long to wait before retrying attempts to retrieve
 	// remote contents.
 	RetryDelay time.Duration
+	// Parents preserve parent directories of source content
+	Parents bool
 }
 
 // gitURLFragmentSuffix matches fragments to use as Git reference and build
@@ -263,6 +265,25 @@ func globbedToGlobbable(glob string) string {
 	return result
 }
 
+// getParentsPrefixToRemoveAndParentsToSkip gets from the pattern the prefix before the "pivot point",
+// the location in the source path marked by the path component named "."
+// (i.e. where "/./" occurs in the path). And list of parents to skip.
+// In case "/./" is not present is returned "/".
+func getParentsPrefixToRemoveAndParentsToSkip(pattern string, contextDir string) (string, []string) {
+	prefix, _, found := strings.Cut(strings.TrimPrefix(pattern, contextDir), "/./")
+	if !found {
+		return string(filepath.Separator), []string{}
+	}
+	prefix = strings.TrimPrefix(filepath.Clean(string(filepath.Separator)+prefix), string(filepath.Separator))
+	out := []string{}
+	parentPath := prefix
+	for parentPath != "/" && parentPath != "." {
+		out = append(out, parentPath)
+		parentPath = filepath.Dir(parentPath)
+	}
+	return prefix, out
+}
+
 // Add copies the contents of the specified sources into the container's root
 // filesystem, optionally extracting contents of local files that look like
 // non-empty archives.
@@ -476,7 +497,6 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 	if err := copier.Mkdir(mountPoint, extractDirectory, mkdirOptions); err != nil {
 		return fmt.Errorf("ensuring target directory exists: %w", err)
 	}
-
 	// Copy each source in turn.
 	for _, src := range sources {
 		var multiErr *multierror.Error
@@ -587,7 +607,6 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 		if localSourceStat == nil {
 			continue
 		}
-
 		// Iterate through every item that matched the glob.
 		itemsCopied := 0
 		for _, globbed := range localSourceStat.Globbed {
@@ -640,6 +659,25 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 						return false, false, nil
 					})
 				}
+
+				if options.Parents {
+					parentsPrefixToRemove, parentsToSkip := getParentsPrefixToRemoveAndParentsToSkip(src, options.ContextDir)
+					writer = newTarFilterer(writer, func(hdr *tar.Header) (bool, bool, io.Reader) {
+						skip := false
+						for _, parent := range parentsToSkip {
+							if hdr.Name == parent && hdr.Typeflag == tar.TypeDir {
+								skip = true
+								break
+							}
+						}
+						hdr.Name = strings.TrimPrefix(hdr.Name, parentsPrefixToRemove)
+						hdr.Name = strings.TrimPrefix(hdr.Name, "/")
+						if hdr.Name == "" {
+							skip = true
+						}
+						return skip, false, nil
+					})
+				}
 				writer = newTarFilterer(writer, func(_ *tar.Header) (bool, bool, io.Reader) {
 					itemsCopied++
 					return false, false, nil
@@ -656,6 +694,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 					StripSetuidBit: options.StripSetuidBit,
 					StripSetgidBit: options.StripSetgidBit,
 					StripStickyBit: options.StripStickyBit,
+					Parents:        options.Parents,
 				}
 				getErr = copier.Get(contextDir, contextDir, getOptions, []string{globbedToGlobbable(globbed)}, writer)
 				closeErr = writer.Close()

cmd/buildah/addcopy.go

@@ -38,6 +38,7 @@ type addCopyResults struct {
 	retry            int
 	retryDelay       string
 	excludes         []string
+	parents          bool
 }
 
 func createCommand(addCopy string, desc string, short string, opts *addCopyResults) *cobra.Command {
@@ -116,6 +117,7 @@ func init() {
 
 	copyFlags := copyCommand.Flags()
 	applyFlagVars(copyFlags, &copyOpts)
+	copyFlags.BoolVar(&copyOpts.parents, "parents", false, "preserve leading directories in the paths of items being copied")
 
 	rootCmd.AddCommand(addCommand)
 	rootCmd.AddCommand(copyCommand)
@@ -246,6 +248,7 @@ func addAndCopyCmd(c *cobra.Command, args []string, verb string, iopts addCopyRe
 		CertPath:              systemContext.DockerCertPath,
 		InsecureSkipTLSVerify: systemContext.DockerInsecureSkipTLSVerify,
 		MaxRetries:            iopts.retry,
+		Parents:               iopts.parents,
 	}
 	if iopts.contextdir != "" {
 		var excludes []string

copier/copier.go

@@ -13,6 +13,8 @@ import (
 	"os/user"
 	"path"
 	"path/filepath"
+	"slices"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -350,6 +352,7 @@ type GetOptions struct {
 	ChmodDirs          *os.FileMode      // set permissions on directories. no effect on archives being extracted
 	ChownFiles         *idtools.IDPair   // set ownership of files. no effect on archives being extracted
 	ChmodFiles         *os.FileMode      // set permissions on files. no effect on archives being extracted
+	Parents            bool              // maintain the sources parent directory in the destination
 	StripSetuidBit     bool              // strip the setuid bit off of items being copied. no effect on archives being extracted
 	StripSetgidBit     bool              // strip the setgid bit off of items being copied. no effect on archives being extracted
 	StripStickyBit     bool              // strip the sticky bit off of items being copied. no effect on archives being extracted
@@ -1182,6 +1185,51 @@ func errorIsPermission(err error) bool {
 	return errors.Is(err, os.ErrPermission) || strings.Contains(err.Error(), "permission denied")
 }
 
+func getParents(path string, stopPath string) []string {
+	out := []string{}
+	for path != "/" && path != "." && path != stopPath {
+		path = filepath.Dir(path)
+		if path == stopPath {
+			continue
+		}
+		out = append(out, path)
+	}
+	sort.Slice(out, func(i, j int) bool {
+		return len(out[i]) < len(out[j])
+	})
+	return out
+}
+
+func checkLinks(item string, req request, info os.FileInfo) (string, os.FileInfo, error) {
+	// chase links. if we hit a dead end, we should just fail
+	oldItem := item
+	followedLinks := 0
+	const maxFollowedLinks = 16
+	for !req.GetOptions.NoDerefSymlinks && info.Mode()&os.ModeType == os.ModeSymlink && followedLinks < maxFollowedLinks {
+		path, err := os.Readlink(item)
+		if err != nil {
+			continue
+		}
+		if filepath.IsAbs(path) || looksLikeAbs(path) {
+			path = filepath.Join(req.Root, path)
+		} else {
+			path = filepath.Join(filepath.Dir(item), path)
+		}
+		item = path
+		if _, err = convertToRelSubdirectory(req.Root, item); err != nil {
+			return "", nil, fmt.Errorf("copier: get: computing path of %q(%q) relative to %q: %w", oldItem, item, req.Root, err)
+		}
+		if info, err = os.Lstat(item); err != nil {
+			return "", nil, fmt.Errorf("copier: get: lstat %q(%q): %w", oldItem, item, err)
+		}
+		followedLinks++
+	}
+	if followedLinks >= maxFollowedLinks {
+		return "", nil, fmt.Errorf("copier: get: resolving symlink %q(%q): %w", oldItem, item, syscall.ELOOP)
+	}
+	return item, info, nil
+}
+
 func copierHandlerGet(bulkWriter io.Writer, req request, pm *fileutils.PatternMatcher, idMappings *idtools.IDMappings) (*response, func() error, error) {
 	statRequest := req
 	statRequest.Request = requestStat
@@ -1196,15 +1244,25 @@ func copierHandlerGet(bulkWriter io.Writer, req request, pm *fileutils.PatternMa
 		return errorResponse("copier: get: expected at least one glob pattern, got 0")
 	}
 	// build a queue of items by globbing
-	var queue []string
+	type queueItem struct {
+		glob    string
+		parents []string
+	}
+	var queue []queueItem
 	globMatchedCount := 0
 	for _, glob := range req.Globs {
 		globMatched, err := extendedGlob(glob)
 		if err != nil {
 			return errorResponse("copier: get: glob %q: %v", glob, err)
 		}
-		globMatchedCount += len(globMatched)
-		queue = append(queue, globMatched...)
+		for _, path := range globMatched {
+			var parents []string
+			if req.GetOptions.Parents {
+				parents = getParents(path, req.Directory)
+			}
+			globMatchedCount++
+			queue = append(queue, queueItem{glob: path, parents: parents})
+		}
 	}
 	// no matches -> error
 	if len(queue) == 0 {
@@ -1219,7 +1277,8 @@ func copierHandlerGet(bulkWriter io.Writer, req request, pm *fileutils.PatternMa
 		defer tw.Close()
 		hardlinkChecker := new(hardlinkChecker)
 		itemsCopied := 0
-		for i, item := range queue {
+		for i, qItem := range queue {
+			item := qItem.glob
 			// if we're not discarding the names of individual directories, keep track of this one
 			relNamePrefix := ""
 			if req.GetOptions.KeepDirectoryNames {
@@ -1230,31 +1289,47 @@ func copierHandlerGet(bulkWriter io.Writer, req request, pm *fileutils.PatternMa
 			if err != nil {
 				return fmt.Errorf("copier: get: lstat %q: %w", item, err)
 			}
-			// chase links. if we hit a dead end, we should just fail
-			followedLinks := 0
-			const maxFollowedLinks = 16
-			for !req.GetOptions.NoDerefSymlinks && info.Mode()&os.ModeType == os.ModeSymlink && followedLinks < maxFollowedLinks {
-				path, err := os.Readlink(item)
+			if req.GetOptions.Parents && info.Mode().IsDir() {
+				if !slices.Contains(qItem.parents, item) {
+					qItem.parents = append(qItem.parents, item)
+				}
+			}
+			// Copy parents in to tarball first if exists
+			for _, parent := range qItem.parents {
+				oldParent := parent
+				parentInfo, err := os.Lstat(parent)
 				if err != nil {
-					continue
+					return fmt.Errorf("copier: get: lstat %q: %w", parent, err)
 				}
-				if filepath.IsAbs(path) || looksLikeAbs(path) {
-					path = filepath.Join(req.Root, path)
-				} else {
-					path = filepath.Join(filepath.Dir(item), path)
+				parent, parentInfo, err = checkLinks(parent, req, parentInfo)
+				if err != nil {
+					return err
+				}
+				parentName, err := convertToRelSubdirectory(req.Directory, oldParent)
+				if err != nil {
+					return fmt.Errorf("copier: get: error computing path of %q relative to %q: %w", parent, req.Directory, err)
 				}
-				item = path
-				if _, err = convertToRelSubdirectory(req.Root, item); err != nil {
-					return fmt.Errorf("copier: get: computing path of %q(%q) relative to %q: %w", queue[i], item, req.Root, err)
+				if parentName == "" || parentName == "." {
+					// skip the "." entry
+					continue
 				}
-				if info, err = os.Lstat(item); err != nil {
-					return fmt.Errorf("copier: get: lstat %q(%q): %w", queue[i], item, err)
+				if err := copierHandlerGetOne(parentInfo, "", parentName, parent, req.GetOptions, tw, hardlinkChecker, idMappings); err != nil {
+					if req.GetOptions.IgnoreUnreadable && errorIsPermission(err) {
+						continue
+					} else if errors.Is(err, os.ErrNotExist) {
+						logrus.Warningf("copier: file disappeared while reading: %q", parent)
+						return nil
+					}
+					return fmt.Errorf("copier: get: %q: %w", queue[i].glob, err)
 				}
-				followedLinks++
+				itemsCopied++
 			}
-			if followedLinks >= maxFollowedLinks {
-				return fmt.Errorf("copier: get: resolving symlink %q(%q): %w", queue[i], item, syscall.ELOOP)
+
+			item, info, err = checkLinks(item, req, info)
+			if err != nil {
+				return err
 			}
+
 			// evaluate excludes relative to the root directory
 			if info.Mode().IsDir() {
 				// we don't expand any of the contents that are archives
@@ -1354,6 +1429,12 @@ func copierHandlerGet(bulkWriter io.Writer, req request, pm *fileutils.PatternMa
 							ok = filepath.SkipDir
 						}
 					}
+					if req.GetOptions.Parents {
+						rel, err = convertToRelSubdirectory(req.Directory, path)
+						if err != nil {
+							return fmt.Errorf("copier: get: error computing path of %q relative to %q: %w", path, req.Root, err)
+						}
+					}
 					// add the item to the outgoing tar stream
 					if err := copierHandlerGetOne(info, symlinkTarget, rel, path, options, tw, hardlinkChecker, idMappings); err != nil {
 						if req.GetOptions.IgnoreUnreadable && errorIsPermission(err) {
@@ -1368,7 +1449,7 @@ func copierHandlerGet(bulkWriter io.Writer, req request, pm *fileutils.PatternMa
 				}
 				// walk the directory tree, checking/adding items individually
 				if err := filepath.WalkDir(item, walkfn); err != nil {
-					return fmt.Errorf("copier: get: %q(%q): %w", queue[i], item, err)
+					return fmt.Errorf("copier: get: %q(%q): %w", queue[i].glob, item, err)
 				}
 				itemsCopied++
 			} else {
@@ -1379,15 +1460,24 @@ func copierHandlerGet(bulkWriter io.Writer, req request, pm *fileutils.PatternMa
 				if skip {
 					continue
 				}
-				// add the item to the outgoing tar stream.  in
-				// cases where this was a symlink that we
-				// dereferenced, be sure to use the name of the
-				// link.
-				if err := copierHandlerGetOne(info, "", filepath.Base(queue[i]), item, req.GetOptions, tw, hardlinkChecker, idMappings); err != nil {
+
+				name := filepath.Base(queue[i].glob)
+				if req.GetOptions.Parents {
+					name, err = convertToRelSubdirectory(req.Directory, queue[i].glob)
+					if err != nil {
+						return fmt.Errorf("copier: get: error computing path of %q relative to %q: %w", item, req.Root, err)
+					}
+					if name == "" || name == "." {
+						// skip the "." entry
+						continue
+					}
+				}
+
+				if err := copierHandlerGetOne(info, "", name, item, req.GetOptions, tw, hardlinkChecker, idMappings); err != nil {
 					if req.GetOptions.IgnoreUnreadable && errorIsPermission(err) {
 						continue
 					}
-					return fmt.Errorf("copier: get: %q: %w", queue[i], err)
+					return fmt.Errorf("copier: get: %q: %w", queue[i].glob, err)
 				}
 				itemsCopied++
 			}