Add --parents option for COPY in Dockerfiles

It also includes an implementation of the --parents flag for the buildah copy command.

Fixes: https://issues.redhat.com/browse/RUN-2193
Fixes: #5557

Signed-off-by: Jan Rodák <[email protected]>

Author: Honny1

add.go

@@ -94,6 +94,8 @@ type AddAndCopyOptions struct {
 	// RetryDelay is how long to wait before retrying attempts to retrieve
 	// remote contents.
 	RetryDelay time.Duration
+	// ParentsPatterns are patterns to preserve parent directories of source content
+	ParentsPatterns map[string]string
 }
 
 // gitURLFragmentSuffix matches fragments to use as Git reference and build
@@ -263,6 +265,18 @@ func globbedToGlobbable(glob string) string {
 	return result
 }
 
+// getParentsPrefixToRemove gets from the pattern the prefix before the "pivot point",
+// the location in the source path marked by the path component named "."
+// (i.e. where "/./" occurs in the path).
+// In case "/./" is not present is returned "/".
+func getParentsPrefixToRemove(pattern string) string {
+	prefix, _, found := strings.Cut(pattern, "/./")
+	if !found {
+		return string(filepath.Separator)
+	}
+	return filepath.Clean(prefix)
+}
+
 // Add copies the contents of the specified sources into the container's root
 // filesystem, optionally extracting contents of local files that look like
 // non-empty archives.
@@ -476,9 +490,12 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 	if err := copier.Mkdir(mountPoint, extractDirectory, mkdirOptions); err != nil {
 		return fmt.Errorf("ensuring target directory exists: %w", err)
 	}
-
 	// Copy each source in turn.
 	for _, src := range sources {
+		parentsPrefixToRemove := ""
+		if pattern, ok := options.ParentsPatterns[src]; ok {
+			parentsPrefixToRemove = getParentsPrefixToRemove(pattern)
+		}
 		var multiErr *multierror.Error
 		var getErr, closeErr, renameErr, putErr error
 		var wg sync.WaitGroup
@@ -498,17 +515,18 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 					var cloneDir, subdir string
 					cloneDir, subdir, getErr = define.TempDirForURL(tmpdir.GetTempDir(), "", src)
 					getOptions := copier.GetOptions{
-						UIDMap:         srcUIDMap,
-						GIDMap:         srcGIDMap,
-						Excludes:       options.Excludes,
-						ExpandArchives: extract,
-						ChownDirs:      chownDirs,
-						ChmodDirs:      chmodDirsFiles,
-						ChownFiles:     chownFiles,
-						ChmodFiles:     chmodDirsFiles,
-						StripSetuidBit: options.StripSetuidBit,
-						StripSetgidBit: options.StripSetgidBit,
-						StripStickyBit: options.StripStickyBit,
+						UIDMap:                srcUIDMap,
+						GIDMap:                srcGIDMap,
+						Excludes:              options.Excludes,
+						ExpandArchives:        extract,
+						ParentsPrefixToRemove: parentsPrefixToRemove,
+						ChownDirs:             chownDirs,
+						ChmodDirs:             chmodDirsFiles,
+						ChownFiles:            chownFiles,
+						ChmodFiles:            chmodDirsFiles,
+						StripSetuidBit:        options.StripSetuidBit,
+						StripSetgidBit:        options.StripSetgidBit,
+						StripStickyBit:        options.StripStickyBit,
 					}
 					writer := io.WriteCloser(pipeWriter)
 					repositoryDir := filepath.Join(cloneDir, subdir)
@@ -645,17 +663,18 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 					return false, false, nil
 				})
 				getOptions := copier.GetOptions{
-					UIDMap:         srcUIDMap,
-					GIDMap:         srcGIDMap,
-					Excludes:       options.Excludes,
-					ExpandArchives: extract,
-					ChownDirs:      chownDirs,
-					ChmodDirs:      chmodDirsFiles,
-					ChownFiles:     chownFiles,
-					ChmodFiles:     chmodDirsFiles,
-					StripSetuidBit: options.StripSetuidBit,
-					StripSetgidBit: options.StripSetgidBit,
-					StripStickyBit: options.StripStickyBit,
+					UIDMap:                srcUIDMap,
+					GIDMap:                srcGIDMap,
+					Excludes:              options.Excludes,
+					ExpandArchives:        extract,
+					ChownDirs:             chownDirs,
+					ChmodDirs:             chmodDirsFiles,
+					ChownFiles:            chownFiles,
+					ChmodFiles:            chmodDirsFiles,
+					ParentsPrefixToRemove: parentsPrefixToRemove,
+					StripSetuidBit:        options.StripSetuidBit,
+					StripSetgidBit:        options.StripSetgidBit,
+					StripStickyBit:        options.StripStickyBit,
 				}
 				getErr = copier.Get(contextDir, contextDir, getOptions, []string{globbedToGlobbable(globbed)}, writer)
 				closeErr = writer.Close()

cmd/buildah/addcopy.go

@@ -38,6 +38,7 @@ type addCopyResults struct {
 	retry            int
 	retryDelay       string
 	excludes         []string
+	parents          bool
 }
 
 func createCommand(addCopy string, desc string, short string, opts *addCopyResults) *cobra.Command {
@@ -116,6 +117,7 @@ func init() {
 
 	copyFlags := copyCommand.Flags()
 	applyFlagVars(copyFlags, &copyOpts)
+	copyFlags.BoolVar(&copyOpts.parents, "parents", false, "preserve leading directories in the paths of items being copied.")
 
 	rootCmd.AddCommand(addCommand)
 	rootCmd.AddCommand(copyCommand)
@@ -247,6 +249,12 @@ func addAndCopyCmd(c *cobra.Command, args []string, verb string, iopts addCopyRe
 		InsecureSkipTLSVerify: systemContext.DockerInsecureSkipTLSVerify,
 		MaxRetries:            iopts.retry,
 	}
+	if iopts.parents {
+		options.ParentsPatterns = map[string]string{}
+		for _, pattern := range args {
+			options.ParentsPatterns[pattern] = pattern
+		}
+	}
 	if iopts.contextdir != "" {
 		var excludes []string
 

copier/copier.go

@@ -343,22 +343,25 @@ func Stat(root string, directory string, options StatOptions, globs []string) ([
 
 // GetOptions controls parts of Get()'s behavior.
 type GetOptions struct {
-	UIDMap, GIDMap     []idtools.IDMap   // map from hostIDs to containerIDs in the output archive
-	Excludes           []string          // contents to pretend don't exist, using the OS-specific path separator
-	ExpandArchives     bool              // extract the contents of named items that are archives
-	ChownDirs          *idtools.IDPair   // set ownership on directories. no effect on archives being extracted
-	ChmodDirs          *os.FileMode      // set permissions on directories. no effect on archives being extracted
-	ChownFiles         *idtools.IDPair   // set ownership of files. no effect on archives being extracted
-	ChmodFiles         *os.FileMode      // set permissions on files. no effect on archives being extracted
-	StripSetuidBit     bool              // strip the setuid bit off of items being copied. no effect on archives being extracted
-	StripSetgidBit     bool              // strip the setgid bit off of items being copied. no effect on archives being extracted
-	StripStickyBit     bool              // strip the sticky bit off of items being copied. no effect on archives being extracted
-	StripXattrs        bool              // don't record extended attributes of items being copied. no effect on archives being extracted
-	KeepDirectoryNames bool              // don't strip the top directory's basename from the paths of items in subdirectories
-	Rename             map[string]string // rename items with the specified names, or under the specified names
-	NoDerefSymlinks    bool              // don't follow symlinks when globs match them
-	IgnoreUnreadable   bool              // ignore errors reading items, instead of returning an error
-	NoCrossDevice      bool              // if a subdirectory is a mountpoint with a different device number, include it but skip its contents
+	UIDMap, GIDMap []idtools.IDMap // map from hostIDs to containerIDs in the output archive
+	Excludes       []string        // contents to pretend don't exist, using the OS-specific path separator
+	ExpandArchives bool            // extract the contents of named items that are archives
+	ChownDirs      *idtools.IDPair // set ownership on directories. no effect on archives being extracted
+	ChmodDirs      *os.FileMode    // set permissions on directories. no effect on archives being extracted
+	ChownFiles     *idtools.IDPair // set ownership of files. no effect on archives being extracted
+	ChmodFiles     *os.FileMode    // set permissions on files. no effect on archives being extracted
+	// ParentsPrefixToRemove is removed from source path and what is left is used as name to maintain
+	// the sources parent directory in the destination if is empty is used only file name as name in tar.
+	ParentsPrefixToRemove string
+	StripSetuidBit        bool              // strip the setuid bit off of items being copied. no effect on archives being extracted
+	StripSetgidBit        bool              // strip the setgid bit off of items being copied. no effect on archives being extracted
+	StripStickyBit        bool              // strip the sticky bit off of items being copied. no effect on archives being extracted
+	StripXattrs           bool              // don't record extended attributes of items being copied. no effect on archives being extracted
+	KeepDirectoryNames    bool              // don't strip the top directory's basename from the paths of items in subdirectories
+	Rename                map[string]string // rename items with the specified names, or under the specified names
+	NoDerefSymlinks       bool              // don't follow symlinks when globs match them
+	IgnoreUnreadable      bool              // ignore errors reading items, instead of returning an error
+	NoCrossDevice         bool              // if a subdirectory is a mountpoint with a different device number, include it but skip its contents
 }
 
 // Get produces an archive containing items that match the specified glob
@@ -1215,6 +1218,7 @@ func copierHandlerGet(bulkWriter io.Writer, req request, pm *fileutils.PatternMa
 		return errorResponse("copier: get: error reading info about directory %q: %v", req.Directory, err)
 	}
 	cb := func() error {
+		alreadyCopied := map[string]struct{}{}
 		tw := tar.NewWriter(bulkWriter)
 		defer tw.Close()
 		hardlinkChecker := new(hardlinkChecker)
@@ -1354,6 +1358,9 @@ func copierHandlerGet(bulkWriter io.Writer, req request, pm *fileutils.PatternMa
 							ok = filepath.SkipDir
 						}
 					}
+					if len(req.GetOptions.ParentsPrefixToRemove) > 0 {
+						rel = filepath.Clean(strings.TrimPrefix(path, filepath.Join(req.Directory, req.GetOptions.ParentsPrefixToRemove)))
+					}
 					// add the item to the outgoing tar stream
 					if err := copierHandlerGetOne(info, symlinkTarget, rel, path, options, tw, hardlinkChecker, idMappings); err != nil {
 						if req.GetOptions.IgnoreUnreadable && errorIsPermission(err) {
@@ -1379,17 +1386,37 @@ func copierHandlerGet(bulkWriter io.Writer, req request, pm *fileutils.PatternMa
 				if skip {
 					continue
 				}
-				// add the item to the outgoing tar stream.  in
-				// cases where this was a symlink that we
-				// dereferenced, be sure to use the name of the
-				// link.
-				if err := copierHandlerGetOne(info, "", filepath.Base(queue[i]), item, req.GetOptions, tw, hardlinkChecker, idMappings); err != nil {
+
+				copyFunc := func(name string, path string, fileInfo os.FileInfo) error {
+					// add the item to the outgoing tar stream.  in
+					// cases where this was a symlink that we
+					// dereferenced, be sure to use the name of the
+					// link.
+					if err := copierHandlerGetOne(fileInfo, "", name, path, req.GetOptions, tw, hardlinkChecker, idMappings); err != nil {
+						return err
+					}
+					itemsCopied++
+					return nil
+				}
+
+				name := filepath.Base(queue[i])
+				if len(req.GetOptions.ParentsPrefixToRemove) > 0 {
+					name = filepath.Clean(strings.TrimPrefix(item, filepath.Join(req.Directory, req.GetOptions.ParentsPrefixToRemove)))
+					alreadyCopied, err = copyParentsDirs(name, item, alreadyCopied, copyFunc)
+					if err != nil {
+						if req.GetOptions.IgnoreUnreadable && errorIsPermission(err) {
+							continue
+						}
+						return fmt.Errorf("copier: get: %q: %w", queue[i], err)
+					}
+				}
+
+				if err := copyFunc(name, item, info); err != nil {
 					if req.GetOptions.IgnoreUnreadable && errorIsPermission(err) {
 						continue
 					}
 					return fmt.Errorf("copier: get: %q: %w", queue[i], err)
 				}
-				itemsCopied++
 			}
 		}
 		if itemsCopied == 0 {
@@ -1400,6 +1427,31 @@ func copierHandlerGet(bulkWriter io.Writer, req request, pm *fileutils.PatternMa
 	return &response{Stat: statResponse.Stat, Get: getResponse{}}, cb, nil
 }
 
+func copyParentsDirs(name string, path string, alreadyCopied map[string]struct{}, copy func(string, string, os.FileInfo) error) (map[string]struct{}, error) {
+	parentName := filepath.Dir(name)
+	parentPath := filepath.Dir(path)
+	for parentName != "/" && parentName != "." {
+		if _, ok := alreadyCopied[parentPath]; ok {
+			parentName = filepath.Dir(parentName)
+			parentPath = filepath.Dir(parentPath)
+			continue
+		}
+
+		parentInfo, err := os.Lstat(parentPath)
+		if err != nil {
+			return alreadyCopied, fmt.Errorf("copier: get: lstat %q: %w", parentPath, err)
+		}
+		if err := copy(strings.TrimPrefix(parentName, "/"), parentPath, parentInfo); err != nil {
+			return alreadyCopied, err
+		}
+
+		alreadyCopied[parentPath] = struct{}{}
+		parentName = filepath.Dir(parentName)
+		parentPath = filepath.Dir(parentPath)
+	}
+	return alreadyCopied, nil
+}
+
 func handleRename(rename map[string]string, name string) string {
 	if rename == nil {
 		return name

docs/buildah-copy.1.md

@@ -65,6 +65,12 @@ is preserved.
 
 Path to an alternative .containerignore (.dockerignore) file. Requires \-\-contextdir be specified.
 
+**--parents**
+
+Preserve leading directories in the paths of items being copied, relative to either the
+top of the build context, or to the "pivot point", a location in the source path marked
+by a path component named "." (i.e., where "/./" occurs in the path).
+
 **--quiet**, **-q**
 
 Refrain from printing a digest of the copied content.
@@ -93,6 +99,8 @@ buildah copy containerID '/myapp/app.conf' '/myapp/app.conf'
 
 buildah copy --exclude=**/*.md docs containerID 'docs' '/docs'
 
+buildah copy --parents containerID './x/a.txt' './y/a.txt' '/parents'
+
 buildah copy --chown myuser:mygroup containerID '/myapp/app.conf' '/myapp/app.conf'
 
 buildah copy --chmod 660 containerID '/myapp/app.conf' '/myapp/app.conf'

imagebuildah/stage_executor.go

@@ -368,9 +368,6 @@ func (s *StageExecutor) Copy(excludes []string, copies ...imagebuilder.Copy) err
 		if cp.Link {
 			return errors.New("COPY --link is not supported")
 		}
-		if cp.Parents {
-			return errors.New("COPY --parents is not supported")
-		}
 		if len(cp.Excludes) > 0 {
 			excludes = append(slices.Clone(excludes), cp.Excludes...)
 		}
@@ -550,6 +547,7 @@ func (s *StageExecutor) performCopy(excludes []string, copies ...imagebuilder.Co
 		} else {
 			logrus.Debugf("COPY %#v, %#v", excludes, copy)
 		}
+		parentsPattern := map[string]string{}
 		for _, src := range copy.Src {
 			if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
 				// Source is a URL, allowed for ADD but not COPY.
@@ -560,7 +558,9 @@ func (s *StageExecutor) performCopy(excludes []string, copies ...imagebuilder.Co
 					return fmt.Errorf("source can't be a URL for COPY")
 				}
 			} else {
-				sources = append(sources, filepath.Join(contextDir, src))
+				source := filepath.Join(contextDir, src)
+				parentsPattern[source] = src
+				sources = append(sources, source)
 			}
 		}
 		options := buildah.AddAndCopyOptions{
@@ -582,6 +582,9 @@ func (s *StageExecutor) performCopy(excludes []string, copies ...imagebuilder.Co
 			MaxRetries:            s.executor.maxPullPushRetries,
 			RetryDelay:            s.executor.retryPullPushDelay,
 		}
+		if copy.Parents {
+			options.ParentsPatterns = parentsPattern
+		}
 		if len(copy.Files) > 0 {
 			// If we are copying heredoc files, we need to temporary place
 			// them in the context dir and then move to container via copier

tests/bud.bats

@@ -394,7 +394,7 @@ _EOF
   assert "$output" !~ "First symlink content"
 
   # Modify the symlink
-  ln -sf samplefile2 $contextdir/tomount 
+  ln -sf samplefile2 $contextdir/tomount
 
   # on third run since we have changed symlink so cache must burst.
   run_buildah build $WITH_POLICY_JSON --layers -t source -f $contextdir/Containerfile $contextdir
@@ -6225,6 +6225,13 @@ _EOF
   assert "$output" !~ "test2.txt"
 }
 
+@test "bud with copy --parents" {
+  run_buildah build -t test $WITH_POLICY_JSON $BUDFILES/copy-parents
+  assert "$output" !~ "./no_parents/a.txt
+  ./parents/x/a.txt
+  ./parents/y/a.txt"
+}
+
 @test "bud with containerfile secret" {
   _prefetch alpine
   mytmpdir=${TEST_SCRATCH_DIR}/my-dir1

tests/bud/copy-parents/Containerfile

@@ -0,0 +1,5 @@
+FROM alpine
+COPY ./x/a.txt ./y/a.txt /no_parents/
+COPY --parents ./x/a.txt ./y/a.txt /parents/
+run find /no_parents
+run find /parents

tests/bud/copy-parents/x/a.txt

@@ -0,0 +1 @@
+A-FILE

tests/bud/copy-parents/x/y/a.txt

@@ -0,0 +1 @@
+A-FILE

tests/bud/copy-parents/x/y/b.txt

@@ -0,0 +1 @@
+Hello

tests/bud/copy-parents/y/a.txt

@@ -0,0 +1 @@
+A-FILE

tests/bud/copy-parents/y/b.txt

@@ -0,0 +1 @@
+Hello