Add reader/writer for oci-archive multi image support

Add reader/writer with helpers to allow podman save/load multi oci-archive images.
Allow read oci-archive using source_index to point to the index from oci-archive manifest.
Also reimplement ociArchiveImage{Source,Destination} to support this.

Signed-off-by: Qi Wang <[email protected]>
Signed-off-by: Urvashi Mohnani <[email protected]>

Author: QiWang19

docs/containers-transports.5.md

@@ -60,11 +60,13 @@ The _algo:digest_ refers to the image ID reported by docker-inspect(1).
 ### **oci:**_path[:reference]_
 
 An image compliant with the "Open Container Image Layout Specification" at _path_.
-Using a _reference_ is optional and allows for storing multiple images at the same _path_.
+Using a @_source-index_ is optional and allows for storing multiple images at the same _path_.
+For reading images, @_source-index_ is a zero-based index in manifest (to access untagged images).
+If neither tag nor @_source_index is specified when reading an image, the path must contain exactly one image.
 
-### **oci-archive:**_path[:reference]_
+### **oci-archive:**_path[:{reference|@source-index}]_
 
-An image compliant with the "Open Container Image Layout Specification" stored as a tar(1) archive at _path_.
+An image compliant with the "Open Container Image Layout Specification" stored as a tar(1) archive at _path_.  For reading archives, @_source-index_ is a zero-based index in archive manifest (to access untagged images).  If neither tag nor @_source_index is specified when reading an archive, the archive must contain exactly one image.
 
 ### **ostree:**_docker-reference[@/absolute/repo/path]_
 

oci/archive/oci_dest.go

@@ -2,38 +2,53 @@ package archive
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"os"
 
+	"github.com/containers/image/v5/oci/layout"
 	"github.com/containers/image/v5/types"
 	"github.com/containers/storage/pkg/archive"
 	digest "github.com/opencontainers/go-digest"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 )
 
 type ociArchiveImageDestination struct {
-	ref          ociArchiveReference
-	unpackedDest types.ImageDestination
-	tempDirRef   tempDirOCIRef
+	ref    ociArchiveReference
+	tmpDir string
+	types.ImageDestination
 }
 
 // newImageDestination returns an ImageDestination for writing to an existing directory.
 func newImageDestination(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (types.ImageDestination, error) {
-	tempDirRef, err := createOCIRef(sys, ref.image)
+	var archive *Writer
+
+	if ref.sourceIndex != -1 {
+		return nil, fmt.Errorf("%w: destination reference must not contain a manifest index @%d", invalidOciArchiveErr, ref.sourceIndex)
+	}
+
+	if ref.archiveWriter != nil {
+		archive = ref.archiveWriter
+	} else {
+		a, err := NewWriter(ctx, sys, ref.file)
+		if err != nil {
+			return nil, err
+		}
+		archive = a
+	}
+	newref, err := layout.NewReference(archive.tempDir, ref.image)
 	if err != nil {
-		return nil, errors.Wrapf(err, "creating oci reference")
+		return nil, err
 	}
-	unpackedDest, err := tempDirRef.ociRefExtracted.NewImageDestination(ctx, sys)
+	dst, err := newref.NewImageDestination(ctx, sys)
 	if err != nil {
-		if err := tempDirRef.deleteTempDir(); err != nil {
-			return nil, errors.Wrapf(err, "deleting temp directory %q", tempDirRef.tempDirectory)
-		}
 		return nil, err
 	}
-	return &ociArchiveImageDestination{ref: ref,
-		unpackedDest: unpackedDest,
-		tempDirRef:   tempDirRef}, nil
+
+	return &ociArchiveImageDestination{
+		ImageDestination: dst,
+		tmpDir:           archive.tempDir,
+		ref:              ref,
+	}, nil
 }
 
 // Reference returns the reference used to set up this destination.
@@ -44,42 +59,41 @@ func (d *ociArchiveImageDestination) Reference() types.ImageReference {
 // Close removes resources associated with an initialized ImageDestination, if any
 // Close deletes the temp directory of the oci-archive image
 func (d *ociArchiveImageDestination) Close() error {
-	defer func() {
-		err := d.tempDirRef.deleteTempDir()
-		logrus.Debugf("Error deleting temporary directory: %v", err)
-	}()
-	return d.unpackedDest.Close()
+	if d.ref.archiveWriter != nil {
+		return nil
+	}
+	return d.ImageDestination.Close()
 }
 
 func (d *ociArchiveImageDestination) SupportedManifestMIMETypes() []string {
-	return d.unpackedDest.SupportedManifestMIMETypes()
+	return d.ImageDestination.SupportedManifestMIMETypes()
 }
 
 // SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures
 func (d *ociArchiveImageDestination) SupportsSignatures(ctx context.Context) error {
-	return d.unpackedDest.SupportsSignatures(ctx)
+	return d.ImageDestination.SupportsSignatures(ctx)
 }
 
 func (d *ociArchiveImageDestination) DesiredLayerCompression() types.LayerCompression {
-	return d.unpackedDest.DesiredLayerCompression()
+	return d.ImageDestination.DesiredLayerCompression()
 }
 
 // AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually
 // uploaded to the image destination, true otherwise.
 func (d *ociArchiveImageDestination) AcceptsForeignLayerURLs() bool {
-	return d.unpackedDest.AcceptsForeignLayerURLs()
+	return d.ImageDestination.AcceptsForeignLayerURLs()
 }
 
 // MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime architecture and OS. False otherwise
 func (d *ociArchiveImageDestination) MustMatchRuntimeOS() bool {
-	return d.unpackedDest.MustMatchRuntimeOS()
+	return d.ImageDestination.MustMatchRuntimeOS()
 }
 
 // IgnoresEmbeddedDockerReference returns true iff the destination does not care about Image.EmbeddedDockerReferenceConflicts(),
 // and would prefer to receive an unmodified manifest instead of one modified for the destination.
 // Does not make a difference if Reference().DockerReference() is nil.
 func (d *ociArchiveImageDestination) IgnoresEmbeddedDockerReference() bool {
-	return d.unpackedDest.IgnoresEmbeddedDockerReference()
+	return d.ImageDestination.IgnoresEmbeddedDockerReference()
 }
 
 // HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently.
@@ -96,7 +110,7 @@ func (d *ociArchiveImageDestination) HasThreadSafePutBlob() bool {
 // to any other readers for download using the supplied digest.
 // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far.
 func (d *ociArchiveImageDestination) PutBlob(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, cache types.BlobInfoCache, isConfig bool) (types.BlobInfo, error) {
-	return d.unpackedDest.PutBlob(ctx, stream, inputInfo, cache, isConfig)
+	return d.ImageDestination.PutBlob(ctx, stream, inputInfo, cache, isConfig)
 }
 
 // TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination
@@ -109,7 +123,7 @@ func (d *ociArchiveImageDestination) PutBlob(ctx context.Context, stream io.Read
 // If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure.
 // May use and/or update cache.
 func (d *ociArchiveImageDestination) TryReusingBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache, canSubstitute bool) (bool, types.BlobInfo, error) {
-	return d.unpackedDest.TryReusingBlob(ctx, info, cache, canSubstitute)
+	return d.ImageDestination.TryReusingBlob(ctx, info, cache, canSubstitute)
 }
 
 // PutManifest writes the manifest to the destination.
@@ -118,14 +132,14 @@ func (d *ociArchiveImageDestination) TryReusingBlob(ctx context.Context, info ty
 // It is expected but not enforced that the instanceDigest, when specified, matches the digest of `manifest` as generated
 // by `manifest.Digest()`.
 func (d *ociArchiveImageDestination) PutManifest(ctx context.Context, m []byte, instanceDigest *digest.Digest) error {
-	return d.unpackedDest.PutManifest(ctx, m, instanceDigest)
+	return d.ImageDestination.PutManifest(ctx, m, instanceDigest)
 }
 
 // PutSignatures writes a set of signatures to the destination.
 // If instanceDigest is not nil, it contains a digest of the specific manifest instance to write or overwrite the signatures for
 // (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list.
 func (d *ociArchiveImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error {
-	return d.unpackedDest.PutSignatures(ctx, signatures, instanceDigest)
+	return d.ImageDestination.PutSignatures(ctx, signatures, instanceDigest)
 }
 
 // Commit marks the process of storing the image as successful and asks for the image to be persisted
@@ -134,12 +148,16 @@ func (d *ociArchiveImageDestination) PutSignatures(ctx context.Context, signatur
 // original manifest list digest, if desired.
 // after the directory is made, it is tarred up into a file and the directory is deleted
 func (d *ociArchiveImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error {
-	if err := d.unpackedDest.Commit(ctx, unparsedToplevel); err != nil {
-		return errors.Wrapf(err, "storing image %q", d.ref.image)
+	if err := d.ImageDestination.Commit(ctx, unparsedToplevel); err != nil {
+		return fmt.Errorf("%w: storing image %q", err, d.ref.image)
+	}
+
+	if d.ref.archiveWriter != nil {
+		return nil
 	}
 
 	// path of directory to tar up
-	src := d.tempDirRef.tempDirectory
+	src := d.tmpDir
 	// path to save tarred up file
 	dst := d.ref.resolvedFile
 	return tarDirectory(src, dst)
@@ -150,13 +168,13 @@ func tarDirectory(src, dst string) error {
 	// input is a stream of bytes from the archive of the directory at path
 	input, err := archive.Tar(src, archive.Uncompressed)
 	if err != nil {
-		return errors.Wrapf(err, "retrieving stream of bytes from %q", src)
+		return fmt.Errorf("%w: retrieving stream of bytes from %q", err, src)
 	}
 
 	// creates the tar file
 	outFile, err := os.Create(dst)
 	if err != nil {
-		return errors.Wrapf(err, "creating tar file %q", dst)
+		return fmt.Errorf("%w: creating tar file %q", err, dst)
 	}
 	defer outFile.Close()
 

oci/archive/oci_src.go

@@ -2,40 +2,51 @@ package archive
 
 import (
 	"context"
+	"fmt"
 	"io"
 
+	"github.com/containers/image/v5/oci/layout"
 	ocilayout "github.com/containers/image/v5/oci/layout"
 	"github.com/containers/image/v5/types"
 	digest "github.com/opencontainers/go-digest"
 	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
 type ociArchiveImageSource struct {
-	ref         ociArchiveReference
-	unpackedSrc types.ImageSource
-	tempDirRef  tempDirOCIRef
+	ref ociArchiveReference
+	types.ImageSource
 }
 
 // newImageSource returns an ImageSource for reading from an existing directory.
-// newImageSource untars the file and saves it in a temp directory
 func newImageSource(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (types.ImageSource, error) {
-	tempDirRef, err := createUntarTempDir(sys, ref)
-	if err != nil {
-		return nil, errors.Wrap(err, "creating temp directory")
+	var archive *Reader
+
+	if ref.archiveReader != nil {
+		archive = ref.archiveReader
+		ref.closeArchive = false
+	} else {
+		a, err := NewReader(ctx, sys, ref)
+		if err != nil {
+			return nil, err
+		}
+		archive = a
+		ref.closeArchive = true
 	}
 
-	unpackedSrc, err := tempDirRef.ociRefExtracted.NewImageSource(ctx, sys)
+	newref, err := layout.NewReference(archive.tempDirectory, ref.image)
+	if err != nil {
+		return nil, err
+	}
+	src, err := newref.NewImageSource(ctx, sys)
 	if err != nil {
-		if err := tempDirRef.deleteTempDir(); err != nil {
-			return nil, errors.Wrapf(err, "deleting temp directory %q", tempDirRef.tempDirectory)
-		}
 		return nil, err
 	}
-	return &ociArchiveImageSource{ref: ref,
-		unpackedSrc: unpackedSrc,
-		tempDirRef:  tempDirRef}, nil
+
+	return &ociArchiveImageSource{
+		ImageSource: src,
+		ref:         ref,
+	}, nil
 }
 
 // LoadManifestDescriptor loads the manifest
@@ -48,11 +59,11 @@ func LoadManifestDescriptor(imgRef types.ImageReference) (imgspecv1.Descriptor,
 func LoadManifestDescriptorWithContext(sys *types.SystemContext, imgRef types.ImageReference) (imgspecv1.Descriptor, error) {
 	ociArchRef, ok := imgRef.(ociArchiveReference)
 	if !ok {
-		return imgspecv1.Descriptor{}, errors.Errorf("error typecasting, need type ociArchiveReference")
+		return imgspecv1.Descriptor{}, fmt.Errorf("error typecasting, need type ociArchiveReference")
 	}
 	tempDirRef, err := createUntarTempDir(sys, ociArchRef)
 	if err != nil {
-		return imgspecv1.Descriptor{}, errors.Wrap(err, "creating temp directory")
+		return imgspecv1.Descriptor{}, fmt.Errorf("%w: creating temp directory", err)
 	}
 	defer func() {
 		err := tempDirRef.deleteTempDir()
@@ -61,7 +72,7 @@ func LoadManifestDescriptorWithContext(sys *types.SystemContext, imgRef types.Im
 
 	descriptor, err := ocilayout.LoadManifestDescriptor(tempDirRef.ociRefExtracted)
 	if err != nil {
-		return imgspecv1.Descriptor{}, errors.Wrap(err, "loading index")
+		return imgspecv1.Descriptor{}, fmt.Errorf("%w: loading index", err)
 	}
 	return descriptor, nil
 }
@@ -72,21 +83,22 @@ func (s *ociArchiveImageSource) Reference() types.ImageReference {
 }
 
 // Close removes resources associated with an initialized ImageSource, if any.
-// Close deletes the temporary directory at dst
 func (s *ociArchiveImageSource) Close() error {
-	defer func() {
-		err := s.tempDirRef.deleteTempDir()
-		logrus.Debugf("error deleting tmp dir: %v", err)
-	}()
-	return s.unpackedSrc.Close()
+	if s.ref.archiveReader != nil {
+		return nil
+	}
+	if s.ref.closeArchive {
+		return s.ImageSource.Close()
+	}
+	return nil
 }
 
 // GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available).
 // It may use a remote (= slow) service.
 // If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list);
 // this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists).
 func (s *ociArchiveImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) {
-	return s.unpackedSrc.GetManifest(ctx, instanceDigest)
+	return s.ImageSource.GetManifest(ctx, instanceDigest)
 }
 
 // HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently.
@@ -98,15 +110,15 @@ func (s *ociArchiveImageSource) HasThreadSafeGetBlob() bool {
 // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided.
 // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location.
 func (s *ociArchiveImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) {
-	return s.unpackedSrc.GetBlob(ctx, info, cache)
+	return s.ImageSource.GetBlob(ctx, info, cache)
 }
 
 // GetSignatures returns the image's signatures.  It may use a remote (= slow) service.
 // If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for
 // (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list
 // (e.g. if the source never returns manifest lists).
 func (s *ociArchiveImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) {
-	return s.unpackedSrc.GetSignatures(ctx, instanceDigest)
+	return s.ImageSource.GetSignatures(ctx, instanceDigest)
 }
 
 // LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer
@@ -118,5 +130,5 @@ func (s *ociArchiveImageSource) GetSignatures(ctx context.Context, instanceDiges
 // The Digest field is guaranteed to be provided; Size may be -1.
 // WARNING: The list may contain duplicates, and they are semantically relevant.
 func (s *ociArchiveImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) {
-	return s.unpackedSrc.LayerInfosForCopy(ctx, instanceDigest)
+	return s.ImageSource.LayerInfosForCopy(ctx, instanceDigest)
 }