diff --git a/Microsoft.NET.Build.Containers.IntegrationTests/EndToEndTests.cs b/Microsoft.NET.Build.Containers.IntegrationTests/EndToEndTests.cs
index 64ace65c..59ddfe68 100644
--- a/Microsoft.NET.Build.Containers.IntegrationTests/EndToEndTests.cs
+++ b/Microsoft.NET.Build.Containers.IntegrationTests/EndToEndTests.cs
@@ -39,25 +39,27 @@ public async Task ApiEndToEndWithRegistryPushAndPull()
 
         Registry registry = new Registry(ContainerHelpers.TryExpandRegistryToUri(DockerRegistryManager.LocalRegistry));
 
-        Image? x = await registry.GetImageManifest(
+        ImageBuilder imageBuilder = await registry.GetImageManifest(
             DockerRegistryManager.BaseImage,
             DockerRegistryManager.Net6ImageTag,
             "linux-x64",
             ToolsetUtils.GetRuntimeGraphFilePath()).ConfigureAwait(false);
 
-        Assert.NotNull(x);
+        Assert.NotNull(imageBuilder);
 
         Layer l = Layer.FromDirectory(publishDirectory, "/app");
 
-        x.AddLayer(l);
+        imageBuilder.AddLayer(l);
 
-        x.SetEntrypoint(new[] { "/app/MinimalTestApp" });
+        imageBuilder.SetEntryPoint(new[] { "/app/MinimalTestApp" });
+
+        BuiltImage builtImage = imageBuilder.Build();
 
         // Push the image back to the local registry
         var sourceReference = new ImageReference(registry, DockerRegistryManager.BaseImage, DockerRegistryManager.Net6ImageTag);
         var destinationReference = new ImageReference(registry, NewImageName(), "latest");
 
-        await registry.Push(x, sourceReference, destinationReference, Console.WriteLine).ConfigureAwait(false);
+        await registry.Push(builtImage, sourceReference, destinationReference, Console.WriteLine).ConfigureAwait(false);
 
         // pull it back locally
         new BasicCommand(_testOutput, "docker", "pull", $"{DockerRegistryManager.LocalRegistry}/{NewImageName()}:latest")
@@ -79,24 +81,26 @@ public async Task ApiEndToEndWithLocalLoad()
 
         Registry registry = new Registry(ContainerHelpers.TryExpandRegistryToUri(DockerRegistryManager.LocalRegistry));
 
-        Image? x = await registry.GetImageManifest(
+        ImageBuilder imageBuilder = await registry.GetImageManifest(
             DockerRegistryManager.BaseImage,
             DockerRegistryManager.Net6ImageTag,
             "linux-x64",
             ToolsetUtils.GetRuntimeGraphFilePath()).ConfigureAwait(false);
-        Assert.NotNull(x);
+        Assert.NotNull(imageBuilder);
 
         Layer l = Layer.FromDirectory(publishDirectory, "/app");
 
-        x.AddLayer(l);
+        imageBuilder.AddLayer(l);
+
+        imageBuilder.SetEntryPoint(new[] { "/app/MinimalTestApp" });
 
-        x.SetEntrypoint(new[] { "/app/MinimalTestApp" });
+        BuiltImage builtImage = imageBuilder.Build();
 
         // Load the image into the local Docker daemon
         var sourceReference = new ImageReference(registry, DockerRegistryManager.BaseImage, DockerRegistryManager.Net6ImageTag);
         var destinationReference = new ImageReference(registry, NewImageName(), "latest");
 
-        await new LocalDocker(Console.WriteLine).Load(x, sourceReference, destinationReference).ConfigureAwait(false);
+        await new LocalDocker(Console.WriteLine).Load(builtImage, sourceReference, destinationReference).ConfigureAwait(false);
 
         // Run the image
         new BasicCommand(_testOutput, "docker", "run", "--rm", "--tty", $"{NewImageName()}:latest")
@@ -289,21 +293,23 @@ public async Task CanPackageForAllSupportedContainerRIDs(string rid, bool isRIDS
         // Build the image
         Registry registry = new Registry(ContainerHelpers.TryExpandRegistryToUri(DockerRegistryManager.BaseImageSource));
 
-        Image? x = await registry.GetImageManifest(DockerRegistryManager.BaseImage, DockerRegistryManager.Net7ImageTag, rid, ToolsetUtils.GetRuntimeGraphFilePath()).ConfigureAwait(false);
-        Assert.NotNull(x);
+        ImageBuilder? imageBuilder = await registry.GetImageManifest(DockerRegistryManager.BaseImage, DockerRegistryManager.Net7ImageTag, rid, ToolsetUtils.GetRuntimeGraphFilePath()).ConfigureAwait(false);
+        Assert.NotNull(imageBuilder);
 
         Layer l = Layer.FromDirectory(publishDirectory, "/app");
 
-        x.AddLayer(l);
-        x.WorkingDirectory = workingDir;
+        imageBuilder.AddLayer(l);
+        imageBuilder.SetWorkingDirectory(workingDir);
+
+        string[] entryPoint = DecideEntrypoint(rid, isRIDSpecific, "MinimalTestApp", workingDir);
+        imageBuilder.SetEntryPoint(entryPoint);
 
-        var entryPoint = DecideEntrypoint(rid, isRIDSpecific, "MinimalTestApp", workingDir);
-        x.SetEntrypoint(entryPoint);
+        BuiltImage builtImage = imageBuilder.Build();
 
         // Load the image into the local Docker daemon
         var sourceReference = new ImageReference(registry, DockerRegistryManager.BaseImage, DockerRegistryManager.Net7ImageTag);
         var destinationReference = new ImageReference(registry, NewImageName(), rid);
-        await new LocalDocker(Console.WriteLine).Load(x, sourceReference, destinationReference).ConfigureAwait(false);
+        await new LocalDocker(Console.WriteLine).Load(builtImage, sourceReference, destinationReference).ConfigureAwait(false);
 
         // Run the image
         new BasicCommand(
diff --git a/Microsoft.NET.Build.Containers.IntegrationTests/RegistryTests.cs b/Microsoft.NET.Build.Containers.IntegrationTests/RegistryTests.cs
index f9ee048f..9469f9cc 100644
--- a/Microsoft.NET.Build.Containers.IntegrationTests/RegistryTests.cs
+++ b/Microsoft.NET.Build.Containers.IntegrationTests/RegistryTests.cs
@@ -16,7 +16,7 @@ public async Task GetFromRegistry()
 
         // Don't need rid graph for local registry image pulls - since we're only pushing single image manifests (not manifest lists)
         // as part of our setup, we could put literally anything in here. The file at the passed-in path would only get read when parsing manifests lists.
-        Image? downloadedImage = await registry.GetImageManifest(DockerRegistryManager.BaseImage, DockerRegistryManager.Net6ImageTag, "linux-x64", ridgraphfile).ConfigureAwait(false);
+        ImageBuilder? downloadedImage = await registry.GetImageManifest(DockerRegistryManager.BaseImage, DockerRegistryManager.Net6ImageTag, "linux-x64", ridgraphfile).ConfigureAwait(false);
 
         Assert.NotNull(downloadedImage);
     }
diff --git a/Microsoft.NET.Build.Containers.UnitTests/ImageBuilderTests.cs b/Microsoft.NET.Build.Containers.UnitTests/ImageBuilderTests.cs
new file mode 100644
index 00000000..d25acf54
--- /dev/null
+++ b/Microsoft.NET.Build.Containers.UnitTests/ImageBuilderTests.cs
@@ -0,0 +1,294 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Text.Json.Nodes;
+using Microsoft.NET.Build.Containers;
+using Xunit;
+
+namespace Test.Microsoft.NET.Build.Containers;
+
+public class ImageBuilderTests
+{
+    [Fact]
+    public void CanAddLabelsToImage()
+    {
+        string simpleImageConfig =
+            """
+                {
+                    "architecture": "amd64",
+                    "config": {
+                      "Hostname": "",
+                      "Domainname": "",
+                      "User": "",
+                      "AttachStdin": false,
+                      "AttachStdout": false,
+                      "AttachStderr": false,
+                      "Tty": false,
+                      "OpenStdin": false,
+                      "StdinOnce": false,
+                      "Env": [
+                        "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+                        "ASPNETCORE_URLS=http://+:80",
+                        "DOTNET_RUNNING_IN_CONTAINER=true",
+                        "DOTNET_VERSION=7.0.2",
+                        "ASPNET_VERSION=7.0.2"
+                      ],
+                      "Cmd": ["bash"],
+                      "Image": "sha256:d772d27ebeec80393349a4770dc37f977be2c776a01c88b624d43f93fa369d69",
+                      "Volumes": null,
+                      "WorkingDir": "",
+                      "Entrypoint": null,
+                      "OnBuild": null,
+                      "Labels": null
+                    },
+                    "created": "2023-02-04T08:14:52.000901321Z",
+                    "os": "linux",
+                    "rootfs": {
+                      "type": "layers",
+                      "diff_ids": [
+                        "sha256:bd2fe8b74db65d82ea10db97368d35b92998d4ea0e7e7dc819481fe4a68f64cf",
+                        "sha256:94100d1041b650c6f7d7848c550cd98c25d0bdc193d30692e5ea5474d7b3b085",
+                        "sha256:53c2a75a33c8f971b4b5036d34764373e134f91ee01d8053b4c3573c42e1cf5d",
+                        "sha256:49a61320e585180286535a2545be5722b09e40ad44c7c190b20ec96c9e42e4a3",
+                        "sha256:8a379cce2ac272aa71aa029a7bbba85c852ba81711d9f90afaefd3bf5036dc48"
+                      ]
+                    }
+                }
+                """;
+
+        JsonNode? node = JsonNode.Parse(simpleImageConfig);
+        Assert.NotNull(node);
+
+        ImageConfig baseConfig = new ImageConfig(node);
+
+        baseConfig.AddLabel("testLabel1", "v1");
+        baseConfig.AddLabel("testLabel2", "v2");
+
+        string readyImage = baseConfig.BuildConfig();
+
+        JsonNode? result = JsonNode.Parse(readyImage);
+
+        var resultLabels = result?["config"]?["Labels"] as JsonObject;
+        Assert.NotNull(resultLabels);
+
+        Assert.Equal(2, resultLabels.Count);
+        Assert.Equal("v1", resultLabels["testLabel1"]?.ToString());
+        Assert.Equal("v2", resultLabels["testLabel2"]?.ToString());
+    }
+
+    [Fact]
+    public void CanPreserveExistingLabels()
+    {
+        string simpleImageConfig =
+            """
+                {
+                    "architecture": "amd64",
+                    "config": {
+                      "Hostname": "",
+                      "Domainname": "",
+                      "User": "",
+                      "AttachStdin": false,
+                      "AttachStdout": false,
+                      "AttachStderr": false,
+                      "Tty": false,
+                      "OpenStdin": false,
+                      "StdinOnce": false,
+                      "Env": [
+                        "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+                        "ASPNETCORE_URLS=http://+:80",
+                        "DOTNET_RUNNING_IN_CONTAINER=true",
+                        "DOTNET_VERSION=7.0.2",
+                        "ASPNET_VERSION=7.0.2"
+                      ],
+                      "Cmd": ["bash"],
+                      "Image": "sha256:d772d27ebeec80393349a4770dc37f977be2c776a01c88b624d43f93fa369d69",
+                      "Volumes": null,
+                      "WorkingDir": "",
+                      "Entrypoint": null,
+                      "OnBuild": null,
+                      "Labels":
+                      {
+                        "existing" : "e1",
+                        "existing2" : "e2"
+                      }
+                    },
+                    "created": "2023-02-04T08:14:52.000901321Z",
+                    "os": "linux",
+                    "rootfs": {
+                      "type": "layers",
+                      "diff_ids": [
+                        "sha256:bd2fe8b74db65d82ea10db97368d35b92998d4ea0e7e7dc819481fe4a68f64cf",
+                        "sha256:94100d1041b650c6f7d7848c550cd98c25d0bdc193d30692e5ea5474d7b3b085",
+                        "sha256:53c2a75a33c8f971b4b5036d34764373e134f91ee01d8053b4c3573c42e1cf5d",
+                        "sha256:49a61320e585180286535a2545be5722b09e40ad44c7c190b20ec96c9e42e4a3",
+                        "sha256:8a379cce2ac272aa71aa029a7bbba85c852ba81711d9f90afaefd3bf5036dc48"
+                      ]
+                    }
+                }
+                """;
+
+        JsonNode? node = JsonNode.Parse(simpleImageConfig);
+        Assert.NotNull(node);
+
+        ImageConfig baseConfig = new ImageConfig(node);
+
+        baseConfig.AddLabel("testLabel1", "v1");
+        baseConfig.AddLabel("existing2", "v2");
+
+        string readyImage = baseConfig.BuildConfig();
+
+        JsonNode? result = JsonNode.Parse(readyImage);
+
+        var resultLabels = result?["config"]?["Labels"] as JsonObject;
+        Assert.NotNull(resultLabels);
+
+        Assert.Equal(3, resultLabels.Count);
+        Assert.Equal("v1", resultLabels["testLabel1"]?.ToString());
+        Assert.Equal("v2", resultLabels["existing2"]?.ToString());
+        Assert.Equal("e1", resultLabels["existing"]?.ToString());
+    }
+
+    [Fact]
+    public void CanAddPortsToImage()
+    {
+        string simpleImageConfig =
+            """
+                {
+                    "architecture": "amd64",
+                    "config": {
+                      "Hostname": "",
+                      "Domainname": "",
+                      "User": "",
+                      "AttachStdin": false,
+                      "AttachStdout": false,
+                      "AttachStderr": false,
+                      "Tty": false,
+                      "OpenStdin": false,
+                      "StdinOnce": false,
+                      "Env": [
+                        "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+                        "ASPNETCORE_URLS=http://+:80",
+                        "DOTNET_RUNNING_IN_CONTAINER=true",
+                        "DOTNET_VERSION=7.0.2",
+                        "ASPNET_VERSION=7.0.2"
+                      ],
+                      "Cmd": ["bash"],
+                      "Image": "sha256:d772d27ebeec80393349a4770dc37f977be2c776a01c88b624d43f93fa369d69",
+                      "Volumes": null,
+                      "WorkingDir": "",
+                      "Entrypoint": null,
+                      "OnBuild": null,
+                      "Labels": null
+                    },
+                    "created": "2023-02-04T08:14:52.000901321Z",
+                    "os": "linux",
+                    "rootfs": {
+                      "type": "layers",
+                      "diff_ids": [
+                        "sha256:bd2fe8b74db65d82ea10db97368d35b92998d4ea0e7e7dc819481fe4a68f64cf",
+                        "sha256:94100d1041b650c6f7d7848c550cd98c25d0bdc193d30692e5ea5474d7b3b085",
+                        "sha256:53c2a75a33c8f971b4b5036d34764373e134f91ee01d8053b4c3573c42e1cf5d",
+                        "sha256:49a61320e585180286535a2545be5722b09e40ad44c7c190b20ec96c9e42e4a3",
+                        "sha256:8a379cce2ac272aa71aa029a7bbba85c852ba81711d9f90afaefd3bf5036dc48"
+                      ]
+                    }
+                }
+                """;
+
+        JsonNode? node = JsonNode.Parse(simpleImageConfig);
+        Assert.NotNull(node);
+
+        ImageConfig baseConfig = new ImageConfig(node);
+
+        baseConfig.ExposePort(6000, PortType.tcp);
+        baseConfig.ExposePort(6010, PortType.udp);
+
+        string readyImage = baseConfig.BuildConfig();
+
+        JsonNode? result = JsonNode.Parse(readyImage);
+
+        var resultPorts = result?["config"]?["ExposedPorts"] as JsonObject;
+        Assert.NotNull(resultPorts);
+
+        Assert.Equal(2, resultPorts.Count);
+        Assert.NotNull(resultPorts["6000/tcp"] as JsonObject);
+        Assert.NotNull( resultPorts["6010/udp"] as JsonObject);
+    }
+
+    [Fact]
+    public void CanPreserveExistingPorts()
+    {
+        string simpleImageConfig =
+            """
+                {
+                    "architecture": "amd64",
+                    "config": {
+                      "Hostname": "",
+                      "Domainname": "",
+                      "User": "",
+                      "AttachStdin": false,
+                      "AttachStdout": false,
+                      "AttachStderr": false,
+                      "Tty": false,
+                      "OpenStdin": false,
+                      "StdinOnce": false,
+                      "Env": [
+                        "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+                        "ASPNETCORE_URLS=http://+:80",
+                        "DOTNET_RUNNING_IN_CONTAINER=true",
+                        "DOTNET_VERSION=7.0.2",
+                        "ASPNET_VERSION=7.0.2"
+                      ],
+                      "Cmd": ["bash"],
+                      "Image": "sha256:d772d27ebeec80393349a4770dc37f977be2c776a01c88b624d43f93fa369d69",
+                      "Volumes": null,
+                      "WorkingDir": "",
+                      "Entrypoint": null,
+                      "OnBuild": null,
+                      "Labels": null,
+                      "ExposedPorts":
+                      {
+                        "6100/tcp": {},
+                        "6200": {}
+                      }
+                    },
+                    "created": "2023-02-04T08:14:52.000901321Z",
+                    "os": "linux",
+                    "rootfs": {
+                      "type": "layers",
+                      "diff_ids": [
+                        "sha256:bd2fe8b74db65d82ea10db97368d35b92998d4ea0e7e7dc819481fe4a68f64cf",
+                        "sha256:94100d1041b650c6f7d7848c550cd98c25d0bdc193d30692e5ea5474d7b3b085",
+                        "sha256:53c2a75a33c8f971b4b5036d34764373e134f91ee01d8053b4c3573c42e1cf5d",
+                        "sha256:49a61320e585180286535a2545be5722b09e40ad44c7c190b20ec96c9e42e4a3",
+                        "sha256:8a379cce2ac272aa71aa029a7bbba85c852ba81711d9f90afaefd3bf5036dc48"
+                      ]
+                    }
+                }
+                """;
+
+        JsonNode? node = JsonNode.Parse(simpleImageConfig);
+        Assert.NotNull(node);
+
+        ImageConfig baseConfig = new ImageConfig(node);
+
+        baseConfig.ExposePort(6000, PortType.tcp);
+        baseConfig.ExposePort(6010, PortType.udp);
+        baseConfig.ExposePort(6100, PortType.udp);
+        baseConfig.ExposePort(6200, PortType.tcp);
+
+        string readyImage = baseConfig.BuildConfig();
+
+        JsonNode? result = JsonNode.Parse(readyImage);
+
+        var resultPorts = result?["config"]?["ExposedPorts"] as JsonObject;
+        Assert.NotNull(resultPorts);
+
+        Assert.Equal(5, resultPorts.Count);
+        Assert.NotNull(resultPorts["6000/tcp"] as JsonObject);
+        Assert.NotNull(resultPorts["6010/udp"] as JsonObject);
+        Assert.NotNull(resultPorts["6100/udp"] as JsonObject);
+        Assert.NotNull(resultPorts["6100/tcp"] as JsonObject);
+        Assert.NotNull(resultPorts["6200/tcp"] as JsonObject);
+    }
+}
diff --git a/Microsoft.NET.Build.Containers/BuiltImage.cs b/Microsoft.NET.Build.Containers/BuiltImage.cs
new file mode 100644
index 00000000..b3bb668f
--- /dev/null
+++ b/Microsoft.NET.Build.Containers/BuiltImage.cs
@@ -0,0 +1,50 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Microsoft.NET.Build.Containers;
+
+/// <summary>
+/// Represents constructed image ready for further processing.
+/// </summary>
+internal readonly struct BuiltImage
+{
+    /// <summary>
+    /// Gets image configuration in JSON format.
+    /// </summary>
+    internal required string Config { get; init; }
+
+    /// <summary>
+    /// Gets image digest.
+    /// </summary>
+    internal required string ImageDigest { get; init; }
+
+    /// <summary>
+    /// Gets image SHA.
+    /// </summary>
+    internal required string ImageSha { get; init; }
+
+    /// <summary>
+    /// Gets image size.
+    /// </summary>
+    internal required long ImageSize { get; init; }
+
+    /// <summary>
+    /// Gets image manifest.
+    /// </summary>
+    internal required ManifestV2 Manifest { get; init; }
+
+    /// <summary>
+    /// Gets layers descriptors.
+    /// </summary>
+    internal IEnumerable<Descriptor> LayerDescriptors
+    {
+        get
+        {
+            List<ManifestLayer> layersNode = Manifest.Layers ?? throw new NotImplementedException("Tried to get layer information but there is no layer node?");
+            foreach (ManifestLayer layer in layersNode)
+            {
+                yield return new(layer.mediaType, layer.digest, layer.size);
+            }
+        }
+    }
+}
diff --git a/Microsoft.NET.Build.Containers/ContainerBuilder.cs b/Microsoft.NET.Build.Containers/ContainerBuilder.cs
index a8b21058..763e0d61 100644
--- a/Microsoft.NET.Build.Containers/ContainerBuilder.cs
+++ b/Microsoft.NET.Build.Containers/ContainerBuilder.cs
@@ -20,9 +20,9 @@ public static async Task Containerize(DirectoryInfo folder, string workingDir, s
         var isDockerPush = String.IsNullOrEmpty(outputRegistry);
         var destinationImageReferences = imageTags.Select(t => new ImageReference(isDockerPush ? null : new Registry(ContainerHelpers.TryExpandRegistryToUri(outputRegistry!)), imageName, t));
 
-        var img = await baseRegistry.GetImageManifest(baseName, baseTag, containerRuntimeIdentifier, ridGraphPath).ConfigureAwait(false);
+        ImageBuilder imageBuilder = await baseRegistry.GetImageManifest(baseName, baseTag, containerRuntimeIdentifier, ridGraphPath).ConfigureAwait(false);
 
-        img.WorkingDirectory = workingDir;
+        imageBuilder.SetWorkingDirectory(workingDir);
 
         JsonSerializerOptions options = new()
         {
@@ -31,38 +31,40 @@ public static async Task Containerize(DirectoryInfo folder, string workingDir, s
 
         Layer l = Layer.FromDirectory(folder.FullName, workingDir);
 
-        img.AddLayer(l);
+        imageBuilder.AddLayer(l);
 
-        img.SetEntrypoint(entrypoint, entrypointArgs);
+        imageBuilder.SetEntryPoint(entrypoint, entrypointArgs);
 
-        foreach (var label in labels)
+        foreach (string label in labels)
         {
             string[] labelPieces = label.Split('=');
 
             // labels are validated by System.CommandLine API
-            img.Label(labelPieces[0], labelPieces[1]);
+            imageBuilder.AddLabel(labelPieces[0], labelPieces[1]);
         }
 
         foreach (string envVar in envVars)
         {
             string[] envPieces = envVar.Split('=', 2);
 
-            img.AddEnvironmentVariable(envPieces[0], envPieces[1]);
+            imageBuilder.AddEnvironmentVariable(envPieces[0], envPieces[1]);
         }
 
-        foreach (var (number, type) in exposedPorts)
+        foreach ((int number, PortType type) in exposedPorts)
         {
             // ports are validated by System.CommandLine API
-            img.ExposePort(number, type);
+            imageBuilder.ExposePort(number, type);
         }
 
+        BuiltImage builtImage = imageBuilder.Build();
+
         foreach (var destinationImageReference in destinationImageReferences)
         {
             if (destinationImageReference.Registry is { } outReg)
             {
                 try
                 {
-                    outReg.Push(img, sourceImageReference, destinationImageReference, (message) => Console.WriteLine($"Containerize: {message}")).Wait();
+                    outReg.Push(builtImage, sourceImageReference, destinationImageReference, (message) => Console.WriteLine($"Containerize: {message}")).Wait();
                     Console.WriteLine($"Containerize: Pushed container '{destinationImageReference.RepositoryAndTag}' to registry '{outputRegistry}'");
                 }
                 catch (Exception e)
@@ -83,7 +85,7 @@ public static async Task Containerize(DirectoryInfo folder, string workingDir, s
                 }
                 try
                 {
-                    localDaemon.Load(img, sourceImageReference, destinationImageReference).Wait();
+                    localDaemon.Load(builtImage, sourceImageReference, destinationImageReference).Wait();
                     Console.WriteLine("Containerize: Pushed container '{0}' to Docker daemon", destinationImageReference.RepositoryAndTag);
                 }
                 catch (Exception e)
diff --git a/Microsoft.NET.Build.Containers/DigestUtils.cs b/Microsoft.NET.Build.Containers/DigestUtils.cs
new file mode 100644
index 00000000..a41b9a6f
--- /dev/null
+++ b/Microsoft.NET.Build.Containers/DigestUtils.cs
@@ -0,0 +1,31 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Security.Cryptography;
+using System.Text;
+
+namespace Microsoft.NET.Build.Containers;
+
+internal sealed class DigestUtils
+{
+    /// <summary>
+    /// Gets digest for string <paramref name="str"/>.
+    /// </summary>
+    internal static string GetDigest(string str) => GetDigestFromSha(GetSha(str));
+
+    /// <summary>
+    /// Formats digest based on ready SHA <paramref name="sha"/>.
+    /// </summary>
+    internal static string GetDigestFromSha(string sha) => $"sha256:{sha}";
+
+    /// <summary>
+    /// Gets the SHA of <paramref name="str"/>.
+    /// </summary>
+    internal static string GetSha(string str)
+    {
+        Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
+        SHA256.HashData(Encoding.UTF8.GetBytes(str), hash);
+
+        return Convert.ToHexString(hash).ToLowerInvariant();
+    }
+}
diff --git a/Microsoft.NET.Build.Containers/Image.cs b/Microsoft.NET.Build.Containers/Image.cs
deleted file mode 100644
index f27350be..00000000
--- a/Microsoft.NET.Build.Containers/Image.cs
+++ /dev/null
@@ -1,266 +0,0 @@
-// Licensed to the .NET Foundation under one or more agreements.
-// The .NET Foundation licenses this file to you under the MIT license.
-
-using System.Security.Cryptography;
-using System.Text;
-using System.Text.Json;
-using System.Text.Json.Nodes;
-
-namespace Microsoft.NET.Build.Containers;
-
-internal class Image
-{
-    private readonly HashSet<Label> _labels;
-
-    private readonly List<Layer> _newLayers = new();
-
-    private readonly HashSet<Port> _exposedPorts;
-
-    private readonly Dictionary<string, string> _environmentVariables;
-
-    public Image(ManifestV2 manifest, JsonNode config)
-    {
-        Manifest = manifest;
-        Config = config;
-        // these next values are inherited from the parent image, so we need to seed our new image with them.
-        _labels = ReadLabelsFromConfig(config);
-        _exposedPorts = ReadPortsFromConfig(config);
-        _environmentVariables = ReadEnvVarsFromConfig(config);
-    }
-
-    public ManifestV2 Manifest { get; private set; }
-
-    public JsonNode Config { get; }
-
-    public IEnumerable<Descriptor> LayerDescriptors
-    {
-        get
-        {
-            var layersNode = Manifest.layers;
-
-            if (layersNode is null)
-            {
-                throw new NotImplementedException("Tried to get layer information but there is no layer node?");
-            }
-
-            foreach (var layer in layersNode)
-            {
-                yield return new(layer.mediaType, layer.digest, layer.size);
-            }
-        }
-    }
-
-    public void AddLayer(Layer l)
-    {
-        _newLayers.Add(l);
-
-        Manifest.layers.Add(new(l.Descriptor.MediaType, l.Descriptor.Size, l.Descriptor.Digest, l.Descriptor.Urls));
-        Config["rootfs"]!["diff_ids"]!.AsArray().Add(l.Descriptor.UncompressedDigest);
-        RecalculateDigest();
-    }
-
-    private void RecalculateDigest()
-    {
-        Config["created"] = DateTime.UtcNow;
-        ManifestConfig newManifestConfig = Manifest.config with
-        {
-            digest = GetDigest(Config),
-            size = Encoding.UTF8.GetBytes(Config.ToJsonString()).Length
-        };
-
-        ManifestV2 currentManifest = Manifest;
-        currentManifest.config = newManifestConfig;
-        Manifest = currentManifest;
-    }
-
-    private JsonObject CreatePortMap()
-    {
-        // ports are entries in a key/value map whose keys are "<number>/<type>" and whose values are an empty object.
-        // yes, this is odd.
-        var container = new JsonObject();
-        foreach (var port in _exposedPorts)
-        {
-            container.Add($"{port.Number}/{port.Type}", new JsonObject());
-        }
-        return container;
-    }
-
-    private JsonArray CreateEnvironmentVarMapping()
-    {
-        // Env is a JSON array where each value is of the format: "VAR=value"
-        var envVarJson = new JsonArray();
-        foreach (var envVar in _environmentVariables)
-        {
-            envVarJson.Add<string>($"{envVar.Key}={envVar.Value}");
-        }
-        return envVarJson;
-    }
-
-    private static HashSet<Label> ReadLabelsFromConfig(JsonNode inputConfig)
-    {
-        if (inputConfig is JsonObject config && config["Labels"] is JsonObject labelsJson)
-        {
-            // read label mappings from object
-            var labels = new HashSet<Label>();
-            foreach (var property in labelsJson)
-            {
-                if (property.Key is { } propertyName && property.Value is JsonValue propertyValue)
-                {
-                    labels.Add(new(propertyName, propertyValue.ToString()));
-                }
-            }
-            return labels;
-        }
-        else
-        {
-            // initialize an empty labels map
-            return new HashSet<Label>();
-        }
-    }
-
-    private static Dictionary<string, string> ReadEnvVarsFromConfig(JsonNode inputConfig)
-    {
-        if (inputConfig is JsonObject config && config["config"]!["Env"] is JsonArray envVarJson)
-        {
-            var envVars = new Dictionary<string, string>();
-            foreach (var entry in envVarJson)
-            {
-                if (entry is null)
-                    continue;
-
-                var val = entry.GetValue<string>().Split('=', 2);
-
-                if (val.Length != 2)
-                    continue;
-
-                envVars.Add(val[0], val[1]);
-            }
-            return envVars;
-        }
-        else
-        {
-            return new Dictionary<string, string>();
-        }
-    }
-
-    private static HashSet<Port> ReadPortsFromConfig(JsonNode inputConfig)
-    {
-        if (inputConfig is JsonObject config && config["ExposedPorts"] is JsonObject portsJson)
-        {
-            // read label mappings from object
-            var ports = new HashSet<Port>();
-            foreach (var property in portsJson)
-            {
-                if (property.Key is { } propertyName
-                    && property.Value is JsonObject propertyValue
-                    && ContainerHelpers.TryParsePort(propertyName, out var parsedPort, out var _))
-                {
-                    ports.Add(parsedPort.Value);
-                }
-            }
-            return ports;
-        }
-        else
-        {
-            // initialize an empty ports map
-            return new HashSet<Port>();
-        }
-    }
-
-    private JsonObject CreateLabelMap()
-    {
-        var container = new JsonObject();
-        foreach (var label in _labels)
-        {
-            container.Add(label.name, label.value);
-        }
-        return container;
-    }
-
-    static JsonArray ToJsonArray(string[] items) => new JsonArray(items.Where(s => !string.IsNullOrEmpty(s)).Select(s => JsonValue.Create(s)).ToArray());
-
-    public void SetEntrypoint(string[] executableArgs, string[]? args = null)
-    {
-        JsonObject? configObject = Config["config"]!.AsObject();
-
-        if (configObject is null)
-        {
-            throw new NotImplementedException("Expected base image to have a config node");
-        }
-
-        configObject["Entrypoint"] = ToJsonArray(executableArgs);
-
-        if (args is null)
-        {
-            configObject.Remove("Cmd");
-        }
-        else
-        {
-            configObject["Cmd"] = ToJsonArray(args);
-        }
-
-        RecalculateDigest();
-    }
-
-    public string WorkingDirectory
-    {
-        get => (string?)Config["config"]!["WorkingDir"] ?? "";
-        set
-        {
-            Config["config"]!["WorkingDir"] = value;
-            RecalculateDigest();
-        }
-    }
-
-    public void Label(string name, string value)
-    {
-        _labels.Add(new(name, value));
-        Config["config"]!["Labels"] = CreateLabelMap();
-        RecalculateDigest();
-    }
-
-    public void AddEnvironmentVariable(string envVarName, string value)
-    {
-        if (!_environmentVariables.ContainsKey(envVarName))
-        {
-            _environmentVariables.Add(envVarName, value);
-        }
-        else
-        {
-            _environmentVariables[envVarName] = value;
-        }
-        Config["config"]!["Env"] = CreateEnvironmentVarMapping();
-        RecalculateDigest();
-    }
-
-    public void ExposePort(int number, PortType type)
-    {
-        _exposedPorts.Add(new(number, type));
-        Config["config"]!["ExposedPorts"] = CreatePortMap();
-        RecalculateDigest();
-    }
-
-    public static string GetDigest(JsonNode json)
-    {
-        string hashString;
-
-        hashString = GetSha(json);
-
-        return $"sha256:{hashString}";
-    }
-
-    public static string GetDigest<T>(T item)
-    {
-        var node = JsonSerializer.SerializeToNode(item);
-        if (node is not null) return GetDigest(node);
-        else return String.Empty;
-    }
-
-    public static string GetSha(JsonNode json)
-    {
-        Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
-        SHA256.HashData(Encoding.UTF8.GetBytes(json.ToJsonString()), hash);
-
-        return Convert.ToHexString(hash).ToLowerInvariant();
-    }
-}
diff --git a/Microsoft.NET.Build.Containers/ImageBuilder.cs b/Microsoft.NET.Build.Containers/ImageBuilder.cs
new file mode 100644
index 00000000..0b2f9adb
--- /dev/null
+++ b/Microsoft.NET.Build.Containers/ImageBuilder.cs
@@ -0,0 +1,86 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Text;
+
+namespace Microsoft.NET.Build.Containers;
+
+/// <summary>
+/// The class builds new image based on the base image.
+/// </summary>
+internal sealed class ImageBuilder
+{
+    private readonly ManifestV2 _manifest;
+    private readonly ImageConfig _baseImageConfig;
+
+    internal ImageBuilder(ManifestV2 manifest, ImageConfig baseImageConfig)
+    {
+        _manifest = manifest;
+        _baseImageConfig = baseImageConfig;
+    }
+
+    /// <summary>
+    /// Builds the image configuration <see cref="BuiltImage"/> ready for further processing.
+    /// </summary>
+    internal BuiltImage Build()
+    {
+        string imageJsonStr = _baseImageConfig.BuildConfig();
+        string imageSha = DigestUtils.GetSha(imageJsonStr);
+        string imageDigest = DigestUtils.GetDigestFromSha(imageSha);
+        long imageSize = Encoding.UTF8.GetBytes(imageJsonStr).Length;
+
+        ManifestConfig newManifestConfig = _manifest.Config with
+        {
+            digest = imageDigest,
+            size = imageSize
+        };
+
+        ManifestV2 newManifest = _manifest with
+        {
+            Config = newManifestConfig
+        };
+
+        return new BuiltImage()
+        {
+            Config = imageJsonStr,
+            ImageDigest = imageDigest,
+            ImageSha = imageSha,
+            ImageSize = imageSize,
+            Manifest = newManifest,
+        };
+    }
+
+    /// <summary>
+    /// Adds a <see cref="Layer"/> to a base image.
+    /// </summary>
+    internal void AddLayer(Layer l)
+    {
+        _manifest.Layers.Add(new(l.Descriptor.MediaType, l.Descriptor.Size, l.Descriptor.Digest, l.Descriptor.Urls));
+        _baseImageConfig.AddLayer(l);
+    }
+
+    /// <summary>
+    /// Adds a label to a base image.
+    /// </summary>
+    internal void AddLabel(string name, string value) => _baseImageConfig.AddLabel(name, value);
+
+    /// <summary>
+    /// Adds environment variables to a base image.
+    /// </summary>
+    internal void AddEnvironmentVariable(string envVarName, string value) => _baseImageConfig.AddEnvironmentVariable(envVarName, value);
+
+    /// <summary>
+    /// Exposes additional port.
+    /// </summary>
+    internal void ExposePort(int number, PortType type) => _baseImageConfig.ExposePort(number, type);
+
+    /// <summary>
+    /// Sets working directory for the image.
+    /// </summary>
+    internal void SetWorkingDirectory(string workingDirectory) => _baseImageConfig.SetWorkingDirectory(workingDirectory);
+
+    /// <summary>
+    /// Sets an entry point for the image.
+    /// </summary>
+    internal void SetEntryPoint(string[] executableArgs, string[]? args = null) => _baseImageConfig.SetEntryPoint(executableArgs, args);
+}
diff --git a/Microsoft.NET.Build.Containers/ImageConfig.cs b/Microsoft.NET.Build.Containers/ImageConfig.cs
new file mode 100644
index 00000000..e201ef29
--- /dev/null
+++ b/Microsoft.NET.Build.Containers/ImageConfig.cs
@@ -0,0 +1,198 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Text.Json.Nodes;
+
+namespace Microsoft.NET.Build.Containers;
+
+/// <summary>
+/// The class allows to modify base image configuration.
+/// </summary>
+internal sealed class ImageConfig
+{
+    private readonly JsonObject _config;
+    private readonly Dictionary<string, string> _labels;
+    private readonly List<Layer> _newLayers = new();
+    private readonly HashSet<Port> _exposedPorts;
+    private readonly Dictionary<string, string> _environmentVariables;
+    private string? _newWorkingDirectory;
+    private (string[] ExecutableArgs, string[]? Args)? _newEntryPoint;
+
+    internal ImageConfig(JsonNode config)
+    {
+        _config = config as JsonObject ?? throw new ArgumentException($"{nameof(config)} should be a JSON object.", nameof(config));
+        if (_config["config"] is not JsonObject)
+        {
+            throw new ArgumentException("Base image configuration should contain a 'config' node.");
+        }
+
+        _labels = GetLabels();
+        _exposedPorts = GetExposedPorts();
+        _environmentVariables = GetEnvironmentVariables();
+    }
+
+    /// <summary>
+    /// Builds in additional configuration and returns updated image configuration in JSON format as string.
+    /// </summary>
+    internal string BuildConfig()
+    {
+        if (_config["config"] is not JsonObject config)
+        {
+            throw new InvalidOperationException("Base image configuration should contain a 'config' node.");
+        }
+
+        //update creation date
+        _config["created"] = DateTime.UtcNow;
+  
+        config["ExposedPorts"] = CreatePortMap();
+        config["Env"] = CreateEnvironmentVariablesMapping();
+        config["Labels"] = CreateLabelMap();
+
+        if (_newWorkingDirectory is not null)
+        {
+            config["WorkingDir"] = _newWorkingDirectory;
+        }
+
+        if (_newEntryPoint.HasValue)
+        {
+            config["Entrypoint"] = ToJsonArray(_newEntryPoint.Value.ExecutableArgs);
+
+            if (_newEntryPoint.Value.Args is null)
+            {
+                config.Remove("Cmd");
+            }
+            else
+            {
+                config["Cmd"] = ToJsonArray(_newEntryPoint.Value.Args);
+            }
+        }
+
+        foreach (Layer l in _newLayers)
+        {
+            _config["rootfs"]!["diff_ids"]!.AsArray().Add(l.Descriptor.UncompressedDigest);
+        }
+
+        return _config.ToJsonString();
+        static JsonArray ToJsonArray(string[] items) => new(items.Where(s => !string.IsNullOrEmpty(s)).Select(s => JsonValue.Create(s)).ToArray());
+    }
+
+    internal void ExposePort(int number, PortType type)
+    {
+        _exposedPorts.Add(new(number, type));
+    }
+
+    internal void AddEnvironmentVariable(string envVarName, string value)
+    {
+        _environmentVariables[envVarName] = value;
+    }
+
+    internal void AddLabel(string name, string value)
+    {
+        _labels[name] = value;
+    }
+
+    internal void SetWorkingDirectory(string workingDirectory)
+    {
+        _newWorkingDirectory = workingDirectory;
+    }
+
+    internal void SetEntryPoint(string[] executableArgs, string[]? args = null)
+    {
+        _newEntryPoint = (executableArgs, args);
+    }
+
+    internal void AddLayer(Layer l)
+    {
+        _newLayers.Add(l);
+    }
+
+    private HashSet<Port> GetExposedPorts()
+    {
+        HashSet<Port> ports = new();
+        if (_config["config"]?["ExposedPorts"] is JsonObject portsJson)
+        {
+            // read label mappings from object
+            foreach (KeyValuePair<string, JsonNode?> property in portsJson)
+            {
+                if (property.Key is { } propertyName
+                    && property.Value is JsonObject propertyValue
+                    && ContainerHelpers.TryParsePort(propertyName, out Port? parsedPort, out ContainerHelpers.ParsePortError? _))
+                {
+                    ports.Add(parsedPort.Value);
+                }
+            }
+        }
+        return ports;
+    }
+
+    private Dictionary<string, string> GetLabels()
+    {
+        Dictionary<string, string> labels = new();
+        if (_config["config"]?["Labels"] is JsonObject labelsJson)
+        {
+            // read label mappings from object
+            foreach (KeyValuePair<string, JsonNode?> property in labelsJson)
+            {
+                if (property.Key is { } propertyName && property.Value is JsonValue propertyValue)
+                {
+                    labels[propertyName] = propertyValue.ToString();
+                }
+            }    
+        }
+        return labels;
+    }
+
+    private Dictionary<string, string> GetEnvironmentVariables()
+    {
+        Dictionary<string, string> envVars = new();
+        if (_config["config"]?["Env"] is JsonArray envVarJson)
+        {
+            foreach (JsonNode? entry in envVarJson)
+            {
+                if (entry is null)
+                    continue;
+
+                string[] val = entry.GetValue<string>().Split('=', 2);
+
+                if (val.Length != 2)
+                    continue;
+
+                envVars.Add(val[0], val[1]);
+            }
+        }
+        return envVars;
+    }
+
+    private JsonObject CreatePortMap()
+    {
+        // ports are entries in a key/value map whose keys are "<number>/<type>" and whose values are an empty object.
+        // yes, this is odd.
+        JsonObject container = new();
+        foreach (Port port in _exposedPorts)
+        {
+            container.Add($"{port.Number}/{port.Type}", new JsonObject());
+        }
+        return container;
+    }
+
+    private JsonArray CreateEnvironmentVariablesMapping()
+    {
+        // Env is a JSON array where each value is of the format: "VAR=value"
+        JsonArray envVarJson = new();
+        foreach (KeyValuePair<string, string> envVar in _environmentVariables)
+        {
+            envVarJson.Add($"{envVar.Key}={envVar.Value}");
+        }
+        return envVarJson;
+    }
+
+    private JsonObject CreateLabelMap()
+    {
+        JsonObject container = new();
+        foreach (KeyValuePair<string, string> label in _labels)
+        {
+            container.Add(label.Key, label.Value);
+        }
+        return container;
+    }
+}
diff --git a/Microsoft.NET.Build.Containers/LocalDaemons/ILocalDaemon.cs b/Microsoft.NET.Build.Containers/LocalDaemons/ILocalDaemon.cs
index 26cc2057..db47a8e6 100644
--- a/Microsoft.NET.Build.Containers/LocalDaemons/ILocalDaemon.cs
+++ b/Microsoft.NET.Build.Containers/LocalDaemons/ILocalDaemon.cs
@@ -12,7 +12,7 @@ internal interface ILocalDaemon {
     /// <summary>
     /// Loads an image (presumably from a tarball) into the local container runtime.
     /// </summary>
-    public Task Load(Image image, ImageReference sourceReference, ImageReference destinationReference);
+    public Task Load(BuiltImage image, ImageReference sourceReference, ImageReference destinationReference);
 
     /// <summary>
     /// Checks to see if the local container runtime is available. This is used to give nice errors to the user.
diff --git a/Microsoft.NET.Build.Containers/LocalDaemons/LocalDocker.cs b/Microsoft.NET.Build.Containers/LocalDaemons/LocalDocker.cs
index 293d4eb7..0d014919 100644
--- a/Microsoft.NET.Build.Containers/LocalDaemons/LocalDocker.cs
+++ b/Microsoft.NET.Build.Containers/LocalDaemons/LocalDocker.cs
@@ -18,7 +18,7 @@ public LocalDocker(Action<string> logger)
         this.logger = logger;
     }
 
-    public async Task Load(Image image, ImageReference sourceReference, ImageReference destinationReference)
+    public async Task Load(BuiltImage image, ImageReference sourceReference, ImageReference destinationReference)
     {
         // call `docker load` and get it ready to receive input
         ProcessStartInfo loadInfo = new("docker", $"load");
@@ -84,7 +84,7 @@ private static async Task<JsonDocument> GetConfig()
         return await JsonDocument.ParseAsync(proc.StandardOutput.BaseStream).ConfigureAwait(false);
     }
 
-    private static async Task WriteImageToStream(Image image, ImageReference sourceReference, ImageReference destinationReference, Stream imageStream)
+    private static async Task WriteImageToStream(BuiltImage image, ImageReference sourceReference, ImageReference destinationReference, Stream imageStream)
     {
         using TarWriter writer = new(imageStream, TarEntryFormat.Pax, leaveOpen: true);
 
@@ -111,9 +111,9 @@ private static async Task WriteImageToStream(Image image, ImageReference sourceR
         }
 
         // add config
-        string configTarballPath = $"{Image.GetSha(image.Config)}.json";
+        string configTarballPath = $"{image.ImageSha}.json";
 
-        using (MemoryStream configStream = new MemoryStream(Encoding.UTF8.GetBytes(image.Config.ToJsonString())))
+        using (MemoryStream configStream = new MemoryStream(Encoding.UTF8.GetBytes(image.Config)))
         {
             PaxTarEntry configEntry = new(TarEntryType.RegularFile, configTarballPath)
             {
diff --git a/Microsoft.NET.Build.Containers/ManifestV2.cs b/Microsoft.NET.Build.Containers/ManifestV2.cs
index 2c215c81..eb001436 100644
--- a/Microsoft.NET.Build.Containers/ManifestV2.cs
+++ b/Microsoft.NET.Build.Containers/ManifestV2.cs
@@ -1,9 +1,54 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
 namespace Microsoft.NET.Build.Containers;
 
-public record struct ManifestV2(int schemaVersion, string mediaType, ManifestConfig config, List<ManifestLayer> layers);
+/// <summary>
+/// The struct represents image manifest specification.
+/// </summary>
+/// <remarks>
+/// https://github.com/opencontainers/image-spec/blob/main/manifest.md
+/// </remarks>
+public readonly record struct ManifestV2
+{
+    /// <summary>
+    /// This REQUIRED property specifies the image manifest schema version.
+    /// For this version of the specification, this MUST be 2 to ensure backward compatibility with older versions of Docker.
+    /// The value of this field will not change. This field MAY be removed in a future version of the specification.
+    /// </summary>
+    [JsonPropertyName("schemaVersion")]
+    public required int SchemaVersion { get; init; }
+
+    /// <summary>
+    /// This property SHOULD be used and remain compatible with earlier versions of this specification and with other similar external formats.
+    /// When used, this field MUST contain the media type application/vnd.oci.image.manifest.v1+json. This field usage differs from the descriptor use of mediaType.
+    /// </summary>
+    [JsonPropertyName("mediaType")]
+    public required string MediaType { get; init; }
+
+    /// <summary>
+    /// This REQUIRED property references a configuration object for a container, by digest.
+    /// </summary>
+    [JsonPropertyName("config")]
+    public required ManifestConfig Config { get; init; }
+
+    /// <summary>
+    /// Each item in the array MUST be a descriptor. The array MUST have the base layer at index 0.
+    /// Subsequent layers MUST then follow in stack order (i.e. from layers[0] to layers[len(layers)-1]).
+    /// The final filesystem layout MUST match the result of applying the layers to an empty directory.
+    /// The ownership, mode, and other attributes of the initial empty directory are unspecified.
+    /// </summary>
+    [JsonPropertyName("layers")]
+    public required List<ManifestLayer> Layers { get; init; }
+
+    /// <summary>
+    /// Gets the digest for this manifest.
+    /// </summary>
+    public string GetDigest() => DigestUtils.GetDigest(JsonSerializer.SerializeToNode(this)?.ToJsonString() ?? string.Empty);
+}
 
 public record struct ManifestConfig(string mediaType, long size, string digest);
 
diff --git a/Microsoft.NET.Build.Containers/PublicAPI/net7.0/PublicAPI.Unshipped.txt b/Microsoft.NET.Build.Containers/PublicAPI/net7.0/PublicAPI.Unshipped.txt
index 51ade4b1..29bdab1f 100644
--- a/Microsoft.NET.Build.Containers/PublicAPI/net7.0/PublicAPI.Unshipped.txt
+++ b/Microsoft.NET.Build.Containers/PublicAPI/net7.0/PublicAPI.Unshipped.txt
@@ -58,16 +58,16 @@ Microsoft.NET.Build.Containers.ManifestListV2.mediaType.set -> void
 Microsoft.NET.Build.Containers.ManifestListV2.schemaVersion.get -> int
 Microsoft.NET.Build.Containers.ManifestListV2.schemaVersion.set -> void
 Microsoft.NET.Build.Containers.ManifestV2
-Microsoft.NET.Build.Containers.ManifestV2.config.get -> Microsoft.NET.Build.Containers.ManifestConfig
-Microsoft.NET.Build.Containers.ManifestV2.config.set -> void
-Microsoft.NET.Build.Containers.ManifestV2.layers.get -> System.Collections.Generic.List<Microsoft.NET.Build.Containers.ManifestLayer>!
-Microsoft.NET.Build.Containers.ManifestV2.layers.set -> void
+Microsoft.NET.Build.Containers.ManifestV2.Config.get -> Microsoft.NET.Build.Containers.ManifestConfig
+Microsoft.NET.Build.Containers.ManifestV2.Config.init -> void
+Microsoft.NET.Build.Containers.ManifestV2.GetDigest() -> string!
+Microsoft.NET.Build.Containers.ManifestV2.Layers.get -> System.Collections.Generic.List<Microsoft.NET.Build.Containers.ManifestLayer>!
+Microsoft.NET.Build.Containers.ManifestV2.Layers.init -> void
 Microsoft.NET.Build.Containers.ManifestV2.ManifestV2() -> void
-Microsoft.NET.Build.Containers.ManifestV2.ManifestV2(int schemaVersion, string! mediaType, Microsoft.NET.Build.Containers.ManifestConfig config, System.Collections.Generic.List<Microsoft.NET.Build.Containers.ManifestLayer>! layers) -> void
-Microsoft.NET.Build.Containers.ManifestV2.mediaType.get -> string!
-Microsoft.NET.Build.Containers.ManifestV2.mediaType.set -> void
-Microsoft.NET.Build.Containers.ManifestV2.schemaVersion.get -> int
-Microsoft.NET.Build.Containers.ManifestV2.schemaVersion.set -> void
+Microsoft.NET.Build.Containers.ManifestV2.MediaType.get -> string!
+Microsoft.NET.Build.Containers.ManifestV2.MediaType.init -> void
+Microsoft.NET.Build.Containers.ManifestV2.SchemaVersion.get -> int
+Microsoft.NET.Build.Containers.ManifestV2.SchemaVersion.init -> void
 Microsoft.NET.Build.Containers.PlatformInformation
 Microsoft.NET.Build.Containers.PlatformInformation.architecture.get -> string!
 Microsoft.NET.Build.Containers.PlatformInformation.architecture.set -> void
diff --git a/Microsoft.NET.Build.Containers/Registry.cs b/Microsoft.NET.Build.Containers/Registry.cs
index c72ee206..b7b86496 100644
--- a/Microsoft.NET.Build.Containers/Registry.cs
+++ b/Microsoft.NET.Build.Containers/Registry.cs
@@ -84,9 +84,8 @@ public bool IsGoogleArtifactRegistry {
     /// </summary>
     private bool SupportsParallelUploads => !IsAmazonECRRegistry;
 
-    public async Task<Image> GetImageManifest(string repositoryName, string reference, string runtimeIdentifier, string runtimeIdentifierGraphPath)
+    public async Task<ImageBuilder> GetImageManifest(string repositoryName, string reference, string runtimeIdentifier, string runtimeIdentifierGraphPath)
     {
-        var client = GetClient();
         var initialManifestResponse = await GetManifest(repositoryName, reference).ConfigureAwait(false);
 
         return initialManifestResponse.Content.Headers.ContentType?.MediaType switch {
@@ -96,8 +95,9 @@ public async Task<Image> GetImageManifest(string repositoryName, string referenc
         };
     }
 
-    private async Task<Image> ReadSingleImage(string repositoryName, ManifestV2 manifest) {
-        var config = manifest.config;
+    private async Task<ImageBuilder> ReadSingleImage(string repositoryName, ManifestV2 manifest)
+    {
+        var config = manifest.Config;
         string configSha = config.digest;
 
         var blobResponse = await GetBlob(repositoryName, configSha).ConfigureAwait(false);
@@ -105,10 +105,10 @@ private async Task<Image> ReadSingleImage(string repositoryName, ManifestV2 mani
         JsonNode? configDoc = JsonNode.Parse(await blobResponse.Content.ReadAsStringAsync().ConfigureAwait(false));
         Debug.Assert(configDoc is not null);
 
-        return new Image(manifest, configDoc);
+        return new ImageBuilder(manifest, new ImageConfig(configDoc));
     }
 
-    async Task<Image> PickBestImageFromManifestList(string repositoryName, string reference, ManifestListV2 manifestList, string runtimeIdentifier, string runtimeIdentifierGraphPath) {
+    async Task<ImageBuilder> PickBestImageFromManifestList(string repositoryName, string reference, ManifestListV2 manifestList, string runtimeIdentifier, string runtimeIdentifierGraphPath) {
         var runtimeGraph = GetRuntimeGraphForDotNet(runtimeIdentifierGraphPath);
         var (ridDict, graphForManifestList) = ConstructRuntimeGraphForManifestList(manifestList, runtimeGraph);
         var bestManifestRid = CheckIfRidExistsInGraph(graphForManifestList, ridDict.Keys, runtimeIdentifier);
@@ -407,7 +407,7 @@ private HttpClient CreateClient()
         return client;
     }
 
-    public async Task Push(Image x, ImageReference source, ImageReference destination, Action<string> logProgressMessage)
+    public async Task Push(BuiltImage builtImage, ImageReference source, ImageReference destination, Action<string> logProgressMessage)
     {
 
         HttpClient client = GetClient();
@@ -447,27 +447,27 @@ public async Task Push(Image x, ImageReference source, ImageReference destinatio
 
         if (SupportsParallelUploads)
         {
-            await Task.WhenAll(x.LayerDescriptors.Select(descriptor => uploadLayerFunc(descriptor))).ConfigureAwait(false);
+            await Task.WhenAll(builtImage.LayerDescriptors.Select(descriptor => uploadLayerFunc(descriptor))).ConfigureAwait(false);
         }
         else
         {
-            foreach(var descriptor in x.LayerDescriptors)
+            foreach(var descriptor in builtImage.LayerDescriptors)
             {
                 await uploadLayerFunc(descriptor).ConfigureAwait(false);
             }
         }
 
-        using (MemoryStream stringStream = new MemoryStream(Encoding.UTF8.GetBytes(x.Config.ToJsonString())))
+        using (MemoryStream stringStream = new MemoryStream(Encoding.UTF8.GetBytes(builtImage.Config)))
         {
-            var configDigest = Image.GetDigest(x.Config);
+            var configDigest = builtImage.ImageDigest;
             logProgressMessage($"Uploading config to registry at blob {configDigest}");
             await UploadBlob(destination.Repository, configDigest, stringStream).ConfigureAwait(false);
             logProgressMessage($"Uploaded config to registry");
         }
 
-        var manifestDigest = Image.GetDigest(x.Manifest);
+        var manifestDigest = builtImage.Manifest.GetDigest();
         logProgressMessage($"Uploading manifest to registry {RegistryName} as blob {manifestDigest}");
-        string jsonString = JsonSerializer.SerializeToNode(x.Manifest)?.ToJsonString() ?? "";
+        string jsonString = JsonSerializer.SerializeToNode(builtImage.Manifest)?.ToJsonString() ?? "";
         HttpContent manifestUploadContent = new StringContent(jsonString);
         manifestUploadContent.Headers.ContentType = new MediaTypeHeaderValue(DockerManifestV2);
         var putResponse = await client.PutAsync(new Uri(BaseUri, $"/v2/{destination.Repository}/manifests/{manifestDigest}"), manifestUploadContent).ConfigureAwait(false);
diff --git a/Microsoft.NET.Build.Containers/Tasks/CreateNewImage.cs b/Microsoft.NET.Build.Containers/Tasks/CreateNewImage.cs
index 15dd2b55..84155a2c 100644
--- a/Microsoft.NET.Build.Containers/Tasks/CreateNewImage.cs
+++ b/Microsoft.NET.Build.Containers/Tasks/CreateNewImage.cs
@@ -32,9 +32,9 @@ public override bool Execute()
         ImageReference sourceImageReference = new(SourceRegistry.Value, BaseImageName, BaseImageTag);
         var destinationImageReferences = ImageTags.Select(t => new ImageReference(DestinationRegistry.Value, ImageName, t));
 
-        var image = GetBaseImage();
+        ImageBuilder imageBuilder = GetBaseImage();
 
-        if (image is null)
+        if (imageBuilder is null)
         {
             Log.LogError($"Couldn't find matching base image for {0} that matches RuntimeIdentifier {1}", sourceImageReference.RepositoryAndTag, ContainerRuntimeIdentifier);
             return !Log.HasLoggedErrors;
@@ -43,18 +43,18 @@ public override bool Execute()
         SafeLog("Building image '{0}' with tags {1} on top of base image {2}", ImageName, String.Join(",", ImageTags), sourceImageReference);
 
         Layer newLayer = Layer.FromDirectory(PublishDirectory, WorkingDirectory);
-        image.AddLayer(newLayer);
-        image.WorkingDirectory = WorkingDirectory;
-        image.SetEntrypoint(Entrypoint.Select(i => i.ItemSpec).ToArray(), EntrypointArgs.Select(i => i.ItemSpec).ToArray());
+        imageBuilder.AddLayer(newLayer);
+        imageBuilder.SetWorkingDirectory(WorkingDirectory);
+        imageBuilder.SetEntryPoint(Entrypoint.Select(i => i.ItemSpec).ToArray(), EntrypointArgs.Select(i => i.ItemSpec).ToArray());
 
         foreach (var label in Labels)
         {
-            image.Label(label.ItemSpec, label.GetMetadata("Value"));
+            imageBuilder.AddLabel(label.ItemSpec, label.GetMetadata("Value"));
         }
 
-        SetEnvironmentVariables(image, ContainerEnvironmentVariables);
+        SetEnvironmentVariables(imageBuilder, ContainerEnvironmentVariables);
 
-        SetPorts(image, ExposedPorts);
+        SetPorts(imageBuilder, ExposedPorts);
 
         // at the end of this step, if any failed then bail out.
         if (Log.HasLoggedErrors)
@@ -62,9 +62,11 @@ public override bool Execute()
             return false;
         }
 
+        BuiltImage builtImage = imageBuilder.Build();
+
         // at this point we're done with modifications and are just pushing the data other places
-        GeneratedContainerManifest = JsonSerializer.Serialize(image.Manifest);
-        GeneratedContainerConfiguration = image.Config.ToJsonString();
+        GeneratedContainerManifest = JsonSerializer.Serialize(builtImage.Manifest);
+        GeneratedContainerConfiguration = builtImage.Config;
 
         foreach (var destinationImageReference in destinationImageReferences)
         {
@@ -78,7 +80,7 @@ public override bool Execute()
                 }
                 try
                 {
-                    localDaemon.Load(image, sourceImageReference, destinationImageReference).Wait();
+                    localDaemon.Load(builtImage, sourceImageReference, destinationImageReference).Wait();
                     SafeLog("Pushed container '{0}' to local daemon", destinationImageReference.RepositoryAndTag);
                 }
                 catch (AggregateException ex) when (ex.InnerException is DockerLoadException dle)
@@ -90,7 +92,7 @@ public override bool Execute()
             {
                 try
                 {
-                    destinationImageReference.Registry?.Push(image, sourceImageReference, destinationImageReference, message => SafeLog(message)).Wait();
+                    destinationImageReference.Registry?.Push(builtImage, sourceImageReference, destinationImageReference, message => SafeLog(message)).Wait();
                     SafeLog("Pushed container '{0}' to registry '{2}'", destinationImageReference.RepositoryAndTag, OutputRegistry);
                 }
                 catch (ContainerHttpException e)
@@ -113,7 +115,7 @@ public override bool Execute()
         return !Log.HasLoggedErrors;
     }
 
-    private void SetPorts(Image image, ITaskItem[] exposedPorts)
+    private void SetPorts(ImageBuilder image, ITaskItem[] exposedPorts)
     {
         foreach (var port in exposedPorts)
         {
@@ -188,7 +190,7 @@ private Lazy<Registry?> DestinationRegistry {
         }
     }
 
-    private static void SetEnvironmentVariables(Image img, ITaskItem[] envVars)
+    private static void SetEnvironmentVariables(ImageBuilder img, ITaskItem[] envVars)
     {
         foreach (ITaskItem envVar in envVars)
         {
@@ -196,10 +198,14 @@ private static void SetEnvironmentVariables(Image img, ITaskItem[] envVars)
         }
     }
 
-    private Image? GetBaseImage() {
-        if (SourceRegistry.Value is {} registry) {
+    private ImageBuilder GetBaseImage()
+    {
+        if (SourceRegistry.Value is {} registry)
+        {
             return registry.GetImageManifest(BaseImageName, BaseImageTag, ContainerRuntimeIdentifier, RuntimeIdentifierGraphPath).Result;
-        } else {
+        }
+        else
+        {
             throw new ArgumentException("Don't know how to pull images from local daemons at the moment");
         }
     }
diff --git a/docs/ReleaseNotes/v0.4.0.md b/docs/ReleaseNotes/v0.4.0.md
index 68d23bce..5b9213cf 100644
--- a/docs/ReleaseNotes/v0.4.0.md
+++ b/docs/ReleaseNotes/v0.4.0.md
@@ -6,6 +6,7 @@ This is the fourth public release of the .NET SDK containerization feature. This
 * [Predefine Conventional Labels from project metadata](https://github.com/dotnet/sdk-container-builds/pull/307)
 * [Better error messages when the local daemon isn't available](https://github.com/dotnet/sdk-container-builds/pull/319)
 * [Defined `NetSdkOCIImageBuild` project capability for projects building the image](https://github.com/dotnet/sdk-container-builds/issues/320)
+* [Labels and Ports are now read from the underlying config correctly](https://github.com/dotnet/sdk-container-builds/issues/332)
 
 ### Special thanks