base/v0_6_exp: add parent directory sugar

prestist · yasminvalim · jmarrero · yasminvalim · commit 3ea71ea3485e · 2024-07-05T18:59:40.000-03:00
Add a field called 'Parent' which is used to specify a file's parent directory. When a parent is specified, all directories from the parent to the file will be created, with the 'mode' supplied in the parent directory. resolves: coreos#380 Co-authored-by: Yasmin Valim <ydesouza@redhat.com> Co-authored-by: Joseph Corchado <jmarrero@redhat.com> Co-authored-by: Adam Piasecki <apiaseck@redhat.com>
diff --git a/base/v0_6_exp/schema.go b/base/v0_6_exp/schema.go
@@ -67,6 +67,12 @@ type File struct {
 	Append    []Resource `yaml:"append"`
 	Contents  Resource   `yaml:"contents"`
 	Mode      *int       `yaml:"mode"`
+	Parent    Parent     `yaml:"parent"`
+}
+
+type Parent struct {
+	Path *string `yaml:"path,omitempty"`
+	Mode *int    `yaml:"mode,omitempty"`
 }
 
 type Filesystem struct {
diff --git a/base/v0_6_exp/translate.go b/base/v0_6_exp/translate.go
@@ -20,6 +20,7 @@ import (
 	slashpath "path"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"strings"
 	"text/template"
 
@@ -83,9 +84,7 @@ func (c Config) ToIgn3_5Unvalidated(options common.TranslateOptions) (types.Conf
 
 	tr := translate.NewTranslator("yaml", "json", options)
 	tr.AddCustomTranslator(translateIgnition)
-	tr.AddCustomTranslator(translateFile)
-	tr.AddCustomTranslator(translateDirectory)
-	tr.AddCustomTranslator(translateLink)
+	tr.AddCustomTranslator(translateStorage)
 	tr.AddCustomTranslator(translateResource)
 	tr.AddCustomTranslator(translatePasswdUser)
 	tr.AddCustomTranslator(translateUnit)
@@ -99,7 +98,6 @@ func (c Config) ToIgn3_5Unvalidated(options common.TranslateOptions) (types.Conf
 	translate.MergeP(tr, tm, &r, "systemd", &c.Systemd, &ret.Systemd)
 
 	c.addMountUnits(&ret, &tm)
-
 	tm2, r2 := c.processTrees(&ret, options)
 	tm.Merge(tm2)
 	r.Merge(r2)
@@ -121,6 +119,82 @@ func translateIgnition(from Ignition, options common.TranslateOptions) (to types
 	return
 }
 
+func translateStorage(from Storage, options common.TranslateOptions) (to types.Storage, tm translate.TranslationSet, r report.Report) {
+	tr := translate.NewTranslator("yaml", "json", options)
+	tr.AddCustomTranslator(translateFile)
+	tr.AddCustomTranslator(translateDirectory)
+	tr.AddCustomTranslator(translateLink)
+	tr.AddCustomTranslator(translateLuks)
+	tm, r = translate.Prefixed(tr, "directories", &from.Directories, &to.Directories)
+	translate.MergeP(tr, tm, &r, "disks", &from.Disks, &to.Disks)
+	translate.MergeP(tr, tm, &r, "files", &from.Files, &to.Files)
+	translate.MergeP(tr, tm, &r, "filesystems", &from.Filesystems, &to.Filesystems)
+	translate.MergeP(tr, tm, &r, "links", &from.Links, &to.Links)
+	translate.MergeP(tr, tm, &r, "luks", &from.Luks, &to.Luks)
+	translate.MergeP(tr, tm, &r, "raid", &from.Raid, &to.Raid)
+	for i, file := range from.Files {
+		if util.NotEmpty(file.Parent.Path) {
+			var dir string
+
+			yamlPath := path.New("yaml", "files", i, "parent")
+
+			if !strings.Contains(file.Path, *file.Parent.Path) {
+				r.AddOnError(yamlPath, common.ErrInvalidParent)
+				continue
+			}
+
+			if runtime.GOOS == "windows" {
+				dir = normalizeWindowsPath(file.Path)
+			} else {
+				dir = filepath.Dir(file.Path)
+			}
+
+			// Clean the path to avoid consistency issues
+			dir = filepath.Clean(dir)
+
+			fmt.Println("Final Directory:", dir)
+
+			for dir != "" {
+				renderedDir := types.Directory{
+					Node: types.Node{
+						Path:  dir,
+						Group: types.NodeGroup{ID: file.Group.ID, Name: file.Group.Name},
+						User:  types.NodeUser{ID: file.User.ID, Name: file.User.Name},
+					},
+					DirectoryEmbedded1: types.DirectoryEmbedded1{
+						Mode: file.Parent.Mode,
+					},
+				}
+				to.Directories = append(to.Directories, renderedDir)
+				nextDir, _ := filepath.Split(dir)
+				// make sure to clean the path to avoid consistency issues
+				nextDir = filepath.Clean(nextDir)
+
+				if dir == *file.Parent.Path || nextDir == dir {
+					// we have reached the parent directory or the end of the path
+					break
+				}
+				dir = nextDir
+			}
+			tm.AddFromCommonSource(yamlPath, path.New("json", "directories"), to.Directories)
+		}
+
+	}
+	return
+}
+
+func normalizeWindowsPath(path string) string {
+	// Convert to absolute path if necessary
+	absPath, err := filepath.Abs(path)
+	if err != nil {
+		fmt.Println("Error converting path to absolute:", err)
+		return path // Return original path in case of error
+	}
+
+	// Clean and return normalized path
+	return filepath.ToSlash(filepath.Clean(absPath))
+}
+
 func translateFile(from File, options common.TranslateOptions) (to types.File, tm translate.TranslationSet, r report.Report) {
 	tr := translate.NewTranslator("yaml", "json", options)
 	tr.AddCustomTranslator(translateResource)
@@ -134,6 +208,22 @@ func translateFile(from File, options common.TranslateOptions) (to types.File, t
 	return
 }
 
+func translateLuks(from Luks, options common.TranslateOptions) (to types.Luks, tm translate.TranslationSet, r report.Report) {
+	tr := translate.NewTranslator("yaml", "json", options)
+	tr.AddCustomTranslator(translateResource)
+	tm, r = translate.Prefixed(tr, "clevis", &from.Clevis, &to.Clevis)
+	translate.MergeP(tr, tm, &r, "device", &from.Device, &to.Device)
+	translate.MergeP(tr, tm, &r, "discard", &from.Discard, &to.Discard)
+	translate.MergeP2(tr, tm, &r, "key_file", &from.KeyFile, "keyFile", &to.KeyFile)
+	translate.MergeP(tr, tm, &r, "label", &from.Label, &to.Label)
+	translate.MergeP(tr, tm, &r, "name", &from.Name, &to.Name)
+	translate.MergeP2(tr, tm, &r, "open_options", &from.OpenOptions, "openOptions", &to.OpenOptions)
+	translate.MergeP(tr, tm, &r, "options", &from.Options, &to.Options)
+	translate.MergeP(tr, tm, &r, "uuid", &from.UUID, &to.UUID)
+	translate.MergeP2(tr, tm, &r, "wipe_volume", &from.WipeVolume, "wipeVolume", &to.WipeVolume)
+	return
+}
+
 func translateResource(from Resource, options common.TranslateOptions) (to types.Resource, tm translate.TranslationSet, r report.Report) {
 	tr := translate.NewTranslator("yaml", "json", options)
 	tm, r = translate.Prefixed(tr, "verification", &from.Verification, &to.Verification)
@@ -294,7 +384,7 @@ func (c Config) processTrees(ret *types.Config, options common.TranslateOptions)
 		return ts, r
 	}
 	t := newNodeTracker(ret)
-
+	ts.AddTranslation(path.New("yaml", "storage"), path.New("json", "storage"))
 	for i, tree := range c.Storage.Trees {
 		yamlPath := path.New("yaml", "storage", "trees", i)
 		if options.FilesDir == "" {
diff --git a/base/v0_6_exp/translate_test.go b/base/v0_6_exp/translate_test.go
@@ -599,6 +599,195 @@ func TestTranslateFile(t *testing.T) {
 		})
 	}
 }
+func TestTranslateStorage(t *testing.T) {
+	tests := []struct {
+		in      Storage
+		out     types.Storage
+		errPath path.ContextPath
+		errors  error
+		skip    func(t *testing.T)
+	}{
+		// Basic parent file directory
+		{
+			in: Storage{
+				Files: []File{
+					{
+						Path:     "/foo/bar/txt.txt",
+						Contents: Resource{},
+						Mode:     util.IntToPtr(420),
+						Parent: Parent{
+							Path: util.StrToPtr("/foo"),
+							Mode: util.IntToPtr(420),
+						},
+					},
+				},
+			},
+			out: types.Storage{
+				Files: []types.File{
+					{
+						Node: types.Node{
+							Path: "/foo/bar/txt.txt",
+						},
+						FileEmbedded1: types.FileEmbedded1{
+							Mode:     util.IntToPtr(420),
+							Contents: types.Resource{},
+						},
+					},
+				},
+				Directories: []types.Directory{
+					{
+						Node: types.Node{
+							Path: "/foo/bar",
+						},
+						DirectoryEmbedded1: types.DirectoryEmbedded1{
+							Mode: util.IntToPtr(420),
+						},
+					},
+					{
+						Node: types.Node{
+							Path: "/foo",
+						},
+						DirectoryEmbedded1: types.DirectoryEmbedded1{
+							Mode: util.IntToPtr(420),
+						},
+					},
+				},
+			},
+			errPath: path.ContextPath{},
+			errors:  nil,
+			skip: func(t *testing.T) {
+				if runtime.GOOS == "windows" {
+					t.Skip("skipping test on Windows")
+				}
+			},
+		},
+		// Empty parent file directory
+		{
+			in: Storage{
+				Files: []File{
+					{
+						Path:     "/foo/bar/txt.txt",
+						Contents: Resource{},
+						Mode:     util.IntToPtr(420),
+						Parent: Parent{
+							Path: util.StrToPtr(""),
+							Mode: util.IntToPtr(420),
+						},
+					},
+				},
+			},
+			out: types.Storage{
+				Files: []types.File{
+					{
+						Node: types.Node{
+							Path: "/foo/bar/txt.txt",
+						},
+						FileEmbedded1: types.FileEmbedded1{
+							Mode:     util.IntToPtr(420),
+							Contents: types.Resource{},
+						},
+					},
+				},
+			},
+			errPath: path.ContextPath{},
+			errors:  nil,
+			skip: func(t *testing.T) {
+				if runtime.GOOS == "windows" {
+					t.Skip("skipping test on Windows")
+				}
+			},
+		},
+		// Parent not defined
+		{
+			in: Storage{
+				Files: []File{
+					{
+						Path:     "/foo/bar/txt.txt",
+						Contents: Resource{},
+						Mode:     util.IntToPtr(420),
+					},
+				},
+			},
+			out: types.Storage{
+				Files: []types.File{
+					{
+						Node: types.Node{
+							Path: "/foo/bar/txt.txt",
+						},
+						FileEmbedded1: types.FileEmbedded1{
+							Mode:     util.IntToPtr(420),
+							Contents: types.Resource{},
+						},
+					},
+				},
+			},
+			errPath: path.ContextPath{},
+			errors:  nil,
+			skip: func(t *testing.T) {
+				if runtime.GOOS == "windows" {
+					t.Skip("skipping test on Windows")
+				}
+			},
+		},
+		// Parent path is not related to file path
+		{
+			in: Storage{
+				Files: []File{
+					{
+						Path:     "/foo/bar/txt.txt",
+						Contents: Resource{},
+						Mode:     util.IntToPtr(420),
+						Parent: Parent{
+							Path: util.StrToPtr("/godzilla"),
+							Mode: util.IntToPtr(420),
+						},
+					},
+				},
+			},
+			out: types.Storage{
+				Files: []types.File{
+					{
+						Node: types.Node{
+							Path: "/foo/bar/txt.txt",
+						},
+						FileEmbedded1: types.FileEmbedded1{
+							Mode:     util.IntToPtr(420),
+							Contents: types.Resource{},
+						},
+					},
+				},
+			},
+			errPath: path.New("yaml", "files", 0, "parent"),
+			errors:  common.ErrInvalidParent,
+			skip: func(t *testing.T) {
+				if runtime.GOOS == "windows" {
+					t.Skip("skipping test on Windows")
+				}
+			},
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("translate %d", i), func(t *testing.T) {
+			if test.skip != nil {
+				// give the test an opportunity to skip
+				test.skip(t)
+			}
+			actual, translations, r := translateStorage(test.in, common.TranslateOptions{})
+			r = confutil.TranslateReportPaths(r, translations)
+			baseutil.VerifyReport(t, test.in, r)
+			assert.Equal(t, test.out, actual, "translation mismatch")
+			assert.NoError(t, translations.DebugVerifyCoverage(actual), "incomplete TranslationSet coverage")
+			if test.errors != nil {
+				expected := report.Report{}
+				expected.AddOnError(test.errPath, test.errors)
+				assert.Equal(t, expected, r, "bad report for test case %d", i)
+			} else {
+				assert.Equal(t, report.Report{}, r, "non-empty report")
+			}
+		})
+	}
+}
 
 // TestTranslateDirectory tests translating the ct storage.directories.[i] entries to ignition storage.directories.[i] entires.
 func TestTranslateDirectory(t *testing.T) {
diff --git a/config/common/errors.go b/config/common/errors.go
@@ -89,6 +89,7 @@ var (
 	ErrLinkSupport       = errors.New("links are not supported in this spec version")
 	ErrLuksSupport       = errors.New("luks is not supported in this spec version")
 	ErrRaidSupport       = errors.New("raid is not supported in this spec version")
+	ErrInvalidParent     = errors.New("parent must be included in the file path")
 
 	// Grub
 	ErrGrubUserNameNotSpecified = errors.New("field \"name\" is required")
diff --git a/docs/config-fcos-v1_6-exp.md b/docs/config-fcos-v1_6-exp.md
@@ -117,6 +117,9 @@ The Fedora CoreOS configuration is a YAML document conforming to the following s
     * **_group_** (object): specifies the file's group.
       * **_id_** (integer): the group ID of the group.
       * **_name_** (string): the group name of the group.
+    * **_parent_** (object): the parent directory for the specified file, by declaring a parent the directories from the parent to the file's target destination.
+      * **_path_** (string): the path of the directory within the file's 'path'.
+      * **_mode_** (integer): directory modes are set to 0755 as a default if not specified and directory does not exist prior to the specified file.
   * **_directories_** (list of objects): the list of directories to be created. Every file, directory, and link must have a unique `path`.
     * **path** (string): the absolute path to the directory.
     * **_overwrite_** (boolean): whether to delete preexisting nodes at the path. If false and a directory already exists at the path, Ignition will only set its permissions. If false and a non-directory exists at that path, Ignition will fail. Defaults to false.
diff --git a/docs/config-fiot-v1_1-exp.md b/docs/config-fiot-v1_1-exp.md
@@ -88,6 +88,9 @@ The Fedora IoT configuration is a YAML document conforming to the following spec
     * **_group_** (object): specifies the file's group.
       * **_id_** (integer): the group ID of the group.
       * **_name_** (string): the group name of the group.
+    * **_parent_** (object): the parent directory for the specified file, by declaring a parent the directories from the parent to the file's target destination.
+      * **_path_** (string): the path of the directory within the file's 'path'.
+      * **_mode_** (integer): directory modes are set to 0755 as a default if not specified and directory does not exist prior to the specified file.
   * **_directories_** (list of objects): the list of directories to be created. Every file, directory, and link must have a unique `path`.
     * **path** (string): the absolute path to the directory.
     * **_overwrite_** (boolean): whether to delete preexisting nodes at the path. If false and a directory already exists at the path, Ignition will only set its permissions. If false and a non-directory exists at that path, Ignition will fail. Defaults to false.
diff --git a/docs/config-flatcar-v1_2-exp.md b/docs/config-flatcar-v1_2-exp.md
@@ -117,6 +117,9 @@ The Flatcar configuration is a YAML document conforming to the following specifi
     * **_group_** (object): specifies the file's group.
       * **_id_** (integer): the group ID of the group.
       * **_name_** (string): the group name of the group.
+    * **_parent_** (object): the parent directory for the specified file, by declaring a parent the directories from the parent to the file's target destination.
+      * **_path_** (string): the path of the directory within the file's 'path'.
+      * **_mode_** (integer): directory modes are set to 0755 as a default if not specified and directory does not exist prior to the specified file.
   * **_directories_** (list of objects): the list of directories to be created. Every file, directory, and link must have a unique `path`.
     * **path** (string): the absolute path to the directory.
     * **_overwrite_** (boolean): whether to delete preexisting nodes at the path. If false and a directory already exists at the path, Ignition will only set its permissions. If false and a non-directory exists at that path, Ignition will fail. Defaults to false.
diff --git a/docs/config-openshift-v4_17-exp.md b/docs/config-openshift-v4_17-exp.md
@@ -107,6 +107,9 @@ The OpenShift configuration is a YAML document conforming to the following speci
     * **_group_** (object): specifies the file's group.
       * **_id_** (integer): the group ID of the group.
       * **_name_** (string): the group name of the group.
+    * **_parent_** (object): the parent directory for the specified file, by declaring a parent the directories from the parent to the file's target destination.
+      * **_path_** (string): the path of the directory within the file's 'path'.
+      * **_mode_** (integer): directory modes are set to 0755 as a default if not specified and directory does not exist prior to the specified file.
   * **_luks_** (list of objects): the list of luks devices to be created. Every device must have a unique `name`.
     * **name** (string): the name of the luks device.
     * **device** (string): the absolute path to the device. Devices are typically referenced by the `/dev/disk/by-*` symlinks.
diff --git a/docs/config-r4e-v1_2-exp.md b/docs/config-r4e-v1_2-exp.md
diff --git a/docs/examples.md b/docs/examples.md
diff --git a/docs/release-notes.md b/docs/release-notes.md
diff --git a/internal/doc/butane.yaml b/internal/doc/butane.yaml
