hof/runtime: add support for loading user files

Author: Tony Worm

.hof/shadow/cli/cmd/hof/flags/root.go

@@ -18,6 +18,7 @@ type RootPflagpole struct {
 	IncludeData  bool
 	WithContext  bool
 	InjectEnv    bool
+	UserFiles    []string
 	AllErrors    bool
 	IngoreErrors bool
 	Stats        bool
@@ -37,6 +38,7 @@ func SetupRootPflags(fset *pflag.FlagSet, fpole *RootPflagpole) {
 	fset.BoolVarP(&(fpole.IncludeData), "include-data", "D", false, "auto include all data files found with cue files")
 	fset.BoolVarP(&(fpole.WithContext), "with-context", "", false, "add extra context for data files, usable in the -l/path flag")
 	fset.BoolVarP(&(fpole.InjectEnv), "inject-env", "V", false, "inject all ENV VARs as default tag vars")
+	fset.StringArrayVarP(&(fpole.UserFiles), "user-files", "U", nil, "file globs to embed into the root value")
 	fset.BoolVarP(&(fpole.AllErrors), "all-errors", "E", false, "print all available errors")
 	fset.BoolVarP(&(fpole.IngoreErrors), "ignore-errors", "i", false, "turn off output and assume defaults at prompts")
 	fset.BoolVarP(&(fpole.Stats), "stats", "", false, "print generator statistics")

cmd/hof/flags/root.go

@@ -18,6 +18,7 @@ type RootPflagpole struct {
 	IncludeData  bool
 	WithContext  bool
 	InjectEnv    bool
+	UserFiles    []string
 	AllErrors    bool
 	IngoreErrors bool
 	Stats        bool
@@ -37,6 +38,7 @@ func SetupRootPflags(fset *pflag.FlagSet, fpole *RootPflagpole) {
 	fset.BoolVarP(&(fpole.IncludeData), "include-data", "D", false, "auto include all data files found with cue files")
 	fset.BoolVarP(&(fpole.WithContext), "with-context", "", false, "add extra context for data files, usable in the -l/path flag")
 	fset.BoolVarP(&(fpole.InjectEnv), "inject-env", "V", false, "inject all ENV VARs as default tag vars")
+	fset.StringArrayVarP(&(fpole.UserFiles), "user-files", "U", nil, "file globs to embed into the root value")
 	fset.BoolVarP(&(fpole.AllErrors), "all-errors", "E", false, "print all available errors")
 	fset.BoolVarP(&(fpole.IngoreErrors), "ignore-errors", "i", false, "turn off output and assume defaults at prompts")
 	fset.BoolVarP(&(fpole.Stats), "stats", "", false, "print generator statistics")

design/pflags.cue

@@ -67,6 +67,13 @@ CliPflags: [...schema.Flag] & [ {
 	Type:    "bool"
 	Default: ""
 	Help:    "inject all ENV VARs as default tag vars"
+}, {
+	Name:    "UserFiles"
+	Long:    "user-files"
+	Short:   "U"
+	Type:    "[]string"
+	Default: "nil"
+	Help:    "file globs to embed into the root value"
 }, {
 	Name:    "AllErrors"
 	Long:    "all-errors"

lib/runtime/load.go

@@ -47,6 +47,11 @@ func (R *Runtime) Load() (err error) {
 		return err
 	}
 
+	err = R.prepPlacedUserfiles()
+	if err != nil {
+		return err
+	}
+
 	if !R.Flags.IngoreErrors {
 		err = R.Value.Validate()
 		if err != nil {
@@ -93,7 +98,7 @@ func (R *Runtime) prepPlacedDatafiles() {
 				fname, fpath := parts[0], parts[1]
 				R.dataMappings[fname] = fpath
 				entries = append(entries, fname)
-			continue
+				continue
 			}
 		}
 
@@ -108,6 +113,113 @@ func (R *Runtime) prepPlacedDatafiles() {
 	R.Entrypoints = entries
 }
 
+func (R *Runtime) prepPlacedUserfiles() error {
+	start := time.Now()
+	defer func() {
+		end := time.Now()
+		R.Stats.Add("files/load", end.Sub(start))
+	}()
+
+	buildFile := func (trimPath, filePath string) (*ast.Field, error) {
+		// prep inputs
+		d, err := os.ReadFile(filePath)
+		if err != nil {
+			return nil, fmt.Errorf("while loading user file: %w", err)
+		}
+		s := string(d)
+		l := strings.TrimPrefix(filePath, trimPath)
+
+		// build file field
+		ff := new(ast.Field)
+		ff.Constraint = token.STRING
+		ff.Label = ast.NewIdent(l)
+		ff.Value = ast.NewString(s)
+
+		return ff, nil
+	}
+
+	embedFiles := func (cuePath, trimPath, filePath string) error {
+		files := []string{filePath}
+		// expand globs
+		if strings.Contains(filePath, "*") {
+			fs, err := yagu.FilesFromGlobs([]string{filePath})
+			if err != nil {
+				return fmt.Errorf("warning: error while globing %q: %v", filePath, err)
+			}
+
+			files = fs
+		} else {
+			// maybe handle directory
+			stat, err := os.Stat(filePath)
+			if err != nil {
+				return err
+			}
+
+			if stat.IsDir() {
+				fs, err := yagu.FilesFromGlobs([]string{filePath + "/*"})
+				if err != nil {
+					return fmt.Errorf("warning: error while loading dir %q: %v", filePath, err)
+				}
+
+				files = fs
+			}
+		}
+
+		fields := []interface{}{}
+		for _, file := range files {
+			fld, err := buildFile(trimPath, file)
+			if err != nil {
+				return err
+			}
+			fields = append(fields, fld)
+		}
+
+		// build CUE struct
+		st := ast.NewStruct(fields...)
+		// fmt.Sprintf("%+v\n", st)
+
+		cp := cue.ParsePath(cuePath)
+		R.Value = R.Value.FillPath(cp, st)
+
+		return nil
+	}
+
+	// handle the flag for placing user files
+	for _, U := range R.Flags.UserFiles {
+		// fmt.Println(U)
+		parts := strings.Split(U, "=")
+		if len(parts) != 2 {
+			return fmt.Errorf("-U/--user-files flag should have format <cue-path>=<glob>")
+		}
+
+		cuePath, filePath := parts[0], parts[1]
+
+		trimPath := ""
+		if strings.Contains(filePath, "%") {
+			parts := strings.Split(filePath, "%")
+			if len(parts) != 2 {
+				return fmt.Errorf("-U/--user-files only supports one % to trim prefix")
+			}
+			trimPath = parts[0] + "/"
+			filePath = strings.Replace(filePath, "%", "/", 1)
+		}
+
+		err := embedFiles(cuePath, trimPath, filePath)
+		if err != nil {
+			return err
+		}
+	}
+
+	// look for @userfiles attributes and do similar
+
+	if R.Flags.Verbosity > 1 {
+		fmt.Println("user files:", R.userFiles)
+		fmt.Println("mod files: ", R.modFiles)
+	}
+
+	return nil
+}
+
 func (R *Runtime) load() (err error) {
 	beg := time.Now()
 	defer func() {
@@ -133,28 +245,6 @@ func (R *Runtime) load() (err error) {
 		R.Entrypoints = append(R.Entrypoints, "-")
 	}
 
-	// fi, err := os.Stdin.Stat()
-
-	//if fi.Size() == 0 {
-	//  fmt.Println("stdin(0)...")
-	//  R.CueConfig.Stdin = &bytes.Buffer{}
-
-	//} else {
-	//  d, err := io.ReadAll(os.Stdin)
-	//  if err != nil {
-	//    return err
-	//  }
-	//  r := bytes.NewReader(d)
-	//  fmt.Println(len(d), r.Len(), r.Size(), fi.Size())
-	//  R.CueConfig.Stdin = r
-
-	//  //if .... {
-	//    //fmt.Println("no stdin...")
-	//  //}
-	//  // R.CueConfig.Stdin = os.Stdin
-	//}
-
-
 	//
 	//
 	// load instances from entrypoints and config, this is the main CUE loader step

lib/runtime/runtime.go

@@ -57,7 +57,11 @@ type Runtime struct {
 	dataMappings    map[string]string
 
 	// internal bookkeeping
-	loadedFiles []string
+	loadedFiles []string  // cue+data (?)
+
+	// non-data files loaded <cue-path> => <file-path>
+	userFiles   map[string]string  // non-data files loaded by user
+	modFiles    map[string]string  // non-data files loaded by modules
 
 	// The CUE value after all loading
 	Value    cue.Value
@@ -112,6 +116,8 @@ func New(entrypoints []string, rflags flags.RootPflagpole) (*Runtime, error) {
 		origEntrypoints: entrypoints,
 		CueConfig:   cfg,
 		dataMappings: make(map[string]string),
+		userFiles: make(map[string]string),
+		modFiles: make(map[string]string),
 		Stats: make(RuntimeStats),
 	}
 

test/userfiles/content/cow.txt

@@ -0,0 +1 @@
+mooooo

test/userfiles/content/foo.md

@@ -0,0 +1,3 @@
+# Foo
+
+bar

test/userfiles/main.cue

@@ -0,0 +1,5 @@
+package main
+
+hello: "world"
+
+files: {}