Skip to content

Commit ffb42b9

Browse files
jahvonjahvon
authored
(feat) read flowfile and ws description from file (#140)
1 parent 4c682b8 commit ffb42b9

18 files changed

+164
-71
lines changed

‎docs/types/flowfile.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@ A workspace can have multiple flow files located anywhere in the workspace direc
1414

1515
| Field | Description | Type | Default | Required |
1616
| ----- | ----------- | ---- | ------- | -------- |
17-
| `description` | A description of the executables defined within the flow file. This description will be set as the executables' description if not defined at the executable level. | `string` | | [] |
17+
| `description` | A description of the executables defined within the flow file. This description will used as a shared description for all executables in the flow file. | `string` | | [] |
18+
| `descriptionFile` | A path to a markdown file that contains the description of the executables defined within the flow file. | `string` | | [] |
1819
| `executables` | | `array` ([Executable](#Executable)) | [] | [] |
1920
| `fromFile` | | [FromFile](#FromFile) | [] | [] |
2021
| `namespace` | The namespace to be given to all executables in the flow file. If not set, the executables in the file will be grouped into the root (*) namespace. Namespaces can be reused across multiple flow files. Namespaces are used to reference executables in the CLI using the format `workspace:namespace/name`. | `string` | | [] |

‎docs/types/workspace.md

+1
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ Every workspace has a workspace config file named `flow.yaml` in the root of the
1515
| Field | Description | Type | Default | Required |
1616
| ----- | ----------- | ---- | ------- | -------- |
1717
| `description` | A description of the workspace. This description is rendered as markdown in the interactive UI. | `string` | | [] |
18+
| `descriptionFile` | A path to a markdown file that contains the description of the workspace. | `string` | | [] |
1819
| `displayName` | The display name of the workspace. This is used in the interactive UI. | `string` | | [] |
1920
| `executables` | | [ExecutableFilter](#ExecutableFilter) | <no value> | [] |
2021
| `tags` | | [CommonTags](#CommonTags) | [] | [] |

‎flow.yaml

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
11
# yaml-language-server: $schema=https://raw.githubusercontent.com/jahvon/flow/HEAD/schemas/workspace_schema.json
2-
displayName: flow
3-
git:
4-
enabled: false
2+
displayName: flow repository
3+
descriptionFile: README.md

‎internal/cache/executable_generator.go

+7-3
Original file line numberDiff line numberDiff line change
@@ -16,18 +16,22 @@ import (
1616
const generatedTag = "generated"
1717

1818
func generatedExecutables(
19-
logger io.Logger,
20-
wsName, wsPath, flowFileNs, flowFilePath string,
21-
files []string,
19+
logger io.Logger, wsName string, flowFile *executable.FlowFile,
2220
) (executable.ExecutableList, error) {
2321
executables := make(executable.ExecutableList, 0)
22+
wsPath := flowFile.WorkspacePath()
23+
flowFilePath := flowFile.ConfigPath()
24+
flowFileNs := flowFile.Namespace
25+
files := flowFile.FromFile
26+
2427
for _, file := range files {
2528
expandedFile := utils.ExpandDirectory(logger, file, wsPath, flowFilePath, nil)
2629
exec, err := executablesFromFile(logger, file, expandedFile)
2730
if err != nil {
2831
return nil, err
2932
}
3033
exec.SetContext(wsName, wsPath, flowFileNs, flowFilePath)
34+
exec.SetInheritedFields(flowFile)
3135
executables = append(executables, exec)
3236
}
3337

‎internal/cache/executables_cache.go

+3-24
Original file line numberDiff line numberDiff line change
@@ -69,14 +69,7 @@ func (c *ExecutableCacheImpl) Update(logger io.Logger) error { //nolint:gocognit
6969
}
7070
for _, flowFile := range flowFiles {
7171
if len(flowFile.FromFile) > 0 {
72-
generated, err := generatedExecutables(
73-
logger,
74-
name,
75-
wsCfg.Location(),
76-
flowFile.Namespace,
77-
flowFile.ConfigPath(),
78-
flowFile.FromFile,
79-
)
72+
generated, err := generatedExecutables(logger, name, flowFile)
8073
if err != nil {
8174
logger.Errorx(
8275
"failed to generate executables from files",
@@ -160,14 +153,7 @@ func (c *ExecutableCacheImpl) GetExecutableByRef(logger io.Logger, ref executabl
160153
cfg.SetDefaults()
161154
cfg.SetContext(wsInfo.WorkspaceName, wsInfo.WorkspacePath, cfgPath)
162155

163-
generated, err := generatedExecutables(
164-
logger,
165-
wsInfo.WorkspaceName,
166-
wsInfo.WorkspacePath,
167-
cfg.Namespace,
168-
cfg.ConfigPath(),
169-
cfg.FromFile,
170-
)
156+
generated, err := generatedExecutables(logger, wsInfo.WorkspaceName, cfg)
171157
if err != nil {
172158
logger.Warnx(
173159
"failed to generate executables from files",
@@ -213,14 +199,7 @@ func (c *ExecutableCacheImpl) GetExecutableList(logger io.Logger) (executable.Ex
213199
cfg.SetDefaults()
214200
cfg.SetContext(wsInfo.WorkspaceName, wsInfo.WorkspacePath, cfgPath)
215201

216-
generated, err := generatedExecutables(
217-
logger,
218-
wsInfo.WorkspaceName,
219-
wsInfo.WorkspacePath,
220-
cfg.Namespace,
221-
cfg.ConfigPath(),
222-
cfg.FromFile,
223-
)
202+
generated, err := generatedExecutables(logger, wsInfo.WorkspaceName, cfg)
224203
if err != nil {
225204
logger.Warnx(
226205
"failed to generate executables from files",

‎schemas/flowfile_schema.json

+6-1
Original file line numberDiff line numberDiff line change
@@ -478,7 +478,12 @@
478478
},
479479
"properties": {
480480
"description": {
481-
"description": "A description of the executables defined within the flow file. This description will be set as the executables'\ndescription if not defined at the executable level.\n",
481+
"description": "A description of the executables defined within the flow file. This description will used as a shared description\nfor all executables in the flow file.\n",
482+
"type": "string",
483+
"default": ""
484+
},
485+
"descriptionFile": {
486+
"description": "A path to a markdown file that contains the description of the executables defined within the flow file.",
482487
"type": "string",
483488
"default": ""
484489
},

‎schemas/workspace_schema.json

+5
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,11 @@
3939
"type": "string",
4040
"default": ""
4141
},
42+
"descriptionFile": {
43+
"description": "A path to a markdown file that contains the description of the workspace.",
44+
"type": "string",
45+
"default": ""
46+
},
4247
"displayName": {
4348
"description": "The display name of the workspace. This is used in the interactive UI.",
4449
"type": "string",

‎types/executable/executable.gen.go

+4
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎types/executable/executable.go

+18
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,24 @@ func (e *Executable) SetContext(workspaceName, workspacePath, namespace, flowFil
7575
e.flowFilePath = flowFilePath
7676
}
7777

78+
func (e *Executable) SetInheritedFields(flowFile *FlowFile) {
79+
e.MergeTags(flowFile.Tags)
80+
if e.Visibility == nil && flowFile.Visibility != nil {
81+
v := ExecutableVisibility(*flowFile.Visibility)
82+
e.Visibility = &v
83+
}
84+
var descFromFIle string
85+
if flowFile.DescriptionFile != "" {
86+
mdBytes, err := os.ReadFile(flowFile.DescriptionFile)
87+
if err != nil {
88+
descFromFIle += fmt.Sprintf("**error rendering description file**: %s", err)
89+
} else {
90+
descFromFIle += string(mdBytes)
91+
}
92+
}
93+
e.inheritedDescription = strings.Join([]string{flowFile.Description, descFromFIle}, "\n")
94+
}
95+
7896
func (e *Executable) YAML() (string, error) {
7997
enriched := &enrichedExecutable{
8098
ID: e.ID(),

‎types/executable/executable_md.go

+22-8
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,7 @@ import (
88
func execMarkdown(e *Executable) string {
99
var mkdwn string
1010
mkdwn += fmt.Sprintf("# [Executable] %s\n", e.Ref())
11-
if e.Description != "" {
12-
mkdwn += "| \n"
13-
lines := strings.Split(e.Description, "\n")
14-
for _, line := range lines {
15-
mkdwn += fmt.Sprintf("| %s\n", line)
16-
}
17-
mkdwn += "| \n\n"
18-
}
11+
mkdwn += execDescriptionMarkdown(e)
1912
if e.Visibility != nil {
2013
mkdwn += fmt.Sprintf("**Visibility:** %s\n", *e.Visibility)
2114
}
@@ -43,6 +36,27 @@ func execMarkdown(e *Executable) string {
4336
return mkdwn
4437
}
4538

39+
func execDescriptionMarkdown(e *Executable) string {
40+
var mkdwn string
41+
const descSpacer = "| \n"
42+
if e.Description != "" {
43+
mkdwn += descSpacer
44+
lines := strings.Split(e.Description, "\n")
45+
for _, line := range lines {
46+
mkdwn += fmt.Sprintf("| %s\n", line)
47+
}
48+
mkdwn += descSpacer
49+
}
50+
if e.inheritedDescription != "" {
51+
for _, line := range strings.Split(e.inheritedDescription, "\n") {
52+
mkdwn += fmt.Sprintf("| %s\n", line)
53+
}
54+
mkdwn += descSpacer
55+
}
56+
mkdwn += "\n"
57+
return mkdwn
58+
}
59+
4660
func execTypeMarkdown(spec *Executable) string {
4761
var mkdwn string
4862
switch {

‎types/executable/executable_schema.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -389,6 +389,11 @@ properties:
389389
default: ""
390390
goJSONSchema:
391391
identifier: flowFilePath
392+
inheritedDescription:
393+
type: string
394+
default: ""
395+
goJSONSchema:
396+
identifier: inheritedDescription
392397
#### Executable runner type fields
393398
#### go-jsonschema does not support oneOf, so we need to define the types separately and validate them in go.
394399
exec:

‎types/executable/flowfile.gen.go

+6-2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎types/executable/flowfile.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ func (f *FlowFile) SetContext(workspaceName, workspacePath, configPath string) {
2222
exec.Visibility = &v
2323
}
2424
exec.SetDefaults()
25-
exec.MergeTags(f.Tags)
25+
exec.SetInheritedFields(f)
2626
}
2727
}
2828

‎types/executable/flowfile_schema.yaml

+6-2
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,12 @@ properties:
5050
description:
5151
type: string
5252
description: |
53-
A description of the executables defined within the flow file. This description will be set as the executables'
54-
description if not defined at the executable level.
53+
A description of the executables defined within the flow file. This description will used as a shared description
54+
for all executables in the flow file.
55+
default: ""
56+
descriptionFile:
57+
type: string
58+
description: A path to a markdown file that contains the description of the executables defined within the flow file.
5559
default: ""
5660
#### Executable config context fields
5761
workspaceName:

‎types/workspace/schema.yaml

+4
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,10 @@ properties:
4242
type: string
4343
description: A description of the workspace. This description is rendered as markdown in the interactive UI.
4444
default: ""
45+
descriptionFile:
46+
type: string
47+
description: A path to a markdown file that contains the description of the workspace.
48+
default: ""
4549
assignedName:
4650
type: string
4751
goJSONSchema:

‎types/workspace/workspace.gen.go

+3
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎types/workspace/workspace.go

+1-26
Original file line numberDiff line numberDiff line change
@@ -49,32 +49,7 @@ func (w *Workspace) JSON() (string, error) {
4949
}
5050

5151
func (w *Workspace) Markdown() string {
52-
var mkdwn string
53-
if w.DisplayName != "" {
54-
mkdwn = fmt.Sprintf("# [Workspace] %s\n", w.DisplayName)
55-
} else {
56-
mkdwn = fmt.Sprintf("# [Workspace] %s\n", w.AssignedName())
57-
}
58-
59-
mkdwn += fmt.Sprintf("## Location\n%s\n", w.Location())
60-
if w.Description != "" {
61-
mkdwn += fmt.Sprintf("## Description\n%s\n", w.Description)
62-
}
63-
if w.Tags != nil && len(w.Tags) > 0 {
64-
mkdwn += "## Tags\n"
65-
for _, tag := range w.Tags {
66-
mkdwn += fmt.Sprintf("- %s\n", tag)
67-
}
68-
}
69-
if w.Executables != nil {
70-
execs, err := yaml.Marshal(w.Executables)
71-
if err != nil {
72-
mkdwn += "## Executables\nerror\n"
73-
} else {
74-
mkdwn += fmt.Sprintf("## Executables\n```yaml\n%s```\n", string(execs))
75-
}
76-
}
77-
return mkdwn
52+
return workspaceMarkdown(w)
7853
}
7954

8055
func DefaultWorkspaceConfig(name string) *Workspace {

‎types/workspace/workspace_md.go

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
package workspace
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"path/filepath"
7+
"strings"
8+
)
9+
10+
func workspaceMarkdown(w *Workspace) string {
11+
var mkdwn string
12+
if w.DisplayName != "" {
13+
mkdwn = fmt.Sprintf("# [Workspace] %s\n", w.DisplayName)
14+
} else {
15+
mkdwn = fmt.Sprintf("# [Workspace] %s\n", w.AssignedName())
16+
}
17+
mkdwn += workspaceDescription(w)
18+
if len(w.Tags) > 0 {
19+
mkdwn += "**Tags**\n"
20+
for _, tag := range w.Tags {
21+
mkdwn += fmt.Sprintf("- %s\n", tag)
22+
}
23+
}
24+
if w.Executables != nil {
25+
mkdwn += "**Executable Filter**\n"
26+
if len(w.Executables.Included) > 0 {
27+
mkdwn += "Included\n"
28+
for _, line := range w.Executables.Included {
29+
mkdwn += fmt.Sprintf(" %s\n", line)
30+
}
31+
}
32+
if len(w.Executables.Excluded) > 0 {
33+
mkdwn += "Excluded\n"
34+
for _, line := range w.Executables.Excluded {
35+
mkdwn += fmt.Sprintf(" %s\n", line)
36+
}
37+
}
38+
}
39+
mkdwn += fmt.Sprintf("\n\n_Workspace can be found in_ [%s](%s)\n", w.Location(), w.Location())
40+
return mkdwn
41+
}
42+
43+
func workspaceDescription(w *Workspace) string {
44+
var mkdwn string
45+
const descSpacer = "| \n"
46+
if w.Description != "" {
47+
mkdwn += descSpacer
48+
lines := strings.Split(w.Description, "\n")
49+
for _, line := range lines {
50+
mkdwn += fmt.Sprintf("| %s\n", line)
51+
}
52+
mkdwn += descSpacer
53+
}
54+
if w.DescriptionFile != "" {
55+
mdBytes, err := os.ReadFile(filepath.Clean(w.DescriptionFile))
56+
if err != nil {
57+
mkdwn += fmt.Sprintf("| **error rendering description file**: %s\n", err)
58+
} else {
59+
lines := strings.Split(string(mdBytes), "\n")
60+
for _, line := range lines {
61+
mkdwn += fmt.Sprintf("| %s\n", line)
62+
}
63+
}
64+
mkdwn += descSpacer
65+
}
66+
mkdwn += "\n"
67+
return mkdwn
68+
}

0 commit comments

Comments
 (0)