Skip to content

Commit a033543

Browse files
vmarcellavmarcella
authored
feat(test): Add test reports that can be generated when executing markdown with ie test --report <file-path> (#212)
This PR adds test reporting to `ie test` when invoked with `--report=<report-name>`. The report format is currently in JSON, and an executable specification which dives more into the problem can now be found under `docs/specs/test-reporting.md`
1 parent f9cea7f commit a033543

File tree

18 files changed

+670
-331
lines changed

18 files changed

+670
-331
lines changed

‎cmd/ie/commands/test.go

+4
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ func init() {
2020
String("subscription", "", "Sets the subscription ID used by a scenarios azure-cli commands. Will rely on the default subscription if not set.")
2121
testCommand.PersistentFlags().
2222
String("working-directory", ".", "Sets the working directory for innovation engine to operate out of. Restores the current working directory when finished.")
23+
testCommand.PersistentFlags().
24+
String("report", "", "The path to generate a report of the scenario execution. The contents of the report are in JSON and will only be generated when this flag is set.")
2325

2426
testCommand.PersistentFlags().
2527
StringArray("var", []string{}, "Sets an environment variable for the scenario. Format: --var <key>=<value>")
@@ -40,6 +42,7 @@ var testCommand = &cobra.Command{
4042
subscription, _ := cmd.Flags().GetString("subscription")
4143
workingDirectory, _ := cmd.Flags().GetString("working-directory")
4244
environment, _ := cmd.Flags().GetString("environment")
45+
generateReport, _ := cmd.Flags().GetString("report")
4346

4447
environmentVariables, _ := cmd.Flags().GetStringArray("var")
4548

@@ -67,6 +70,7 @@ var testCommand = &cobra.Command{
6770
CorrelationId: "",
6871
WorkingDirectory: workingDirectory,
6972
Environment: environment,
73+
ReportFile: generateReport,
7074
})
7175
if err != nil {
7276
logging.GlobalLogger.Errorf("Error creating engine %s", err)

‎docs/specs/test-reporting.md

+216
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
# Test Reports
2+
3+
## Summary
4+
5+
When users are testing their executable documentation using `ie test`, being
6+
able to see the results of their execution is important, especially in instances
7+
where the test fails. At the moment, there are only two ways to see what
8+
happened during the test execution:
9+
10+
1. The user can look through the standard output from `ie test <scenario>`
11+
(Most common).
12+
1. The user can look through `ie.log` (Not as common).
13+
14+
While these methods are effective for troubleshooting most issues, they also have
15+
a few issues:
16+
17+
1. Storing the output of `ie test` in a file doesn't provide a good way to
18+
navigate the output and see the results of the test.
19+
1. The log file `ie.log` is not user-friendly and can be difficult to navigate, especially
20+
so if the user is invoking multiple `ie test` commands as the log file is cumulative.
21+
1. It's not easy to reproduce the execution of a specific scenario, as most of
22+
the variables declared by the scenario are randomized and don't have their
23+
values rendered in the output.
24+
1. The output of `ie test` is not easily shareable with others.
25+
26+
To address these issues, we propose the introduction of test reports, a feature
27+
for `ie test` that will generate a report of the scenario execution in JSON
28+
format so that users can easily navigate the results of the test, reproduce
29+
specific runs, and share the results with others.
30+
31+
## Requirements
32+
33+
- [x] The user can generate a test report by running `ie test <scenario> //report=<path>`
34+
- [x] Reports capture the yaml metadata of the scenario.
35+
- [x] Reports store the variables declared in the scenario and their values.
36+
- [x] The report is generated in JSON format.
37+
- [ ] Just like the scenarios that generated them, Reports are executable.
38+
- [x] Outputs of the codeblocks executed are stored in the report.
39+
- [x] Expected outputs for codeblocks are stored in the report.
40+
41+
## Technical specifications
42+
43+
- The report will be generated in JSON format, but in the future we may consider
44+
other formats like yaml or HTML. JSON format was chosen for v1 because it is
45+
easy to parse, and it is a common format for sharing data.
46+
- Users must specify `//report=<path>` to generate a report. If the path is not
47+
specified, the report will not be generated.
48+
49+
### Report schema
50+
51+
The actual JSON schema is a work in progress, and will not be released with
52+
the initial implementation, so we will list out the actual JSON with
53+
documentation about each field until then.
54+
55+
```json
56+
{
57+
// Name of the scenario
58+
"name": "Test reporting doc",
59+
// Properties found in the yaml header
60+
"properties": {
61+
"ms.author": "vmarcella",
62+
"otherProperty": "otherValue"
63+
},
64+
65+
// Variables declared in the scenario
66+
"environmentVariables": {
67+
"NEW_VAR": "1"
68+
},
69+
// Whether the test was successful or not
70+
"success": true,
71+
// Error message if the test failed
72+
"error": "",
73+
// The step number where the test failed (-1 if successful)
74+
"failedAtStep": -1,
75+
"steps": [
76+
// The entire step
77+
{
78+
// The codeblock for the step
79+
"codeBlock": {
80+
// The language of the codeblock
81+
"language": "bash",
82+
// The content of the codeblock
83+
"content": "echo \"Hello, world!\"\n",
84+
// The header paired with the codeblock
85+
"header": "First step",
86+
// The paragraph paired with the codeblock
87+
"description": "This step will show you how to do something.",
88+
// The expected output for the codeblock
89+
"resultBlock": {
90+
// The language of the expected output
91+
"language": "text",
92+
// The content of the expected output
93+
"content": "Hello, world!\n",
94+
// The expected similarity score of the output (between 0 - 1)
95+
"expectedSimilarityScore": 1,
96+
// The expected regex pattern of the output
97+
"expectedRegexPattern": null
98+
}
99+
},
100+
// Codeblock number underneath the step (Should be ignored for now)
101+
"codeBlockNumber": 0,
102+
// Error message if the step failed (Would be same as top level error)
103+
"error": null,
104+
// Standard error output from executing the step
105+
"stdErr": "",
106+
// Standard output from executing the step
107+
"stdOut": "Hello, world!\n",
108+
// The name of the step
109+
"stepName": "First step",
110+
// The step number
111+
"stepNumber": 0,
112+
// Whether the step was successful or not
113+
"success": true,
114+
// The computed similarity score of the output (between 0 - 1)
115+
"similarityScore": 0
116+
},
117+
{
118+
"codeBlock": {
119+
"language": "bash",
120+
"content": "export NEW_VAR=1\n",
121+
"header": "Second step",
122+
"description": "This step will show you how to do something else.",
123+
"resultBlock": {
124+
"language": "",
125+
"content": "",
126+
"expectedSimilarityScore": 0,
127+
"expectedRegexPattern": null
128+
}
129+
},
130+
"codeBlockNumber": 0,
131+
"error": null,
132+
"stdErr": "",
133+
"stdOut": "",
134+
"stepName": "Second step",
135+
"stepNumber": 1,
136+
"success": true,
137+
"similarityScore": 0
138+
}
139+
]
140+
}
141+
```
142+
143+
## Examples
144+
145+
Assuming you're running this command from the root of the repository:
146+
147+
```bash
148+
ie test scenarios/testing/reporting.md --report=report.json >/dev/null && cat report.json
149+
```
150+
151+
The output of the command above should look like this:
152+
153+
<!-- Need to increase this score once I fix issue #214 -->
154+
<!-- expected_similarity=0.8 -->
155+
156+
```json
157+
{
158+
"name": "Test reporting doc",
159+
"properties": {
160+
"ms.author": "vmarcella",
161+
"otherProperty": "otherValue"
162+
},
163+
"environmentVariables": {
164+
"NEW_VAR": "1"
165+
},
166+
"success": true,
167+
"error": "",
168+
"failedAtStep": -1,
169+
"steps": [
170+
{
171+
"codeBlock": {
172+
"language": "bash",
173+
"content": "echo \"Hello, world!\"\n",
174+
"header": "First step",
175+
"description": "This step will show you how to do something.",
176+
"resultBlock": {
177+
"language": "text",
178+
"content": "Hello, world!\n",
179+
"expectedSimilarityScore": 1,
180+
"expectedRegexPattern": null
181+
}
182+
},
183+
"codeBlockNumber": 0,
184+
"error": null,
185+
"stdErr": "",
186+
"stdOut": "Hello, world!\n",
187+
"stepName": "First step",
188+
"stepNumber": 0,
189+
"success": true,
190+
"similarityScore": 1
191+
},
192+
{
193+
"codeBlock": {
194+
"language": "bash",
195+
"content": "export NEW_VAR=1\n",
196+
"header": "Second step",
197+
"description": "This step will show you how to do something else.",
198+
"resultBlock": {
199+
"language": "",
200+
"content": "",
201+
"expectedSimilarityScore": 0,
202+
"expectedRegexPattern": null
203+
}
204+
},
205+
"codeBlockNumber": 0,
206+
"error": null,
207+
"stdErr": "",
208+
"stdOut": "",
209+
"stepName": "Second step",
210+
"stepNumber": 1,
211+
"success": true,
212+
"similarityScore": 1
213+
}
214+
]
215+
}
216+
```

‎internal/engine/common/codeblock.go

+9-8
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,15 @@ import "github.com/Azure/InnovationEngine/internal/parsers"
55
// State for the codeblock in interactive mode. Used to keep track of the
66
// state of each codeblock.
77
type StatefulCodeBlock struct {
8-
CodeBlock parsers.CodeBlock
9-
CodeBlockNumber int
10-
Error error
11-
StdErr string
12-
StdOut string
13-
StepName string
14-
StepNumber int
15-
Success bool
8+
CodeBlock parsers.CodeBlock `json:"codeBlock"`
9+
CodeBlockNumber int `json:"codeBlockNumber"`
10+
Error error `json:"error"`
11+
StdErr string `json:"stdErr"`
12+
StdOut string `json:"stdOut"`
13+
StepName string `json:"stepName"`
14+
StepNumber int `json:"stepNumber"`
15+
Success bool `json:"success"`
16+
SimilarityScore float64 `json:"similarityScore"`
1617
}
1718

1819
// Checks if a codeblock was executed by looking at the

‎internal/engine/common/commands.go

+19-14
Original file line numberDiff line numberDiff line change
@@ -12,15 +12,17 @@ import (
1212

1313
// Emitted when a command has been executed successfully.
1414
type SuccessfulCommandMessage struct {
15-
StdOut string
16-
StdErr string
15+
StdOut string
16+
StdErr string
17+
SimilarityScore float64
1718
}
1819

1920
// Emitted when a command has failed to execute.
2021
type FailedCommandMessage struct {
21-
StdOut string
22-
StdErr string
23-
Error error
22+
StdOut string
23+
StdErr string
24+
Error error
25+
SimilarityScore float64
2426
}
2527

2628
type ExitMessage struct {
@@ -49,9 +51,10 @@ func ExecuteCodeBlockAsync(codeBlock parsers.CodeBlock, env map[string]string) t
4951
if err != nil {
5052
logging.GlobalLogger.Errorf("Error executing command:\n %s", err.Error())
5153
return FailedCommandMessage{
52-
StdOut: output.StdOut,
53-
StdErr: output.StdErr,
54-
Error: err,
54+
StdOut: output.StdOut,
55+
StdErr: output.StdErr,
56+
Error: err,
57+
SimilarityScore: 0,
5558
}
5659
}
5760

@@ -62,7 +65,7 @@ func ExecuteCodeBlockAsync(codeBlock parsers.CodeBlock, env map[string]string) t
6265
expectedRegex := codeBlock.ExpectedOutput.ExpectedRegex
6366
expectedOutputLanguage := codeBlock.ExpectedOutput.Language
6467

65-
outputComparisonError := CompareCommandOutputs(
68+
score, outputComparisonError := CompareCommandOutputs(
6669
actualOutput,
6770
expectedOutput,
6871
expectedSimilarity,
@@ -77,17 +80,19 @@ func ExecuteCodeBlockAsync(codeBlock parsers.CodeBlock, env map[string]string) t
7780
)
7881

7982
return FailedCommandMessage{
80-
StdOut: output.StdOut,
81-
StdErr: output.StdErr,
82-
Error: outputComparisonError,
83+
StdOut: output.StdOut,
84+
StdErr: output.StdErr,
85+
Error: outputComparisonError,
86+
SimilarityScore: score,
8387
}
8488

8589
}
8690

8791
logging.GlobalLogger.Infof("Command output to stdout:\n %s", output.StdOut)
8892
return SuccessfulCommandMessage{
89-
StdOut: output.StdOut,
90-
StdErr: output.StdErr,
93+
StdOut: output.StdOut,
94+
StdErr: output.StdErr,
95+
SimilarityScore: score,
9196
}
9297
}
9398
}

0 commit comments

Comments
 (0)