Skip to content

Commit f4abc82

Browse files
dongjoon-hyundongjoon-hyun
committed
ORC-1607: Fix testDoubleNaNAndInfinite to use TestFileDump.checkOutput
This PR aims to fix a unit test failure. ``` [ERROR] Failures: [ERROR] TestJsonFileDump.testDoubleNaNAndInfinite:167 expected: <{"fileName":"TestFileDump.testDump.orc","fileVersion":"0.12","writerVersion":"ORC_14","softwareVersion":"ORC Java unknown","numberOfRows":3,"compression":"ZSTD","compressionBufferSize":262144,"schemaString":"struct<x:double>","schema":{"columnId":0,"columnType":"STRUCT","children":{"x":{"columnId":1,"columnType":"DOUBLE"}}},"calendar":"Julian/Gregorian","stripeStatistics":[{"stripeNumber":1,"columnStatistics":[{"columnId":0,"count":3,"hasNull":false},{"columnId":1,"count":3,"hasNull":false,"bytesOnDisk":27,"min":NaN,"max":NaN,"sum":NaN,"type":"DOUBLE"}]}],"fileStatistics":[{"columnId":0,"count":3,"hasNull":false},{"columnId":1,"count":3,"hasNull":false,"bytesOnDisk":27,"min":NaN,"max":NaN,"sum":NaN,"type":"DOUBLE"}],"stripes":[{"stripeNumber":1,"stripeInformation":{"offset":3,"indexLength":55,"dataLength":27,"footerLength":35,"rowCount":3},"streams":[{"columnId":0,"section":"ROW_INDEX","startOffset":3,"length":11},{"columnId":1,"section":"ROW_INDEX","startOffset":14,"length":44},{"columnId":1,"section":"DATA","startOffset":58,"length":27}],"encodings":[{"columnId":0,"kind":"DIRECT"},{"columnId":1,"kind":"DIRECT"}]}],"fileLength":286,"rawDataSize":36,"paddingLength":0,"paddingRatio":0.0,"status":"OK"}> but was: <{"fileName":"TestFileDump.testDump.orc","fileVersion":"0.12","writerVersion":"ORC_14","softwareVersion":"ORC Java 2.1.0-SNAPSHOT","numberOfRows":3,"compression":"ZSTD","compressionBufferSize":262144,"schemaString":"struct<x:double>","schema":{"columnId":0,"columnType":"STRUCT","children":{"x":{"columnId":1,"columnType":"DOUBLE"}}},"calendar":"Julian/Gregorian","stripeStatistics":[{"stripeNumber":1,"columnStatistics":[{"columnId":0,"count":3,"hasNull":false},{"columnId":1,"count":3,"hasNull":false,"bytesOnDisk":27,"min":NaN,"max":NaN,"sum":NaN,"type":"DOUBLE"}]}],"fileStatistics":[{"columnId":0,"count":3,"hasNull":false},{"columnId":1,"count":3,"hasNull":false,"bytesOnDisk":27,"min":NaN,"max":NaN,"sum":NaN,"type":"DOUBLE"}],"stripes":[{"stripeNumber":1,"stripeInformation":{"offset":3,"indexLength":55,"dataLength":27,"footerLength":35,"rowCount":3},"streams":[{"columnId":0,"section":"ROW_INDEX","startOffset":3,"length":11},{"columnId":1,"section":"ROW_INDEX","startOffset":14,"length":44},{"columnId":1,"section":"DATA","startOffset":58,"length":27}],"encodings":[{"columnId":0,"kind":"DIRECT"},{"columnId":1,"kind":"DIRECT"}]}],"fileLength":293,"rawDataSize":36,"paddingLength":0,"paddingRatio":0.0,"status":"OK"}> [INFO] [ERROR] Tests run: 51, Failures: 1, Errors: 0, Skipped: 0 ``` Manually check on Apple Silicon MacOS. No. Closes #1781 from dongjoon-hyun/ORC-1607. Authored-by: Dongjoon Hyun <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]> (cherry picked from commit 0dee947) Signed-off-by: Dongjoon Hyun <[email protected]>
1 parent b3cc073 commit f4abc82

File tree

2 files changed

+111
-6
lines changed

2 files changed

+111
-6
lines changed

‎java/tools/src/test/org/apache/orc/tools/TestJsonFileDump.java

+5-6
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333
import org.junit.jupiter.api.BeforeEach;
3434
import org.junit.jupiter.api.Test;
3535

36-
import java.io.ByteArrayOutputStream;
3736
import java.io.File;
3837
import java.io.FileOutputStream;
3938
import java.io.PrintStream;
@@ -156,14 +155,14 @@ public void testDoubleNaNAndInfinite() throws Exception {
156155
assertEquals(3, writer.getNumberOfRows());
157156

158157
PrintStream origOut = System.out;
159-
ByteArrayOutputStream myOut = new ByteArrayOutputStream();
158+
String outputFilename = "orc-file-dump-nan-and-infinite.json";
159+
FileOutputStream myOut = new FileOutputStream(workDir + File.separator + outputFilename);
160160

161161
// replace stdout and run command
162-
System.setOut(new PrintStream(myOut, false, StandardCharsets.UTF_8.toString()));
163-
FileDump.main(new String[]{testFilePath.toString(), "-j"});
162+
System.setOut(new PrintStream(myOut, true, StandardCharsets.UTF_8));
163+
FileDump.main(new String[]{testFilePath.toString(), "-j", "-p"});
164164
System.out.flush();
165165
System.setOut(origOut);
166-
String[] lines = myOut.toString(StandardCharsets.UTF_8.toString()).split("\n");
167-
assertEquals("{\"fileName\":\"TestFileDump.testDump.orc\",\"fileVersion\":\"0.12\",\"writerVersion\":\"ORC_14\",\"softwareVersion\":\"ORC Java unknown\",\"numberOfRows\":3,\"compression\":\"ZLIB\",\"compressionBufferSize\":262144,\"schemaString\":\"struct<x:double>\",\"schema\":{\"columnId\":0,\"columnType\":\"STRUCT\",\"children\":{\"x\":{\"columnId\":1,\"columnType\":\"DOUBLE\"}}},\"calendar\":\"Julian/Gregorian\",\"stripeStatistics\":[{\"stripeNumber\":1,\"columnStatistics\":[{\"columnId\":0,\"count\":3,\"hasNull\":false},{\"columnId\":1,\"count\":3,\"hasNull\":false,\"bytesOnDisk\":23,\"min\":NaN,\"max\":NaN,\"sum\":NaN,\"type\":\"DOUBLE\"}]}],\"fileStatistics\":[{\"columnId\":0,\"count\":3,\"hasNull\":false},{\"columnId\":1,\"count\":3,\"hasNull\":false,\"bytesOnDisk\":23,\"min\":NaN,\"max\":NaN,\"sum\":NaN,\"type\":\"DOUBLE\"}],\"stripes\":[{\"stripeNumber\":1,\"stripeInformation\":{\"offset\":3,\"indexLength\":41,\"dataLength\":23,\"footerLength\":29,\"rowCount\":3},\"streams\":[{\"columnId\":0,\"section\":\"ROW_INDEX\",\"startOffset\":3,\"length\":11},{\"columnId\":1,\"section\":\"ROW_INDEX\",\"startOffset\":14,\"length\":30},{\"columnId\":1,\"section\":\"DATA\",\"startOffset\":44,\"length\":23}],\"encodings\":[{\"columnId\":0,\"kind\":\"DIRECT\"},{\"columnId\":1,\"kind\":\"DIRECT\"}]}],\"fileLength\":234,\"paddingLength\":0,\"paddingRatio\":0.0,\"status\":\"OK\"}", lines[0]);
166+
TestFileDump.checkOutput(outputFilename, workDir + File.separator + outputFilename);
168167
}
169168
}

‎java/tools/src/test/resources/orc-file-dump-nan-and-infinite.json

+106
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
{
2+
"fileName": "TestFileDump.testDump.orc",
3+
"fileVersion": "0.12",
4+
"writerVersion": "ORC_14",
5+
"softwareVersion": "ORC Java 2.1.0-SNAPSHOT",
6+
"numberOfRows": 3,
7+
"compression": "ZLIB",
8+
"compressionBufferSize": 262144,
9+
"schemaString": "struct<x:double>",
10+
"schema": {
11+
"columnId": 0,
12+
"columnType": "STRUCT",
13+
"children": {
14+
"x": {
15+
"columnId": 1,
16+
"columnType": "DOUBLE"
17+
}
18+
}
19+
},
20+
"calendar": "Julian/Gregorian",
21+
"stripeStatistics": [
22+
{
23+
"stripeNumber": 1,
24+
"columnStatistics": [
25+
{
26+
"columnId": 0,
27+
"count": 3,
28+
"hasNull": false
29+
},
30+
{
31+
"columnId": 1,
32+
"count": 3,
33+
"hasNull": false,
34+
"bytesOnDisk": 23,
35+
"min": NaN,
36+
"max": NaN,
37+
"sum": NaN,
38+
"type": "DOUBLE"
39+
}
40+
]
41+
}
42+
],
43+
"fileStatistics": [
44+
{
45+
"columnId": 0,
46+
"count": 3,
47+
"hasNull": false
48+
},
49+
{
50+
"columnId": 1,
51+
"count": 3,
52+
"hasNull": false,
53+
"bytesOnDisk": 23,
54+
"min": NaN,
55+
"max": NaN,
56+
"sum": NaN,
57+
"type": "DOUBLE"
58+
}
59+
],
60+
"stripes": [
61+
{
62+
"stripeNumber": 1,
63+
"stripeInformation": {
64+
"offset": 3,
65+
"indexLength": 41,
66+
"dataLength": 23,
67+
"footerLength": 29,
68+
"rowCount": 3
69+
},
70+
"streams": [
71+
{
72+
"columnId": 0,
73+
"section": "ROW_INDEX",
74+
"startOffset": 3,
75+
"length": 11
76+
},
77+
{
78+
"columnId": 1,
79+
"section": "ROW_INDEX",
80+
"startOffset": 14,
81+
"length": 30
82+
},
83+
{
84+
"columnId": 1,
85+
"section": "DATA",
86+
"startOffset": 44,
87+
"length": 23
88+
}
89+
],
90+
"encodings": [
91+
{
92+
"columnId": 0,
93+
"kind": "DIRECT"
94+
},
95+
{
96+
"columnId": 1,
97+
"kind": "DIRECT"
98+
}
99+
]
100+
}
101+
],
102+
"fileLength": 293,
103+
"paddingLength": 0,
104+
"paddingRatio": 0.0,
105+
"status": "OK"
106+
}

0 commit comments

Comments
 (0)