Skip to content

Commit 12b3fea

Browse files
oschwaldoschwald
committed
First commit where the reader actually works.
* Added reader tests. * Fixed pointer bug in the decoder and added tests. * Fixed many small issues in the reader.
1 parent 0465d9a commit 12b3fea

File tree

14 files changed

+317
-57
lines changed

14 files changed

+317
-57
lines changed

Diff for: ‎src/main/java/com/maxmind/maxminddb/Decoder.java

+18-28
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ public class Decoder {
2020

2121
class Result {
2222
private final Object obj;
23-
private final long new_offset;
23+
private long new_offset;
2424
private final Type type;
2525

2626
Result(Object obj, Type t, long new_offset) {
@@ -37,6 +37,10 @@ long getOffset() {
3737
return this.new_offset;
3838
}
3939

40+
void setOffset(long offset) {
41+
this.new_offset = offset;
42+
}
43+
4044
Type getType() {
4145
return this.type;
4246
}
@@ -49,18 +53,18 @@ public Decoder(FileChannel in, long pointerBase) {
4953

5054
// FIXME - Move most of this method to a switch statement
5155
public Result decode(long offset) throws MaxMindDbException, IOException {
52-
56+
this.in.position(offset);
5357
if (this.DEBUG) {
5458
Log.debug("Offset", String.valueOf(offset));
5559
}
5660

5761
ByteBuffer buffer = ByteBuffer.wrap(new byte[1]);
5862
this.in.read(buffer);
59-
int ctrlByte = buffer.get(0);
63+
int ctrlByte = 0xFF & buffer.get(0);
6064
offset++;
6165

6266
if (this.DEBUG) {
63-
Log.debug("Control byte", ctrlByte);
67+
Log.debugBinary("Control byte", ctrlByte);
6468
}
6569

6670
Type type = Type.fromControlByte(ctrlByte);
@@ -78,8 +82,10 @@ public Result decode(long offset) throws MaxMindDbException, IOException {
7882
if (this.POINTER_TEST_HACK) {
7983
return pointer;
8084
}
81-
82-
return this.decode(((Long) pointer.getObject()).longValue());
85+
Result result = this.decode(((Long) pointer.getObject())
86+
.longValue());
87+
result.setOffset(pointer.getOffset());
88+
return result;
8389
}
8490

8591
if (type.equals(Type.EXTENDED)) {
@@ -194,7 +200,7 @@ private Result decodePointer(int ctrlByte, long offset) throws IOException {
194200
buffer.put(0, pointerSize == 4 ? (byte) 0
195201
: (byte) (ctrlByte & 0b00000111));
196202

197-
long packed = decodeLong(buffer.array());
203+
long packed = Util.decodeLong(buffer.array());
198204

199205
if (this.DEBUG) {
200206
Log.debug("Packed pointer", String.valueOf(packed));
@@ -227,11 +233,11 @@ static int decodeUint16(byte[] bytes) {
227233
}
228234

229235
static int decodeInt32(byte[] bytes) {
230-
return decodeInteger(bytes);
236+
return Util.decodeInteger(bytes);
231237
}
232238

233239
static long decodeUint32(byte[] bytes) {
234-
return decodeLong(bytes);
240+
return Util.decodeLong(bytes);
235241
}
236242

237243
static BigInteger decodeUint64(byte[] bytes) {
@@ -242,22 +248,6 @@ static BigInteger decodeUint128(byte[] bytes) {
242248
return new BigInteger(1, bytes);
243249
}
244250

245-
static int decodeInteger(byte[] bytes) {
246-
int i = 0;
247-
for (byte b : bytes) {
248-
i = (i << 8) | (b & 0xFF);
249-
}
250-
return i;
251-
}
252-
253-
static long decodeLong(byte[] bytes) {
254-
long i = 0;
255-
for (byte b : bytes) {
256-
i = (i << 8) | (b & 0xFF);
257-
}
258-
return i;
259-
}
260-
261251
private double decodeDouble(byte[] bytes) {
262252
return new Double(new String(bytes, Charset.forName("US-ASCII")));
263253
}
@@ -339,14 +329,14 @@ private long[] sizeFromCtrlByte(int ctrlByte, long offset)
339329
this.in.read(buffer);
340330

341331
if (size == 29) {
342-
int i = decodeInteger(buffer.array());
332+
int i = Util.decodeInteger(buffer.array());
343333
size = 29 + i;
344334
} else if (size == 30) {
345-
int i = decodeInteger(buffer.array());
335+
int i = Util.decodeInteger(buffer.array());
346336
size = 285 + i;
347337
} else {
348338
buffer.put(0, (byte) (buffer.get(0) & 0x0F));
349-
int i = decodeInteger(buffer.array());
339+
int i = Util.decodeInteger(buffer.array());
350340
size = 65821 + i;
351341
}
352342

Diff for: ‎src/main/java/com/maxmind/maxminddb/Log.java

+4
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ static void debug(String name, int value) {
3030
Log.debug(name, String.valueOf(value));
3131
}
3232

33+
static void debugBinary(String name, int value) {
34+
Log.debug(name, Integer.toBinaryString(value));
35+
}
36+
3337
private static void debug(String string, byte b) {
3438
Log.debug(string, Integer.toBinaryString(b & 0xFF));
3539
}

Diff for: ‎src/main/java/com/maxmind/maxminddb/Metadata.java

+9-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.maxmind.maxminddb;
22

33
import java.math.BigInteger;
4+
import java.util.Arrays;
45
import java.util.Map;
56

67
// XXX - if we make this public, add getters
@@ -13,6 +14,9 @@ class Metadata {
1314
Integer ipVersion;
1415
Long nodeCount;
1516
Long recordSize;
17+
int nodeByteSize;
18+
long searchTreeSize;
19+
String[] languages;
1620

1721
// XXX - think about how I want to construct this. Maybe look at how JSON
1822
// parsers deal with types
@@ -23,10 +27,15 @@ public Metadata(Map<String, Object> metadata) {
2327
.get("binary_format_minor_version");
2428
this.buildEpoch = (BigInteger) metadata.get("build_epoch");
2529
this.databaseType = (String) metadata.get("database_type");
30+
Object[] langs = (Object[]) metadata.get("languages");
31+
this.languages = Arrays.copyOf(langs, langs.length, String[].class);
2632
this.description = (Map<String, Object>) metadata.get("description");
2733
this.ipVersion = (Integer) metadata.get("ip_version");
2834
this.nodeCount = (Long) metadata.get("node_count");
2935
this.recordSize = (Long) metadata.get("record_size");
36+
this.nodeByteSize = (int) (this.recordSize / 4);
37+
this.searchTreeSize = this.nodeCount * this.nodeByteSize;
38+
3039
}
3140

3241
/*
@@ -44,5 +53,4 @@ public String toString() {
4453
+ this.ipVersion + ", nodeCount=" + this.nodeCount
4554
+ ", recordSize=" + this.recordSize + "]";
4655
}
47-
4856
}

Diff for: ‎src/main/java/com/maxmind/maxminddb/Reader.java

+83-28
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,18 @@
66
import java.net.InetAddress;
77
import java.nio.ByteBuffer;
88
import java.nio.channels.FileChannel;
9+
import java.util.Arrays;
910
import java.util.Map;
1011

1112
public class Reader {
12-
private static final boolean DEBUG = true;
13-
private Decoder decoder;
14-
private long nodeCount;
15-
private final long dataSectionEnd;
13+
private static int DATA_SECTION_SEPARATOR_SIZE = 16;
1614
private static byte METADATE_START_MARKER[] = { (byte) 0xAB, (byte) 0xCD,
1715
(byte) 0xEF, 'M', 'a', 'x', 'M', 'i', 'n', 'd', '.', 'c', 'o', 'm' };
16+
17+
private static final boolean DEBUG = true;
18+
private final Decoder decoder;
19+
private final Metadata metadata;
20+
private final long dataSectionEnd;
1821
private final FileChannel fc;
1922

2023
public Reader(File dataSource) throws MaxMindDbException, IOException {
@@ -51,16 +54,16 @@ public Reader(File dataSource) throws MaxMindDbException, IOException {
5154
// XXX - right?
5255
this.dataSectionEnd = start - METADATE_START_MARKER.length;
5356

54-
Decoder decoder = new Decoder(this.fc, 0);
57+
Decoder metadataDecoder = new Decoder(this.fc, 0);
58+
59+
this.metadata = new Metadata((Map<String, Object>) metadataDecoder
60+
.decode(start).getObject());
61+
62+
this.decoder = new Decoder(this.fc, this.metadata.searchTreeSize
63+
+ DATA_SECTION_SEPARATOR_SIZE);
5564

56-
// FIXME - pretty ugly that I am setting the position outside of the
57-
// decoder. Move this all into
58-
// the decoder and make sure it is thread safe
59-
this.fc.position(start);
60-
Metadata metadata = new Metadata((Map<String, Object>) decoder
61-
.decode(0).getObject());
6265
if (DEBUG) {
63-
Log.debug(metadata.toString());
66+
Log.debug(this.metadata.toString());
6467
}
6568
}
6669

@@ -74,12 +77,21 @@ Object dataForAddress(InetAddress address) throws MaxMindDbException,
7477
return null;
7578
}
7679

80+
this.fc.position(pointer);
7781
return this.resolveDataPointer(pointer);
7882
}
7983

80-
long findAddressInTree(InetAddress address) throws MaxMindDbException {
84+
long findAddressInTree(InetAddress address) throws MaxMindDbException,
85+
IOException {
8186
byte[] rawAddress = address.getAddress();
8287

88+
// XXX sort of wasteful
89+
if (rawAddress.length == 4 && this.metadata.ipVersion == 6) {
90+
byte[] newAddress = new byte[16];
91+
System.arraycopy(rawAddress, 0, newAddress, 12, rawAddress.length);
92+
rawAddress = newAddress;
93+
}
94+
8395
if (DEBUG) {
8496
Log.debugNewLine();
8597
Log.debug("IP address", address);
@@ -92,27 +104,29 @@ long findAddressInTree(InetAddress address) throws MaxMindDbException {
92104
long nodeNum = 0;
93105

94106
for (int i = 0; i < rawAddress.length * 8; i++) {
95-
byte b = rawAddress[i / 8];
96-
int bit = 1 & (b >> 7 - i);
97-
int[] nodes = this.readNode(nodeNum);
107+
int b = 0xFF & rawAddress[i / 8];
108+
int bit = 1 & (b >> 7 - (i % 8));
109+
long[] nodes = this.readNode(nodeNum);
98110

99-
int record = nodes[bit];
111+
long record = nodes[bit];
100112

101113
if (DEBUG) {
114+
Log.debug("Nodes", Arrays.toString(nodes));
102115
Log.debug("Bit #", i);
103116
Log.debug("Bit value", bit);
104117
Log.debug("Record", bit == 1 ? "right" : "left");
118+
// Log.debug("Node count", this.metadata.nodeCount);
105119
Log.debug("Record value", record);
106120
}
107121

108-
if (record == this.nodeCount) {
122+
if (record == this.metadata.nodeCount) {
109123
if (DEBUG) {
110124
Log.debug("Record is empty");
111125
}
112126
return 0;
113127
}
114128

115-
if (record >= this.nodeCount) {
129+
if (record >= this.metadata.nodeCount) {
116130
if (DEBUG) {
117131
Log.debug("Record is a data pointer");
118132
}
@@ -130,20 +144,61 @@ int record = nodes[bit];
130144
throw new MaxMindDbException("Something bad happened");
131145
}
132146

133-
private int[] readNode(long nodeNum) {
134-
throw new AssertionError("not implemented");
147+
private long[] readNode(long nodeNumber) throws IOException,
148+
MaxMindDbException {
149+
ByteBuffer buffer = ByteBuffer
150+
.wrap(new byte[this.metadata.nodeByteSize]);
151+
this.fc.position(nodeNumber * this.metadata.nodeByteSize);
135152

153+
this.fc.read(buffer);
154+
155+
if (DEBUG) {
156+
Log.debug("Node bytes", buffer);
157+
}
158+
return this.splitNodeIntoRecords(buffer);
159+
}
160+
161+
private long[] splitNodeIntoRecords(ByteBuffer bytes)
162+
throws MaxMindDbException {
163+
long[] nodes = new long[2];
164+
switch (this.metadata.recordSize.intValue()) {
165+
case 24:
166+
nodes[0] = Util.decodeLong(Arrays.copyOfRange(bytes.array(), 0,
167+
3));
168+
nodes[1] = Util.decodeLong(Arrays.copyOfRange(bytes.array(), 3,
169+
6));
170+
return nodes;
171+
case 28:
172+
nodes[0] = Util.decodeLong(Arrays.copyOfRange(bytes.array(), 0,
173+
3));
174+
nodes[1] = Util.decodeLong(Arrays.copyOfRange(bytes.array(), 4,
175+
7));
176+
nodes[0] = ((0xF0 & bytes.get(3)) << 24) | nodes[0];
177+
nodes[1] = ((0x0F & bytes.get(3)) << 24) | nodes[1];
178+
return nodes;
179+
case 32:
180+
nodes[0] = Util.decodeLong(Arrays.copyOfRange(bytes.array(), 0,
181+
4));
182+
nodes[1] = Util.decodeLong(Arrays.copyOfRange(bytes.array(), 4,
183+
8));
184+
return nodes;
185+
default:
186+
throw new MaxMindDbException("Unknown record size: "
187+
+ this.metadata.recordSize);
188+
}
136189
}
137190

138191
private Object resolveDataPointer(long pointer) throws MaxMindDbException,
139192
IOException {
140-
long resolved = (pointer - this.nodeCount) + this.searchTreeSize();
193+
long resolved = (pointer - this.metadata.nodeCount)
194+
+ this.metadata.searchTreeSize;
141195

142196
if (DEBUG) {
143-
long treeSize = this.searchTreeSize();
197+
long treeSize = this.metadata.searchTreeSize;
144198

145199
Log.debug("Resolved data pointer", "( " + pointer + " - "
146-
+ this.nodeCount + " ) + " + treeSize + " = " + resolved);
200+
+ this.metadata.nodeCount + " ) + " + treeSize + " = "
201+
+ resolved);
147202

148203
}
149204

@@ -152,10 +207,6 @@ private Object resolveDataPointer(long pointer) throws MaxMindDbException,
152207
return this.decoder.decode(resolved).getObject();
153208
}
154209

155-
private long searchTreeSize() {
156-
throw new AssertionError("not implemented");
157-
}
158-
159210
/*
160211
* And here I though searching a file was a solved problem.
161212
*
@@ -181,4 +232,8 @@ private long findMetadataStart() throws IOException {
181232
}
182233
return -1;
183234
}
235+
236+
public Metadata getMetadata() {
237+
return this.metadata;
238+
}
184239
}

Diff for: ‎src/main/java/com/maxmind/maxminddb/Util.java

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package com.maxmind.maxminddb;
2+
3+
class Util {
4+
5+
public Util() {
6+
// TODO Auto-generated constructor stub
7+
}
8+
9+
static int decodeInteger(byte[] bytes) {
10+
int i = 0;
11+
for (byte b : bytes) {
12+
i = (i << 8) | (b & 0xFF);
13+
}
14+
return i;
15+
}
16+
17+
static long decodeLong(byte[] bytes) {
18+
long i = 0;
19+
for (byte b : bytes) {
20+
i = (i << 8) | (b & 0xFF);
21+
}
22+
return i;
23+
}
24+
25+
}

0 commit comments

Comments
 (0)