diff --git a/README.md b/README.md
index 22c1f5a95..c270d966b 100755
--- a/README.md
+++ b/README.md
@@ -74,6 +74,13 @@ This has meant that the spatial library needed a major refactoring to work with
and simply add on the rights to create tokens and indexes. In 0.27.2 we instead use `RestrictedAccessMode`
to restrict the users access right to the built in `AccessModel.Static.SCHEMA` and then boost to enable
index and token writes. The difference is subtle and should only be possible to notice in Enterprise Edition.
+* 0.28.0 tackles the ability to import multiple OSM files. The initial solution for Neo4j 4.x made use
+ of schema indexes keyed by the label and property. However, that means that all OSM imports would share
+ the same index. If they are completely disjointed data sets, this would not matter. But if you import
+ overlapping OSM files or different versions of the same file file, a mangled partial merger would result.
+ 0.28.0 solves this by using different indexes, and keeping all imports completely separate.
+ The more complex problems of importing newer versions, and stitching together overlapping areas, are not
+ yet solved.
Consequences of the port to Neo4j 4.x:
@@ -347,6 +354,7 @@ The Neo4j Spatial Plugin is available for inclusion in the server version of Neo
* [v0.27.0 for Neo4j 4.0.3](https://github.com/neo4j-contrib/m2/blob/master/releases/org/neo4j/neo4j-spatial/0.27.0-neo4j-4.0.3/neo4j-spatial-0.27.0-neo4j-4.0.3-server-plugin.jar?raw=true)
* [v0.27.1 for Neo4j 4.1.7](https://github.com/neo4j-contrib/m2/blob/master/releases/org/neo4j/neo4j-spatial/0.27.1-neo4j-4.1.7/neo4j-spatial-0.27.1-neo4j-4.1.7-server-plugin.jar?raw=true)
* [v0.27.2 for Neo4j 4.2.3](https://github.com/neo4j-contrib/m2/blob/master/releases/org/neo4j/neo4j-spatial/0.27.2-neo4j-4.2.3/neo4j-spatial-0.27.2-neo4j-4.2.3-server-plugin.jar?raw=true)
+ * [v0.28.0 for Neo4j 4.2.3](https://github.com/neo4j-contrib/m2/blob/master/releases/org/neo4j/neo4j-spatial/0.28.0-neo4j-4.2.3/neo4j-spatial-0.28.0-neo4j-4.2.3-server-plugin.jar?raw=true)
For versions up to 0.15-neo4j-2.3.4:
@@ -463,7 +471,7 @@ Add the following repositories and dependency to your project's pom.xml:
org.neo4j
neo4j-spatial
- 0.27.2-neo4j-4.2.3
+ 0.28.0-neo4j-4.2.3
~~~
diff --git a/pom.xml b/pom.xml
index db64dbce9..e80dea01a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -23,7 +23,7 @@
4.0.0
neo4j-spatial
org.neo4j
- 0.27.2-neo4j-4.2.3
+ 0.28.0-neo4j-4.2.3
Neo4j - Spatial Components
Spatial utilities and components for Neo4j
http://components.neo4j.org/${project.artifactId}/${project.version}
diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java
index 92c07120a..e5f7a029c 100644
--- a/src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java
+++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java
@@ -28,7 +28,6 @@
import org.neo4j.gis.spatial.rtree.Envelope;
import org.neo4j.gis.spatial.rtree.Listener;
import org.neo4j.gis.spatial.rtree.NullListener;
-import org.neo4j.gis.spatial.utilities.ReferenceNodes;
import org.neo4j.graphdb.*;
import org.neo4j.graphdb.schema.IndexDefinition;
import org.neo4j.graphdb.traversal.Evaluators;
@@ -41,10 +40,13 @@
import org.neo4j.kernel.impl.traversal.MonoDirectionalTraversalDescription;
import org.neo4j.kernel.internal.GraphDatabaseAPI;
+import javax.xml.bind.DatatypeConverter;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import java.io.*;
import java.nio.charset.Charset;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
@@ -392,7 +394,7 @@ private OSMWriter(StatsManager statsManager, OSMImporter osmImporter) {
this.osmImporter = osmImporter;
}
- static OSMWriter fromGraphDatabase(GraphDatabaseService graphDb, SecurityContext securityContext, StatsManager stats, OSMImporter osmImporter, int txInterval) {
+ static OSMWriter fromGraphDatabase(GraphDatabaseService graphDb, SecurityContext securityContext, StatsManager stats, OSMImporter osmImporter, int txInterval) throws NoSuchAlgorithmException {
return new OSMGraphWriter(graphDb, securityContext, stats, osmImporter, txInterval);
}
@@ -867,8 +869,10 @@ private static class OSMGraphWriter extends OSMWriter {
private IndexDefinition relationIndex;
private IndexDefinition changesetIndex;
private IndexDefinition userIndex;
+ private final String layerHash;
+ private final HashMap hashedLabels = new HashMap<>();
- private OSMGraphWriter(GraphDatabaseService graphDb, SecurityContext securityContext, StatsManager statsManager, OSMImporter osmImporter, int txInterval) {
+ private OSMGraphWriter(GraphDatabaseService graphDb, SecurityContext securityContext, StatsManager statsManager, OSMImporter osmImporter, int txInterval) throws NoSuchAlgorithmException {
super(statsManager, osmImporter);
this.graphDb = graphDb;
this.securityContext = securityContext;
@@ -876,9 +880,18 @@ private OSMGraphWriter(GraphDatabaseService graphDb, SecurityContext securityCon
if (this.txInterval < 100) {
System.err.println("Warning: Unusually short txInterval, expect bad insert performance");
}
+ this.layerHash = md5Hash(osmImporter.layerName);
checkTx(null); // Opens transaction for future writes
}
+ private static String md5Hash(String text) throws NoSuchAlgorithmException {
+ MessageDigest md = MessageDigest.getInstance("MD5");
+ md.update(text.getBytes());
+ byte[] digest = md.digest();
+ String hashed = DatatypeConverter.printHexBinary(digest).toUpperCase();
+ return hashed;
+ }
+
private void successTx() {
if (tx != null) {
tx.commit();
@@ -930,14 +943,19 @@ private void recoverNode(WrappedNode outOfTx) {
}
}
- private WrappedNode findNode(Label label, String name) {
- Node node = tx.findNode(label, "name", name);
+ private WrappedNode findNodeByName(Label label, String name) {
+ Node node = findNodeByLabelProperty(tx, label, "name", name);
if (node != null) {
return WrappedNode.fromNode(node);
}
return null;
}
+ private WrappedNode createNodeWithLabel(Transaction tx, Label label) {
+ Label hashed = getLabelHashed(label);
+ return WrappedNode.fromNode(tx.createNode(label, hashed));
+ }
+
@Override
protected void startWays() {
System.out.println("About to create node index");
@@ -968,12 +986,29 @@ protected void optimize() {
}
}
+ private Label getLabelHashed(Label label) {
+ if (hashedLabels.containsKey(label)) {
+ return hashedLabels.get(label);
+ } else {
+ Label hashed = Label.label(label.name() + "_" + layerHash);
+ hashedLabels.put(label, hashed);
+ return hashed;
+ }
+ }
+
+ private Node findNodeByLabelProperty(Transaction tx, Label label, String propertyKey, Object value) {
+ Label hashed = getLabelHashed(label);
+ return tx.findNode(hashed, propertyKey, value);
+ }
+
private IndexDefinition createIndex(Label label, String propertyKey) {
- IndexDefinition index = findIndex(tx, label, propertyKey);
+ Label hashed = getLabelHashed(label);
+ String indexName = String.format("OSM-%s-%s-%s", osmImporter.layerName, hashed.name(), propertyKey);
+ IndexDefinition index = findIndex(tx, indexName, hashed, propertyKey);
if (index == null) {
successTx();
try (Transaction indexTx = beginIndexTx(graphDb)) {
- index = indexTx.schema().indexFor(label).on(propertyKey).create();
+ index = indexTx.schema().indexFor(hashed).on(propertyKey).withName(indexName).create();
indexTx.commit();
}
System.out.println("Created index " + index.getName());
@@ -990,11 +1025,15 @@ private IndexDefinition createIndexIfNotNull(IndexDefinition index, Label label,
return index;
}
- private IndexDefinition findIndex(Transaction tx, Label label, String propertyKey) {
+ private IndexDefinition findIndex(Transaction tx, String indexName, Label label, String propertyKey) {
for (IndexDefinition index : tx.schema().getIndexes(label)) {
for (String prop : index.getPropertyKeys()) {
if (prop.equals(propertyKey)) {
- return index;
+ if (index.getName().equals(indexName)) {
+ return index;
+ } else {
+ throw new IllegalStateException(String.format("Found pre-existing index '%s' for index '%s'", index.getName(), indexName));
+ }
}
}
}
@@ -1002,12 +1041,12 @@ private IndexDefinition findIndex(Transaction tx, Label label, String propertyKe
}
private WrappedNode getOrCreateNode(Label label, String name, String type) {
- WrappedNode node = findNode(label, name);
+ WrappedNode node = findNodeByName(label, name);
if (node == null) {
- Node n = tx.createNode(label);
+ WrappedNode n = createNodeWithLabel(tx, label);
n.setProperty("name", name);
n.setProperty("type", type);
- node = checkTx(WrappedNode.fromNode(n));
+ node = checkTx(n);
}
return node;
}
@@ -1038,9 +1077,9 @@ protected void addNodeTags(WrappedNode node, LinkedHashMap tags,
logNodeAddition(tags, type);
if (node != null && tags.size() > 0) {
statsManager.addToTagStats(type, tags.keySet());
- Node tagsNode = tx.createNode(LABEL_TAGS);
- addProperties(tagsNode, tags);
- node.createRelationshipTo(new WrappedNode(tagsNode), OSMRelation.TAGS);
+ WrappedNode tagsNode = createNodeWithLabel(tx, LABEL_TAGS);
+ addProperties(tagsNode.inner, tags);
+ node.createRelationshipTo(tagsNode, OSMRelation.TAGS);
tags.clear();
}
}
@@ -1060,12 +1099,12 @@ protected void addNodeGeometry(WrappedNode node, int gtype, Envelope bbox, int v
@Override
protected WrappedNode addNode(Label label, Map properties, String indexKey) {
- Node node = tx.createNode(label);
+ WrappedNode node = createNodeWithLabel(tx, label);
if (indexKey != null && properties.containsKey(indexKey)) {
properties.put(indexKey, Long.parseLong(properties.get(indexKey).toString()));
}
- addProperties(node, properties);
- return checkTx(WrappedNode.fromNode(node));
+ addProperties(node.inner, properties);
+ return checkTx(node);
}
@Override
@@ -1085,7 +1124,7 @@ protected long getDatasetId() {
@Override
protected WrappedNode getSingleNode(Label label, String property, Object value) {
- Node node = tx.findNode(LABEL_NODE, property, value);
+ Node node = findNodeByLabelProperty(tx, LABEL_NODE, property, value);
return node == null ? null : WrappedNode.fromNode(node);
}
@@ -1116,7 +1155,7 @@ protected WrappedNode getOSMNode(long osmId, WrappedNode changesetNode) {
WrappedNode node = changesetNodes.get(osmId);
if (node == null) {
logNodeFoundFrom("node-index");
- node = WrappedNode.fromNode(tx.findNode(LABEL_NODE, PROP_NODE_ID, osmId));
+ node = WrappedNode.fromNode(findNodeByLabelProperty(tx, LABEL_NODE, PROP_NODE_ID, osmId));
} else {
logNodeFoundFrom(PROP_CHANGESET);
}
@@ -1157,7 +1196,7 @@ protected WrappedNode getChangesetNode(Map nodeProps, WrappedNod
if (changeset != currentChangesetId) {
changesetIndex = createIndexIfNotNull(changesetIndex, LABEL_CHANGESET, PROP_CHANGESET);
currentChangesetId = changeset;
- Node changesetNode = tx.findNode(LABEL_CHANGESET, PROP_CHANGESET, currentChangesetId);
+ Node changesetNode = findNodeByLabelProperty(tx, LABEL_CHANGESET, PROP_CHANGESET, currentChangesetId);
if (changesetNode != null) {
currentChangesetNode = WrappedNode.fromNode(changesetNode);
} else {
@@ -1187,7 +1226,7 @@ protected WrappedNode getUserNode(Map nodeProps) {
if (uid != currentUserId) {
currentUserId = uid;
userIndex = createIndexIfNotNull(userIndex, LABEL_USER, PROP_USER_ID);
- Node userNode = tx.findNode(LABEL_USER, PROP_USER_ID, currentUserId);
+ Node userNode = findNodeByLabelProperty(tx, LABEL_USER, PROP_USER_ID, currentUserId);
if (userNode != null) {
currentUserNode = WrappedNode.fromNode(userNode);
} else {
@@ -1198,7 +1237,7 @@ protected WrappedNode getUserNode(Map nodeProps) {
currentUserNode = addNode(LABEL_USER, userProps, PROP_USER_ID);
userCount++;
if (usersNode == null) {
- usersNode = WrappedNode.fromNode(tx.createNode(LABEL_USER));
+ usersNode = createNodeWithLabel(tx, LABEL_USER);
osm_dataset.createRelationshipTo(usersNode, OSMRelation.USERS);
}
usersNode.createRelationshipTo(currentUserNode, OSMRelation.OSM_USER);
@@ -1218,15 +1257,15 @@ public String toString() {
}
- public void importFile(GraphDatabaseService database, String dataset) throws IOException, XMLStreamException {
+ public void importFile(GraphDatabaseService database, String dataset) throws Exception {
importFile(database, dataset, false, 5000);
}
- public void importFile(GraphDatabaseService database, String dataset, int txInterval) throws IOException, XMLStreamException {
+ public void importFile(GraphDatabaseService database, String dataset, int txInterval) throws Exception {
importFile(database, dataset, false, txInterval);
}
- public void importFile(GraphDatabaseService database, String dataset, boolean allPoints, int txInterval) throws IOException, XMLStreamException {
+ public void importFile(GraphDatabaseService database, String dataset, boolean allPoints, int txInterval) throws Exception {
importFile(OSMWriter.fromGraphDatabase(database, securityContext, stats, this, txInterval), dataset, allPoints, charset);
}
diff --git a/src/test/java/org/neo4j/gis/spatial/Neo4jSpatialDataStoreTest.java b/src/test/java/org/neo4j/gis/spatial/Neo4jSpatialDataStoreTest.java
index f9af5f560..6a2f31406 100644
--- a/src/test/java/org/neo4j/gis/spatial/Neo4jSpatialDataStoreTest.java
+++ b/src/test/java/org/neo4j/gis/spatial/Neo4jSpatialDataStoreTest.java
@@ -35,7 +35,7 @@ public class Neo4jSpatialDataStoreTest {
public GraphDatabaseService graph;
@Before
- public void setup() throws IOException, XMLStreamException {
+ public void setup() throws Exception {
this.databases = new TestDatabaseManagementServiceBuilder(Path.of("target", "test")).impermanent().build();
this.graph = databases.database(DEFAULT_DATABASE_NAME);
OSMImporter importer = new OSMImporter("map", new ConsoleListener());
diff --git a/src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java b/src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java
index 025e13396..5aa740111 100644
--- a/src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java
+++ b/src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java
@@ -929,6 +929,22 @@ public void import_osm_to_layer() {
testCallCount(db, "CALL spatial.layers()", null, 1);
}
+ @Test
+ public void import_osm_twice_should_pass_with_different_layers() {
+ execute("CALL spatial.addLayer('geom1','OSM','')");
+ execute("CALL spatial.addLayer('geom2','OSM','')");
+
+ testCountQuery("importOSM", "CALL spatial.importOSMToLayer('geom1','map.osm')", 55, "count", null);
+ testCallCount(db, "CALL spatial.layers()", null, 2);
+ testCallCount(db, "CALL spatial.withinDistance('geom1',{lon:6.3740429666,lat:50.93676351666},10000)", null, 217);
+ testCallCount(db, "CALL spatial.withinDistance('geom2',{lon:6.3740429666,lat:50.93676351666},10000)", null, 0);
+
+ testCountQuery("importOSM", "CALL spatial.importOSMToLayer('geom2','map.osm')", 55, "count", null);
+ testCallCount(db, "CALL spatial.layers()", null, 2);
+ testCallCount(db, "CALL spatial.withinDistance('geom1',{lon:6.3740429666,lat:50.93676351666},10000)", null, 217);
+ testCallCount(db, "CALL spatial.withinDistance('geom2',{lon:6.3740429666,lat:50.93676351666},10000)", null, 217);
+ }
+
@Ignore
public void import_cracow_to_layer() {
execute("CALL spatial.addLayer('geom','OSM','')");