From 447b35ce3327fd359a35d9995464543c2a2cf4e4 Mon Sep 17 00:00:00 2001 From: yrong Date: Sat, 3 Sep 2016 23:03:27 +0800 Subject: [PATCH 1/2] neo4j 3.1.0 support --- pom.xml | 37 ++++- .../graphdatabases/GraphDatabaseBase.java | 7 +- .../graphdatabases/Neo4jGraphDatabase.java | 134 +++++++++--------- .../graphdatabases/OrientGraphDatabase.java | 21 ++- .../insert/Neo4jSingleInsertion.java | 12 +- .../insert/OrientMassiveInsertion.java | 3 + .../insert/OrientSingleInsertion.java | 2 +- .../main/GraphDatabaseBenchmarkTest.java | 1 + src/test/resources/META-INF/input.properties | 22 +-- 9 files changed, 138 insertions(+), 101 deletions(-) diff --git a/pom.xml b/pom.xml index 90bc562..f04087f 100644 --- a/pom.xml +++ b/pom.xml @@ -51,10 +51,10 @@ 2.6.0 - 2.2.5-SNAPSHOT + 2.2.8 0.5.4 0.98.8-hadoop2 - 2.0.1 + 3.1.0-M07 1.0.0 2.1 2.18.1 @@ -148,22 +148,47 @@ jna 4.0.0 - + + + + org.neo4j + neo4j + ${neo4j.version} + + + com.orientechnologies orientdb-graphdb diff --git a/src/main/java/eu/socialsensor/graphdatabases/GraphDatabaseBase.java b/src/main/java/eu/socialsensor/graphdatabases/GraphDatabaseBase.java index d4992e0..28732b9 100644 --- a/src/main/java/eu/socialsensor/graphdatabases/GraphDatabaseBase.java +++ b/src/main/java/eu/socialsensor/graphdatabases/GraphDatabaseBase.java @@ -4,7 +4,6 @@ import java.util.Set; import org.neo4j.graphdb.Transaction; -import org.neo4j.kernel.GraphDatabaseAPI; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; @@ -98,7 +97,7 @@ public void findAllNodeNeighbours() { } } finally {//TODO fix this if(GraphDatabaseType.NEO4J == type) { - ((Transaction) tx).finish(); + ((Transaction) tx).terminate(); } } } @@ -107,7 +106,7 @@ public void findAllNodeNeighbours() { public void findNodesOfAllEdges() { Object tx = null; if(GraphDatabaseType.NEO4J == type) {//TODO fix this - tx = ((GraphDatabaseAPI) ((Neo4jGraphDatabase) this).neo4jGraph).tx().unforced().begin(); + tx = ((Neo4jGraphDatabase)this).neo4jGraph.beginTx(); } try { @@ -163,7 +162,7 @@ public void shortestPaths(Set nodes) { } } finally {//TODO fix this if(GraphDatabaseType.NEO4J == type) { - ((Transaction) tx).finish(); + ((Transaction) tx).terminate(); } } } diff --git a/src/main/java/eu/socialsensor/graphdatabases/Neo4jGraphDatabase.java b/src/main/java/eu/socialsensor/graphdatabases/Neo4jGraphDatabase.java index d3e86d6..a0f391c 100644 --- a/src/main/java/eu/socialsensor/graphdatabases/Neo4jGraphDatabase.java +++ b/src/main/java/eu/socialsensor/graphdatabases/Neo4jGraphDatabase.java @@ -11,27 +11,21 @@ import org.neo4j.graphalgo.GraphAlgoFactory; import org.neo4j.graphalgo.PathFinder; -import org.neo4j.graphdb.Direction; -import org.neo4j.graphdb.DynamicLabel; -import org.neo4j.graphdb.GraphDatabaseService; -import org.neo4j.graphdb.Label; -import org.neo4j.graphdb.Node; -import org.neo4j.graphdb.Path; -import org.neo4j.graphdb.Relationship; -import org.neo4j.graphdb.RelationshipType; -import org.neo4j.graphdb.ResourceIterable; -import org.neo4j.graphdb.Transaction; +import org.neo4j.graphdb.*; import org.neo4j.graphdb.factory.GraphDatabaseFactory; import org.neo4j.graphdb.schema.Schema; -import org.neo4j.helpers.collection.IteratorUtil; -import org.neo4j.kernel.GraphDatabaseAPI; -import org.neo4j.kernel.TransactionBuilder; -import org.neo4j.kernel.Traversal; -import org.neo4j.tooling.GlobalGraphOperations; +//import org.neo4j.helpers.collection.IteratorUtil; +//import org.neo4j.kernel.GraphDatabaseAPI; +//import org.neo4j.kernel.TransactionBuilder; +//import org.neo4j.kernel.Traversal; +//import org.neo4j.tooling.GlobalGraphOperations; +import org.neo4j.graphdb.traversal.TraversalDescription; +import org.neo4j.helpers.collection.Iterators; import org.neo4j.unsafe.batchinsert.BatchInserter; import org.neo4j.unsafe.batchinsert.BatchInserters; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -69,7 +63,7 @@ public Neo4jGraphDatabase(File dbStorageDirectoryIn) @Override public void open() { - neo4jGraph = new GraphDatabaseFactory().newEmbeddedDatabase(dbStorageDirectory.getAbsolutePath()); + neo4jGraph = new GraphDatabaseFactory().newEmbeddedDatabase(dbStorageDirectory); try (final Transaction tx = beginUnforcedTransaction()) { try @@ -88,7 +82,8 @@ public void open() @Override public void createGraphForSingleLoad() { - neo4jGraph = new GraphDatabaseFactory().newEmbeddedDatabase(dbStorageDirectory.getAbsolutePath()); + neo4jGraph = new GraphDatabaseFactory().newEmbeddedDatabase(dbStorageDirectory); + try (final Transaction tx = beginUnforcedTransaction()) { try @@ -118,7 +113,11 @@ public void createGraphForMassiveLoad() config.put("neostore.propertystore.db.mapped_memory", "250M"); config.put("neostore.propertystore.db.strings.mapped_memory", "250M"); - inserter = BatchInserters.inserter(dbStorageDirectory.getAbsolutePath(), config); + try { + inserter = BatchInserters.inserter(dbStorageDirectory, config); + } catch (IOException e) { + e.printStackTrace(); + } createDeferredSchema(); } @@ -188,8 +187,9 @@ public void shutdownMassiveGraph() @Override public void shortestPath(Node n1, Integer i) { + PathExpander pathExpander = PathExpanderBuilder.allTypesAndDirections().add(Neo4jGraphDatabase.RelTypes.SIMILAR,Direction.BOTH).build(); PathFinder finder - = GraphAlgoFactory.shortestPath(Traversal.expanderForTypes(Neo4jGraphDatabase.RelTypes.SIMILAR), 5); + = GraphAlgoFactory.shortestPath(pathExpander, 5); Node n2 = getVertex(i); Path path = finder.findSinglePath(n1, n2); @@ -197,8 +197,8 @@ public void shortestPath(Node n1, Integer i) //TODO can unforced option be pulled into configuration? private Transaction beginUnforcedTransaction() { - final TransactionBuilder builder = ((GraphDatabaseAPI) neo4jGraph).tx().unforced(); - return builder.begin(); + return neo4jGraph.beginTx(); + } @Override @@ -209,7 +209,7 @@ public int getNodeCount() { try { - nodeCount = IteratorUtil.count(GlobalGraphOperations.at(neo4jGraph).getAllNodes()); + nodeCount = (int)Iterators.count(neo4jGraph.getAllNodes().iterator()); tx.success(); } catch (Exception e) @@ -230,8 +230,7 @@ public Set getNeighborsIds(int nodeId) { try { - Node n = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, NODE_ID, String.valueOf(nodeId)).iterator() - .next(); + Node n = neo4jGraph.findNodes(NODE_LABEL, NODE_ID, String.valueOf(nodeId)).next(); for (Relationship relationship : n.getRelationships(RelTypes.SIMILAR, Direction.OUTGOING)) { Node neighbour = relationship.getOtherNode(n); @@ -258,8 +257,7 @@ public double getNodeWeight(int nodeId) { try { - Node n = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, NODE_ID, String.valueOf(nodeId)).iterator() - .next(); + Node n = neo4jGraph.findNodes(NODE_LABEL, NODE_ID, String.valueOf(nodeId)).next(); weight = getNodeOutDegree(n); tx.success(); } @@ -276,13 +274,13 @@ public double getNodeWeight(int nodeId) public double getNodeInDegree(Node node) { Iterable rel = node.getRelationships(Direction.OUTGOING, RelTypes.SIMILAR); - return (double) (IteratorUtil.count(rel)); + return (double) (Iterators.count(rel.iterator())); } public double getNodeOutDegree(Node node) { Iterable rel = node.getRelationships(Direction.INCOMING, RelTypes.SIMILAR); - return (double) (IteratorUtil.count(rel)); + return (double) (Iterators.count(rel.iterator())); } @Override @@ -295,7 +293,7 @@ public void initCommunityProperty() { try { - for (Node n : GlobalGraphOperations.at(neo4jGraph).getAllNodes()) + for (Node n : neo4jGraph.getAllNodes()) { n.setProperty(NODE_COMMUNITY, communityCounter); n.setProperty(COMMUNITY, communityCounter); @@ -319,10 +317,11 @@ public Set getCommunitiesConnectedToNodeCommunities(int nodeCommunities { try { - ResourceIterable nodes = neo4jGraph.findNodesByLabelAndProperty(Neo4jGraphDatabase.NODE_LABEL, + ResourceIterator nodes = neo4jGraph.findNodes(Neo4jGraphDatabase.NODE_LABEL, NODE_COMMUNITY, nodeCommunities); - for (Node n : nodes) + while(nodes.hasNext()) { + Node n = nodes.next(); for (Relationship r : n.getRelationships(RelTypes.SIMILAR, Direction.OUTGOING)) { Node neighbour = r.getOtherNode(n); @@ -350,9 +349,10 @@ public Set getNodesFromCommunity(int community) { try { - ResourceIterable iter = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, COMMUNITY, community); - for (Node n : iter) + ResourceIterator iter = neo4jGraph.findNodes(NODE_LABEL, COMMUNITY, community); + while(iter.hasNext()) { + Node n = iter.next(); String nodeIdString = (String) (n.getProperty(NODE_ID)); nodes.add(Integer.valueOf(nodeIdString)); } @@ -376,10 +376,11 @@ public Set getNodesFromNodeCommunity(int nodeCommunity) { try { - ResourceIterable iter = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, NODE_COMMUNITY, + ResourceIterator iter = neo4jGraph.findNodes(NODE_LABEL, NODE_COMMUNITY, nodeCommunity); - for (Node n : iter) + while(iter.hasNext()) { + Node n = iter.next(); String nodeIdString = (String) (n.getProperty(NODE_ID)); nodes.add(Integer.valueOf(nodeIdString)); } @@ -403,19 +404,22 @@ public double getEdgesInsideCommunity(int nodeCommunity, int communityNodes) { try { - ResourceIterable nodes = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, NODE_COMMUNITY, + ResourceIterator nodes = neo4jGraph.findNodes(NODE_LABEL, NODE_COMMUNITY, nodeCommunity); - ResourceIterable comNodes = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, COMMUNITY, + ResourceIterator comNodes = neo4jGraph.findNodes(NODE_LABEL, COMMUNITY, communityNodes); - for (Node node : nodes) + while(nodes.hasNext()) { + Node node = nodes.next(); Iterable relationships = node.getRelationships(RelTypes.SIMILAR, Direction.OUTGOING); for (Relationship r : relationships) { Node neighbor = r.getOtherNode(node); - if (Iterables.contains(comNodes, neighbor)) - { - edges++; + while(comNodes.hasNext()) { + Node comNode = comNodes.next(); + if(comNode.equals(neighbor)){ + edges++; + } } } } @@ -439,10 +443,11 @@ public double getCommunityWeight(int community) { try { - ResourceIterable iter = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, COMMUNITY, community); - if (Iterables.size(iter) > 1) + ResourceIterator iter = neo4jGraph.findNodes(NODE_LABEL, COMMUNITY, community); + List list = Iterators.asList(iter); + if (Iterators.asList(iter).size() > 1) { - for (Node n : iter) + for (Node n : list) { communityWeight += getNodeOutDegree(n); } @@ -467,11 +472,12 @@ public double getNodeCommunityWeight(int nodeCommunity) { try { - ResourceIterable iter = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, NODE_COMMUNITY, + ResourceIterator iter = neo4jGraph.findNodes(NODE_LABEL, NODE_COMMUNITY, nodeCommunity); - if (Iterables.size(iter) > 1) + List list = Iterators.asList(iter); + if (list.size() > 1) { - for (Node n : iter) + for (Node n : list) { nodeCommunityWeight += getNodeOutDegree(n); } @@ -495,9 +501,10 @@ public void moveNode(int nodeCommunity, int toCommunity) { try { - ResourceIterable fromIter = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, NODE_COMMUNITY, + ResourceIterator fromIter = neo4jGraph.findNodes(NODE_LABEL, NODE_COMMUNITY, nodeCommunity); - for (Node node : fromIter) + List list = Iterators.asList(fromIter); + for (Node node : list) { node.setProperty(COMMUNITY, toCommunity); } @@ -520,7 +527,7 @@ public double getGraphWeightSum() { try { - edgeCount = IteratorUtil.count(GlobalGraphOperations.at(neo4jGraph).getAllRelationships()); + edgeCount = (int)Iterators.count(neo4jGraph.getAllRelationships().iterator()); tx.success(); } catch (Exception e) @@ -543,7 +550,7 @@ public int reInitializeCommunities() { try { - for (Node n : GlobalGraphOperations.at(neo4jGraph).getAllNodes()) + for (Node n : neo4jGraph.getAllNodes()) { Integer communityId = (Integer) (n.getProperty(COMMUNITY)); if (!initCommunities.containsKey(communityId)) @@ -576,8 +583,7 @@ public int getCommunity(int nodeCommunity) { try { - Node node = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, NODE_COMMUNITY, nodeCommunity).iterator() - .next(); + Node node = neo4jGraph.findNode(NODE_LABEL, NODE_COMMUNITY, nodeCommunity); community = (Integer) (node.getProperty(COMMUNITY)); tx.success(); } @@ -599,8 +605,7 @@ public int getCommunityFromNode(int nodeId) { try { - Node node = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, NODE_ID, String.valueOf(nodeId)).iterator() - .next(); + Node node = neo4jGraph.findNode(NODE_LABEL, NODE_ID, String.valueOf(nodeId)); community = (Integer) (node.getProperty(COMMUNITY)); tx.success(); } @@ -623,8 +628,8 @@ public int getCommunitySize(int community) { try { - ResourceIterable nodes = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, COMMUNITY, community); - for (Node n : nodes) + ResourceIterator nodes = neo4jGraph.findNodes(NODE_LABEL, COMMUNITY, community); + for (Node n : Iterators.asList(nodes)) { Integer nodeCommunity = (Integer) (n.getProperty(COMMUNITY)); nodeCommunities.add(nodeCommunity); @@ -652,9 +657,9 @@ public Map> mapCommunities(int numberOfCommunities) { for (int i = 0; i < numberOfCommunities; i++) { - ResourceIterable nodesIter = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, COMMUNITY, i); + ResourceIterator nodesIter = neo4jGraph.findNodes(NODE_LABEL, COMMUNITY, i); List nodes = new ArrayList(); - for (Node n : nodesIter) + for (Node n : Iterators.asList(nodesIter)) { String nodeIdString = (String) (n.getProperty(NODE_ID)); nodes.add(Integer.valueOf(nodeIdString)); @@ -680,8 +685,8 @@ public boolean nodeExists(int nodeId) { try { - ResourceIterable nodesIter = neo4jGraph.findNodesByLabelAndProperty(NODE_LABEL, NODE_ID, nodeId); - if (nodesIter.iterator().hasNext()) + ResourceIterator nodesIter = neo4jGraph.findNodes(NODE_LABEL, NODE_ID, nodeId); + if (nodesIter.hasNext()) { tx.success(); return true; @@ -700,7 +705,7 @@ public boolean nodeExists(int nodeId) @Override public Iterator getVertexIterator() { - return GlobalGraphOperations.at(neo4jGraph).getAllNodes().iterator(); + return neo4jGraph.getAllNodes().iterator(); } @Override @@ -724,7 +729,7 @@ public Node getOtherVertexFromEdge(Relationship r, Node n) @Override public Iterator getAllEdges() { - return GlobalGraphOperations.at(neo4jGraph).getAllRelationships().iterator(); + return neo4jGraph.getAllRelationships().iterator(); } @Override @@ -773,8 +778,7 @@ public Node nextVertex(Iterator it) public Node getVertex(Integer i) { // note, this probably should be run in the context of an active transaction. - return neo4jGraph.findNodesByLabelAndProperty(Neo4jGraphDatabase.NODE_LABEL, NODE_ID, i.toString()).iterator() - .next(); + return neo4jGraph.findNode(Neo4jGraphDatabase.NODE_LABEL, NODE_ID, i.toString()); } } diff --git a/src/main/java/eu/socialsensor/graphdatabases/OrientGraphDatabase.java b/src/main/java/eu/socialsensor/graphdatabases/OrientGraphDatabase.java index c44c280..585927d 100644 --- a/src/main/java/eu/socialsensor/graphdatabases/OrientGraphDatabase.java +++ b/src/main/java/eu/socialsensor/graphdatabases/OrientGraphDatabase.java @@ -26,6 +26,7 @@ import eu.socialsensor.main.BenchmarkConfiguration; import eu.socialsensor.main.GraphDatabaseType; import eu.socialsensor.utils.Utils; +import org.apache.commons.collections.map.HashedMap; import java.io.File; import java.util.ArrayList; @@ -53,7 +54,7 @@ public OrientGraphDatabase(BenchmarkConfiguration config, File dbStorageDirector { super(GraphDatabaseType.ORIENT_DB, dbStorageDirectoryIn); OGlobalConfiguration.STORAGE_COMPRESSION_METHOD.setValue("nothing"); - this.useLightWeightEdges = config.orientLightweightEdges() == null ? true : config.orientLightweightEdges() + this.useLightWeightEdges = config.orientLightweightEdges() == null ? false : config.orientLightweightEdges() .booleanValue(); } @@ -67,7 +68,7 @@ public void open() @Override public void createGraphForSingleLoad() { - OGlobalConfiguration.STORAGE_KEEP_OPEN.setValue(false); +// OGlobalConfiguration.STORAGE_KEEP_OPEN.setValue(false); graph = getGraph(dbStorageDirectory); createSchema(); } @@ -76,7 +77,7 @@ public void createGraphForSingleLoad() @Override public void createGraphForMassiveLoad() { - OGlobalConfiguration.STORAGE_KEEP_OPEN.setValue(false); +// OGlobalConfiguration.STORAGE_KEEP_OPEN.setValue(false); graph = getGraph(dbStorageDirectory); createSchema(); } @@ -102,6 +103,7 @@ public void shutdown() { return; } + graph.commit(); graph.shutdown(); graph = null; } @@ -125,9 +127,10 @@ public void shutdownMassiveGraph() public void shortestPath(final Vertex v1, Integer i) { final OrientVertex v2 = (OrientVertex) getVertex(i); - + Map attrs = new HashMap(); + attrs.put("maxDepth",5); List result = new OSQLFunctionShortestPath().execute(graph, - null, null, new Object[] { ((OrientVertex) v1).getRecord(), v2.getRecord(), Direction.OUT, 5 }, + null, null, new Object[] { ((OrientVertex) v1).getRecord(), v2.getRecord(), Direction.OUT, "similar", attrs}, new OBasicCommandContext()); result.size(); @@ -384,11 +387,14 @@ public Object call(final OrientBaseGraph g) g.createKeyIndex(NODE_ID, Vertex.class, new Parameter("type", "UNIQUE_HASH_INDEX"), new Parameter( "keytype", "INTEGER")); - v.createEdgeProperty(Direction.OUT, SIMILAR, OType.LINKBAG); - v.createEdgeProperty(Direction.IN, SIMILAR, OType.LINKBAG); OrientEdgeType similar = g.createEdgeType(SIMILAR); similar.createProperty("out", OType.LINK, v); similar.createProperty("in", OType.LINK, v); + + v.createEdgeProperty(Direction.OUT, SIMILAR, OType.LINKBAG); + v.createEdgeProperty(Direction.IN, SIMILAR, OType.LINKBAG); + + g.createKeyIndex(COMMUNITY, Vertex.class, new Parameter("type", "NOTUNIQUE_HASH_INDEX"), new Parameter("keytype", "INTEGER")); g.createKeyIndex(NODE_COMMUNITY, Vertex.class, new Parameter("type", "NOTUNIQUE_HASH_INDEX"), @@ -406,6 +412,7 @@ private OrientGraph getGraph(final File dbPath) OrientGraphFactory graphFactory = new OrientGraphFactory("plocal:" + dbPath.getAbsolutePath()); g = graphFactory.getTx(); g.setUseLightweightEdges(this.useLightWeightEdges); + return g; } diff --git a/src/main/java/eu/socialsensor/insert/Neo4jSingleInsertion.java b/src/main/java/eu/socialsensor/insert/Neo4jSingleInsertion.java index 7ba92f7..62bef0f 100644 --- a/src/main/java/eu/socialsensor/insert/Neo4jSingleInsertion.java +++ b/src/main/java/eu/socialsensor/insert/Neo4jSingleInsertion.java @@ -4,12 +4,10 @@ import java.util.HashMap; import java.util.Map; -import org.neo4j.cypher.javacompat.ExecutionEngine; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.ResourceIterator; import org.neo4j.graphdb.Transaction; -import org.neo4j.kernel.GraphDatabaseAPI; import eu.socialsensor.graphdatabases.Neo4jGraphDatabase; import eu.socialsensor.main.BenchmarkingException; @@ -26,27 +24,27 @@ public class Neo4jSingleInsertion extends InsertionBase { private final GraphDatabaseService neo4jGraph; - private final ExecutionEngine engine; +// private final ExecutionEngine engine; public Neo4jSingleInsertion(GraphDatabaseService neo4jGraph, File resultsPath) { super(GraphDatabaseType.NEO4J, resultsPath); this.neo4jGraph = neo4jGraph; - engine = new ExecutionEngine(this.neo4jGraph); +// engine = new ExecutionEngine(this.neo4jGraph,null); } public Node getOrCreate(String nodeId) { Node result = null; - try(final Transaction tx = ((GraphDatabaseAPI) neo4jGraph).tx().unforced().begin()) + try(final Transaction tx = (neo4jGraph.beginTx())) { try { String queryString = "MERGE (n:Node {nodeId: {nodeId}}) RETURN n"; Map parameters = new HashMap(); parameters.put("nodeId", nodeId); - ResourceIterator resultIterator = engine.execute(queryString, parameters).columnAs("n"); + ResourceIterator resultIterator = neo4jGraph.execute(queryString, parameters).columnAs("n"); result = resultIterator.next(); tx.success(); } @@ -63,7 +61,7 @@ public Node getOrCreate(String nodeId) @Override public void relateNodes(Node src, Node dest) { - try (final Transaction tx = ((GraphDatabaseAPI) neo4jGraph).tx().unforced().begin()) + try (final Transaction tx = (neo4jGraph.beginTx())) { try { diff --git a/src/main/java/eu/socialsensor/insert/OrientMassiveInsertion.java b/src/main/java/eu/socialsensor/insert/OrientMassiveInsertion.java index 6d2a1e7..1b007ce 100644 --- a/src/main/java/eu/socialsensor/insert/OrientMassiveInsertion.java +++ b/src/main/java/eu/socialsensor/insert/OrientMassiveInsertion.java @@ -1,6 +1,7 @@ package eu.socialsensor.insert; import com.orientechnologies.orient.core.config.OGlobalConfiguration; +import com.orientechnologies.orient.core.intent.OIntentMassiveInsert; import com.orientechnologies.orient.graph.batch.OGraphBatchInsertBasic; import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; @@ -25,6 +26,7 @@ public OrientMassiveInsertion(final String url) super(GraphDatabaseType.ORIENT_DB, null /* resultsPath */); OGlobalConfiguration.ENVIRONMENT_CONCURRENT.setValue(false); OrientGraphNoTx transactionlessGraph = new OrientGraphNoTx(url); +// transactionlessGraph.declareIntent(new OIntentMassiveInsert()); for (int i = 0; i < NUMBER_OF_ORIENT_CLUSTERS; ++i) { transactionlessGraph.getVertexBaseType().addCluster("v_" + i); @@ -36,6 +38,7 @@ public OrientMassiveInsertion(final String url) graph.setAverageEdgeNumberPerNode(AVERAGE_NUMBER_OF_EDGES_PER_NODE); graph.setEstimatedEntries(ESTIMATED_ENTRIES); graph.setIdPropertyName("nodeId"); + graph.setEdgeClass("similar"); graph.begin(); } diff --git a/src/main/java/eu/socialsensor/insert/OrientSingleInsertion.java b/src/main/java/eu/socialsensor/insert/OrientSingleInsertion.java index 39d4875..57741d3 100644 --- a/src/main/java/eu/socialsensor/insert/OrientSingleInsertion.java +++ b/src/main/java/eu/socialsensor/insert/OrientSingleInsertion.java @@ -8,6 +8,7 @@ import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.impls.orient.OrientGraph; +import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; import eu.socialsensor.main.GraphDatabaseType; /** @@ -38,7 +39,6 @@ protected void relateNodes(Vertex src, Vertex dest) if (orientGraph instanceof TransactionalGraph) { orientGraph.commit(); - orientGraph.commit(); } } diff --git a/src/test/java/eu/socialsensor/main/GraphDatabaseBenchmarkTest.java b/src/test/java/eu/socialsensor/main/GraphDatabaseBenchmarkTest.java index b00d00e..608ee68 100644 --- a/src/test/java/eu/socialsensor/main/GraphDatabaseBenchmarkTest.java +++ b/src/test/java/eu/socialsensor/main/GraphDatabaseBenchmarkTest.java @@ -3,6 +3,7 @@ import static org.junit.Assert.fail; import org.junit.Test; + public class GraphDatabaseBenchmarkTest { @Test diff --git a/src/test/resources/META-INF/input.properties b/src/test/resources/META-INF/input.properties index 7c271b5..a79170e 100644 --- a/src/test/resources/META-INF/input.properties +++ b/src/test/resources/META-INF/input.properties @@ -1,11 +1,11 @@ # Choose which data sets you want to include in the benchmark by removing the contents. -#eu.socialsensor.dataset=data/Email-Enron.txt +eu.socialsensor.dataset=data/email-Enron.txt #eu.socialsensor.dataset=data/com-youtube.ungraph.txt #eu.socialsensor.dataset=data/Amazon0601.txt #eu.socialsensor.dataset=data/com-lj.ungraph.txt #can change the number in the filename of the synthetic datasets to 1000, 5000, 10000, 20000, 30000, 40000, 50000 -eu.socialsensor.dataset=data/network1000.dat -eu.socialsensor.actual-communities=data/community1000.dat +#eu.socialsensor.dataset=data/network1000.dat +#eu.socialsensor.actual-communities=data/community1000.dat eu.socialsensor.database-storage-directory=storage # Sample meters this frequently (milliseconds) @@ -17,14 +17,14 @@ eu.socialsensor.metrics.csv.directory=metrics # Choose which databases you want to in the benchmark by removing the comments. # Available dbs are: -eu.socialsensor.databases=tbdb -eu.socialsensor.databases=tddb +#eu.socialsensor.databases=tbdb +#eu.socialsensor.databases=tddb #eu.socialsensor.databases=tc #eu.socialsensor.databases=thb #eu.socialsensor.databases=tce #eu.socialsensor.databases=tp -#eu.socialsensor.databases=orient -#eu.socialsensor.databases=neo4j +eu.socialsensor.databases=orient +eu.socialsensor.databases=neo4j #eu.socialsensor.databases=sparksee # Database specific options @@ -63,7 +63,7 @@ eu.socialsensor.dynamodb.endpoint=http://127.0.0.1:4567 #eu.socialsensor.dynamodb.endpoint=https://dynamodb.us-east-1.amazonaws.com # OrientDB options -eu.socialsensor.orient.lightweight-edges=true +eu.socialsensor.orient.lightweight-edges=false # Sparksee options eu.socialsensor.sparksee.license-key=DEADBEEF @@ -75,10 +75,10 @@ eu.socialsensor.permute-benchmarks=false # Choose which benchmark you want to run by removing the comments. Choose one Insertion # workload and then query/clustering workloads afterward. -eu.socialsensor.benchmarks=MASSIVE_INSERTION +#eu.socialsensor.benchmarks=MASSIVE_INSERTION #eu.socialsensor.benchmarks=SINGLE_INSERTION -eu.socialsensor.benchmarks=FIND_NEIGHBOURS -eu.socialsensor.benchmarks=FIND_ADJACENT_NODES +#eu.socialsensor.benchmarks=FIND_NEIGHBOURS +#eu.socialsensor.benchmarks=FIND_ADJACENT_NODES eu.socialsensor.benchmarks=FIND_SHORTEST_PATH eu.socialsensor.shortest-path-random-nodes=100 From fecc54638e8cafa9088f5b31c62857985ac3600a Mon Sep 17 00:00:00 2001 From: yrong Date: Mon, 12 Sep 2016 16:33:37 +0800 Subject: [PATCH 2/2] change per review --- pom.xml | 2 +- .../graphdatabases/Neo4jGraphDatabase.java | 93 +++++++------------ .../graphdatabases/OrientGraphDatabase.java | 3 +- .../insert/OrientMassiveInsertion.java | 75 +++++++++++++-- .../insert/OrientSingleInsertion.java | 1 + src/test/resources/META-INF/input.properties | 4 +- 6 files changed, 102 insertions(+), 76 deletions(-) diff --git a/pom.xml b/pom.xml index f04087f..369f28c 100644 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,7 @@ 2.2.8 0.5.4 0.98.8-hadoop2 - 3.1.0-M07 + 3.0.5 1.0.0 2.1 2.18.1 diff --git a/src/main/java/eu/socialsensor/graphdatabases/Neo4jGraphDatabase.java b/src/main/java/eu/socialsensor/graphdatabases/Neo4jGraphDatabase.java index a0f391c..784b73a 100644 --- a/src/main/java/eu/socialsensor/graphdatabases/Neo4jGraphDatabase.java +++ b/src/main/java/eu/socialsensor/graphdatabases/Neo4jGraphDatabase.java @@ -1,7 +1,5 @@ package eu.socialsensor.graphdatabases; -import com.google.common.collect.Iterables; - import eu.socialsensor.insert.Insertion; import eu.socialsensor.insert.Neo4jMassiveInsertion; import eu.socialsensor.insert.Neo4jSingleInsertion; @@ -14,12 +12,6 @@ import org.neo4j.graphdb.*; import org.neo4j.graphdb.factory.GraphDatabaseFactory; import org.neo4j.graphdb.schema.Schema; -//import org.neo4j.helpers.collection.IteratorUtil; -//import org.neo4j.kernel.GraphDatabaseAPI; -//import org.neo4j.kernel.TransactionBuilder; -//import org.neo4j.kernel.Traversal; -//import org.neo4j.tooling.GlobalGraphOperations; -import org.neo4j.graphdb.traversal.TraversalDescription; import org.neo4j.helpers.collection.Iterators; import org.neo4j.unsafe.batchinsert.BatchInserter; import org.neo4j.unsafe.batchinsert.BatchInserters; @@ -44,6 +36,7 @@ public class Neo4jGraphDatabase extends GraphDatabaseBase, Iterator, Node, Relationship> { protected GraphDatabaseService neo4jGraph = null; + private Schema schema = null; private BatchInserter inserter = null; @@ -64,7 +57,7 @@ public Neo4jGraphDatabase(File dbStorageDirectoryIn) public void open() { neo4jGraph = new GraphDatabaseFactory().newEmbeddedDatabase(dbStorageDirectory); - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { @@ -84,7 +77,7 @@ public void createGraphForSingleLoad() { neo4jGraph = new GraphDatabaseFactory().newEmbeddedDatabase(dbStorageDirectory); - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { @@ -192,20 +185,13 @@ public void shortestPath(Node n1, Integer i) = GraphAlgoFactory.shortestPath(pathExpander, 5); Node n2 = getVertex(i); Path path = finder.findSinglePath(n1, n2); - - } - - //TODO can unforced option be pulled into configuration? - private Transaction beginUnforcedTransaction() { - return neo4jGraph.beginTx(); - } @Override public int getNodeCount() { int nodeCount = 0; - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { @@ -226,7 +212,7 @@ public int getNodeCount() public Set getNeighborsIds(int nodeId) { Set neighbors = new HashSet(); - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { @@ -253,7 +239,7 @@ public Set getNeighborsIds(int nodeId) public double getNodeWeight(int nodeId) { double weight = 0; - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { @@ -289,7 +275,7 @@ public void initCommunityProperty() int communityCounter = 0; // maybe commit changes every 1000 transactions? - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { @@ -313,7 +299,7 @@ public void initCommunityProperty() public Set getCommunitiesConnectedToNodeCommunities(int nodeCommunities) { Set communities = new HashSet(); - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { @@ -345,7 +331,7 @@ public Set getCommunitiesConnectedToNodeCommunities(int nodeCommunities public Set getNodesFromCommunity(int community) { Set nodes = new HashSet(); - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { @@ -372,7 +358,7 @@ public Set getNodesFromNodeCommunity(int nodeCommunity) { Set nodes = new HashSet(); - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { @@ -400,7 +386,7 @@ public Set getNodesFromNodeCommunity(int nodeCommunity) public double getEdgesInsideCommunity(int nodeCommunity, int communityNodes) { double edges = 0; - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { @@ -439,18 +425,13 @@ public double getEdgesInsideCommunity(int nodeCommunity, int communityNodes) public double getCommunityWeight(int community) { double communityWeight = 0; - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { ResourceIterator iter = neo4jGraph.findNodes(NODE_LABEL, COMMUNITY, community); - List list = Iterators.asList(iter); - if (Iterators.asList(iter).size() > 1) - { - for (Node n : list) - { - communityWeight += getNodeOutDegree(n); - } + while(iter.hasNext()){ + communityWeight += getNodeOutDegree(iter.next()); } tx.success(); } @@ -468,19 +449,14 @@ public double getCommunityWeight(int community) public double getNodeCommunityWeight(int nodeCommunity) { double nodeCommunityWeight = 0; - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { ResourceIterator iter = neo4jGraph.findNodes(NODE_LABEL, NODE_COMMUNITY, nodeCommunity); - List list = Iterators.asList(iter); - if (list.size() > 1) - { - for (Node n : list) - { - nodeCommunityWeight += getNodeOutDegree(n); - } + while(iter.hasNext()){ + nodeCommunityWeight += getNodeOutDegree(iter.next()); } tx.success(); } @@ -497,16 +473,14 @@ public double getNodeCommunityWeight(int nodeCommunity) @Override public void moveNode(int nodeCommunity, int toCommunity) { - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { - ResourceIterator fromIter = neo4jGraph.findNodes(NODE_LABEL, NODE_COMMUNITY, + ResourceIterator iter = neo4jGraph.findNodes(NODE_LABEL, NODE_COMMUNITY, nodeCommunity); - List list = Iterators.asList(fromIter); - for (Node node : list) - { - node.setProperty(COMMUNITY, toCommunity); + while(iter.hasNext()){ + iter.next().setProperty(COMMUNITY, toCommunity); } tx.success(); } @@ -523,7 +497,7 @@ public double getGraphWeightSum() { int edgeCount = 0; - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { @@ -546,7 +520,7 @@ public int reInitializeCommunities() Map initCommunities = new HashMap(); int communityCounter = 0; - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { @@ -579,7 +553,7 @@ public int getCommunity(int nodeCommunity) { Integer community = 0; - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { @@ -601,7 +575,7 @@ public int getCommunity(int nodeCommunity) public int getCommunityFromNode(int nodeId) { Integer community = 0; - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { @@ -624,14 +598,13 @@ public int getCommunitySize(int community) { Set nodeCommunities = new HashSet(); - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { ResourceIterator nodes = neo4jGraph.findNodes(NODE_LABEL, COMMUNITY, community); - for (Node n : Iterators.asList(nodes)) - { - Integer nodeCommunity = (Integer) (n.getProperty(COMMUNITY)); + while(nodes.hasNext()){ + Integer nodeCommunity = (Integer) (nodes.next().getProperty(COMMUNITY)); nodeCommunities.add(nodeCommunity); } tx.success(); @@ -651,7 +624,7 @@ public Map> mapCommunities(int numberOfCommunities) { Map> communities = new HashMap>(); - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { @@ -659,9 +632,8 @@ public Map> mapCommunities(int numberOfCommunities) { ResourceIterator nodesIter = neo4jGraph.findNodes(NODE_LABEL, COMMUNITY, i); List nodes = new ArrayList(); - for (Node n : Iterators.asList(nodesIter)) - { - String nodeIdString = (String) (n.getProperty(NODE_ID)); + while(nodesIter.hasNext()){ + String nodeIdString = (String) (nodesIter.next().getProperty(NODE_ID)); nodes.add(Integer.valueOf(nodeIdString)); } communities.put(i, nodes); @@ -674,14 +646,13 @@ public Map> mapCommunities(int numberOfCommunities) throw new BenchmarkingException("unable to map communities", e); } } - return communities; } @Override public boolean nodeExists(int nodeId) { - try (final Transaction tx = beginUnforcedTransaction()) + try (final Transaction tx = neo4jGraph.beginTx()) { try { diff --git a/src/main/java/eu/socialsensor/graphdatabases/OrientGraphDatabase.java b/src/main/java/eu/socialsensor/graphdatabases/OrientGraphDatabase.java index 585927d..5dea988 100644 --- a/src/main/java/eu/socialsensor/graphdatabases/OrientGraphDatabase.java +++ b/src/main/java/eu/socialsensor/graphdatabases/OrientGraphDatabase.java @@ -77,7 +77,7 @@ public void createGraphForSingleLoad() @Override public void createGraphForMassiveLoad() { -// OGlobalConfiguration.STORAGE_KEEP_OPEN.setValue(false); + OGlobalConfiguration.STORAGE_KEEP_OPEN.setValue(false); graph = getGraph(dbStorageDirectory); createSchema(); } @@ -412,7 +412,6 @@ private OrientGraph getGraph(final File dbPath) OrientGraphFactory graphFactory = new OrientGraphFactory("plocal:" + dbPath.getAbsolutePath()); g = graphFactory.getTx(); g.setUseLightweightEdges(this.useLightWeightEdges); - return g; } diff --git a/src/main/java/eu/socialsensor/insert/OrientMassiveInsertion.java b/src/main/java/eu/socialsensor/insert/OrientMassiveInsertion.java index 1b007ce..357b0a0 100644 --- a/src/main/java/eu/socialsensor/insert/OrientMassiveInsertion.java +++ b/src/main/java/eu/socialsensor/insert/OrientMassiveInsertion.java @@ -1,8 +1,11 @@ package eu.socialsensor.insert; import com.orientechnologies.orient.core.config.OGlobalConfiguration; -import com.orientechnologies.orient.core.intent.OIntentMassiveInsert; +import com.orientechnologies.orient.core.db.record.OIdentifiable; +import com.orientechnologies.orient.core.index.OIndex; import com.orientechnologies.orient.graph.batch.OGraphBatchInsertBasic; +import com.tinkerpop.blueprints.TransactionalGraph; +import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; import eu.socialsensor.main.GraphDatabaseType; @@ -14,19 +17,21 @@ * @author Alexander Patrikalakis * */ -public class OrientMassiveInsertion extends InsertionBase implements Insertion +public class OrientMassiveInsertion extends InsertionBase implements Insertion { private static final int ESTIMATED_ENTRIES = 1000000; private static final int AVERAGE_NUMBER_OF_EDGES_PER_NODE = 40; private static final int NUMBER_OF_ORIENT_CLUSTERS = 16; private final OGraphBatchInsertBasic graph; + protected final OrientGraphNoTx orientGraph; + protected final OIndex index; + public OrientMassiveInsertion(final String url) { super(GraphDatabaseType.ORIENT_DB, null /* resultsPath */); OGlobalConfiguration.ENVIRONMENT_CONCURRENT.setValue(false); OrientGraphNoTx transactionlessGraph = new OrientGraphNoTx(url); -// transactionlessGraph.declareIntent(new OIntentMassiveInsert()); for (int i = 0; i < NUMBER_OF_ORIENT_CLUSTERS; ++i) { transactionlessGraph.getVertexBaseType().addCluster("v_" + i); @@ -34,6 +39,9 @@ public OrientMassiveInsertion(final String url) } transactionlessGraph.shutdown(); + orientGraph = new OrientGraphNoTx(url); + this.index = this.orientGraph.getRawGraph().getMetadata().getIndexManager().getIndex("V.nodeId"); + graph = new OGraphBatchInsertBasic(url); graph.setAverageEdgeNumberPerNode(AVERAGE_NUMBER_OF_EDGES_PER_NODE); graph.setEstimatedEntries(ESTIMATED_ENTRIES); @@ -42,22 +50,69 @@ public OrientMassiveInsertion(final String url) graph.begin(); } +// @Override +// protected void post() { +// graph.end(); +// } + +// @Override +// protected Long getOrCreate(String value) +// { +// final long v = Long.parseLong(value); +// graph.createVertex(v); +// return v; +// } + +// @Override +// protected void relateNodes(Long src, Long dest) +// { +// graph.createEdge(src, dest); +// } + + + @Override - protected void post() { - graph.end(); + protected void relateNodes(Vertex src, Vertex dest) + { + orientGraph.addEdge(null, src, dest, "similar"); + + // TODO why commit twice? is this a nested transaction? + if (orientGraph instanceof TransactionalGraph) + { + orientGraph.commit(); + } } @Override - protected Long getOrCreate(String value) + protected Vertex getOrCreate(final String value) { - final long v = Long.parseLong(value); - graph.createVertex(v); + final int key = Integer.parseInt(value); + + Vertex v; + final OIdentifiable rec = (OIdentifiable) index.get(key); + if (rec != null) + { + return orientGraph.getVertex(rec); + } + + v = orientGraph.addVertex(key, "nodeId", key); + + + if (orientGraph instanceof TransactionalGraph) + { + orientGraph.commit(); + } + return v; } @Override - protected void relateNodes(Long src, Long dest) + protected void post() { - graph.createEdge(src, dest); + super.post(); + if (orientGraph instanceof TransactionalGraph) + { + orientGraph.commit(); + } } } diff --git a/src/main/java/eu/socialsensor/insert/OrientSingleInsertion.java b/src/main/java/eu/socialsensor/insert/OrientSingleInsertion.java index 57741d3..31dbaec 100644 --- a/src/main/java/eu/socialsensor/insert/OrientSingleInsertion.java +++ b/src/main/java/eu/socialsensor/insert/OrientSingleInsertion.java @@ -56,6 +56,7 @@ protected Vertex getOrCreate(final String value) v = orientGraph.addVertex(key, "nodeId", key); + if (orientGraph instanceof TransactionalGraph) { orientGraph.commit(); diff --git a/src/test/resources/META-INF/input.properties b/src/test/resources/META-INF/input.properties index a79170e..5f90750 100644 --- a/src/test/resources/META-INF/input.properties +++ b/src/test/resources/META-INF/input.properties @@ -1,5 +1,5 @@ # Choose which data sets you want to include in the benchmark by removing the contents. -eu.socialsensor.dataset=data/email-Enron.txt +eu.socialsensor.dataset=data/Email-Enron.txt #eu.socialsensor.dataset=data/com-youtube.ungraph.txt #eu.socialsensor.dataset=data/Amazon0601.txt #eu.socialsensor.dataset=data/com-lj.ungraph.txt @@ -63,7 +63,7 @@ eu.socialsensor.dynamodb.endpoint=http://127.0.0.1:4567 #eu.socialsensor.dynamodb.endpoint=https://dynamodb.us-east-1.amazonaws.com # OrientDB options -eu.socialsensor.orient.lightweight-edges=false +eu.socialsensor.orient.lightweight-edges=true # Sparksee options eu.socialsensor.sparksee.license-key=DEADBEEF