I am trying to iterate through the results from the first query and execute
an extra cypher query for each result to either create or find the existing
relationship. However I encountered GC overhead error and the stack trace
is as followed. I thought running the queries in batch of 100 will solve
the memory problem but it seems that either it's using up the memory too
quickly or not releasing the memory quick enough. It broke down after
processing 200000 queries after I increased the batch size to 1000. Is
there a way to optimise my query to reduce the memory used, or is there an
alternative way to achieve the same result? Thanks.
Exception in thread "main" java.lang.OutOfMemoryError: GC overhead limit
exceeded
at
scala.collection.mutable.OpenHashMap.foreachUndeletedEntry(OpenHashMap.scala:226)
at scala.collection.mutable.OpenHashMap.foreach(OpenHashMap.scala:219)
at
org.neo4j.cypher.internal.compiler.v2_3.ExecutionContext.foreach(ExecutionContext.scala:46)
at
scala.collection.TraversableOnce$class.foldLeft(TraversableOnce.scala:155)
at
org.neo4j.cypher.internal.compiler.v2_3.ExecutionContext.foldLeft(ExecutionContext.scala:33)
at
org.neo4j.cypher.internal.frontend.v2_3.helpers.Eagerly$.mapToBuilder(Eagerly.scala:44)
at
org.neo4j.cypher.internal.frontend.v2_3.helpers.Eagerly$.immutableMapValues(Eagerly.scala:37)
at
org.neo4j.cypher.internal.compiler.v2_3.ClosingIterator$$anonfun$next$1.apply(ResultIterator.scala:76)
at
org.neo4j.cypher.internal.compiler.v2_3.ClosingIterator$$anonfun$next$1.apply(ResultIterator.scala:72)
at
org.neo4j.cypher.internal.compiler.v2_3.ClosingIterator$$anonfun$failIfThrows$1.apply(ResultIterator.scala:121)
at
org.neo4j.cypher.internal.compiler.v2_3.ClosingIterator.decoratedCypherException(ResultIterator.scala:130)
at
org.neo4j.cypher.internal.compiler.v2_3.ClosingIterator.failIfThrows(ResultIterator.scala:119)
at
org.neo4j.cypher.internal.compiler.v2_3.ClosingIterator.next(ResultIterator.scala:72)
at
org.neo4j.cypher.internal.compiler.v2_3.ClosingIterator.next(ResultIterator.scala:50)
at
org.neo4j.cypher.internal.compiler.v2_3.PipeExecutionResult.next(PipeExecutionResult.scala:77)
at
org.neo4j.cypher.internal.compiler.v2_3.PipeExecutionResult$$anon$2.next(PipeExecutionResult.scala:70)
at
org.neo4j.cypher.internal.compiler.v2_3.PipeExecutionResult$$anon$2.next(PipeExecutionResult.scala:68)
at
org.neo4j.cypher.internal.compatibility.ExecutionResultWrapperFor2_3$$anon$1$$anonfun$next$1.apply(CompatibilityFor2_3.scala:234)
at
org.neo4j.cypher.internal.compatibility.ExecutionResultWrapperFor2_3$$anon$1$$anonfun$next$1.apply(CompatibilityFor2_3.scala:234)
at
org.neo4j.cypher.internal.compatibility.exceptionHandlerFor2_3$.runSafely(CompatibilityFor2_3.scala:116)
at
org.neo4j.cypher.internal.compatibility.ExecutionResultWrapperFor2_3$$anon$1.next(CompatibilityFor2_3.scala:234)
at
org.neo4j.cypher.internal.compatibility.ExecutionResultWrapperFor2_3$$anon$1.next(CompatibilityFor2_3.scala:229)
at
org.neo4j.cypher.javacompat.ExecutionResult.next(ExecutionResult.java:233)
at org.neo4j.cypher.javacompat.ExecutionResult.next(ExecutionResult.java:55)
at java.util.Iterator.forEachRemaining(Iterator.java:116)
at
Neo4j.TrustCalculator.iterateAndExecuteBatchedInSeparateThread(TrustCalculator.java:239)
at
Neo4j.TrustCalculator.insertSimpleTransitiveTrust(TrustCalculator.java:127)
at Neo4j.TrustCalculator.main(TrustCalculator.java:265)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:483)
private final String CREATE_UNIQUE_TRUST_RELATION_QUERY =
"MATCH (a:" + TwitterLabels.USERS + "),(b:" + TwitterLabels.USERS +
")\n" +
"WHERE id(a) = {startNode} AND id(b) = {endNode} \n" +
"CREATE UNIQUE (a) - [r:" + TwitterRelationships.TRUST + "] ->
(b) \n" +
"return r";
private final String FIND_TRANSITIVE_RELATION_QUERY =
"MATCH (a:" + TwitterLabels.USERS + ") " +
"- [ab:" + TwitterRelationships.TRUST + "] -> " +
"(b:" + TwitterLabels.USERS + ") " +
"- [bc:" + TwitterRelationships.TRUST + "] -> " +
"(c:" + TwitterLabels.USERS + ")\n" +
"WHERE a <> c\n" +
"return ab, bc";
private ExecutorService createSinglePool() {
return Executors.newSingleThreadExecutor();
}
private void iterateAndExecuteBatchedInSeparateThread(int batchsize, Iterator
iterator, Consumer consumer) {
final int[] opsCount = {0};
final int[] batchesCount = {0};
ExecutorService executorService = createSinglePool();
try {
final Transaction[] workerTransaction = {executorService.submit(() ->
graphDb.beginTx()).get()};
iterator.forEachRemaining(t -> executorService.submit(() -> {
consumer.accept(t);
if ((++opsCount[0]) % batchsize == 0) {
batchesCount[0]++;
workerTransaction[0].success();
workerTransaction[0].close();
workerTransaction[0] = graphDb.beginTx();
}
}));
executorService.submit(() -> {
workerTransaction[0].success();
workerTransaction[0].close();
}).get();
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
}
}
public void insertSimpleTransitiveTrust(){
try ( Transaction tx = graphDb.beginTx() )
{
Result results = graphDb.execute( FIND_TRANSITIVE_RELATION_QUERY, new
HashMap<String, Object>() );
iterateAndExecuteBatchedInSeparateThread(1000, results, result->
insertOneSimpleTransitiveTrust((Map<String, Object>) result));
tx.success();
}
}
private void insertOneSimpleTransitiveTrust(Map<String, Object> map){
Object object = map.get("ab");
Relationship trustAb = (Relationship) object;
object = map.get("bc");
Relationship trustBc = (Relationship) object;
if(trustAb.hasProperty(TwitterProperties.CONVERSATIONAL_TRUST) &&
trustBc.hasProperty(TwitterProperties.CONVERSATIONAL_TRUST)){
Node nodeA = trustAb.getStartNode();
Node nodeB = trustAb.getEndNode();
Node nodeC = trustBc.getEndNode();
Map<String, Object> params = new HashMap<String, Object>();
params.put("startNode", nodeA.getId());
params.put("endNode", nodeC.getId());
Result results = graphDb.execute( CREATE_UNIQUE_TRUST_RELATION_QUERY,
params );
if(results.hasNext()){
Map<String, Object> result = results.next();
object = result.get("r");
Relationship transitiveTrust = (Relationship) object;
transitiveTrust.setProperty(TwitterProperties.SIMPLE_TRANSITIVE_TRUST,
trustBc.getProperty(TwitterProperties.CONVERSATIONAL_TRUST));
}
}
}
--
You received this message because you are subscribed to the Google Groups
"Neo4j" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/d/optout.
