Can you share your actual code and queries Von meinem iPhone gesendet
> Am 12.05.2016 um 18:52 schrieb Cherie Pun <[email protected]>: > > I am trying to iterate through the results from the first query and execute > an extra cypher query for each result to either create or find the existing > relationship. However I encountered GC overhead error and the stack trace is > as followed. I thought running the queries in batch of 100 will solve the > memory problem but it seems that either it's using up the memory too quickly > or not releasing the memory quick enough. It broke down after processing > 200000 queries after I increased the batch size to 1000. Is there a way to > optimise my query to reduce the memory used, or is there an alternative way > to achieve the same result? Thanks. > > Exception in thread "main" java.lang.OutOfMemoryError: GC overhead limit > exceeded > at > scala.collection.mutable.OpenHashMap.foreachUndeletedEntry(OpenHashMap.scala:226) > at scala.collection.mutable.OpenHashMap.foreach(OpenHashMap.scala:219) > at > org.neo4j.cypher.internal.compiler.v2_3.ExecutionContext.foreach(ExecutionContext.scala:46) > at > scala.collection.TraversableOnce$class.foldLeft(TraversableOnce.scala:155) > at > org.neo4j.cypher.internal.compiler.v2_3.ExecutionContext.foldLeft(ExecutionContext.scala:33) > at > org.neo4j.cypher.internal.frontend.v2_3.helpers.Eagerly$.mapToBuilder(Eagerly.scala:44) > at > org.neo4j.cypher.internal.frontend.v2_3.helpers.Eagerly$.immutableMapValues(Eagerly.scala:37) > at > org.neo4j.cypher.internal.compiler.v2_3.ClosingIterator$$anonfun$next$1.apply(ResultIterator.scala:76) > at > org.neo4j.cypher.internal.compiler.v2_3.ClosingIterator$$anonfun$next$1.apply(ResultIterator.scala:72) > at > org.neo4j.cypher.internal.compiler.v2_3.ClosingIterator$$anonfun$failIfThrows$1.apply(ResultIterator.scala:121) > at > org.neo4j.cypher.internal.compiler.v2_3.ClosingIterator.decoratedCypherException(ResultIterator.scala:130) > at > org.neo4j.cypher.internal.compiler.v2_3.ClosingIterator.failIfThrows(ResultIterator.scala:119) > at > org.neo4j.cypher.internal.compiler.v2_3.ClosingIterator.next(ResultIterator.scala:72) > at > org.neo4j.cypher.internal.compiler.v2_3.ClosingIterator.next(ResultIterator.scala:50) > at > org.neo4j.cypher.internal.compiler.v2_3.PipeExecutionResult.next(PipeExecutionResult.scala:77) > at > org.neo4j.cypher.internal.compiler.v2_3.PipeExecutionResult$$anon$2.next(PipeExecutionResult.scala:70) > at > org.neo4j.cypher.internal.compiler.v2_3.PipeExecutionResult$$anon$2.next(PipeExecutionResult.scala:68) > at > org.neo4j.cypher.internal.compatibility.ExecutionResultWrapperFor2_3$$anon$1$$anonfun$next$1.apply(CompatibilityFor2_3.scala:234) > at > org.neo4j.cypher.internal.compatibility.ExecutionResultWrapperFor2_3$$anon$1$$anonfun$next$1.apply(CompatibilityFor2_3.scala:234) > at > org.neo4j.cypher.internal.compatibility.exceptionHandlerFor2_3$.runSafely(CompatibilityFor2_3.scala:116) > at > org.neo4j.cypher.internal.compatibility.ExecutionResultWrapperFor2_3$$anon$1.next(CompatibilityFor2_3.scala:234) > at > org.neo4j.cypher.internal.compatibility.ExecutionResultWrapperFor2_3$$anon$1.next(CompatibilityFor2_3.scala:229) > at > org.neo4j.cypher.javacompat.ExecutionResult.next(ExecutionResult.java:233) > at > org.neo4j.cypher.javacompat.ExecutionResult.next(ExecutionResult.java:55) > at java.util.Iterator.forEachRemaining(Iterator.java:116) > at > Neo4j.TrustCalculator.iterateAndExecuteBatchedInSeparateThread(TrustCalculator.java:239) > at > Neo4j.TrustCalculator.insertSimpleTransitiveTrust(TrustCalculator.java:127) > at Neo4j.TrustCalculator.main(TrustCalculator.java:265) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:483) > > private final String CREATE_UNIQUE_TRUST_RELATION_QUERY = > "MATCH (a:" + TwitterLabels.USERS + "),(b:" + TwitterLabels.USERS + > ")\n" + > "WHERE id(a) = {startNode} AND id(b) = {endNode} \n" + > "CREATE UNIQUE (a) - [r:" + TwitterRelationships.TRUST + "] > -> (b) \n" + > "return r"; > > private final String FIND_TRANSITIVE_RELATION_QUERY = > "MATCH (a:" + TwitterLabels.USERS + ") " + > "- [ab:" + TwitterRelationships.TRUST + "] -> " + > "(b:" + TwitterLabels.USERS + ") " + > "- [bc:" + TwitterRelationships.TRUST + "] -> " + > "(c:" + TwitterLabels.USERS + ")\n" + > "WHERE a <> c\n" + > "return ab, bc"; > > private ExecutorService createSinglePool() { > return Executors.newSingleThreadExecutor(); > } > > private void iterateAndExecuteBatchedInSeparateThread(int batchsize, Iterator > iterator, Consumer consumer) { > final int[] opsCount = {0}; > final int[] batchesCount = {0}; > ExecutorService executorService = createSinglePool(); > try { > final Transaction[] workerTransaction = {executorService.submit(() -> > graphDb.beginTx()).get()}; > > iterator.forEachRemaining(t -> executorService.submit(() -> { > consumer.accept(t); > if ((++opsCount[0]) % batchsize == 0) { > batchesCount[0]++; > workerTransaction[0].success(); > workerTransaction[0].close(); > workerTransaction[0] = graphDb.beginTx(); > } > })); > executorService.submit(() -> { > workerTransaction[0].success(); > workerTransaction[0].close(); > }).get(); > > } catch (InterruptedException | ExecutionException e) { > throw new RuntimeException(e); > } > } > > > public void insertSimpleTransitiveTrust(){ > try ( Transaction tx = graphDb.beginTx() ) > { > Result results = graphDb.execute( FIND_TRANSITIVE_RELATION_QUERY, new > HashMap<String, Object>() ); > > iterateAndExecuteBatchedInSeparateThread(1000, results, result-> > insertOneSimpleTransitiveTrust((Map<String, Object>) result)); > > tx.success(); > } > } > > private void insertOneSimpleTransitiveTrust(Map<String, Object> map){ > Object object = map.get("ab"); > Relationship trustAb = (Relationship) object; > object = map.get("bc"); > Relationship trustBc = (Relationship) object; > if(trustAb.hasProperty(TwitterProperties.CONVERSATIONAL_TRUST) && > trustBc.hasProperty(TwitterProperties.CONVERSATIONAL_TRUST)){ > Node nodeA = trustAb.getStartNode(); > Node nodeB = trustAb.getEndNode(); > Node nodeC = trustBc.getEndNode(); > > Map<String, Object> params = new HashMap<String, Object>(); > params.put("startNode", nodeA.getId()); > params.put("endNode", nodeC.getId()); > Result results = graphDb.execute( CREATE_UNIQUE_TRUST_RELATION_QUERY, > params ); > if(results.hasNext()){ > Map<String, Object> result = results.next(); > object = result.get("r"); > Relationship transitiveTrust = (Relationship) object; > > transitiveTrust.setProperty(TwitterProperties.SIMPLE_TRANSITIVE_TRUST, > trustBc.getProperty(TwitterProperties.CONVERSATIONAL_TRUST)); > } > } > } > > > -- > You received this message because you are subscribed to the Google Groups > "Neo4j" group. > To unsubscribe from this group and stop receiving emails from it, send an > email to [email protected]. > For more options, visit https://groups.google.com/d/optout. -- You received this message because you are subscribed to the Google Groups "Neo4j" group. To unsubscribe from this group and stop receiving emails from it, send an email to [email protected]. For more options, visit https://groups.google.com/d/optout.
