coderzc commented on code in PR #248:
URL:
https://github.com/apache/incubator-hugegraph-computer/pull/248#discussion_r1208053074
##########
computer-core/src/main/java/org/apache/hugegraph/computer/core/input/WorkerInputManager.java:
##########
@@ -61,35 +89,56 @@ public void init(Config config) {
@Override
public void close(Config config) {
- this.loadService.close();
this.sendManager.close(config);
+ this.sendExecutor.shutdown();
}
public void service(InputSplitRpcService rpcService) {
this.loadService.rpcService(rpcService);
}
/**
- * TODO: Load vertices and edges parallel.
* When this method finish, it means that all vertices and edges are sent,
* but there is no guarantee that all of them has been received.
*/
public void loadGraph() {
+ List<CompletableFuture<?>> futures = new ArrayList<>();
+ CompletableFuture<?> future;
this.sendManager.startSend(MessageType.VERTEX);
- Iterator<Vertex> iterator =
this.loadService.createIteratorFromVertex();
- while (iterator.hasNext()) {
- Vertex vertex = iterator.next();
- this.sendManager.sendVertex(vertex);
+ for (int i = 0; i < this.sendThreadNum; i++) {
+ future = send(this.sendManager::sendVertex,
this.loadService::createIteratorFromVertex);
+ futures.add(future);
}
+ CompletableFuture.allOf(futures.toArray(new
CompletableFuture[0])).exceptionally(e -> {
+ throw new ComputerException("An exception occurred during parallel
" +
+ "sending vertices", e);
+ }).join();
this.sendManager.finishSend(MessageType.VERTEX);
+ futures.clear();
+
this.sendManager.startSend(MessageType.EDGE);
- iterator = this.loadService.createIteratorFromEdge();
- while (iterator.hasNext()) {
- Vertex vertex = iterator.next();
- this.sendManager.sendEdge(vertex);
+ for (int i = 0; i < this.sendThreadNum; i++) {
+ future = send(this.sendManager::sendEdge,
this.loadService::createIteratorFromEdge);
+ futures.add(future);
}
+ CompletableFuture.allOf(futures.toArray(new
CompletableFuture[0])).exceptionally(e -> {
+ throw new ComputerException("An exception occurred during parallel
" +
+ "sending edges", e);
+ }).join();
this.sendManager.finishSend(MessageType.EDGE);
this.sendManager.clearBuffer();
}
+
+ private CompletableFuture<?> send(Consumer<Vertex> sendConsumer,
+ Supplier<Iterator<Vertex>>
iteratorSupplier) {
+ return CompletableFuture.runAsync(() -> {
+ Iterator<Vertex> iterator = iteratorSupplier.get();
+ while (iterator.hasNext()) {
+ Vertex vertex = iterator.next();
+ sendConsumer.accept(vertex);
+ }
+ }, this.sendExecutor);
Review Comment:
It seems that `iterator` is not closed, use `try-with-resources` statement?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
