javeme commented on code in PR #248:
URL:
https://github.com/apache/incubator-hugegraph-computer/pull/248#discussion_r1205717331
##########
computer-core/src/main/java/org/apache/hugegraph/computer/core/input/WorkerInputManager.java:
##########
@@ -46,6 +62,15 @@ public WorkerInputManager(ComputerContext context,
MessageSendManager sendManager) {
this.loadService = new LoadService(context);
this.sendManager = sendManager;
+
+ int sendThreadNum = this.inputLoaderThreadNum(context.config());
+ this.sendExecutor = ExecutorUtil.newFixedThreadPool(sendThreadNum,
PREFIX);
+ LOG.info("Created parallel sending thread pool, thread num: {}",
+ sendThreadNum);
+ }
+
+ private Integer inputLoaderThreadNum(Config config) {
Review Comment:
inputLoaderThreadNum => inputSendThreadNum ?
##########
computer-core/src/main/java/org/apache/hugegraph/computer/core/input/WorkerInputManager.java:
##########
@@ -63,33 +88,42 @@ public void init(Config config) {
public void close(Config config) {
this.loadService.close();
this.sendManager.close(config);
+ this.sendExecutor.shutdown();
}
public void service(InputSplitRpcService rpcService) {
this.loadService.rpcService(rpcService);
}
/**
- * TODO: Load vertices and edges parallel.
* When this method finish, it means that all vertices and edges are sent,
* but there is no guarantee that all of them has been received.
*/
public void loadGraph() {
this.sendManager.startSend(MessageType.VERTEX);
Iterator<Vertex> iterator =
this.loadService.createIteratorFromVertex();
- while (iterator.hasNext()) {
- Vertex vertex = iterator.next();
- this.sendManager.sendVertex(vertex);
- }
+ this.send(iterator, this.sendManager::sendVertex);
this.sendManager.finishSend(MessageType.VERTEX);
this.sendManager.startSend(MessageType.EDGE);
iterator = this.loadService.createIteratorFromEdge();
+ this.send(iterator, this.sendManager::sendEdge);
+ this.sendManager.finishSend(MessageType.EDGE);
+ this.sendManager.clearBuffer();
+ }
+
+ private void send(Iterator<Vertex> iterator, Consumer<Vertex> consumer) {
+ List<CompletableFuture<?>> futures = new ArrayList<>();
while (iterator.hasNext()) {
Vertex vertex = iterator.next();
- this.sendManager.sendEdge(vertex);
+ futures.add(CompletableFuture.runAsync(() ->
consumer.accept(vertex),
+ this.sendExecutor));
Review Comment:
prefer to align with CompletableFuture
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
