cloud-fan commented on a change in pull request #27665: [SPARK-30623][Core]
Spark external shuffle allow disable of separate event loop group
URL: https://github.com/apache/spark/pull/27665#discussion_r396407166
##########
File path:
common/network-common/src/main/java/org/apache/spark/network/server/ChunkFetchRequestHandler.java
##########
@@ -17,46 +17,33 @@
package org.apache.spark.network.server;
-import java.net.SocketAddress;
-
import com.google.common.base.Throwables;
-import io.netty.channel.Channel;
-import io.netty.channel.ChannelFuture;
-import io.netty.channel.ChannelFutureListener;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.SimpleChannelInboundHandler;
+import io.netty.channel.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.spark.network.buffer.ManagedBuffer;
import org.apache.spark.network.client.TransportClient;
-import org.apache.spark.network.protocol.ChunkFetchFailure;
-import org.apache.spark.network.protocol.ChunkFetchRequest;
-import org.apache.spark.network.protocol.ChunkFetchSuccess;
-import org.apache.spark.network.protocol.Encodable;
+import org.apache.spark.network.protocol.*;
-import static org.apache.spark.network.util.NettyUtils.*;
+import static org.apache.spark.network.util.NettyUtils.getRemoteAddress;
/**
- * A dedicated ChannelHandler for processing ChunkFetchRequest messages. When
sending response
- * of ChunkFetchRequest messages to the clients, the thread performing the I/O
on the underlying
- * channel could potentially be blocked due to disk contentions. If several
hundreds of clients
- * send ChunkFetchRequest to the server at the same time, it could potentially
occupying all
- * threads from TransportServer's default EventLoopGroup for waiting for disk
reads before it
- * can send the block data back to the client as part of the ChunkFetchSuccess
messages. As a
- * result, it would leave no threads left to process other RPC messages, which
takes much less
- * time to process, and could lead to client timing out on either performing
SASL authentication,
- * registering executors, or waiting for response for an OpenBlocks messages.
+ * Base class for custom ChannelHandler for FetchRequest messages.
+ * The messages can be processed in a separate event loop group or the one
shared with other
+ * {@link Message.Type}s. The behavior is decided by whether the config
+ * `spark.shuffle.server.chunkFetchHandlerThreadsPercent` is set or not.
+ * See more details in SPARK-30623.
*/
-public class ChunkFetchRequestHandler extends
SimpleChannelInboundHandler<ChunkFetchRequest> {
+public abstract class ChunkFetchRequestHandler
+ extends SimpleChannelInboundHandler<ChunkFetchRequest> {
private static final Logger logger =
LoggerFactory.getLogger(ChunkFetchRequestHandler.class);
private final TransportClient client;
private final StreamManager streamManager;
- /** The max number of chunks being transferred and not finished yet. */
Review comment:
shall we keep the comment?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
