FMX commented on code in PR #3177:
URL: https://github.com/apache/celeborn/pull/3177#discussion_r2022250672
##########
common/src/main/scala/org/apache/celeborn/common/serializer/JavaSerializer.scala:
##########
@@ -98,19 +99,51 @@ private[celeborn] class JavaSerializerInstance(
override def serialize[T: ClassTag](t: T): ByteBuffer = {
val bos = new ByteBufferOutputStream()
+ val msg = Utils.toTransportMessage(t)
+ msg match {
+ case transMsg: TransportMessage =>
+ // Check if the msg is a TransportMessage with CPP languageType.
+ // If so, write the marker and the body explicitly.
+ if (transMsg.getLanguageType == LanguageType.CPP) {
+ val out = new DataOutputStream(bos)
+ out.writeByte(LanguageType.CPP.getMarker)
+ out.write(transMsg.toByteBuffer.array)
+ out.close()
+ return bos.toByteBuffer
+ }
+ case _ =>
+ }
val out = serializeStream(bos)
out.writeObject(Utils.toTransportMessage(t))
out.close()
bos.toByteBuffer
}
override def deserialize[T: ClassTag](bytes: ByteBuffer): T = {
+ bytes.mark
+ val languageMarker = bytes.get
+ bytes.reset
+ // If the languageMarker byte is CPP, deserialize directly.
+ if (languageMarker == LanguageType.CPP.getMarker) {
+ bytes.get
+ return Utils.fromTransportMessage(
+ TransportMessage.fromByteBuffer(bytes,
LanguageType.CPP)).asInstanceOf[T]
+ }
val bis = new ByteBufferInputStream(bytes)
Review Comment:
I think you can move `bytes.reset` to this line.
##########
common/src/main/scala/org/apache/celeborn/common/serializer/JavaSerializer.scala:
##########
@@ -98,19 +99,51 @@ private[celeborn] class JavaSerializerInstance(
override def serialize[T: ClassTag](t: T): ByteBuffer = {
val bos = new ByteBufferOutputStream()
+ val msg = Utils.toTransportMessage(t)
+ msg match {
+ case transMsg: TransportMessage =>
+ // Check if the msg is a TransportMessage with CPP languageType.
+ // If so, write the marker and the body explicitly.
+ if (transMsg.getLanguageType == LanguageType.CPP) {
+ val out = new DataOutputStream(bos)
+ out.writeByte(LanguageType.CPP.getMarker)
+ out.write(transMsg.toByteBuffer.array)
+ out.close()
+ return bos.toByteBuffer
+ }
+ case _ =>
+ }
val out = serializeStream(bos)
out.writeObject(Utils.toTransportMessage(t))
out.close()
bos.toByteBuffer
}
override def deserialize[T: ClassTag](bytes: ByteBuffer): T = {
+ bytes.mark
+ val languageMarker = bytes.get
+ bytes.reset
+ // If the languageMarker byte is CPP, deserialize directly.
+ if (languageMarker == LanguageType.CPP.getMarker) {
+ bytes.get
+ return Utils.fromTransportMessage(
+ TransportMessage.fromByteBuffer(bytes,
LanguageType.CPP)).asInstanceOf[T]
+ }
val bis = new ByteBufferInputStream(bytes)
val in = deserializeStream(bis)
Utils.fromTransportMessage(in.readObject()).asInstanceOf[T]
}
override def deserialize[T: ClassTag](bytes: ByteBuffer, loader:
ClassLoader): T = {
+ bytes.mark
+ val languageMarker = bytes.get
+ bytes.reset
+ // If the languageMarker byte is CPP, deserialize directly.
+ if (languageMarker == LanguageType.CPP.getMarker) {
+ bytes.get
+ return Utils.fromTransportMessage(
+ TransportMessage.fromByteBuffer(bytes,
LanguageType.CPP)).asInstanceOf[T]
+ }
val bis = new ByteBufferInputStream(bytes)
Review Comment:
I think you can move bytes.reset to this line.
##########
client/src/main/scala/org/apache/celeborn/client/commit/ReducePartitionCommitHandler.scala:
##########
@@ -64,8 +65,10 @@ class ReducePartitionCommitHandler(
sharedRpcPool)
with Logging {
+ class Record(val ctx: RpcCallContext, val languageType: LanguageType) {}
Review Comment:
```suggestion
class MultiLanguageRpcContext(val ctx: RpcCallContext, val languageType:
LanguageType) {}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
