amaliujia commented on code in PR #41107:
URL: https://github.com/apache/spark/pull/41107#discussion_r1189290739
##########
connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/connect/client/SparkConnectClient.scala:
##########
@@ -235,14 +233,14 @@ private[sql] class SparkConnectClient(
/**
* Cache the given local relation at the server, and return its key in the
remote cache.
*/
- def cacheLocalRelation(size: Int, data: ByteString, schema: String): String
= {
- val schemaBytes = schema.getBytes(StandardCharsets.UTF_8)
- val locRelData = data.toByteArray
- val locRel = ByteBuffer.allocate(4 + locRelData.length +
schemaBytes.length)
- locRel.putInt(size)
- locRel.put(locRelData)
- locRel.put(schemaBytes)
- artifactManager.cacheArtifact(locRel.array())
+ def cacheLocalRelation(data: ByteString, schema: String): String = {
+ val localRelation = proto.Relation
+ .newBuilder()
+ .getLocalRelationBuilder
+ .setSchema(schema)
+ .setData(data)
+ .build()
+ artifactManager.cacheArtifact(localRelation.toByteArray)
Review Comment:
is the byte format of the proto LocalRelation compatible with existing code?
FYI @vicennial
##########
connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/connect/client/SparkConnectClient.scala:
##########
@@ -235,14 +233,14 @@ private[sql] class SparkConnectClient(
/**
* Cache the given local relation at the server, and return its key in the
remote cache.
*/
- def cacheLocalRelation(size: Int, data: ByteString, schema: String): String
= {
- val schemaBytes = schema.getBytes(StandardCharsets.UTF_8)
- val locRelData = data.toByteArray
- val locRel = ByteBuffer.allocate(4 + locRelData.length +
schemaBytes.length)
- locRel.putInt(size)
- locRel.put(locRelData)
- locRel.put(schemaBytes)
- artifactManager.cacheArtifact(locRel.array())
+ def cacheLocalRelation(data: ByteString, schema: String): String = {
+ val localRelation = proto.Relation
+ .newBuilder()
+ .getLocalRelationBuilder
+ .setSchema(schema)
+ .setData(data)
+ .build()
+ artifactManager.cacheArtifact(localRelation.toByteArray)
Review Comment:
Oh I see so from the server side it also re-construct LocalRelation proto
from the bytes. Probably @vicennial can confirm if all the places are updated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
