jingz-db commented on code in PR #47107:
URL: https://github.com/apache/spark/pull/47107#discussion_r1664939251
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateEncoder.scala:
##########
@@ -491,52 +522,80 @@ class RangeKeyScanStateEncoder(
writer.getRow()
}
- override def encodeKey(row: UnsafeRow): Array[Byte] = {
+ override def encodeKey(row: UnsafeRow, vcfId: Option[Short]): Array[Byte] = {
+ val hasVirtualColFamilyPrefix: Boolean = vcfId.isDefined
// This prefix key has the columns specified by orderingOrdinals
val prefixKey = extractPrefixKey(row)
val rangeScanKeyEncoded =
encodeUnsafeRow(encodePrefixKeyForRangeScan(prefixKey))
+ val offSetForColFamilyPrefix =
+ if (hasVirtualColFamilyPrefix) VIRTUAL_COL_FAMILY_PREFIX_BYTES else 0
+
val result = if (orderingOrdinals.length < keySchema.length) {
val remainingEncoded = encodeUnsafeRow(remainingKeyProjection(row))
- val encodedBytes = new Array[Byte](rangeScanKeyEncoded.length +
remainingEncoded.length + 4)
- Platform.putInt(encodedBytes, Platform.BYTE_ARRAY_OFFSET,
rangeScanKeyEncoded.length)
+ val encodedBytes = new Array[Byte](rangeScanKeyEncoded.length +
Review Comment:
I am not sure if we can do this very clean. The put of the vcf prefix
happens only once at the beginning of `encode(Prefix)Key` and it has to be
happen after the correct length of `encodedBytes` is allocated (I am not sure
if there is any clean way to move this outside of `encodeKey` function and
unify it somewhere in the base class).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
