Zand100 commented on code in PR #7902:
URL: https://github.com/apache/incubator-gluten/pull/7902#discussion_r1837347383
##########
backends-velox/src/main/scala/org/apache/spark/sql/execution/UnsafeArray.scala:
##########
@@ -0,0 +1,58 @@
+package org.apache.spark.sql.execution
+
+import org.apache.spark.memory.{MemoryConsumer, SparkOutOfMemoryError,
TaskMemoryManager}
+import org.apache.spark.sql.catalyst.expressions.UnsafeArrayData
+import org.apache.spark.unsafe.{Platform, UnsafeAlignedOffset}
+import org.apache.spark.unsafe.memory.MemoryBlock
+
+class UnsafeArray(taskMemoryManager: TaskMemoryManager) extends
MemoryConsumer {
+
+ protected var page: MemoryBlock = null
+ acquirePage(taskMemoryManager.pageSizeBytes)
+ protected var base: AnyRef = page.getBaseObject
+ protected var pageCursor = 0
+ private var keyOffsets: Array[Long] = null
+ protected var numRows = 0
+
+ def iterator() {}
+
+ private def acquirePage(requiredSize: Long): Boolean = {
+ try page = allocatePage(requiredSize)
+
+ catch {
+ case SparkOutOfMemoryError =>
+ return false
+ }
+ base = page.getBaseObject
+ pageCursor = 0
+ true
+ }
+
+ def get(rowId: Int): UnsafeArrayData = {
+ val offset = keyOffsets(rowId)
+ val klen = UnsafeAlignedOffset.getSize(base, offset -
UnsafeAlignedOffset.getUaoSize)
+ val result = new UnsafeArrayData
+ result.pointTo (base, offset, klen)
+ result
+ }
+
+ def write(bytes: Array[Byte], inputOffset: Long, inputLength: Int): Unit = {
+ var offset: Long = page.getBaseOffset + pageCursor
+ val recordOffset = offset
+
+ val uaoSize = UnsafeAlignedOffset.getUaoSize
+
+ val recordLength = 2L * uaoSize + inputLength + 8L
+
+ UnsafeAlignedOffset.putSize(base, offset, inputLength + uaoSize)
+ offset += 2L * uaoSize
Review Comment:
I'm not sure if I'm calculating the `offset` correctly
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
