malinjawi commented on code in PR #13595: URL: https://github.com/apache/gluten/pull/13595#discussion_r4613536729
########## backends-velox/src-lance/main/scala/org/apache/gluten/connector/write/LanceColumnarBatchDataWriter.scala: ########## @@ -0,0 +1,424 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.connector.write + +import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.columnarbatch.{ColumnarBatches, ColumnarBatchJniWrapper} +import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators +import org.apache.gluten.metrics.BatchWriteMetrics + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.connector.metric.CustomTaskMetric +import org.apache.spark.sql.connector.write.{DataWriter, WriterCommitMessage} +import org.apache.spark.sql.execution.metric.CustomMetrics +import org.apache.spark.sql.vectorized.ColumnarBatch + +import org.apache.arrow.c.{ArrowArray, ArrowArrayStream, ArrowSchema, CDataDictionaryProvider, Data} +import org.apache.arrow.memory.BufferAllocator +import org.apache.arrow.vector.VectorSchemaRoot +import org.apache.arrow.vector.ipc.ArrowReader +import org.apache.arrow.vector.types.pojo.{ArrowType, Field, Schema} +import org.lance.{Fragment, FragmentMetadata} +import org.lance.spark.write.{FragmentWriteSpec, LanceBatchWrite} +import org.lance.spark.write.metric.LanceWriteMetrics + +import java.io.IOException +import java.util.{Collections, List => JList} +import java.util.concurrent.{ArrayBlockingQueue, TimeUnit} +import java.util.concurrent.atomic.AtomicLong + +import scala.collection.JavaConverters._ + +/** + * Writes one task's Velox batches into uncommitted Lance fragments. + * + * Spark pushes batches through [[write]], while Lance pulls batches from an Arrow C stream until it + * ends. A background thread bridges the two, as lance-spark's own row writer does: each batch is + * exported to Arrow C structs on the task thread and queued, and the thread writes fragments with + * the parameters of the [[FragmentWriteSpec]] from a stream that drains the queue. + * + * The stream advertises the commit schema, so its names, nullability and timestamp zone are the + * ones `LanceBatchWrite.commit` uses. Every exported batch is checked against it, so a type Velox + * exports differently fails the task instead of writing a corrupt table. Fixed-size list columns + * arrive as plain lists and are converted by [[LanceFixedSizeLists]], and the values Velox leaves + * under nulls are zeroed by [[LanceNullSlots]] so that Lance encodes as compactly as it does for + * the row writer. + * + * The writer thread imports the batches with the task's Arrow allocator, so they count as task + * memory while Lance holds them. That reserves task memory off the task thread, which Gluten's + * memory targets allow and Velox's I/O threads do as well. A reservation that fails makes the + * stream fail, which fails the task. + * + * It reports the metrics lance-spark's row writer reports, under the same names, and fills the + * native write metrics of the write node. + */ +class LanceColumnarBatchDataWriter(spec: FragmentWriteSpec, commitSchema: Schema) + extends DataWriter[ColumnarBatch] + with Logging { + import LanceColumnarBatchDataWriter._ + + private val allocator: BufferAllocator = + ArrowBufferAllocators.contextInstance(classOf[LanceColumnarBatchDataWriter].getSimpleName) + private val queue = new ArrayBlockingQueue[Item](QueueCapacity) + private val exportedSchema = LanceFixedSizeLists.exportedSchema(commitSchema) + + private var stream: ArrowArrayStream = _ + private var thread: Thread = _ + @volatile private var fragments: JList[FragmentMetadata] = _ + @volatile private var failure: Throwable = _ + @volatile private var writeWallNs = 0L + // Time the stream waits for the task to queue its next batch, which is not time spent writing. + private val queueWaitNs = new AtomicLong() + // Set by the stream when Lance reads its end. + @volatile private var streamEnded = false + // Set by the task to make the stream fail, so Lance stops writing. + @volatile private var stopRequested = false + private var stopWaited = false + + private var recordsWritten = 0L + private var bytesWritten = 0L + private var filesWritten = 0L + + // Spark forwards bytesWritten and recordsWritten to the task's output metrics, as for the row + // writer. Lance only reports file sizes once its fragments complete, inside commit(). + private val taskMetrics: Array[CustomTaskMetric] = Array( + taskMetric(LanceWriteMetrics.RECORDS_WRITTEN, recordsWritten), + taskMetric(LanceWriteMetrics.BYTES_WRITTEN, bytesWritten), + taskMetric(BatchWriteMetrics.NUM_WRITTEN_BYTES, bytesWritten), + taskMetric(BatchWriteMetrics.NUM_WRITTEN_FILES, filesWritten), + taskMetric(BatchWriteMetrics.WRITE_WALL_NS, writeWallNs) + ) + + override def write(batch: ColumnarBatch): Unit = { + val numRows = batch.numRows() + if (numRows == 0) { + return + } + if (thread == null) { + start() + } else { + // Once Lance has failed, it reads no more batches, so do not export them. + checkWriter() + } + enqueue(exportBatch(batch)) + recordsWritten += numRows + } + + override def commit(): WriterCommitMessage = { + if (thread == null) { + // No rows in this task: commit nothing, as lance-spark's row writer does. + return LanceBatchWrite.taskCommit(Collections.emptyList()) + } + enqueue(End) + thread.join() + throwIfFailed() + if (!streamEnded) { + throw new IllegalStateException("Lance finished writing before the end of the stream") + } + fragments.asScala.foreach { + fragment => + fragment.getFiles.asScala.foreach { + file => + filesWritten += 1 + Option(file.getFileSizeBytes).foreach(size => bytesWritten += size) + } + } + LanceBatchWrite.taskCommit(fragments) + } + + override def currentMetricsValues(): Array[CustomTaskMetric] = taskMetrics + + override def abort(): Unit = { + // A failed stream makes Lance delete the files it wrote for this task, finished or not, as + // Spark expects of abort(). lance-spark's row writer ends its stream instead, which leaves + // them. + stopWriter() + // Spark adds a failed attempt's output metrics to the stage totals, so the retry would count + // these rows twice. Clear them, as lance-spark's row writer does. + recordsWritten = 0 + bytesWritten = 0 + CustomMetrics.updateMetrics(taskMetrics.toSeq, Map.empty) + } + + override def close(): Unit = { + stopWriter() + // A writer thread that did not stop may still read the stream, so leave it rather than free it. + // The task then reports the stream's struct as leaked Arrow memory. + if (stream != null && (thread == null || !thread.isAlive)) { + // Lance has taken the stream by now, unless it failed before reading it; release() then + // closes the reader, and frees nothing otherwise. + stream.release() + stream.close() + stream = null + } + } + + private def start(): Unit = { + stream = ArrowArrayStream.allocateNew(allocator) + Data.exportArrayStream(allocator, new QueueReader, stream) + thread = new Thread(() => runWriter(), "gluten-lance-write") + thread.setDaemon(true) + thread.start() + } + + /** + * Runs on the writer thread until Lance has read the stream to its end, or failed. The write time + * leaves out the time the stream waits for batches, which the operators producing them spend, as + * the time of write of Velox's own writers does. + */ + private def runWriter(): Unit = { + val startNs = System.nanoTime() + try { + fragments = Fragment + .write() + .datasetUri(spec.getDatasetUri) + .data(stream) + .writeParams(spec.toWriteParams()) + .execute() + } catch { + case t: Throwable => failure = t + } finally { + writeWallNs = System.nanoTime() - startNs - queueWaitNs.get() + } + } + + /** Exports a Velox batch to Arrow C structs, checking the schema Velox exported it with. */ + private def exportBatch(batch: ColumnarBatch): ExportedBatch = { + val handle = ColumnarBatches.getNativeHandle(BackendsApiManager.getBackendName, batch) + val exported = new ExportedBatch(ArrowArray.allocateNew(allocator)) + val cSchema = ArrowSchema.allocateNew(allocator) Review Comment: Good catch, fixed. The schema struct is now allocated inside the array's cleanup block, so if that allocation fails the array is released. ########## backends-velox/src-lance/main/scala/org/apache/gluten/connector/write/LanceFixedSizeLists.scala: ########## @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.connector.write + +import org.apache.arrow.vector.{BaseFixedWidthVector, BitVectorHelper, VectorSchemaRoot} +import org.apache.arrow.vector.complex.{FixedSizeListVector, ListVector} +import org.apache.arrow.vector.types.pojo.{ArrowType, Field, FieldType, Schema} + +import scala.collection.JavaConverters._ + +/** + * Velox has no fixed-size list type, so it exports Lance's fixed-size list columns, such as + * vectors, as plain lists. A batch with such columns is imported with the schema Velox exports and + * its lists are copied into the fixed-size list vectors of the commit schema. Only top-level + * fixed-size lists are converted; the write falls back for nested ones. + */ +object LanceFixedSizeLists { + + /** The schema Velox exports for `commitSchema`: its top-level fixed-size lists as plain lists. */ + def exportedSchema(commitSchema: Schema): Schema = { + val fields = commitSchema.getFields.asScala.map { + field => + field.getType match { + case _: ArrowType.FixedSizeList => + val fieldType = + new FieldType(field.isNullable, ArrowType.List.INSTANCE, null, field.getMetadata) + new Field(field.getName, fieldType, field.getChildren) + case _ => field + } + } + new Schema(fields.asJava, commitSchema.getCustomMetadata) + } + + /** + * Moves the rows of `from`, shaped like [[exportedSchema]], into `to`, shaped like the commit + * schema. Plain lists are copied into fixed-size lists, and every other column is transferred. + */ + def moveInto(from: VectorSchemaRoot, to: VectorSchemaRoot): Unit = { + val rows = from.getRowCount + from.getFieldVectors.asScala.zip(to.getFieldVectors.asScala).foreach { + case (list: ListVector, fixed: FixedSizeListVector) => + copyLists(list, fixed, rows) + // Free the exported lists now rather than at the next import, so a batch does not hold + // both copies of its vectors while Lance encodes it. + list.clear() + case (source, target) => source.makeTransferPair(target).transfer() + } + to.setRowCount(rows) + } + + private def copyLists(source: ListVector, target: FixedSizeListVector, rows: Int): Unit = { + val size = target.getListSize + target.clear() + target.setInitialCapacity(rows) + target.allocateNew() Review Comment: Fixed. When no row is null it now allocates only the validity bitmap and moves the elements over. I added a check to the test that no element buffer gets allocated on that path. ########## backends-velox/src-lance/main/scala/org/apache/gluten/connector/write/LanceNullSlots.scala: ########## @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.connector.write + +import org.apache.arrow.memory.{ArrowBuf, BufferAllocator} +import org.apache.arrow.vector.{BaseFixedWidthVector, BitVector, FieldVector, VectorSchemaRoot} +import org.apache.arrow.vector.complex.StructVector +import org.apache.arrow.vector.ipc.message.ArrowFieldNode + +import scala.collection.JavaConverters._ + +/** + * Zeroes the values under nulls in fixed-width columns, including the values of a struct's children + * in the rows where the struct is null. + * + * Arrow leaves the value of a null slot undefined, and Velox exports whatever its buffers hold + * there. Lance counts those slots when it picks an encoding, so leftover values widen bit-packed + * integers and defeat dictionaries, and the files grow. Lance also pushes a struct's nulls down to + * its children before encoding them. Spark's ArrowWriter, which lance-spark's row writer uses, + * leaves zeros, and nulls a null struct's children. The exported buffers belong to Velox and may be + * shared with other vectors, so they are never written: a column with a non-zero null slot is + * copied, and the copy is zeroed. + */ +object LanceNullSlots { + + /** Zeroes the null slots of the fixed-width vectors in `root`, nested ones included. */ + def zero(root: VectorSchemaRoot, allocator: BufferAllocator): Unit = + root.getFieldVectors.asScala.foreach(zero(_, Nil, allocator)) + + /** + * The bytes of a validity bitmap of `bits` bits. Arrow 19 renamed BitVectorHelper's + * getValidityBufferSize, so no one name works across the Arrow versions Gluten builds with. + */ + private[write] def validityBytes(bits: Int): Int = ((bits + 7L) / 8).toInt + + /** + * `parentNulls` holds the validity bitmaps of the structs above `vector` that have nulls. They + * line up with its rows, and a value in a row where one of them is null is a null slot too. + */ + private def zero( + vector: FieldVector, + parentNulls: List[ArrowBuf], + allocator: BufferAllocator): Unit = { + val nulls = + if (vector.getNullCount > 0) vector.getValidityBuffer :: parentNulls else parentNulls + vector match { + // Booleans are single bits, so leftover values cost Lance next to nothing. + case _: BitVector => + case fixed: BaseFixedWidthVector if nulls.nonEmpty && hasNonZeroNullSlot(fixed, nulls) => + replaceWithZeroedCopy(fixed, nulls, allocator) + case _ => + } + vector.getChildrenFromFields.asScala.foreach { + child => + // A struct's children line up with its rows; the elements of lists and maps do not. + val childParentNulls = vector match { + case struct: StructVector if child.getValueCount == struct.getValueCount => nulls + case _ => Nil + } + zero(child, childParentNulls, allocator) + } + } + + private def hasNonZeroNullSlot(vector: BaseFixedWidthVector, nulls: List[ArrowBuf]): Boolean = { + val data = vector.getDataBuffer + val width = vector.getTypeWidth + var found = false + foreachNull(nulls, vector.getValueCount) { + row => + found = !isZero(data, row.toLong * width, width) + !found + } + found + } + + private def isZero(buffer: ArrowBuf, offset: Long, width: Int): Boolean = { + var i = 0 + while (i + 8 <= width) { + if (buffer.getLong(offset + i) != 0) { + return false + } + i += 8 + } + while (i < width) { + if (buffer.getByte(offset + i) != 0) { + return false + } + i += 1 + } + true + } + + /** Swaps the buffers of `vector` for a copy whose slots that `nulls` marks null are zero. */ + private def replaceWithZeroedCopy( + vector: BaseFixedWidthVector, + nulls: List[ArrowBuf], + allocator: BufferAllocator): Unit = { + val rows = vector.getValueCount + val width = vector.getTypeWidth.toLong + val bitmapBytes = validityBytes(rows) + val copy = vector.getField.createVector(allocator).asInstanceOf[BaseFixedWidthVector] + // Raw buffers rather than allocateNew(), which would zero the data only for it to be copied over. + val validity = allocator.buffer(bitmapBytes) + val data = allocator.buffer(rows * width) Review Comment: Fixed, the validity buffer is freed if the data buffer can't be allocated. There's a new test that forces that allocation to fail and checks nothing is leaked. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
