This is an automated email from the ASF dual-hosted git repository.
viirya pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 627c1fb17587 [SPARK-60367][4.1][SQL] Fix sliced Arrow output
processors and disable slicing for runners that need whole batches
627c1fb17587 is described below
commit 627c1fb17587c79b5debc2043d8019ba5bc08328
Author: Liang-Chi Hsieh <[email protected]>
AuthorDate: Sat Oct 24 13:55:04 2026 -0700
[SPARK-60367][4.1][SQL] Fix sliced Arrow output processors and disable
slicing for runners that need whole batches
### What changes were proposed in this pull request?
This is a backport of #59616 to branch-4.1.
It fixes the sliced Arrow output processors of the Python runners, which
are used when the internal config
`spark.sql.execution.arrow.maxRecordsPerOutputBatch` or
`spark.sql.execution.arrow.maxBytesPerOutputBatch` is set, and disables the
slicing for `ApplyInPandasWithStatePythonRunner` and `ArrowPythonUDTFRunner`,
whose consumers need whole batches. See #59616 for the details.
Differences from the master commit:
- branch-4.1 has no columnar input path for Arrow Python UDFs (SPARK-56350,
4.2.0), so the change to `ColumnarArrowPythonRunner.scala` and the new test in
`test_arrow_python_udf_cached.py` are dropped, and the docs of both configs say
"applyInPandasWithState and Arrow Python UDTFs ignore this config."
- `ApplyInPandasWithStatePythonRunner`, `ArrowPythonUDTFRunner` and the
imports of `test_pandas_grouped_map_with_state.py` are resolved against the
branch-4.1 code; the overrides and tests are the same as on master.
### Why are the changes needed?
With either config set, Python UDF output could contain duplicated rows or
wrong values, or fail with `IndexOutOfBoundsException`, and
`applyInPandasWithState` and Arrow Python UDTFs could fail or return wrong
results. See #59616.
### Does this PR introduce _any_ user-facing change?
Yes, only for users who set one of the internal configs above (both are
disabled by default): the affected operators no longer return duplicated rows
or wrong values, or fail.
### How was this patch tested?
The tests from #59616, except the one in `test_arrow_python_udf_cached.py`:
`PythonArrowOutputSuite`, and the new tests in `test_arrow_map.py`,
`test_arrow_python_udf.py`, `test_udtf.py` and
`test_pandas_grouped_map_with_state.py`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Claude Opus 5.5)
This pull request and its description were written by Isaac.
Closes #59770 from viirya/SPARK-60367-4.1.
Authored-by: Liang-Chi Hsieh <[email protected]>
Signed-off-by: Liang-Chi Hsieh <[email protected]>
---
python/pyspark/sql/tests/arrow/test_arrow_map.py | 3 +-
.../sql/tests/arrow/test_arrow_python_udf.py | 8 ++
.../pandas/test_pandas_grouped_map_with_state.py | 38 +++++-
python/pyspark/sql/tests/test_udtf.py | 21 +++
.../org/apache/spark/sql/internal/SQLConf.scala | 6 +-
.../execution/python/ArrowPythonUDTFRunner.scala | 5 +
.../sql/execution/python/PythonArrowOutput.scala | 58 ++++----
.../ApplyInPandasWithStatePythonRunner.scala | 5 +
.../execution/python/PythonArrowOutputSuite.scala | 151 +++++++++++++++++++++
9 files changed, 258 insertions(+), 37 deletions(-)
diff --git a/python/pyspark/sql/tests/arrow/test_arrow_map.py
b/python/pyspark/sql/tests/arrow/test_arrow_map.py
index 5ef80df0ec77..22a8645a26ab 100644
--- a/python/pyspark/sql/tests/arrow/test_arrow_map.py
+++ b/python/pyspark/sql/tests/arrow/test_arrow_map.py
@@ -114,7 +114,8 @@ class MapInArrowTestsMixin(object):
df = self.spark.range(10)
actual = df.repartition(1).mapInArrow(func, "a long").collect()
- self.assertEqual(set((r.a for r in actual)), set(range(100)))
+ # Each input batch produces 0..99 once, in order, since there is one
partition.
+ self.assertEqual([r.a for r in actual], list(range(100)) * max(1,
len(actual) // 100))
def test_other_than_recordbatch_iter(self):
with self.quiet():
diff --git a/python/pyspark/sql/tests/arrow/test_arrow_python_udf.py
b/python/pyspark/sql/tests/arrow/test_arrow_python_udf.py
index c315151d4d75..ee813b679e4d 100644
--- a/python/pyspark/sql/tests/arrow/test_arrow_python_udf.py
+++ b/python/pyspark/sql/tests/arrow/test_arrow_python_udf.py
@@ -375,6 +375,14 @@ class ArrowPythonUDFTestsMixin(BaseUDFTestsMixin):
self.assertIsNotNone(result["interval_field"])
self.assertEqual(result["name"], "test_name")
+ def test_udf_with_output_batch_slicing(self):
+ # Each result row is joined with the next queued input row, so the
sliced 10-row result
+ # batch must be output once and in order.
+ df = self.spark.range(0, 10, 1, 1)
+ with
self.sql_conf({"spark.sql.execution.arrow.maxRecordsPerOutputBatch": "3"}):
+ result = df.select(col("id"), udf(lambda x: x + 1,
"long")(col("id")))
+ assertDataFrameEqual(result, [Row(i, i + 1) for i in range(10)])
+
@unittest.skipIf(
not have_pandas or not have_pyarrow, pandas_requirement_message or
pyarrow_requirement_message
diff --git
a/python/pyspark/sql/tests/pandas/test_pandas_grouped_map_with_state.py
b/python/pyspark/sql/tests/pandas/test_pandas_grouped_map_with_state.py
index e4f307ebc96a..6d53e9028be8 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_grouped_map_with_state.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_grouped_map_with_state.py
@@ -41,7 +41,7 @@ from pyspark.testing.sqlutils import (
pandas_requirement_message,
pyarrow_requirement_message,
)
-from pyspark.testing.utils import eventually
+from pyspark.testing.utils import assertDataFrameEqual, eventually
if have_pandas:
import pandas as pd
@@ -95,10 +95,14 @@ class GroupedApplyInPandasWithStateTestsMixin:
.start()
)
- self.assertEqual(q.name, "this_query")
- self.assertTrue(q.isActive)
- q.processAllAvailable()
- self.assertTrue(q.exception() is None)
+ try:
+ self.assertEqual(q.name, "this_query")
+ self.assertTrue(q.isActive)
+ q.processAllAvailable()
+ self.assertTrue(q.exception() is None)
+ finally:
+ q.stop()
+ shutil.rmtree(input_path, ignore_errors=True)
def test_apply_in_pandas_with_state_basic(self):
def func(key, pdf_iter, state):
@@ -196,6 +200,30 @@ class GroupedApplyInPandasWithStateTestsMixin:
self._test_apply_in_pandas_with_state_basic(func, check_results)
+ def test_apply_in_pandas_with_state_output_batch_slicing(self):
+ # The worker packs the data rows (5 per group) and state rows (1 per
group) of the groups
+ # in a partition into one output batch. Its data and state counts
cover the whole batch,
+ # so it must not be sliced by the configs below.
+ def func(key, pdf_iter, state):
+ state.update((1,))
+ yield pd.DataFrame(
+ {"key": [f"{key[0]}_{i}" for i in range(5)], "countAsString":
list("01234")}
+ )
+
+ expected = [
+ Row(key=f"{k}_{i}", countAsString=str(i)) for k in ["hello",
"this"] for i in range(5)
+ ]
+
+ def check_results(batch_df, _):
+ assertDataFrameEqual(batch_df, expected)
+
+ for conf in [
+ {"spark.sql.execution.arrow.maxRecordsPerOutputBatch": "3"},
+ {"spark.sql.execution.arrow.maxBytesPerOutputBatch": "3"},
+ ]:
+ with self.subTest(conf=conf), self.sql_conf(conf):
+ self._test_apply_in_pandas_with_state_basic(func,
check_results)
+
def test_apply_in_pandas_with_state_python_worker_random_failure(self):
input_path = tempfile.mkdtemp()
output_path = tempfile.mkdtemp()
diff --git a/python/pyspark/sql/tests/test_udtf.py
b/python/pyspark/sql/tests/test_udtf.py
index 2d4bc854d7f2..afec25558270 100644
--- a/python/pyspark/sql/tests/test_udtf.py
+++ b/python/pyspark/sql/tests/test_udtf.py
@@ -3520,6 +3520,27 @@ class LegacyUDTFArrowTestsMixin(BaseUDTFTestsMixin):
):
udtf(TestUDTF, returnType=ret_type)().collect()
+ def test_udtf_lateral_join_with_output_batch_slicing(self):
+ # Each output batch is joined with one input row, so it must not be
sliced by the
+ # configs below.
+ @udtf(returnType="x: int")
+ class TestUDTF:
+ def eval(self, n: int):
+ for i in range(5):
+ yield (n * 10 + i,)
+
+ self.spark.udtf.register("test_udtf_slicing", TestUDTF)
+ expected = [Row(id=n, x=n * 10 + i) for n in range(3) for i in
range(5)]
+ for conf in [
+ {"spark.sql.execution.arrow.maxRecordsPerOutputBatch": "3"},
+ {"spark.sql.execution.arrow.maxBytesPerOutputBatch": "3"},
+ ]:
+ with self.subTest(conf=conf), self.sql_conf(conf):
+ df = self.spark.sql(
+ "SELECT t.id, u.x FROM range(0, 3, 1, 1) t, LATERAL
test_udtf_slicing(t.id) u"
+ )
+ assertDataFrameEqual(df, expected)
+
class LegacyUDTFArrowTests(LegacyUDTFArrowTestsMixin, ReusedSQLTestCase):
@classmethod
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 708903c72897..e9c9c89c297c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -4046,7 +4046,8 @@ object SQLConf {
"there is no limit. Note that the complete ArrowRecordBatch is
actually created but " +
"the number of records is limited when sending it to the downstream
operator. This is " +
"used to avoid large batches being sent to the downstream operator
including " +
- "the columnar-based operator implemented by third-party libraries.")
+ "the columnar-based operator implemented by third-party libraries. " +
+ "applyInPandasWithState and Arrow Python UDTFs ignore this config.")
.version("4.1.0")
.internal()
.intConf
@@ -4062,7 +4063,8 @@ object SQLConf {
"the columnar-based operator implemented by third-party libraries.
Spark will try to " +
"create batches with the size equal or less than this value. Normally
it should not " +
"happen, but if in extreme case that even one record is still larger
than this value, " +
- "Spark will create a batch with one record.")
+ "Spark will create a batch with one record. applyInPandasWithState and
Arrow Python " +
+ "UDTFs ignore this config.")
.version("4.1.0")
.internal()
.bytesConf(ByteUnit.BYTE)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonUDTFRunner.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonUDTFRunner.scala
index 1d5df9bad924..6c43515310cc 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonUDTFRunner.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonUDTFRunner.scala
@@ -49,6 +49,11 @@ class ArrowPythonUDTFRunner(
with BatchedPythonArrowInput
with BasicPythonArrowOutput {
+ // `EvalPythonUDTFExec` joins each output batch with one input row, so the
batches returned
+ // from Python must not be sliced.
+ override protected def arrowMaxRecordsPerOutputBatch: Int = -1
+ override protected def arrowMaxBytesPerOutputBatch: Int = -1
+
override protected def writeUDF(dataOut: DataOutputStream): Unit = {
// For arrow-optimized Python UDTFs (@udtf(useArrow=True)), we need to
write
// the schema to the worker to support UDT (user-defined type).
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonArrowOutput.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonArrowOutput.scala
index 1e8f4ebfd1fe..19884b9a70c8 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonArrowOutput.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonArrowOutput.scala
@@ -44,6 +44,9 @@ private[python] trait PythonArrowOutput[OUT <: AnyRef] {
self: BasePythonRunner[
protected def deserializeColumnarBatch(batch: ColumnarBatch, schema:
StructType): OUT
+ // The limits for slicing the batches returned from Python. A runner whose
consumer cannot take
+ // a batch split into several, e.g. one that pairs each batch with one input
row, overrides both
+ // to -1, as `ArrowPythonUDTFRunner` does.
protected def arrowMaxRecordsPerOutputBatch: Int =
SQLConf.get.arrowMaxRecordsPerOutputBatch
protected def arrowMaxBytesPerOutputBatch: Int =
SQLConf.get.arrowMaxBytesPerOutputBatch.toInt
@@ -201,8 +204,7 @@ abstract class BaseSliceArrowOutputProcessor(
extends ArrowOutputProcessorImpl(reader, pythonMetrics) {
protected var currentRowIdx = -1
- protected var prevRoot: VectorSchemaRoot = null
- protected var prevVectors: Array[ColumnVector] = _
+ protected var prevRoot: VectorSchemaRoot = root
override def loadBatch(): Boolean = {
if (rowCount > 0 && currentRowIdx < rowCount) {
@@ -216,34 +218,34 @@ abstract class BaseSliceArrowOutputProcessor(
override def produceBatch(): ColumnarBatch = {
val batchRoot = getRoot
+ closePrevSlice()
+ prevRoot = batchRoot
- if (batchRoot != prevRoot) {
- if (prevRoot != null) {
- prevVectors.foreach(_.close())
- prevRoot.close()
- }
- prevRoot = batchRoot
- }
-
- val vectors = getVectors(batchRoot)
- prevVectors = vectors
-
- val batch = new ColumnarBatch(vectors)
+ val batch = new ColumnarBatch(getVectors(batchRoot))
batch.setNumRows(batchRoot.getRowCount)
batch
}
protected override def getVectors(root: VectorSchemaRoot):
Array[ColumnVector] = {
- root.getFieldVectors.asScala.map { vector =>
- new ArrowColumnVector(vector)
- }.toArray[ColumnVector]
+ if (root eq this.root) {
+ super.getVectors(root)
+ } else {
+ root.getFieldVectors.asScala.map { vector =>
+ new ArrowColumnVector(vector)
+ }.toArray[ColumnVector]
+ }
}
- override def close(): Unit = {
- if (prevRoot != null) {
- prevVectors.foreach(_.close())
+ // Releases the previous output batch if it is a slice. The reader loads
every batch from Python
+ // into the same root, so it must not be closed here;
`ArrowOutputProcessorImpl.close` closes it.
+ private def closePrevSlice(): Unit = {
+ if (prevRoot ne root) {
prevRoot.close()
}
+ }
+
+ override def close(): Unit = {
+ closePrevSlice()
super.close()
}
}
@@ -255,8 +257,9 @@ class SliceBytesArrowOutputProcessorImpl(
extends BaseSliceArrowOutputProcessor(reader, pythonMetrics) {
protected override def getRoot: VectorSchemaRoot = {
- if (getBatchBytes(root) < arrowMaxBytesPerOutputBatch) {
- currentRowIdx = currentRowIdx + root.getRowCount
+ // Output the whole root only if no row of this batch has been output yet.
+ if (currentRowIdx == 0 && getBatchBytes(root) <
arrowMaxBytesPerOutputBatch) {
+ currentRowIdx = rowCount
root
} else {
// We iteratively slice the batch by 90% until we reach a batch with
required byte size.
@@ -304,14 +307,11 @@ class SliceRecordsArrowOutputProcessorImpl(
extends BaseSliceArrowOutputProcessor(reader, pythonMetrics) {
protected override def getRoot: VectorSchemaRoot = {
- val remainingRows = rowCount - currentRowIdx
- val rootSlice = if (remainingRows > arrowMaxRecordsPerOutputBatch) {
- root.slice(currentRowIdx, arrowMaxRecordsPerOutputBatch)
- } else {
- root
- }
+ val numRows = math.min(rowCount - currentRowIdx,
arrowMaxRecordsPerOutputBatch)
+ // Output the whole root only if this chunk is the whole batch.
+ val rootSlice = if (numRows == rowCount) root else
root.slice(currentRowIdx, numRows)
- currentRowIdx = currentRowIdx + rootSlice.getRowCount
+ currentRowIdx = currentRowIdx + numRows
rootSlice
}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/streaming/ApplyInPandasWithStatePythonRunner.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/streaming/ApplyInPandasWithStatePythonRunner.scala
index 14054ba89a94..299e4bfe5bc4 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/streaming/ApplyInPandasWithStatePythonRunner.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/streaming/ApplyInPandasWithStatePythonRunner.scala
@@ -70,6 +70,11 @@ class ApplyInPandasWithStatePythonRunner(
with PythonArrowInput[InType]
with PythonArrowOutput[OutType] {
+ // The data and state counts of each batch returned from Python cover the
whole batch and are
+ // read from its first row, so the batch must not be sliced.
+ override protected def arrowMaxRecordsPerOutputBatch: Int = -1
+ override protected def arrowMaxBytesPerOutputBatch: Int = -1
+
override val pythonExec: String =
SQLConf.get.pysparkWorkerPythonExecutable.getOrElse(
funcs.head._1.funcs.head.pythonExec)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/python/PythonArrowOutputSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/PythonArrowOutputSuite.scala
new file mode 100644
index 000000000000..6ee8fcba9db9
--- /dev/null
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/PythonArrowOutputSuite.scala
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.python
+
+import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
+
+import scala.collection.mutable.ArrayBuffer
+import scala.jdk.CollectionConverters._
+
+import org.apache.arrow.vector.ipc.{ArrowStreamReader, ArrowStreamWriter}
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.execution.arrow.ArrowWriter
+import org.apache.spark.sql.execution.metric.SQLMetric
+import org.apache.spark.sql.types._
+import org.apache.spark.sql.util.ArrowUtils
+import org.apache.spark.sql.vectorized.ArrowColumnVector
+import org.apache.spark.unsafe.types.UTF8String
+
+class PythonArrowOutputSuite extends SparkFunSuite {
+
+ private val schema = new StructType()
+ .add("id", LongType)
+ .add("s", StringType)
+ .add("st", new StructType().add("a", IntegerType).add("b", StringType))
+
+ private def row(i: Long): InternalRow = {
+ val s = if (i % 3 == 0) null else UTF8String.fromString(s"str_$i")
+ val st = if (i % 5 == 0) {
+ null
+ } else {
+ InternalRow(i.toInt * 2, if (i % 2 == 0) null else
UTF8String.fromString(s"b_$i"))
+ }
+ InternalRow(i, s, st)
+ }
+
+ /** Writes an Arrow IPC stream with one record batch per entry of
`batchSizes`. */
+ private def writeStream(batchSizes: Seq[Int]): Array[Byte] = {
+ val writer = ArrowWriter.create(schema, "UTC")
+ val out = new ByteArrayOutputStream()
+ try {
+ val streamWriter = new ArrowStreamWriter(writer.root, null, out)
+ streamWriter.start()
+ var next = 0L
+ batchSizes.foreach { size =>
+ (0 until size).foreach { _ =>
+ writer.write(row(next))
+ next += 1
+ }
+ writer.finish()
+ streamWriter.writeBatch()
+ writer.reset()
+ }
+ streamWriter.end()
+ } finally {
+ writer.root.close()
+ }
+ out.toByteArray
+ }
+
+ /**
+ * Drives `processor` the way `PythonArrowOutput.newReaderIterator.read()`
does, and returns
+ * the rows and the buffer size in bytes of every output batch. Rows are
copied before the
+ * next batch is produced, since the processor releases the previous output
batch then.
+ */
+ private def readAll(
+ batchSizes: Seq[Int],
+ createProcessor: (ArrowStreamReader, Map[String, SQLMetric]) =>
ArrowOutputProcessor)
+ : Seq[(Seq[InternalRow], Long)] = {
+ val allocator = ArrowUtils.rootAllocator.newChildAllocator("reader", 0,
Long.MaxValue)
+ val reader = new ArrowStreamReader(new
ByteArrayInputStream(writeStream(batchSizes)), allocator)
+ val metrics = Map(
+ "pythonNumRowsReceived" -> new SQLMetric("sum"),
+ "pythonDataReceived" -> new SQLMetric("size"))
+ val processor = createProcessor(reader, metrics)
+ val output = ArrayBuffer.empty[(Seq[InternalRow], Long)]
+ while (processor.loadBatch()) {
+ val batch = processor.produceBatch()
+ val bytes = (0 until batch.numCols()).map { i =>
+
batch.column(i).asInstanceOf[ArrowColumnVector].getValueVector.getBufferSize.toLong
+ }.sum
+ output += ((batch.rowIterator().asScala.map(_.copy()).toSeq, bytes))
+ }
+ processor.close()
+ reader.close(false)
+ // Fails if any slice made by the processor was leaked.
+ allocator.close()
+ // `PythonArrowOutput` closes the processor again in its task completion
listener.
+ processor.close()
+ assert(metrics("pythonNumRowsReceived").value === batchSizes.sum)
+ output.toSeq
+ }
+
+ private def checkAllRowsOnceInOrder(
+ batchSizes: Seq[Int],
+ output: Seq[(Seq[InternalRow], Long)]): Unit = {
+ assert(output.flatMap(_._1) === (0L until batchSizes.sum).map(row))
+ }
+
+ Seq(
+ (Seq(3, 3, 3), 2),
+ (Seq(4, 4, 4), 2),
+ (Seq(2, 5, 2, 7), 2),
+ (Seq(1, 0, 6, 1, 3), 3),
+ (Seq(10), 3)
+ ).foreach { case (batchSizes, maxRecords) =>
+ test(s"slice by records: batches $batchSizes, max $maxRecords records per
batch") {
+ val output = readAll(batchSizes, (reader, metrics) =>
+ new SliceRecordsArrowOutputProcessorImpl(reader, metrics, maxRecords))
+ checkAllRowsOnceInOrder(batchSizes, output)
+ val expectedSizes = batchSizes.flatMap { n =>
+ val sizes = Seq.fill(n / maxRecords)(maxRecords) ++ Seq(n %
maxRecords).filter(_ > 0)
+ if (sizes.isEmpty) Seq(0) else sizes
+ }
+ assert(output.map(_._1.size) === expectedSizes)
+ }
+ }
+
+ Seq(
+ (Seq(1, 100, 1), 512),
+ (Seq(100, 1, 100), 512),
+ (Seq(1, 0, 57, 2, 33), 512),
+ // Even a single row is larger than the limit, so every non-empty output
batch has one row.
+ (Seq(2, 0, 5, 1), 3)
+ ).foreach { case (batchSizes, maxBytes) =>
+ test(s"slice by bytes: batches $batchSizes, max $maxBytes bytes per
batch") {
+ val output = readAll(batchSizes, (reader, metrics) =>
+ new SliceBytesArrowOutputProcessorImpl(reader, metrics, maxBytes))
+ checkAllRowsOnceInOrder(batchSizes, output)
+ // Guards the choice of cases: each has a batch that does not fit, which
must be sliced.
+ assert(output.size > batchSizes.size)
+ assert(output.forall { case (rows, bytes) => rows.size <= 1 || bytes <
maxBytes })
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]