LuciferYang commented on PR #37353:
URL: https://github.com/apache/spark/pull/37353#issuecomment-1200630210
Write a mirco-bench to test `Utils.getIteratorSize` vs `Iterator.size`:
```
object IteratorSizeBenchmark extends BenchmarkBase {
private def testRangeIteratorSize(valuesPerIteration: Int, range: Range):
Unit = {
val benchmark = new Benchmark(s"Test Range iterator size ${range.size}",
valuesPerIteration,
output = output)
benchmark.addCase("Use Iterator.size") { _: Int =>
for (_ <- 0L until valuesPerIteration) {
range.iterator.size.toLong
}
}
benchmark.addCase("Use Utils.getIteratorSize") { _: Int =>
for (_ <- 0L until valuesPerIteration) {
Utils.getIteratorSize(range.iterator)
}
}
benchmark.run()
}
private def testSeqIteratorSize(valuesPerIteration: Int, seq: Seq[Int]):
Unit = {
val benchmark = new Benchmark(s"Test Seq iterator size ${seq.size}",
valuesPerIteration,
output = output)
benchmark.addCase("Use Iterator.size") { _: Int =>
for (_ <- 0L until valuesPerIteration) {
seq.iterator.size.toLong
}
}
benchmark.addCase("Use Utils.getIteratorSize") { _: Int =>
for (_ <- 0L until valuesPerIteration) {
Utils.getIteratorSize(seq.iterator)
}
}
benchmark.run()
}
private def testArrayIteratorSize(valuesPerIteration: Int, array:
Array[Int]): Unit = {
val benchmark = new Benchmark(s"Test Array iterator size
${array.length}",
valuesPerIteration,
output = output)
benchmark.addCase("Use Iterator.size") { _: Int =>
for (_ <- 0L until valuesPerIteration) {
array.iterator.size.toLong
}
}
benchmark.addCase("Use Utils.getIteratorSize") { _: Int =>
for (_ <- 0L until valuesPerIteration) {
Utils.getIteratorSize(array.iterator)
}
}
benchmark.run()
}
override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
val valuesPerIteration = 100000
// Test Range
testRangeIteratorSize(valuesPerIteration, Range(0, 100))
testRangeIteratorSize(valuesPerIteration, Range(0, 1000))
testRangeIteratorSize(valuesPerIteration, Range(0, 10000))
testRangeIteratorSize(valuesPerIteration, Range(0, 30000))
// Test Seq
testSeqIteratorSize(valuesPerIteration, Seq.range(0, 100))
testSeqIteratorSize(valuesPerIteration, Seq.range(0, 1000))
testSeqIteratorSize(valuesPerIteration, Seq.range(0, 10000))
testSeqIteratorSize(valuesPerIteration, Seq.range(0, 30000))
// Test Array
testArrayIteratorSize(valuesPerIteration, Array.range(0, 100))
testArrayIteratorSize(valuesPerIteration, Array.range(0, 1000))
testArrayIteratorSize(valuesPerIteration, Array.range(0, 10000))
testArrayIteratorSize(valuesPerIteration, Array.range(0, 30000))
}
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
