huaxingao commented on a change in pull request #32473:
URL: https://github.com/apache/spark/pull/32473#discussion_r629757951
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/BloomFilterBenchmark.scala
##########
@@ -81,8 +80,57 @@ object BloomFilterBenchmark extends SqlBasedBenchmark {
}
}
+ private def writeParquetBenchmark(): Unit = {
+ withTempPath { dir =>
+ val path = dir.getCanonicalPath
+
+ runBenchmark(s"Parquet Write") {
+ val benchmark = new Benchmark(s"Write ${scaleFactor}M rows", N, output
= output)
+ benchmark.addCase("Without bloom filter") { _ =>
+ df.write.mode("overwrite").parquet(path + "/withoutBF")
+ }
+ benchmark.addCase("With bloom filter") { _ =>
+ df.write.mode("overwrite")
+ .option(ParquetOutputFormat.BLOOM_FILTER_ENABLED + "#value", true)
+ .parquet(path + "/withBF")
+ }
+ benchmark.run()
+ }
+ }
+ }
+
+ private def readParquetBenchmark(): Unit = {
+ val blockSizes = Seq(512 * 1024, 1024 * 1024, 2 * 1024 * 1024, 3 * 1024 *
1024,
+ 4 * 1024 * 1024, 5 * 1024 * 1024, 6 * 1024 * 1024, 7 * 1024 * 1024,
+ 8 * 1024 * 1024, 9 * 1024 * 1024, 10 * 1024 * 1024)
+ for (blocksize <- blockSizes) {
+ withTempPath { dir =>
+ val path = dir.getCanonicalPath
+
+ df.write.option("parquet.block.size", blocksize).parquet(path +
"/withoutBF")
Review comment:
@wangyum That's how I got confused. I thought you were asking me to use
the same value for parquet block size and parquet compress size. Since parquet
doesn't have a compress size, I didn't get what you wanted me to do. Or do you
mean that for each of the block size (.5, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10MB) I
tested for parquet, I need to use the same size to set ORC compress size and
get a benchmark result for ORC?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
