beliefer commented on code in PR #41856:
URL: https://github.com/apache/spark/pull/41856#discussion_r1258218998
##########
sql/core/src/test/scala/org/apache/spark/sql/GenTPCDSData.scala:
##########
@@ -17,390 +17,18 @@
package org.apache.spark.sql
-import java.util.concurrent.LinkedBlockingQueue
-
-import scala.collection.immutable.Stream
-import scala.sys.process._
-import scala.util.Try
-
-import org.apache.spark.SparkContext
import org.apache.spark.internal.Logging
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.functions.{col, rpad}
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{CharType, StringType, StructField,
StructType, VarcharType}
-
-// The classes in this file are basically moved from
https://github.com/databricks/spark-sql-perf
-
-/**
- * Using ProcessBuilder.lineStream produces a stream, that uses
- * a LinkedBlockingQueue with a default capacity of Integer.MAX_VALUE.
- *
- * This causes OOM if the consumer cannot keep up with the producer.
- *
- * See scala.sys.process.ProcessBuilderImpl.lineStream
- */
-object BlockingLineStream {
-
- // See scala.sys.process.Streamed
- private final class BlockingStreamed[T](
- val process: T => Unit,
- val done: Int => Unit,
- val stream: () => Stream[T])
+import org.apache.spark.sql.types.StructType
- // See scala.sys.process.Streamed
- private object BlockingStreamed {
- // scala.process.sys.Streamed uses default of Integer.MAX_VALUE,
- // which causes OOMs if the consumer cannot keep up with producer.
- val maxQueueSize = 65536
+class TPCDSTables(val spark: SparkSession, val dsdgenDir: String, val
scaleFactor: Int)
Review Comment:
`dsdgenDir` is not extended from the parent trait, so remove the `val`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
