Ngone51 commented on a change in pull request #25670: [SPARK-28869][CORE] Roll
over event log files
URL: https://github.com/apache/spark/pull/25670#discussion_r331051103
##########
File path:
core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
##########
@@ -17,38 +17,33 @@
package org.apache.spark.scheduler
-import java.io._
import java.net.URI
-import java.nio.charset.StandardCharsets
-import scala.collection.mutable.{ArrayBuffer, Map}
+import scala.collection.mutable
import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.{FileSystem, FSDataOutputStream, Path}
-import org.apache.hadoop.fs.permission.FsPermission
import org.json4s.JsonAST.JValue
import org.json4s.jackson.JsonMethods._
import org.apache.spark.{SPARK_VERSION, SparkConf}
import org.apache.spark.deploy.SparkHadoopUtil
+import org.apache.spark.deploy.history.EventLogFileWriter
import org.apache.spark.executor.ExecutorMetrics
import org.apache.spark.internal.Logging
import org.apache.spark.internal.config._
-import org.apache.spark.io.CompressionCodec
import org.apache.spark.util.{JsonProtocol, Utils}
/**
* A SparkListener that logs events to persistent storage.
*
* Event logging is specified by the following configurable parameters:
* spark.eventLog.enabled - Whether event logging is enabled.
- * spark.eventLog.logBlockUpdates.enabled - Whether to log block updates
- * spark.eventLog.compress - Whether to compress logged events
- * spark.eventLog.compression.codec - The codec to compress logged events
- * spark.eventLog.overwrite - Whether to overwrite any existing files.
* spark.eventLog.dir - Path to the directory in which events are logged.
- * spark.eventLog.buffer.kb - Buffer size to use when writing to output
streams
+ * spark.eventLog.logBlockUpdates.enabled - Whether to log block updates
* spark.eventLog.logStageExecutorMetrics.enabled - Whether to log stage
executor metrics
+ *
+ * Event log file writer maintains its own parameters: refer the javadoc of
[[EventLogFileWriter]]
Review comment:
`javadoc` or `scaladoc` ? Not sure what's the correct one.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
