Github user andrewor14 commented on a diff in the pull request:
https://github.com/apache/spark/pull/1222#discussion_r21765774
--- Diff:
core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
---
@@ -110,129 +114,18 @@ class EventLoggingListenerSuite extends FunSuite
with BeforeAndAfter {
import EventLoggingListenerSuite._
/**
- * Test whether names of special files are correctly identified and
parsed.
- */
- private def testParsingFileName() {
- val logPrefix = EventLoggingListener.LOG_PREFIX
- val sparkVersionPrefix = EventLoggingListener.SPARK_VERSION_PREFIX
- val compressionCodecPrefix =
EventLoggingListener.COMPRESSION_CODEC_PREFIX
- val applicationComplete = EventLoggingListener.APPLICATION_COMPLETE
- assert(EventLoggingListener.isEventLogFile(logPrefix + "0"))
- assert(EventLoggingListener.isEventLogFile(logPrefix + "100"))
- assert(EventLoggingListener.isEventLogFile(logPrefix + "ANYTHING"))
- assert(EventLoggingListener.isSparkVersionFile(sparkVersionPrefix +
"0.9.1"))
- assert(EventLoggingListener.isSparkVersionFile(sparkVersionPrefix +
"1.0.0"))
- assert(EventLoggingListener.isSparkVersionFile(sparkVersionPrefix +
"ANYTHING"))
-
assert(EventLoggingListener.isApplicationCompleteFile(applicationComplete))
- allCompressionCodecs.foreach { codec =>
-
assert(EventLoggingListener.isCompressionCodecFile(compressionCodecPrefix +
codec))
- }
-
- // Negatives
- assert(!EventLoggingListener.isEventLogFile("The greatest man of all
mankind"))
- assert(!EventLoggingListener.isSparkVersionFile("Will never falter in
the face of death!"))
- assert(!EventLoggingListener.isCompressionCodecFile("Unless he chooses
to leave behind"))
- assert(!EventLoggingListener.isApplicationCompleteFile("The very
treasure he calls Macbeth"))
-
- // Verify that parsing is correct
- assert(EventLoggingListener.parseSparkVersion(sparkVersionPrefix +
"1.0.0") === "1.0.0")
- allCompressionCodecs.foreach { codec =>
-
assert(EventLoggingListener.parseCompressionCodec(compressionCodecPrefix +
codec) === codec)
- }
- }
-
- /**
- * Test whether the special files produced by EventLoggingListener exist.
- *
- * There should be exactly one event log and one spark version file
throughout the entire
- * execution. If a compression codec is specified, then the compression
codec file should
- * also exist. Only after the application has completed does the test
expect the application
- * completed file to be present.
- */
- private def testSpecialFilesExist(compressionCodec: Option[String] =
None) {
-
- def assertFilesExist(logFiles: Array[FileStatus], loggerStopped:
Boolean) {
- val numCompressionCodecFiles = if (compressionCodec.isDefined) 1
else 0
- val numApplicationCompleteFiles = if (loggerStopped) 1 else 0
- assert(logFiles.size === 2 + numCompressionCodecFiles +
numApplicationCompleteFiles)
- assert(eventLogsExist(logFiles))
- assert(sparkVersionExists(logFiles))
- assert(compressionCodecExists(logFiles) ===
compressionCodec.isDefined)
- assert(applicationCompleteExists(logFiles) === loggerStopped)
- assertSparkVersionIsValid(logFiles)
- compressionCodec.foreach { codec =>
- assertCompressionCodecIsValid(logFiles, codec)
- }
- }
-
- // Verify logging directory exists
- val conf = getLoggingConf(logDirPath, compressionCodec)
- val logBaseDir = conf.get("spark.eventLog.dir")
- val appId = EventLoggingListenerSuite.getUniqueApplicationId
- val eventLogger = new EventLoggingListener(appId, logBaseDir, conf)
- eventLogger.start()
- val logPath = new Path(eventLogger.logDir)
- assert(fileSystem.exists(logPath))
- val logDir = fileSystem.getFileStatus(logPath)
- assert(logDir.isDir)
-
- // Verify special files are as expected before stop()
- var logFiles = fileSystem.listStatus(logPath)
- assert(logFiles != null)
- assertFilesExist(logFiles, loggerStopped = false)
-
- // Verify special files are as expected after stop()
- eventLogger.stop()
- logFiles = fileSystem.listStatus(logPath)
- assertFilesExist(logFiles, loggerStopped = true)
- }
-
- /**
- * Test whether EventLoggingListener correctly parses the correct
information from the logs.
- *
- * This includes whether it returns the correct Spark version,
compression codec (if any),
- * and the application's completion status.
- */
- private def testParsingLogInfo(compressionCodec: Option[String] = None) {
-
- def assertInfoCorrect(info: EventLoggingInfo, loggerStopped: Boolean) {
- assert(info.logPaths.size > 0)
- assert(info.sparkVersion === SPARK_VERSION)
- assert(info.compressionCodec.isDefined ===
compressionCodec.isDefined)
- info.compressionCodec.foreach { codec =>
- assert(compressionCodec.isDefined)
- val expectedCodec = compressionCodec.get.split('.').last
- assert(codec.getClass.getSimpleName === expectedCodec)
- }
- assert(info.applicationComplete === loggerStopped)
- }
-
- // Verify that all information is correctly parsed before stop()
- val conf = getLoggingConf(logDirPath, compressionCodec)
- val logBaseDir = conf.get("spark.eventLog.dir")
- val appId = EventLoggingListenerSuite.getUniqueApplicationId
- val eventLogger = new EventLoggingListener(appId, logBaseDir, conf)
- eventLogger.start()
- var eventLoggingInfo =
EventLoggingListener.parseLoggingInfo(eventLogger.logDir, fileSystem)
- assertInfoCorrect(eventLoggingInfo, loggerStopped = false)
-
- // Verify that all information is correctly parsed after stop()
- eventLogger.stop()
- eventLoggingInfo =
EventLoggingListener.parseLoggingInfo(eventLogger.logDir, fileSystem)
- assertInfoCorrect(eventLoggingInfo, loggerStopped = true)
- }
-
--- End diff --
shouldn't we keep these tests for old logs? Right now there are no tests
for them
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
