Repository: spark Updated Branches: refs/heads/master 7ced88b0d -> 24b818b97
[SPARK-3944][Core] Using Option[String] where value of String can be null Author: shitis <ssaxena....@gmail.com> Closes #2795 from Shiti/master and squashes the following commits: 46897d7 [shitis] Using Option Wrapper to convert String with value null to None Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/24b818b9 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/24b818b9 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/24b818b9 Branch: refs/heads/master Commit: 24b818b971ba715b6796518e4c6afdecb1b16f15 Parents: 7ced88b Author: shitis <ssaxena....@gmail.com> Authored: Tue Oct 14 14:16:45 2014 -0700 Committer: Andrew Or <andrewo...@gmail.com> Committed: Tue Oct 14 14:16:45 2014 -0700 ---------------------------------------------------------------------- .../scala/org/apache/spark/util/Utils.scala | 26 +++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/24b818b9/core/src/main/scala/org/apache/spark/util/Utils.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 07477dd..aad9016 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -340,8 +340,8 @@ private[spark] object Utils extends Logging { val targetFile = new File(targetDir, filename) val uri = new URI(url) val fileOverwrite = conf.getBoolean("spark.files.overwrite", defaultValue = false) - uri.getScheme match { - case "http" | "https" | "ftp" => + Option(uri.getScheme) match { + case Some("http") | Some("https") | Some("ftp") => logInfo("Fetching " + url + " to " + tempFile) var uc: URLConnection = null @@ -374,7 +374,7 @@ private[spark] object Utils extends Logging { } } Files.move(tempFile, targetFile) - case "file" | null => + case Some("file") | None => // In the case of a local file, copy the local file to the target directory. // Note the difference between uri vs url. val sourceFile = if (uri.isAbsolute) new File(uri) else new File(url) @@ -403,7 +403,7 @@ private[spark] object Utils extends Logging { logInfo("Copying " + sourceFile.getAbsolutePath + " to " + targetFile.getAbsolutePath) Files.copy(sourceFile, targetFile) } - case _ => + case Some(other) => // Use the Hadoop filesystem library, which supports file://, hdfs://, s3://, and others val fs = getHadoopFileSystem(uri, hadoopConf) val in = fs.open(new Path(uri)) @@ -1368,16 +1368,17 @@ private[spark] object Utils extends Logging { if (uri.getPath == null) { throw new IllegalArgumentException(s"Given path is malformed: $uri") } - uri.getScheme match { - case windowsDrive(d) if windows => + + Option(uri.getScheme) match { + case Some(windowsDrive(d)) if windows => new URI("file:/" + uri.toString.stripPrefix("/")) - case null => + case None => // Preserve fragments for HDFS file name substitution (denoted by "#") // For instance, in "abc.py#xyz.py", "xyz.py" is the name observed by the application val fragment = uri.getFragment val part = new File(uri.getPath).toURI new URI(part.getScheme, part.getPath, fragment) - case _ => + case Some(other) => uri } } @@ -1399,10 +1400,11 @@ private[spark] object Utils extends Logging { } else { paths.split(",").filter { p => val formattedPath = if (windows) formatWindowsPath(p) else p - new URI(formattedPath).getScheme match { - case windowsDrive(d) if windows => false - case "local" | "file" | null => false - case _ => true + val uri = new URI(formattedPath) + Option(uri.getScheme) match { + case Some(windowsDrive(d)) if windows => false + case Some("local") | Some("file") | None => false + case Some(other) => true } } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
