Github user tgravescs commented on a diff in the pull request: https://github.com/apache/spark/pull/969#discussion_r13866068 --- Diff: yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala --- @@ -45,6 +44,25 @@ class ClientArguments(val args: Array[String], val sparkConf: SparkConf) { parseArgs(args.toList) + // -archives/--files via spark submit or yarn-client defaults to use file:// if not specified + if (sys.props.contains("SPARK_SUBMIT") || (sparkConf.getOption("spark.master").isDefined && + sparkConf.get("spark.master") == "yarn-client")) { + files = Option(files).map(p => Utils.resolveURIs(p)).orNull + archives = Option(archives).map(p => Utils.resolveURIs(p)).orNull + } + + // env variable SPARK_YARN_DIST_ARCHIVES/SPARK_YARN_DIST_FILES set in yarn-client then + // it should default to hdfs:// + files = Option(files).getOrElse(sys.env.get("SPARK_YARN_DIST_FILES").orNull) --- End diff -- again this is already handled in the YarnClientSchedulerBackend. It reads the env variables and passes in the --files/--archives without being resolveURI extended. The issue with that code is that it also looks at spark.yarn.dist.archives and spark.yarn.dist.files and doesn't resolveURI extend them.
--- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---