Repository: incubator-livy Updated Branches: refs/heads/master e54318314 -> 85e8fba3c
[LIVY-429] preserve url fragment in spark.yarn.dist.archives ## What changes were proposed in this pull request? Preserve url fragment identifier in spark.yarn.dist.archives setting when there is no schema specified. Current code truncates url fragment, as a result Spark just copies archives to job local dir without extracting it. ## How was this patch tested? Added case to existing unit test Author: Artem Plotnikov <aplotni...@joom.com> Closes #71 from broartem/LIVY-429. Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/85e8fba3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/85e8fba3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/85e8fba3 Branch: refs/heads/master Commit: 85e8fba3ce5d0e180655fd9ee239cb320d550780 Parents: e543183 Author: Artem Plotnikov <aplotni...@joom.com> Authored: Thu Jan 4 10:22:40 2018 +0800 Committer: jerryshao <ss...@hortonworks.com> Committed: Thu Jan 4 10:22:40 2018 +0800 ---------------------------------------------------------------------- server/src/main/scala/org/apache/livy/sessions/Session.scala | 7 +++++-- .../src/test/scala/org/apache/livy/sessions/SessionSpec.scala | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/85e8fba3/server/src/main/scala/org/apache/livy/sessions/Session.scala ---------------------------------------------------------------------- diff --git a/server/src/main/scala/org/apache/livy/sessions/Session.scala b/server/src/main/scala/org/apache/livy/sessions/Session.scala index d467076..0b54779 100644 --- a/server/src/main/scala/org/apache/livy/sessions/Session.scala +++ b/server/src/main/scala/org/apache/livy/sessions/Session.scala @@ -116,8 +116,11 @@ object Session { val defaultFS = livyConf.hadoopConf.get("fs.defaultFS").stripSuffix("/") val resolved = if (uri.getScheme() == null) { - require(uri.getPath().startsWith("/"), s"Path '${uri.getPath()}' is not absolute.") - new URI(defaultFS + uri.getPath()) + val pathWithSegment = + if (uri.getFragment() != null) uri.getPath() + '#' + uri.getFragment() else uri.getPath() + + require(pathWithSegment.startsWith("/"), s"Path '${uri.getPath()}' is not absolute.") + new URI(defaultFS + pathWithSegment) } else { uri } http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/85e8fba3/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala ---------------------------------------------------------------------- diff --git a/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala b/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala index a7bfaaa..05b41bb 100644 --- a/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala +++ b/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala @@ -29,8 +29,8 @@ class SessionSpec extends FunSuite with LivyBaseUnitTestSuite { val conf = new LivyConf(false) conf.hadoopConf.set("fs.defaultFS", "dummy:///") - val uris = Seq("http://example.com/foo", "hdfs:/bar", "/baz") - val expected = Seq(uris(0), uris(1), "dummy:///baz") + val uris = Seq("http://example.com/foo", "hdfs:/bar", "/baz", "/foo#bar") + val expected = Seq(uris(0), uris(1), "dummy:///baz", "dummy:///foo#bar") assert(Session.resolveURIs(uris, conf) === expected) intercept[IllegalArgumentException] {