Repository: falcon Updated Branches: refs/heads/master c12c999b4 -> 57fa2ff12
FALCON-2048 Cluster submission failed in yarn-cluster mode Author: peeyush b <[email protected]> Reviewers: "Venkat Ranganathan <[email protected]>" Closes #210 from peeyushb/FALCON-2048 Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/57fa2ff1 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/57fa2ff1 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/57fa2ff1 Branch: refs/heads/master Commit: 57fa2ff125602706b26b6d168a84b828c490ff74 Parents: c12c999 Author: peeyush b <[email protected]> Authored: Fri Jul 1 10:31:15 2016 -0700 Committer: bvellanki <[email protected]> Committed: Fri Jul 1 10:31:15 2016 -0700 ---------------------------------------------------------------------- .../falcon/entity/parser/ClusterEntityParser.java | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/57fa2ff1/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java b/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java index 96ba748..3f35962 100644 --- a/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java +++ b/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java @@ -45,8 +45,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaSparkContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -236,14 +234,13 @@ public class ClusterEntityParser extends EntityParser<Cluster> { } protected void validateSparkMasterInterface(Cluster cluster) throws ValidationException { - final String sparkMasterUrl = ClusterHelper.getSparkMasterEndPoint(cluster); - if (StringUtils.isNotEmpty(sparkMasterUrl)) { - SparkConf sparkConf = new SparkConf(); - sparkConf.setMaster(sparkMasterUrl).setAppName("Falcon Spark"); - - JavaSparkContext sc = new JavaSparkContext(sparkConf); - if (sc.startTime() == null) { - throw new ValidationException("Unable to reach Spark master URL:" + sparkMasterUrl); + final String sparkMasterEndPoint = ClusterHelper.getSparkMasterEndPoint(cluster); + LOG.info("Validating spark interface: {}", sparkMasterEndPoint); + if (StringUtils.isNotEmpty(sparkMasterEndPoint)) { + if (!("yarn-cluster".equalsIgnoreCase(sparkMasterEndPoint) + || "yarn-client".equalsIgnoreCase(sparkMasterEndPoint) + || "local".equalsIgnoreCase(sparkMasterEndPoint))) { + throw new ValidationException("Invalid Spark Interface End Point:" + sparkMasterEndPoint); } } }
