FALCON-1343 Fix validation of read/write endpoints in ClusterEntityParser. Contributed by Balu Vellanki.
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/48d0723c Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/48d0723c Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/48d0723c Branch: refs/heads/master Commit: 48d0723ccae09fc26b0e6294214e032acd96a115 Parents: 11f20f4 Author: Ajay Yadava <[email protected]> Authored: Tue Sep 22 21:55:40 2015 +0530 Committer: Ajay Yadava <[email protected]> Committed: Tue Sep 22 21:55:40 2015 +0530 ---------------------------------------------------------------------- CHANGES.txt | 2 + .../entity/parser/ClusterEntityParser.java | 8 ++-- .../apache/falcon/entity/AbstractTestBase.java | 2 + .../entity/parser/ClusterEntityParserTest.java | 14 +++++-- .../cluster/cluster-bad-write-endpoint.xml | 44 ++++++++++++++++++++ 5 files changed, 64 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/48d0723c/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 511c4ec..79d7548 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -33,6 +33,8 @@ Trunk (Unreleased) FALCON-1403 Revisit IT cleanup and teardown(Narayan Periwal via Pallavi Rao) BUG FIXES + FALCON-1343 Fix validation of read/write endpoints in ClusterEntityParser(Balu Vellanki via Ajay Yadava) + FALCON-1373 HiveDR does not work when job is run on destination cluster(Balu Vellanki via Ajay Yadava) FALCON-1401 MetadataMappingService fails to add an edge for a process instance(Pallavi Rao) http://git-wip-us.apache.org/repos/asf/falcon/blob/48d0723c/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java b/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java index 6bfcb98..b4f61d7 100644 --- a/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java +++ b/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java @@ -138,9 +138,11 @@ public class ClusterEntityParser extends EntityParser<Cluster> { conf.set(SecurityUtil.NN_PRINCIPAL, nameNodePrincipal); } - HadoopClientFactory.get().createProxiedFileSystem(conf); - } catch (FalconException e) { - throw new ValidationException("Invalid storage server or port: " + storageUrl, e); + FileSystem fs = HadoopClientFactory.get().createProxiedFileSystem(conf); + fs.exists(new Path("/")); + } catch (Exception e) { + throw new ValidationException("Invalid storage server or port: " + storageUrl + + ", " + e.getMessage(), e); } } http://git-wip-us.apache.org/repos/asf/falcon/blob/48d0723c/common/src/test/java/org/apache/falcon/entity/AbstractTestBase.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/falcon/entity/AbstractTestBase.java b/common/src/test/java/org/apache/falcon/entity/AbstractTestBase.java index 6179855..e9946c4 100644 --- a/common/src/test/java/org/apache/falcon/entity/AbstractTestBase.java +++ b/common/src/test/java/org/apache/falcon/entity/AbstractTestBase.java @@ -35,6 +35,7 @@ import org.apache.falcon.util.StartupProperties; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; import org.testng.annotations.BeforeClass; import javax.xml.bind.JAXBException; @@ -79,6 +80,7 @@ public class AbstractTestBase { store.init(); CurrentUser.authenticate(FalconTestUtil.TEST_USER_2); + UserGroupInformation.createUserForTesting(FalconTestUtil.TEST_USER_2, new String[]{"testgroup"}); } protected void cleanupStore() throws FalconException { http://git-wip-us.apache.org/repos/asf/falcon/blob/48d0723c/common/src/test/java/org/apache/falcon/entity/parser/ClusterEntityParserTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/falcon/entity/parser/ClusterEntityParserTest.java b/common/src/test/java/org/apache/falcon/entity/parser/ClusterEntityParserTest.java index 2bafac9..cd61a8c 100644 --- a/common/src/test/java/org/apache/falcon/entity/parser/ClusterEntityParserTest.java +++ b/common/src/test/java/org/apache/falcon/entity/parser/ClusterEntityParserTest.java @@ -137,6 +137,14 @@ public class ClusterEntityParserTest extends AbstractTestBase { Assert.assertEquals(ClusterHelper.getMessageBrokerUrl(cluster), ClusterHelper.NO_USER_BROKER_URL); } + @Test(expectedExceptions = ValidationException.class, + expectedExceptionsMessageRegExp = ".*java.net.UnknownHostException.*") + public void testParseClusterWithBadWriteInterface() throws Exception { + InputStream stream = this.getClass().getResourceAsStream("/config/cluster/cluster-bad-write-endpoint.xml"); + Cluster cluster = parser.parse(stream); + parser.validate(cluster); + } + @Test public void testParseClusterWithBadRegistry() throws Exception { // disable catalog service @@ -164,7 +172,7 @@ public class ClusterEntityParserTest extends AbstractTestBase { Mockito.doNothing().when(clusterEntityParser).validateLocations(cluster); // Good set of properties, should work - clusterEntityParser.validate(cluster); + clusterEntityParser.validateProperties(cluster); // add duplicate property, should throw validation exception. Property property1 = new Property(); @@ -180,13 +188,13 @@ public class ClusterEntityParserTest extends AbstractTestBase { // Remove duplicate property. It should not throw exception anymore cluster.getProperties().getProperties().remove(property1); - clusterEntityParser.validate(cluster); + clusterEntityParser.validateProperties(cluster); // add empty property name, should throw validation exception. property1.setName(""); cluster.getProperties().getProperties().add(property1); try { - clusterEntityParser.validate(cluster); + clusterEntityParser.validateProperties(cluster); Assert.fail(); // should not reach here } catch (ValidationException e) { // Do nothing http://git-wip-us.apache.org/repos/asf/falcon/blob/48d0723c/common/src/test/resources/config/cluster/cluster-bad-write-endpoint.xml ---------------------------------------------------------------------- diff --git a/common/src/test/resources/config/cluster/cluster-bad-write-endpoint.xml b/common/src/test/resources/config/cluster/cluster-bad-write-endpoint.xml new file mode 100644 index 0000000..fc709a2 --- /dev/null +++ b/common/src/test/resources/config/cluster/cluster-bad-write-endpoint.xml @@ -0,0 +1,44 @@ +<?xml version="1.0"?> +<!--~ + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<cluster colo="default" description="" name="testCluster" xmlns="uri:falcon:cluster:0.1"> + <tags>[email protected], [email protected], department=forecasting</tags> + <interfaces> + <interface type="readonly" endpoint="jail://testcluster:00" + version="0.20.2"/> + <interface type="write" endpoint="hdfs://bad-end-point:8020" + version="0.20.2"/> + <interface type="execute" endpoint="localhost:8021" version="0.20.2"/> + <interface type="workflow" endpoint="http://localhost:11000/oozie/"; + version="4.0"/> + <interface type="registry" endpoint="http://localhost:48080/templeton/v1"; + version="0.11.0"/> + <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" + version="5.1.6"/> + </interfaces> + <locations> + <location name="staging" path="/projects/falcon/staging"/> + <location name="temp" path="/tmp"/> + <location name="working" path="/projects/falcon/working"/> + </locations> + <properties> + <property name="field1" value="value1"/> + <property name="field2" value="value2"/> + </properties> +</cluster>
