Repository: falcon Updated Branches: refs/heads/0.6.1 c6ff55f80 -> bd1ac1e2e
FALCON-1195 ClusterEntityParserTest fails intermittently & fixed Checkstyle errors. Contributed by Balu Vellanki Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/bd1ac1e2 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/bd1ac1e2 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/bd1ac1e2 Branch: refs/heads/0.6.1 Commit: bd1ac1e2eefe1e252006b43210e8c8c1b75aadb0 Parents: c6ff55f Author: shaik.idris <[email protected]> Authored: Wed May 13 15:30:04 2015 +0530 Committer: shaik.idris <[email protected]> Committed: Wed May 13 15:30:04 2015 +0530 ---------------------------------------------------------------------- CHANGES.txt | 3 + .../entity/parser/ClusterEntityParser.java | 2 +- .../entity/parser/ClusterEntityParserTest.java | 93 ++++++++++++-------- 3 files changed, 62 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/bd1ac1e2/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index e6e4d41..8057105 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -142,6 +142,9 @@ Branch: 0.6.1 (Proposed Release Version: 0.6.1) (Suhas vasu) BUG FIXES + FALCON-1195 ClusterEntityParserTest fails intermittently. + (Balu Vellanki via Shaik Idris Ali) + FALCON-1162 Cluster submit succeeds when staging HDFS dir does not have 777 (ALL) permission (Venkat Ramachandran via Suhas Vasu) http://git-wip-us.apache.org/repos/asf/falcon/blob/bd1ac1e2/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java b/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java index 4555cb0..5c3ce4f 100644 --- a/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java +++ b/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java @@ -294,7 +294,7 @@ public class ClusterEntityParser extends EntityParser<Cluster> { if (stagingLocation.getPath().equals(workingLocation.getPath())) { throw new ValidationException( "Location with name: " + stagingLocation.getName().value() + " and " + workingLocation - .getName().value() + " cannot of same path: " + stagingLocation.getPath() + .getName().value() + " cannot have same path: " + stagingLocation.getPath() + " for cluster :" + cluster.getName()); } else { http://git-wip-us.apache.org/repos/asf/falcon/blob/bd1ac1e2/common/src/test/java/org/apache/falcon/entity/parser/ClusterEntityParserTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/falcon/entity/parser/ClusterEntityParserTest.java b/common/src/test/java/org/apache/falcon/entity/parser/ClusterEntityParserTest.java index 4920d03..69230b2 100644 --- a/common/src/test/java/org/apache/falcon/entity/parser/ClusterEntityParserTest.java +++ b/common/src/test/java/org/apache/falcon/entity/parser/ClusterEntityParserTest.java @@ -18,6 +18,7 @@ package org.apache.falcon.entity.parser; +import org.apache.commons.lang3.StringUtils; import org.apache.falcon.FalconException; import org.apache.falcon.catalog.CatalogServiceFactory; import org.apache.falcon.cluster.util.EmbeddedCluster; @@ -53,6 +54,7 @@ import java.io.StringWriter; public class ClusterEntityParserTest extends AbstractTestBase { private final ClusterEntityParser parser = (ClusterEntityParser) EntityParserFactory.getParser(EntityType.CLUSTER); + private static final String CLUSTER_LOCATIONS_BASE_DIR = "/projects/falcon/ClusterEntityParserTestLocations/"; @Test public void testParse() throws IOException, FalconException, JAXBException { @@ -62,6 +64,8 @@ public class ClusterEntityParserTest extends AbstractTestBase { Cluster cluster = parser.parse(stream); ClusterHelper.getInterface(cluster, Interfacetype.WRITE) .setEndpoint(conf.get(HadoopClientFactory.FS_DEFAULT_NAME_KEY)); + Locations locations = getClusterLocations("staging0", "working0"); + cluster.setLocations(locations); Assert.assertNotNull(cluster); Assert.assertEquals(cluster.getName(), "testCluster"); @@ -84,7 +88,9 @@ public class ClusterEntityParserTest extends AbstractTestBase { Assert.assertEquals(workflow.getVersion(), "4.0"); Assert.assertEquals(ClusterHelper.getLocation(cluster, ClusterLocationType.STAGING).getPath(), - "/projects/falcon/staging"); + CLUSTER_LOCATIONS_BASE_DIR + "staging0"); + Assert.assertEquals(ClusterHelper.getLocation(cluster, ClusterLocationType.WORKING).getPath(), + CLUSTER_LOCATIONS_BASE_DIR + "working0"); StringWriter stringWriter = new StringWriter(); Marshaller marshaller = EntityType.CLUSTER.getMarshaller(); @@ -95,8 +101,7 @@ public class ClusterEntityParserTest extends AbstractTestBase { Assert.assertEquals(catalog.getEndpoint(), "http://localhost:48080/templeton/v1";); Assert.assertEquals(catalog.getVersion(), "0.11.0"); - Assert.assertEquals(ClusterHelper.getLocation(cluster, ClusterLocationType.STAGING).getPath(), - "/projects/falcon/staging"); + } @Test @@ -205,18 +210,15 @@ public class ClusterEntityParserTest extends AbstractTestBase { * * @throws ValidationException */ - @Test(expectedExceptions = ValidationException.class) public void testClusterWithoutStaging() throws Exception { + @Test(expectedExceptions = ValidationException.class, expectedExceptionsMessageRegExp = ".*Unable to find.*") + public void testClusterWithoutStaging() throws Exception { ClusterEntityParser clusterEntityParser = Mockito .spy((ClusterEntityParser) EntityParserFactory.getParser(EntityType.CLUSTER)); Cluster cluster = (Cluster) this.dfsCluster.getCluster().copy(); Mockito.doNothing().when(clusterEntityParser).validateWorkflowInterface(cluster); Mockito.doNothing().when(clusterEntityParser).validateMessagingInterface(cluster); Mockito.doNothing().when(clusterEntityParser).validateRegistryInterface(cluster); - Location location = new Location(); - location.setName(ClusterLocationType.WORKING); - location.setPath("/apps/non/existent/path"); - Locations locations = new Locations(); - locations.getLocations().add(location); + Locations locations = getClusterLocations(null, "non/existent/path"); cluster.setLocations(locations); clusterEntityParser.validate(cluster); Assert.fail("Should have thrown a validation exception"); @@ -228,28 +230,17 @@ public class ClusterEntityParserTest extends AbstractTestBase { * * @throws ValidationException */ - @Test(expectedExceptions = ValidationException.class) + @Test(expectedExceptions = ValidationException.class, expectedExceptionsMessageRegExp = ".*Location.*must exist.") public void testClusterWithInvalidLocationsPaths() throws Exception { ClusterEntityParser clusterEntityParser = Mockito .spy((ClusterEntityParser) EntityParserFactory.getParser(EntityType.CLUSTER)); Cluster cluster = (Cluster)this.dfsCluster.getCluster().copy(); - Location location = new Location(); - location.setName(ClusterLocationType.STAGING); - location.setPath("/apps/non/existent/path"); - Locations locations = new Locations(); - locations.getLocations().add(location); + Locations locations = getClusterLocations("non/existent/path", null); cluster.setLocations(locations); Mockito.doNothing().when(clusterEntityParser).validateWorkflowInterface(cluster); Mockito.doNothing().when(clusterEntityParser).validateMessagingInterface(cluster); Mockito.doNothing().when(clusterEntityParser).validateRegistryInterface(cluster); - try { - clusterEntityParser.validate(cluster); - } catch (ValidationException e) { - String errorMessage = - "Location " + location.getPath() + " for cluster " + cluster.getName() + " must exist."; - Assert.assertEquals(e.getMessage(), errorMessage); - throw e; - } + clusterEntityParser.validate(cluster); Assert.fail("Should have thrown a validation exception"); } @@ -259,12 +250,15 @@ public class ClusterEntityParserTest extends AbstractTestBase { * * @throws ValidationException */ - @Test(expectedExceptions = ValidationException.class) + @Test(expectedExceptions = ValidationException.class, expectedExceptionsMessageRegExp = ".*same path:.*") public void testClusterWithSameWorkingAndStaging() throws Exception { ClusterEntityParser clusterEntityParser = Mockito .spy((ClusterEntityParser) EntityParserFactory.getParser(EntityType.CLUSTER)); Cluster cluster = (Cluster) this.dfsCluster.getCluster().copy(); - cluster.getLocations().getLocations().get(1).setPath("/projects/falcon/staging"); + Locations locations = getClusterLocations("staging1", "staging1"); + cluster.setLocations(locations); + this.dfsCluster.getFileSystem().mkdirs(new Path(cluster.getLocations().getLocations().get(0).getPath()), + HadoopClientFactory.ALL_PERMISSION); Mockito.doNothing().when(clusterEntityParser).validateWorkflowInterface(cluster); Mockito.doNothing().when(clusterEntityParser).validateMessagingInterface(cluster); Mockito.doNothing().when(clusterEntityParser).validateRegistryInterface(cluster); @@ -277,11 +271,13 @@ public class ClusterEntityParserTest extends AbstractTestBase { * It should automatically get generated * Extensive tests are found in ClusterEntityValidationIT. */ - @Test public void testClusterWithOnlyStaging() throws Exception { + @Test + public void testClusterWithOnlyStaging() throws Exception { ClusterEntityParser clusterEntityParser = Mockito .spy((ClusterEntityParser) EntityParserFactory.getParser(EntityType.CLUSTER)); Cluster cluster = (Cluster) this.dfsCluster.getCluster().copy(); - cluster.getLocations().getLocations().remove(1); + Locations locations = getClusterLocations("staging2", null); + cluster.setLocations(locations); Mockito.doNothing().when(clusterEntityParser).validateWorkflowInterface(cluster); Mockito.doNothing().when(clusterEntityParser).validateMessagingInterface(cluster); Mockito.doNothing().when(clusterEntityParser).validateRegistryInterface(cluster); @@ -303,16 +299,21 @@ public class ClusterEntityParserTest extends AbstractTestBase { * * @throws ValidationException */ - @Test(expectedExceptions = ValidationException.class) + @Test(expectedExceptions = ValidationException.class, expectedExceptionsMessageRegExp = ".*rwxr-xr-x.*rwxrwxrwx") public void testClusterWithSubdirInStaging() throws Exception { ClusterEntityParser clusterEntityParser = Mockito .spy((ClusterEntityParser) EntityParserFactory.getParser(EntityType.CLUSTER)); Cluster cluster = (Cluster) this.dfsCluster.getCluster().copy(); - cluster.getLocations().getLocations().get(1).setPath("/projects/falcon/staging"); - cluster.getLocations().getLocations().remove(1); + Locations locations = getClusterLocations("staging3", null); + cluster.setLocations(locations); + + HadoopClientFactory.mkdirs(this.dfsCluster.getFileSystem(), + new Path(cluster.getLocations().getLocations().get(0).getPath()), + HadoopClientFactory.ALL_PERMISSION); HadoopClientFactory.mkdirs(this.dfsCluster.getFileSystem(), - new Path(ClusterHelper.getLocation(cluster, ClusterLocationType.WORKING).getPath()), + new Path(cluster.getLocations().getLocations().get(0).getPath() + "/working"), HadoopClientFactory.ALL_PERMISSION); + Mockito.doNothing().when(clusterEntityParser).validateWorkflowInterface(cluster); Mockito.doNothing().when(clusterEntityParser).validateMessagingInterface(cluster); Mockito.doNothing().when(clusterEntityParser).validateRegistryInterface(cluster); @@ -328,13 +329,13 @@ public class ClusterEntityParserTest extends AbstractTestBase { * * @throws ValidationException */ - @Test(expectedExceptions = ValidationException.class) + @Test(expectedExceptions = ValidationException.class, expectedExceptionsMessageRegExp = ".*rwxr-xr-x.*rwxrwxrwx") public void testClusterWithStagingPermission() throws Exception { ClusterEntityParser clusterEntityParser = Mockito .spy((ClusterEntityParser) EntityParserFactory.getParser(EntityType.CLUSTER)); Cluster cluster = (Cluster) this.dfsCluster.getCluster().copy(); - cluster.getLocations().getLocations().get(0).setPath("/projects/falcon/staging2"); - cluster.getLocations().getLocations().remove(1); + Locations locations = getClusterLocations("staging4", null); + cluster.setLocations(locations); Mockito.doNothing().when(clusterEntityParser).validateWorkflowInterface(cluster); Mockito.doNothing().when(clusterEntityParser).validateMessagingInterface(cluster); Mockito.doNothing().when(clusterEntityParser).validateRegistryInterface(cluster); @@ -348,10 +349,32 @@ public class ClusterEntityParserTest extends AbstractTestBase { public void init() throws Exception { this.dfsCluster = EmbeddedCluster.newCluster("testCluster"); this.conf = dfsCluster.getConf(); + this.dfsCluster.getFileSystem().mkdirs(new Path(CLUSTER_LOCATIONS_BASE_DIR)); } @AfterClass - public void tearDown() { + public void tearDown() throws IOException { + this.dfsCluster.getFileSystem().delete(new Path(CLUSTER_LOCATIONS_BASE_DIR), true); this.dfsCluster.shutdown(); } + + private Locations getClusterLocations(String staging, String working) { + Locations locations = new Locations(); + + Location loc = new Location(); + loc.setName(ClusterLocationType.STAGING); + if (StringUtils.isNotEmpty(staging)) { + loc.setPath(CLUSTER_LOCATIONS_BASE_DIR + staging); + locations.getLocations().add(loc); + } + + loc = new Location(); + loc.setName(ClusterLocationType.WORKING); + if (StringUtils.isNotEmpty(working)) { + loc.setPath(CLUSTER_LOCATIONS_BASE_DIR + working); + locations.getLocations().add(loc); + } + + return locations; + } }
