Repository: falcon Updated Branches: refs/heads/master c790761e0 -> 0c6f3911d
FALCON-1020 validate command produces different results when run via prism and server. Contributed by pavan kumar kolamuri Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/0c6f3911 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/0c6f3911 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/0c6f3911 Branch: refs/heads/master Commit: 0c6f3911d2972f266f7e0aca003560b5d530ca7a Parents: c790761 Author: Suhas Vasu <[email protected]> Authored: Mon Mar 9 20:44:10 2015 +0530 Committer: Suhas Vasu <[email protected]> Committed: Mon Mar 9 20:44:10 2015 +0530 ---------------------------------------------------------------------- CHANGES.txt | 6 + FALCON-1063.patch | 204 ------------------- client/src/main/resources/feed-0.1.xsd | 7 +- .../apache/falcon/entity/FileSystemStorage.java | 2 +- .../proxy/SchedulableEntityManagerProxy.java | 23 ++- 5 files changed, 34 insertions(+), 208 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/0c6f3911/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 453e44c..3b9b243 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -81,6 +81,9 @@ Trunk (Unreleased) Seetharam) OPTIMIZATIONS + FALCON-1063 Falcon CLI list entities operation throws NullPointerException + (Pallavi Rao via Suhas Vasu) + FALCON-987 params command inconsistent behavior (pavan kumar kolamuri via Suhas Vasu) @@ -98,6 +101,9 @@ Trunk (Unreleased) (Suhas vasu) BUG FIXES + FALCON-1020 validate command produces different results when run + via prism and server (pavan kumar kolamuri via Suhas Vasu) + FALCON-950 Rerun does not work on succeeded instances (Suhas Vasu) FALCON-1048 Incorrect documentation for feed instacnce listing api. (Suhas http://git-wip-us.apache.org/repos/asf/falcon/blob/0c6f3911/FALCON-1063.patch ---------------------------------------------------------------------- diff --git a/FALCON-1063.patch b/FALCON-1063.patch deleted file mode 100644 index 5aea017..0000000 --- a/FALCON-1063.patch +++ /dev/null @@ -1,204 +0,0 @@ -diff --git a/common/src/main/java/org/apache/falcon/cleanup/AbstractCleanupHandler.java b/common/src/main/java/org/apache/falcon/cleanup/AbstractCleanupHandler.java -index 5a4dce6..85d7263 100644 ---- a/common/src/main/java/org/apache/falcon/cleanup/AbstractCleanupHandler.java -+++ b/common/src/main/java/org/apache/falcon/cleanup/AbstractCleanupHandler.java -@@ -116,11 +116,11 @@ public abstract class AbstractCleanupHandler { - Entity entity) throws FalconException { - try { - final AccessControlList acl = entity.getACL(); -- if (acl == null) { -- throw new FalconException("ACL for entity " + entity.getName() + " is empty"); -+ // To support backward compatibility, will only use the ACL owner only if present -+ if (acl != null) { -+ CurrentUser.authenticate(acl.getOwner()); // proxy user - } - -- CurrentUser.authenticate(acl.getOwner()); // proxy user - return HadoopClientFactory.get().createProxiedFileSystem( - ClusterHelper.getConfiguration(cluster)); - } catch (Exception e) { -diff --git a/common/src/main/java/org/apache/falcon/security/SecurityUtil.java b/common/src/main/java/org/apache/falcon/security/SecurityUtil.java -index b9fd37e..861f80f 100644 ---- a/common/src/main/java/org/apache/falcon/security/SecurityUtil.java -+++ b/common/src/main/java/org/apache/falcon/security/SecurityUtil.java -@@ -19,11 +19,13 @@ - package org.apache.falcon.security; - - import org.apache.falcon.FalconException; -+import org.apache.falcon.entity.v0.Entity; - import org.apache.falcon.util.ReflectionUtils; - import org.apache.falcon.util.StartupProperties; - import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; - import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; - -+import java.io.IOException; - import java.net.InetAddress; - import java.net.UnknownHostException; - -@@ -104,4 +106,16 @@ public final class SecurityUtil { - "org.apache.falcon.security.DefaultAuthorizationProvider"); - return ReflectionUtils.getInstanceByClassName(providerClassName); - } -+ -+ public static void tryProxy(Entity entity) throws IOException, FalconException { -+ if (entity != null && entity.getACL() != null && SecurityUtil.isAuthorizationEnabled()) { -+ final String aclOwner = entity.getACL().getOwner(); -+ final String aclGroup = entity.getACL().getGroup(); -+ -+ if (SecurityUtil.getAuthorizationProvider().shouldProxy( -+ CurrentUser.getAuthenticatedUGI(), aclOwner, aclGroup)) { -+ CurrentUser.proxy(aclOwner, aclGroup); -+ } -+ } -+ } - } -diff --git a/common/src/test/java/org/apache/falcon/cleanup/LogCleanupServiceTest.java b/common/src/test/java/org/apache/falcon/cleanup/LogCleanupServiceTest.java -index 8e2e544..0df59b2 100644 ---- a/common/src/test/java/org/apache/falcon/cleanup/LogCleanupServiceTest.java -+++ b/common/src/test/java/org/apache/falcon/cleanup/LogCleanupServiceTest.java -@@ -55,6 +55,8 @@ public class LogCleanupServiceTest extends AbstractTestBase { - + "sample2" + "/logs/job-2010-01-01-01-00/000"); - private final Path instanceLogPath4 = new Path("/projects/falcon/staging/falcon/workflows/process/" - + "sample" + "/logs/latedata/2010-01-01-01-00"); -+ private final Path instanceLogPath5 = new Path("/projects/falcon/staging/falcon/workflows/process/" -+ + "sample3" + "/logs/job-2010-01-01-01-00/000"); - private final Path feedInstanceLogPath = new Path("/projects/falcon/staging/falcon/workflows/feed/" - + "impressionFeed" + "/logs/job-2010-01-01-01-00/testCluster/000"); - private final Path feedInstanceLogPath1 = new Path("/projects/falcon/staging/falcon/workflows/feed/" -@@ -90,15 +92,22 @@ public class LogCleanupServiceTest extends AbstractTestBase { - Process otherProcess = (Process) process.copy(); - otherProcess.setName("sample2"); - otherProcess.setFrequency(new Frequency("days(1)")); -+ Process noACLProcess = (Process) process.copy(); -+ noACLProcess.setName("sample3"); -+ noACLProcess.setACL(null); - ConfigurationStore.get().remove(EntityType.PROCESS, - otherProcess.getName()); - ConfigurationStore.get().publish(EntityType.PROCESS, otherProcess); -+ ConfigurationStore.get().remove(EntityType.PROCESS, -+ noACLProcess.getName()); -+ ConfigurationStore.get().publish(EntityType.PROCESS, noACLProcess); - - fs.mkdirs(instanceLogPath); - fs.mkdirs(instanceLogPath1); - fs.mkdirs(instanceLogPath2); - fs.mkdirs(instanceLogPath3); - fs.mkdirs(instanceLogPath4); -+ fs.mkdirs(instanceLogPath5); - - // fs.setTimes wont work on dirs - fs.createNewFile(new Path(instanceLogPath, "oozie.log")); -@@ -138,6 +147,7 @@ public class LogCleanupServiceTest extends AbstractTestBase { - Assert.assertFalse(fs.exists(instanceLogPath)); - Assert.assertFalse(fs.exists(instanceLogPath1)); - Assert.assertFalse(fs.exists(instanceLogPath2)); -+ Assert.assertFalse(fs.exists(instanceLogPath5)); - Assert.assertTrue(fs.exists(instanceLogPath3)); - } - -diff --git a/common/src/test/java/org/apache/falcon/security/SecurityUtilTest.java b/common/src/test/java/org/apache/falcon/security/SecurityUtilTest.java -index a36d916..7f9b405 100644 ---- a/common/src/test/java/org/apache/falcon/security/SecurityUtilTest.java -+++ b/common/src/test/java/org/apache/falcon/security/SecurityUtilTest.java -@@ -18,10 +18,17 @@ - - package org.apache.falcon.security; - -+ -+import org.apache.falcon.FalconException; -+import org.apache.falcon.entity.v0.process.*; -+import org.apache.falcon.entity.v0.process.Process; - import org.apache.falcon.util.StartupProperties; -+import org.mockito.Mockito; - import org.testng.Assert; - import org.testng.annotations.Test; - -+import java.io.IOException; -+ - /** - * Unit test for Security utils. - */ -@@ -81,4 +88,25 @@ public class SecurityUtilTest { - Assert.assertEquals(SecurityUtil.getAuthorizationProvider().getClass(), - DefaultAuthorizationProvider.class); - } -+ -+ @Test -+ public void testTryProxy() throws IOException, FalconException { -+ Process process = Mockito.mock(Process.class); -+ StartupProperties.get().setProperty("falcon.security.authorization.enabled", "true"); -+ final String currentUser = System.getProperty("user.name"); -+ -+ // When ACL not specified -+ CurrentUser.authenticate(currentUser); -+ SecurityUtil.tryProxy(process); -+ Assert.assertEquals(CurrentUser.getUser(), currentUser); -+ -+ ACL acl = new ACL(); -+ acl.setOwner("testuser"); -+ acl.setGroup("users"); -+ Mockito.when(process.getACL()).thenReturn(acl); -+ -+ // When ACL is specified -+ SecurityUtil.tryProxy(process); -+ Assert.assertEquals(CurrentUser.getUser(), "testuser"); -+ } - } -diff --git a/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java b/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java -index 9a044d9..67a66c6 100644 ---- a/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java -+++ b/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java -@@ -406,23 +406,13 @@ public abstract class AbstractEntityManager { - + "Can't be submitted again. Try removing before submitting."); - } - -- tryProxy(entity); // proxy before validating since FS/Oozie needs to be proxied -+ SecurityUtil.tryProxy(entity); // proxy before validating since FS/Oozie needs to be proxied - validate(entity); - configStore.publish(entityType, entity); - LOG.info("Submit successful: ({}): {}", type, entity.getName()); - return entity; - } - -- private void tryProxy(Entity entity) throws IOException, FalconException { -- final String aclOwner = entity.getACL().getOwner(); -- final String aclGroup = entity.getACL().getGroup(); -- if (SecurityUtil.isAuthorizationEnabled() -- && SecurityUtil.getAuthorizationProvider().shouldProxy( -- CurrentUser.getAuthenticatedUGI(), aclOwner, aclGroup)) { -- CurrentUser.proxy(aclOwner, aclGroup); -- } -- } -- - /** - * KLUDGE - Until ACL is mandated entity passed should be decorated for equals check to pass. - * existingEntity in config store will have teh decoration and equals check fails -@@ -646,7 +636,7 @@ public abstract class AbstractEntityManager { - // the user who requested list query has no permission to access this entity. Skip this entity - continue; - } -- tryProxy(entity); -+ SecurityUtil.tryProxy(entity); - - List<String> tags = EntityUtil.getTags(entity); - List<String> pipelines = EntityUtil.getPipelines(entity); -diff --git a/prism/src/main/java/org/apache/falcon/security/FalconAuthorizationFilter.java b/prism/src/main/java/org/apache/falcon/security/FalconAuthorizationFilter.java -index 6b022c9..15e94cd 100644 ---- a/prism/src/main/java/org/apache/falcon/security/FalconAuthorizationFilter.java -+++ b/prism/src/main/java/org/apache/falcon/security/FalconAuthorizationFilter.java -@@ -141,14 +141,7 @@ public class FalconAuthorizationFilter implements Filter { - try { - EntityType type = EntityType.getEnum(entityType); - Entity entity = EntityUtil.getEntity(type, entityName); -- if (entity != null && entity.getACL() != null) { -- final String aclOwner = entity.getACL().getOwner(); -- final String aclGroup = entity.getACL().getGroup(); -- if (authorizationProvider.shouldProxy( -- authenticatedUGI, aclOwner, aclGroup)) { -- CurrentUser.proxy(aclOwner, aclGroup); -- } -- } -+ SecurityUtil.tryProxy(entity); - } catch (FalconException ignore) { - // do nothing - } http://git-wip-us.apache.org/repos/asf/falcon/blob/0c6f3911/client/src/main/resources/feed-0.1.xsd ---------------------------------------------------------------------- diff --git a/client/src/main/resources/feed-0.1.xsd b/client/src/main/resources/feed-0.1.xsd index 0168a35..a228b29 100644 --- a/client/src/main/resources/feed-0.1.xsd +++ b/client/src/main/resources/feed-0.1.xsd @@ -286,7 +286,7 @@ </xs:documentation> </xs:annotation> <xs:attribute type="location-type" name="type" use="required"/> - <xs:attribute type="xs:string" name="path" use="required"/> + <xs:attribute type="non-empty-string" name="path" use="required"/> </xs:complexType> <xs:complexType name="partition"> <xs:attribute type="IDENTIFIER" name="name" use="required"/> @@ -391,4 +391,9 @@ </xs:annotation> <xs:attribute type="xs:string" name="uri" use="required"/> </xs:complexType> + <xs:simpleType name="non-empty-string"> + <xs:restriction base="xs:string"> + <xs:minLength value="1"/> + </xs:restriction> + </xs:simpleType> </xs:schema> http://git-wip-us.apache.org/repos/asf/falcon/blob/0c6f3911/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java index fe93048..1ba7b9d 100644 --- a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java +++ b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java @@ -199,7 +199,7 @@ public class FileSystemStorage extends Configured implements Storage { } } - if (locationForType == null) { + if (locationForType == null || StringUtils.isEmpty(locationForType.getPath())) { return null; } http://git-wip-us.apache.org/repos/asf/falcon/blob/0c6f3911/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java ---------------------------------------------------------------------- diff --git a/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java b/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java index 5f711ee..7ba289a 100644 --- a/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java +++ b/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java @@ -163,8 +163,27 @@ public class SchedulableEntityManagerProxy extends AbstractSchedulableEntityMana @Consumes({MediaType.TEXT_XML, MediaType.TEXT_PLAIN}) @Produces({MediaType.TEXT_XML, MediaType.TEXT_PLAIN, MediaType.APPLICATION_JSON}) @Override - public APIResult validate(@Context HttpServletRequest request, @PathParam("type") String type) { - return super.validate(request, type); + public APIResult validate(@Context final HttpServletRequest request, @PathParam("type") final String type) { + final HttpServletRequest bufferedRequest = getBufferedRequest(request); + EntityType entityType = EntityType.getEnum(type); + final Entity entity; + try { + entity = deserializeEntity(bufferedRequest, entityType); + bufferedRequest.getInputStream().reset(); + } catch (Exception e) { + throw FalconWebException.newException("Unable to parse the request", Response.Status.BAD_REQUEST); + } + return new EntityProxy(type, entity.getName()) { + @Override + protected Set<String> getColosToApply() { + return getApplicableColos(type, entity); + } + + @Override + protected APIResult doExecute(String colo) throws FalconException { + return getEntityManager(colo).invoke("validate", bufferedRequest, type); + } + }.execute(); } @DELETE
