Repository: falcon Updated Branches: refs/heads/master 14b1bb8f4 -> fa66c17e9
FALCON-1881 Database export does not require fields tag in entity specification Database export does not require fields tag in entity specification Author: Venkatesan Ramachandran <[email protected]> Reviewers: "Peeyush Bishnoi <[email protected]>, Pallavi Rao <[email protected]>, Balu Vellanki <[email protected]>" Closes #83 from vramachan/FALCON-1881.20160331-1.ExportFieldsList Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/fa66c17e Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/fa66c17e Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/fa66c17e Branch: refs/heads/master Commit: fa66c17e967359c519c59717f52a705297595d3f Parents: 14b1bb8 Author: Venkatesan Ramachandran <[email protected]> Authored: Wed Apr 6 13:39:42 2016 -0700 Committer: bvellanki <[email protected]> Committed: Wed Apr 6 13:39:42 2016 -0700 ---------------------------------------------------------------------- .../falcon/entity/parser/FeedEntityParser.java | 16 +++-- .../entity/parser/FeedEntityParserTest.java | 4 +- .../feed/feed-export-exclude-fields-0.1.xml | 66 -------------------- .../config/feed/feed-export-fields-0.1.xml | 66 ++++++++++++++++++++ .../test/resources/feed-export-template6.xml | 6 -- 5 files changed, 79 insertions(+), 79 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/fa66c17e/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java index c942862..2a9a852 100644 --- a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java +++ b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java @@ -110,7 +110,7 @@ public class FeedEntityParser extends EntityParser<Feed> { if (FeedHelper.isExportEnabled(cluster)) { validateEntityExists(EntityType.DATASOURCE, FeedHelper.getExportDatasourceName(cluster)); validateFeedExportArgs(cluster); - validateFeedExportFieldExcludes(cluster); + validateFeedExportFields(cluster); } } @@ -709,10 +709,16 @@ public class FeedEntityParser extends EntityParser<Feed> { } } - private void validateFeedExportFieldExcludes(Cluster feedCluster) throws FalconException { - if (FeedHelper.isFieldExcludes(feedCluster.getExport().getTarget())) { - throw new ValidationException(String.format("Field excludes are not supported " - + "currently in Feed import policy")); + /** + * Export infers the target fields from the destination. There is no need to enumerate or exclude the fields + * in the feed entity definition. + * + * @param feedCluster feed's cluster + * @throws FalconException + */ + private void validateFeedExportFields(Cluster feedCluster) throws FalconException { + if (feedCluster.getExport().getTarget().getFields() != null) { + throw new ValidationException(String.format("Feed Export does not expect Fields specification")); } } http://git-wip-us.apache.org/repos/asf/falcon/blob/fa66c17e/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java index 382a8c4..ceec3c4 100644 --- a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java +++ b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java @@ -1213,9 +1213,9 @@ public class FeedEntityParserTest extends AbstractTestBase { public void testExportFeedSqoopExcludeFields() throws Exception { storeEntity(EntityType.CLUSTER, "testCluster"); - InputStream feedStream = this.getClass().getResourceAsStream("/config/feed/feed-export-exclude-fields-0.1.xml"); + InputStream feedStream = this.getClass().getResourceAsStream("/config/feed/feed-export-fields-0.1.xml"); Feed feed = parser.parseAndValidate(feedStream); - Assert.fail("An exception should have been thrown: Feed Export policy not yet implement Field exclusion."); + Assert.fail("An exception should have been thrown: Feed Export policy does not require Fields specification."); } @Test (expectedExceptions = ValidationException.class) http://git-wip-us.apache.org/repos/asf/falcon/blob/fa66c17e/common/src/test/resources/config/feed/feed-export-exclude-fields-0.1.xml ---------------------------------------------------------------------- diff --git a/common/src/test/resources/config/feed/feed-export-exclude-fields-0.1.xml b/common/src/test/resources/config/feed/feed-export-exclude-fields-0.1.xml deleted file mode 100644 index 6753a00..0000000 --- a/common/src/test/resources/config/feed/feed-export-exclude-fields-0.1.xml +++ /dev/null @@ -1,66 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> -<feed description="Customer data" name="CustomerFeed" xmlns="uri:falcon:feed:0.1"> - <tags>[email protected], [email protected], _department_type=forecasting</tags> - <partitions> - <partition name="fraud"/> - <partition name="good"/> - </partitions> - - <groups>online,bi</groups> - <availabilityFlag>_SUCCESS</availabilityFlag> - - <frequency>hours(1)</frequency> - <sla slaLow="hours(2)" slaHigh="hours(3)"/> - <timezone>UTC</timezone> - - <late-arrival cut-off="hours(6)"/> - - <clusters> - <cluster name="testCluster" type="source"> - <validity start="2011-11-01T00:00Z" end="2011-12-31T00:00Z"/> - <retention limit="hours(48)" action="delete"/> - <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE --> - <sla slaLow="hours(3)" slaHigh="hours(4)"/> - <export> - <target name="test-hsql-db" tableName="customer"> - <load type="updateonly"/> - <fields> - <excludes> - <field>id</field> - <field>name</field> - </excludes> - </fields> - </target> - <arguments> - <argument name="--update-key" value="id"/> - </arguments> - </export> - </cluster> - </clusters> - - <locations> - <location type="data" path="/projects/falcon/clicks"/> - <location type="stats" path="/projects/falcon/clicksStats"/> - <location type="meta" path="/projects/falcon/clicksMetaData"/> - </locations> - - <ACL owner="testuser" group="group" permission="0x755"/> - <schema location="/schema/clicks" provider="protobuf"/> -</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/fa66c17e/common/src/test/resources/config/feed/feed-export-fields-0.1.xml ---------------------------------------------------------------------- diff --git a/common/src/test/resources/config/feed/feed-export-fields-0.1.xml b/common/src/test/resources/config/feed/feed-export-fields-0.1.xml new file mode 100644 index 0000000..8fda43d --- /dev/null +++ b/common/src/test/resources/config/feed/feed-export-fields-0.1.xml @@ -0,0 +1,66 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> +<feed description="Customer data" name="CustomerFeed" xmlns="uri:falcon:feed:0.1"> + <tags>[email protected], [email protected], _department_type=forecasting</tags> + <partitions> + <partition name="fraud"/> + <partition name="good"/> + </partitions> + + <groups>online,bi</groups> + <availabilityFlag>_SUCCESS</availabilityFlag> + + <frequency>hours(1)</frequency> + <sla slaLow="hours(2)" slaHigh="hours(3)"/> + <timezone>UTC</timezone> + + <late-arrival cut-off="hours(6)"/> + + <clusters> + <cluster name="testCluster" type="source"> + <validity start="2011-11-01T00:00Z" end="2011-12-31T00:00Z"/> + <retention limit="hours(48)" action="delete"/> + <!-- Limit can be in Time or Instances 100, Action ENUM DELETE,ARCHIVE --> + <sla slaLow="hours(3)" slaHigh="hours(4)"/> + <export> + <target name="test-hsql-db" tableName="customer"> + <load type="updateonly"/> + <fields> + <includes> + <field>id</field> + <field>name</field> + </includes> + </fields> + </target> + <arguments> + <argument name="--update-key" value="id"/> + </arguments> + </export> + </cluster> + </clusters> + + <locations> + <location type="data" path="/projects/falcon/clicks"/> + <location type="stats" path="/projects/falcon/clicksStats"/> + <location type="meta" path="/projects/falcon/clicksMetaData"/> + </locations> + + <ACL owner="testuser" group="group" permission="0x755"/> + <schema location="/schema/clicks" provider="protobuf"/> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/fa66c17e/webapp/src/test/resources/feed-export-template6.xml ---------------------------------------------------------------------- diff --git a/webapp/src/test/resources/feed-export-template6.xml b/webapp/src/test/resources/feed-export-template6.xml index 0eb748b..0534257 100644 --- a/webapp/src/test/resources/feed-export-template6.xml +++ b/webapp/src/test/resources/feed-export-template6.xml @@ -31,12 +31,6 @@ <export> <target name="##datasourcename##" tableName="simple_export"> <load type="allowinsert"/> - <fields> - <includes> - <field>id</field> - <field>name</field> - </includes> - </fields> </target> <arguments> <argument name="--update-key" value="id"/>
