Repository: falcon Updated Branches: refs/heads/master 969d10e96 -> 1aac3a502
http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile4.txt ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile4.txt b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile4.txt new file mode 100644 index 0000000..64434fb --- /dev/null +++ b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/dataFile4.txt @@ -0,0 +1,174 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +package com.prism; + +import java.lang.reflect.Method; +import java.util.ArrayList; + +import org.testng.Assert; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import com.core.Bundle; +import com.core.ColoHelper; +import com.core.PrismHelper; +import com.core.ServiceResponse; +import com.core.Util; +import com.core.Util.URLS; +import com.core.instanceUtil; +import com.core.xmlUtil; +import com.generated.feed.ActionType; +import com.generated.feed.ClusterType; +import com.generated.feed.TimezoneType; + +public class PrismFeedLateReplicationTest { + + + @BeforeMethod(alwaysRun=true) + public void testName(Method method) throws Exception + { + Util.print("test name: "+method.getName()); + //restart server as precaution + Util.restartService(ua1.getClusterHelper()); + Util.restartService(ua2.getClusterHelper()); + Util.restartService(ua3.getClusterHelper()); + + + } + + public PrismFeedLateReplicationTest() throws Exception{ + + } + + PrismHelper prismHelper=new PrismHelper("prism.properties"); + + ColoHelper ua1=new ColoHelper("mk-qa.config.properties"); + + ColoHelper ua2 = new ColoHelper("ivoryqa-1.config.properties"); + + ColoHelper ua3 = new ColoHelper("gs1001.config.properties"); + + @SuppressWarnings("deprecation") + @Test + public void multipleSourceOneTarget() throws Exception + { + + Bundle b1 = (Bundle)Util.readELBundles()[0][0]; + b1.generateUniqueBundle(); + Bundle b2 = (Bundle)Util.readELBundles()[0][0]; + b2.generateUniqueBundle(); + Bundle b3 = (Bundle)Util.readELBundles()[0][0]; + b3.generateUniqueBundle(); + + try{ + b1 = new Bundle(b1,ua1.getEnvFileName()); + b2 = new Bundle(b2,ua2.getEnvFileName()); + b3 = new Bundle(b3,ua3.getEnvFileName()); + + + b1.setInputFeedDataPath("/samarthRetention/input-data/rawLogs/oozieExample/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}/"); + + b1.setCLusterColo("ua1"); + Util.print("cluster b1: "+b1.getClusters().get(0)); + + ServiceResponse r = prismHelper.getClusterHelper().submitEntity(URLS.SUBMIT_URL,b1.getClusters().get(0)); + Assert.assertTrue(r.getMessage().contains("SUCCEEDED")); + + + b2.setCLusterColo("ua2"); + Util.print("cluster b2: "+b2.getClusters().get(0)); + r = prismHelper.getClusterHelper().submitEntity(URLS.SUBMIT_URL,b2.getClusters().get(0)); + Assert.assertTrue(r.getMessage().contains("SUCCEEDED")); + + + b3.setCLusterColo("ua3"); + Util.print("cluster b3: "+b3.getClusters().get(0)); + r = prismHelper.getClusterHelper().submitEntity(URLS.SUBMIT_URL,b3.getClusters().get(0)); + Assert.assertTrue(r.getMessage().contains("SUCCEEDED")); + + + String feed = b1.getDataSets().get(0); + feed = instanceUtil.setFeedCluster(feed,xmlUtil.createValidity("2009-02-01T00:00Z","2012-01-01T00:00Z",TimezoneType.UTC),xmlUtil.createRtention("hours(10)",ActionType.DELETE),null,ClusterType.SOURCE,null); + + String postFix = "/US/ua2" ; + String prefix = b1.getFeedDataPathPrefix(); + Util.HDFSCleanup(ua2,prefix.substring(1)); + Util.lateDataReplenish(ua2,90,0,1,prefix,postFix); + + + postFix = "/UK/ua3" ; + prefix = b1.getFeedDataPathPrefix(); + Util.HDFSCleanup(ua3,prefix.substring(1)); + Util.lateDataReplenish(ua3,90,0,1,prefix,postFix); + + String startTime = instanceUtil.getTimeWrtSystemTime(-30); + + feed = instanceUtil.setFeedCluster(feed,xmlUtil.createValidity(startTime,"2099-01-01T00:00Z",TimezoneType.UTC),xmlUtil.createRtention("hours(10)",ActionType.DELETE),Util.readClusterName(b2.getClusters().get(0)),ClusterType.SOURCE,"US/${cluster.colo}"); + feed = instanceUtil.setFeedCluster(feed,xmlUtil.createValidity(startTime,"2099-01-01T00:00Z",TimezoneType.UTC),xmlUtil.createRtention("hours(10)",ActionType.DELETE),Util.readClusterName(b1.getClusters().get(0)),ClusterType.TARGET,null); + feed = instanceUtil.setFeedCluster(feed,xmlUtil.createValidity(startTime,"2099-01-01T00:00Z",TimezoneType.UTC),xmlUtil.createRtention("hours(10)",ActionType.DELETE),Util.readClusterName(b3.getClusters().get(0)),ClusterType.SOURCE,"UK/${cluster.colo}"); + + + Util.print("feed: "+feed); + + r= prismHelper.getFeedHelper().submitAndSchedule(URLS.SUBMIT_AND_SCHEDULE_URL, feed); + Thread.sleep(10000); + + String TargetBundleID = instanceUtil.getLatestBundleID(Util.readDatasetName(feed),"FEED",ua1.getFeedHelper()); + + //wait till 1st instance of replication coord is SUCCEEDED + ArrayList<String> replicationCoordIDTarget = instanceUtil.getReplicationCoordID(TargetBundleID,ua1.getFeedHelper()); + + for(int i = 0 ; i < 30 ; i++) + { + if(instanceUtil.getInstanceStatusFromCoord(ua1,replicationCoordIDTarget.get(0),0).toString().equals("SUCCEEDED") && instanceUtil.getInstanceStatusFromCoord(ua1,replicationCoordIDTarget.get(1),0).toString().equals("SUCCEEDED")) + break; + Thread.sleep(20000); + } + + Thread.sleep(15000); + + ArrayList<String> inputFolderListForColo1= instanceUtil.getInputFoldersForInstanceForReplication(ua1,replicationCoordIDTarget.get(0),0); + ArrayList<String> inputFolderListForColo2= instanceUtil.getInputFoldersForInstanceForReplication(ua1,replicationCoordIDTarget.get(1),0); + + instanceUtil.putDataInFolders(ua2, inputFolderListForColo1); + instanceUtil.putDataInFolders(ua3, inputFolderListForColo2); + + + + Util.print("folder list 1: "+inputFolderListForColo1.toString()); + Util.print("folder list 2: "+inputFolderListForColo2.toString()); + + + + + } + + finally{ + prismHelper.getProcessHelper().delete(URLS.DELETE_URL,b1.getProcessData()); + prismHelper.getFeedHelper().delete(URLS.DELETE_URL, b1.getDataSets().get(0)); + prismHelper.getFeedHelper().delete(URLS.DELETE_URL, b1.getDataSets().get(1)); + prismHelper.getClusterHelper().delete(URLS.DELETE_URL,b1.getClusters().get(0)); + + } + } + + + + +} http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/jms-config.properties ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/jms-config.properties b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/jms-config.properties deleted file mode 100644 index 4d3d972..0000000 --- a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/jms-config.properties +++ /dev/null @@ -1,31 +0,0 @@ - -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - - -#Global configuration for Ivory JMS, default to use embedded ActiveMQ. - -#Defualt Active MQ url -#ivory.broker.url = tcp://localhost:61616?daemon=true - -#Embedded -ivory.broker.url = vm://localhost?broker.useJmx=false&broker.persistent=true - - - http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/log4j.xml ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/log4j.xml b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/log4j.xml deleted file mode 100644 index 045002c..0000000 --- a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/log4j.xml +++ /dev/null @@ -1,63 +0,0 @@ -<?xml version="1.0" encoding="UTF-8" ?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> - -<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd"> - -<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/";> - <appender name="console" class="org.apache.log4j.ConsoleAppender"> - <param name="Target" value="System.out"/> - <layout class="org.apache.log4j.PatternLayout"> - <param name="ConversionPattern" value="%d %-5p - %m (%c{1}:%L)%n"/> - </layout> - </appender> - - <appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender"> - <param name="File" value="/var/log/ivory/application.log"/> - <param name="Append" value="true"/> - <param name="Threshold" value="debug"/> - <layout class="org.apache.log4j.PatternLayout"> - <param name="ConversionPattern" value="%d %-5p - %m (%c{1}:%L)%n"/> - </layout> - </appender> - - <appender name="AUDIT" class="org.apache.log4j.DailyRollingFileAppender"> - <param name="File" value="/var/log/ivory/audit.log"/> - <param name="Append" value="true"/> - <param name="Threshold" value="debug"/> - <layout class="org.apache.log4j.PatternLayout"> - <param name="ConversionPattern" value="%d %-5p - %m%n"/> - </layout> - </appender> - - <logger name="org.apache.ivory" additivity="false"> - <level value="debug"/> - <appender-ref ref="console" /> - </logger> - - <logger name="AUDIT"> - <level value="info"/> - <appender-ref ref="AUDIT" /> - </logger> - - <root> - <priority value ="info" /> - <appender-ref ref="console" /> - </root> - -</log4j:configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/log_01.txt ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/log_01.txt b/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/log_01.txt deleted file mode 100644 index 64434fb..0000000 --- a/falcon-regression/merlin/src/test/resources/OozieExampleInputData/normalInput/log_01.txt +++ /dev/null @@ -1,174 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -package com.prism; - -import java.lang.reflect.Method; -import java.util.ArrayList; - -import org.testng.Assert; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import com.core.Bundle; -import com.core.ColoHelper; -import com.core.PrismHelper; -import com.core.ServiceResponse; -import com.core.Util; -import com.core.Util.URLS; -import com.core.instanceUtil; -import com.core.xmlUtil; -import com.generated.feed.ActionType; -import com.generated.feed.ClusterType; -import com.generated.feed.TimezoneType; - -public class PrismFeedLateReplicationTest { - - - @BeforeMethod(alwaysRun=true) - public void testName(Method method) throws Exception - { - Util.print("test name: "+method.getName()); - //restart server as precaution - Util.restartService(ua1.getClusterHelper()); - Util.restartService(ua2.getClusterHelper()); - Util.restartService(ua3.getClusterHelper()); - - - } - - public PrismFeedLateReplicationTest() throws Exception{ - - } - - PrismHelper prismHelper=new PrismHelper("prism.properties"); - - ColoHelper ua1=new ColoHelper("mk-qa.config.properties"); - - ColoHelper ua2 = new ColoHelper("ivoryqa-1.config.properties"); - - ColoHelper ua3 = new ColoHelper("gs1001.config.properties"); - - @SuppressWarnings("deprecation") - @Test - public void multipleSourceOneTarget() throws Exception - { - - Bundle b1 = (Bundle)Util.readELBundles()[0][0]; - b1.generateUniqueBundle(); - Bundle b2 = (Bundle)Util.readELBundles()[0][0]; - b2.generateUniqueBundle(); - Bundle b3 = (Bundle)Util.readELBundles()[0][0]; - b3.generateUniqueBundle(); - - try{ - b1 = new Bundle(b1,ua1.getEnvFileName()); - b2 = new Bundle(b2,ua2.getEnvFileName()); - b3 = new Bundle(b3,ua3.getEnvFileName()); - - - b1.setInputFeedDataPath("/samarthRetention/input-data/rawLogs/oozieExample/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}/"); - - b1.setCLusterColo("ua1"); - Util.print("cluster b1: "+b1.getClusters().get(0)); - - ServiceResponse r = prismHelper.getClusterHelper().submitEntity(URLS.SUBMIT_URL,b1.getClusters().get(0)); - Assert.assertTrue(r.getMessage().contains("SUCCEEDED")); - - - b2.setCLusterColo("ua2"); - Util.print("cluster b2: "+b2.getClusters().get(0)); - r = prismHelper.getClusterHelper().submitEntity(URLS.SUBMIT_URL,b2.getClusters().get(0)); - Assert.assertTrue(r.getMessage().contains("SUCCEEDED")); - - - b3.setCLusterColo("ua3"); - Util.print("cluster b3: "+b3.getClusters().get(0)); - r = prismHelper.getClusterHelper().submitEntity(URLS.SUBMIT_URL,b3.getClusters().get(0)); - Assert.assertTrue(r.getMessage().contains("SUCCEEDED")); - - - String feed = b1.getDataSets().get(0); - feed = instanceUtil.setFeedCluster(feed,xmlUtil.createValidity("2009-02-01T00:00Z","2012-01-01T00:00Z",TimezoneType.UTC),xmlUtil.createRtention("hours(10)",ActionType.DELETE),null,ClusterType.SOURCE,null); - - String postFix = "/US/ua2" ; - String prefix = b1.getFeedDataPathPrefix(); - Util.HDFSCleanup(ua2,prefix.substring(1)); - Util.lateDataReplenish(ua2,90,0,1,prefix,postFix); - - - postFix = "/UK/ua3" ; - prefix = b1.getFeedDataPathPrefix(); - Util.HDFSCleanup(ua3,prefix.substring(1)); - Util.lateDataReplenish(ua3,90,0,1,prefix,postFix); - - String startTime = instanceUtil.getTimeWrtSystemTime(-30); - - feed = instanceUtil.setFeedCluster(feed,xmlUtil.createValidity(startTime,"2099-01-01T00:00Z",TimezoneType.UTC),xmlUtil.createRtention("hours(10)",ActionType.DELETE),Util.readClusterName(b2.getClusters().get(0)),ClusterType.SOURCE,"US/${cluster.colo}"); - feed = instanceUtil.setFeedCluster(feed,xmlUtil.createValidity(startTime,"2099-01-01T00:00Z",TimezoneType.UTC),xmlUtil.createRtention("hours(10)",ActionType.DELETE),Util.readClusterName(b1.getClusters().get(0)),ClusterType.TARGET,null); - feed = instanceUtil.setFeedCluster(feed,xmlUtil.createValidity(startTime,"2099-01-01T00:00Z",TimezoneType.UTC),xmlUtil.createRtention("hours(10)",ActionType.DELETE),Util.readClusterName(b3.getClusters().get(0)),ClusterType.SOURCE,"UK/${cluster.colo}"); - - - Util.print("feed: "+feed); - - r= prismHelper.getFeedHelper().submitAndSchedule(URLS.SUBMIT_AND_SCHEDULE_URL, feed); - Thread.sleep(10000); - - String TargetBundleID = instanceUtil.getLatestBundleID(Util.readDatasetName(feed),"FEED",ua1.getFeedHelper()); - - //wait till 1st instance of replication coord is SUCCEEDED - ArrayList<String> replicationCoordIDTarget = instanceUtil.getReplicationCoordID(TargetBundleID,ua1.getFeedHelper()); - - for(int i = 0 ; i < 30 ; i++) - { - if(instanceUtil.getInstanceStatusFromCoord(ua1,replicationCoordIDTarget.get(0),0).toString().equals("SUCCEEDED") && instanceUtil.getInstanceStatusFromCoord(ua1,replicationCoordIDTarget.get(1),0).toString().equals("SUCCEEDED")) - break; - Thread.sleep(20000); - } - - Thread.sleep(15000); - - ArrayList<String> inputFolderListForColo1= instanceUtil.getInputFoldersForInstanceForReplication(ua1,replicationCoordIDTarget.get(0),0); - ArrayList<String> inputFolderListForColo2= instanceUtil.getInputFoldersForInstanceForReplication(ua1,replicationCoordIDTarget.get(1),0); - - instanceUtil.putDataInFolders(ua2, inputFolderListForColo1); - instanceUtil.putDataInFolders(ua3, inputFolderListForColo2); - - - - Util.print("folder list 1: "+inputFolderListForColo1.toString()); - Util.print("folder list 2: "+inputFolderListForColo2.toString()); - - - - - } - - finally{ - prismHelper.getProcessHelper().delete(URLS.DELETE_URL,b1.getProcessData()); - prismHelper.getFeedHelper().delete(URLS.DELETE_URL, b1.getDataSets().get(0)); - prismHelper.getFeedHelper().delete(URLS.DELETE_URL, b1.getDataSets().get(1)); - prismHelper.getClusterHelper().delete(URLS.DELETE_URL,b1.getClusters().get(0)); - - } - } - - - - -} http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/ReplicationResources/cluster-0.1.xml ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/ReplicationResources/cluster-0.1.xml b/falcon-regression/merlin/src/test/resources/ReplicationResources/cluster-0.1.xml deleted file mode 100644 index 6478384..0000000 --- a/falcon-regression/merlin/src/test/resources/ReplicationResources/cluster-0.1.xml +++ /dev/null @@ -1,41 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> - -<cluster colo="ua1" description="" name="corp" xmlns="uri:falcon:cluster:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> - <interfaces> - <interface type="readonly" endpoint="http://host:50070"; - version="0.20.2" /> - <interface type="write" endpoint="hdfs://host:54310" - version="0.20.2" /> - <interface type="execute" endpoint="hdfs://host:54311" version="0.20.2" /> - <interface type="workflow" endpoint="http://host:11000/oozie/"; - version="3.1" /> - <interface type="messaging" endpoint="tcp://host:61616?daemon=true" - version="5.1.6" /> - </interfaces> - <locations> - <location name="staging" path="/projects/falcon/staging" /> - <location name="temp" path="/tmp" /> - <location name="working" path="/projects/falcon/working" /> - </locations> - <properties> - <property name="field1" value="value1" /> - <property name="field2" value="value2" /> - </properties> -</cluster> http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/ReplicationResources/feed-s4Replication.xml ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/ReplicationResources/feed-s4Replication.xml b/falcon-regression/merlin/src/test/resources/ReplicationResources/feed-s4Replication.xml deleted file mode 100644 index bbda0c5..0000000 --- a/falcon-regression/merlin/src/test/resources/ReplicationResources/feed-s4Replication.xml +++ /dev/null @@ -1,38 +0,0 @@ -<?xml version="1.0" encoding="UTF-8" standalone="yes"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> - -<feed name="iat-download-data-1" description="iat-download-data-1" xmlns="uri:falcon:feed:0.1"> - <availabilityFlag>_SUCCESS</availabilityFlag> - <frequency>hours(1)</frequency> - <timezone>UTC</timezone> - <clusters> - <cluster name="corp" type="source"> - <validity start="2012-12-26T05:00Z" end="2034-12-20T08:00Z"/> - <retention limit="months(36)" action="delete"/> - <locations> - <location type="data" path="s4://data/userplatform/${YEAR}/${MONTH}/${DAY}/${HOUR}"/> - </locations> - </cluster> - </clusters> - <locations> - <location type="data" path="/projects/userplatform/iatdownload/${YEAR}/${MONTH}/${DAY}/${HOUR}"/> - </locations> - <ACL owner="rmcuser" group="group" permission="0x755"/> - <schema location="/schema/clicks" provider="protobuf"/> -</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/ReplicationResources/id.pig ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/ReplicationResources/id.pig b/falcon-regression/merlin/src/test/resources/ReplicationResources/id.pig deleted file mode 100644 index 11f227e..0000000 --- a/falcon-regression/merlin/src/test/resources/ReplicationResources/id.pig +++ /dev/null @@ -1,20 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"); you may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- -A = load '$INPUT' using PigStorage(':'); -B = foreach A generate $0 as id; -store B into '$OUTPUT' USING PigStorage(); http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/ReplicationResources/log4testng.properties ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/ReplicationResources/log4testng.properties b/falcon-regression/merlin/src/test/resources/ReplicationResources/log4testng.properties deleted file mode 100644 index 34b4bde..0000000 --- a/falcon-regression/merlin/src/test/resources/ReplicationResources/log4testng.properties +++ /dev/null @@ -1,29 +0,0 @@ -## -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# log4testng will log its own behavior (generally used for debugging this package only). - log4testng.debug=false - - # Specifies the root Loggers logging level. Will log DEBUG level and above - log4testng.rootLogger=DEBUG - - # The org.testng.reporters.EmailableReporter Logger will log TRACE level and above - log4testng.logger.org.testng.reporters.EmailableReporter=TRACE - - # All Logger in packages below org.testng will log WARN level and above - log4testng.logger.org.testng=INFO - http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/ReplicationResources/process-agg.xml ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/ReplicationResources/process-agg.xml b/falcon-regression/merlin/src/test/resources/ReplicationResources/process-agg.xml deleted file mode 100644 index edfc1af..0000000 --- a/falcon-regression/merlin/src/test/resources/ReplicationResources/process-agg.xml +++ /dev/null @@ -1,52 +0,0 @@ -<?xml version="1.0" encoding="UTF-8" standalone="yes"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> - -<process name="agregator-coord16" xmlns="uri:falcon:process:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> - - <clusters> - <cluster name="corp"> - <validity end="2011-01-03T03:00Z" start="2010-01-02T01:00Z" /> - </cluster> - </clusters> - <parallel>1</parallel> - <order>FIFO</order> - <frequency>minutes(5)</frequency> - <timezone>UTC</timezone> - - <inputs> - <input end="now(0,0)" start="now(0,-20)" - feed="raaw-logs16" name="inputData"/> - </inputs> - <outputs> - <output instance="now(0,0)" feed="agregated-logs16" - name="outputData" /> - </outputs> - <properties> - <property name="queueName" value="default"/> - - <property name="fileTime" value="${formatTime(dateOffset(instanceTime(), 1, 'DAY'), 'yyyy-MMM-dd')}"/> - <property name="user" value="${user()}"/> - <property name="baseTime" value="${today(0,0)}"/> - - </properties> - <workflow path="/examples/apps/aggregator" /> - <retry policy="periodic" delay="minutes(3)" attempts="3" /> - -</process> - http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/feed-s4Replication.xml ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/feed-s4Replication.xml b/falcon-regression/merlin/src/test/resources/feed-s4Replication.xml deleted file mode 100644 index 719c684..0000000 --- a/falcon-regression/merlin/src/test/resources/feed-s4Replication.xml +++ /dev/null @@ -1,42 +0,0 @@ -<?xml version="1.0" encoding="UTF-8" standalone="yes"?> -<!--~ - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> - -<feed name="iat-download-data-1" description="iat-download-data-1" xmlns="uri:ivory:feed:0.1"> - <availabilityFlag>_SUCCESS</availabilityFlag> - <frequency>hours(1)</frequency> - <timezone>UTC</timezone> - <clusters> - <cluster name="ua2-ruby" type="source"> - <validity start="2012-12-26T05:00Z" end="2034-12-20T08:00Z"/> - <retention limit="months(36)" action="delete"/> - <locations> - <location type="data" path="s4://data/userplatform/${YEAR}/${MONTH}/${DAY}/${HOUR}"/> - </locations> - </cluster> - <cluster name="prod-global" type="target"> - <validity start="2012-12-24T10:00Z" end="2034-12-20T08:00Z"/> - <retention limit="days(4)" action="delete"/> - </cluster> - </clusters> - <locations> - <location type="data" path="/projects/userplatform/iatdownload/${YEAR}/${MONTH}/${DAY}/${HOUR}"/> - </locations> - <ACL owner="rmcuser" group="group" permission="0x755"/> - <schema location="/schema/clicks" provider="protobuf"/> -</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/1aac3a50/falcon-regression/merlin/src/test/resources/log_01.txt ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/log_01.txt b/falcon-regression/merlin/src/test/resources/log_01.txt deleted file mode 100644 index 29a116d..0000000 --- a/falcon-regression/merlin/src/test/resources/log_01.txt +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -[01] LOG!!!!!!
