FALCON-1899 Create examples artifact module in Falcon Author: peeyush b <[email protected]>
Reviewers: Balu <[email protected]>, Venkatesan <[email protected]>, Venkat Ranganathan <[email protected]> Closes #100 from peeyushb/FALCON-1899 and squashes the following commits: 0824204 [peeyush b] changed examples path in assembly-standalone.xml cdcb3cd [peeyush b] removed extra line 26bc4f8 [peeyush b] FALCON-1899 : Create examples artifact module in Falcon Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/2945fa80 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/2945fa80 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/2945fa80 Branch: refs/heads/master Commit: 2945fa80b9a46b003d64d2cf2dbe10232e4b4e55 Parents: b49056d Author: Peeyush Bishnoi <[email protected]> Authored: Sat Apr 23 10:23:44 2016 +0530 Committer: peeyush b <[email protected]> Committed: Sat Apr 23 10:23:44 2016 +0530 ---------------------------------------------------------------------- examples/app/hive/wordcount.hql | 19 ++++++ examples/app/oozie-mr/workflow.xml | 63 ++++++++++++++++++++ examples/app/pig/hcat-wordcount.pig | 23 +++++++ examples/app/pig/wordcount.pig | 23 +++++++ examples/data/generate.sh | 53 ++++++++++++++++ examples/data/hcat-generate.sh | 45 ++++++++++++++ examples/entity/filesystem/embedded-cluster.xml | 51 ++++++++++++++++ examples/entity/filesystem/in-feed.xml | 39 ++++++++++++ examples/entity/filesystem/oozie-mr-process.xml | 50 ++++++++++++++++ examples/entity/filesystem/out-feed.xml | 39 ++++++++++++ examples/entity/filesystem/pig-process.xml | 43 +++++++++++++ examples/entity/filesystem/replication-feed.xml | 46 ++++++++++++++ .../entity/filesystem/standalone-cluster.xml | 43 +++++++++++++ .../filesystem/standalone-target-cluster.xml | 43 +++++++++++++ examples/entity/hcat/hcat-in-feed.xml | 37 ++++++++++++ examples/entity/hcat/hcat-out-feed.xml | 35 +++++++++++ examples/entity/hcat/hcat-pig-process.xml | 43 +++++++++++++ examples/entity/hcat/hcat-replication-feed.xml | 42 +++++++++++++ .../entity/hcat/hcat-standalone-cluster.xml | 45 ++++++++++++++ .../hcat/hcat-standalone-target-cluster.xml | 45 ++++++++++++++ examples/entity/hcat/hive-process.xml | 43 +++++++++++++ examples/pom.xml | 47 +++++++++++++++ pom.xml | 1 + src/main/assemblies/assembly-standalone.xml | 4 +- src/main/assemblies/distributed-package.xml | 17 +++++- src/main/assemblies/standalone-package.xml | 16 ++++- src/main/examples/app/hive/wordcount.hql | 19 ------ src/main/examples/app/oozie-mr/workflow.xml | 63 -------------------- src/main/examples/app/pig/hcat-wordcount.pig | 23 ------- src/main/examples/app/pig/wordcount.pig | 23 ------- src/main/examples/data/generate.sh | 53 ---------------- src/main/examples/data/hcat-generate.sh | 45 -------------- .../entity/filesystem/embedded-cluster.xml | 51 ---------------- src/main/examples/entity/filesystem/in-feed.xml | 39 ------------ .../entity/filesystem/oozie-mr-process.xml | 50 ---------------- .../examples/entity/filesystem/out-feed.xml | 39 ------------ .../examples/entity/filesystem/pig-process.xml | 43 ------------- .../entity/filesystem/replication-feed.xml | 46 -------------- .../entity/filesystem/standalone-cluster.xml | 43 ------------- .../filesystem/standalone-target-cluster.xml | 43 ------------- src/main/examples/entity/hcat/hcat-in-feed.xml | 37 ------------ src/main/examples/entity/hcat/hcat-out-feed.xml | 35 ----------- .../examples/entity/hcat/hcat-pig-process.xml | 43 ------------- .../entity/hcat/hcat-replication-feed.xml | 42 ------------- .../entity/hcat/hcat-standalone-cluster.xml | 45 -------------- .../hcat/hcat-standalone-target-cluster.xml | 45 -------------- src/main/examples/entity/hcat/hive-process.xml | 43 ------------- 47 files changed, 949 insertions(+), 876 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/app/hive/wordcount.hql ---------------------------------------------------------------------- diff --git a/examples/app/hive/wordcount.hql b/examples/app/hive/wordcount.hql new file mode 100644 index 0000000..4c101c5 --- /dev/null +++ b/examples/app/hive/wordcount.hql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +FROM ${falcon_inparts_table} INSERT OVERWRITE TABLE ${falcon_outpart_table} PARTITION(${falcon_outpart_partitions}) SELECT word, SUM(cnt) as cnt WHERE ${falcon_inparts_filter} GROUP BY word; http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/app/oozie-mr/workflow.xml ---------------------------------------------------------------------- diff --git a/examples/app/oozie-mr/workflow.xml b/examples/app/oozie-mr/workflow.xml new file mode 100644 index 0000000..c715c91 --- /dev/null +++ b/examples/app/oozie-mr/workflow.xml @@ -0,0 +1,63 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<workflow-app xmlns="uri:oozie:workflow:0.2" name="map-reduce-wf"> + <start to="mr-node"/> + <action name="mr-node"> + <map-reduce> + <job-tracker>${jobTracker}</job-tracker> + <name-node>${nameNode}</name-node> + <prepare> + <delete path="${outpath}"/> + </prepare> + <configuration> + <property> + <name>mapred.job.queue.name</name> + <value>${queueName}</value> + </property> + <property> + <name>mapred.mapper.class</name> + <value>org.apache.hadoop.mapred.lib.IdentityMapper</value> + </property> + <property> + <name>mapred.reducer.class</name> + <value>org.apache.hadoop.mapred.lib.IdentityReducer</value> + </property> + <property> + <name>mapred.map.tasks</name> + <value>1</value> + </property> + <property> + <name>mapred.input.dir</name> + <value>${inpaths}</value> + </property> + <property> + <name>mapred.output.dir</name> + <value>${outpath}</value> + </property> + </configuration> + </map-reduce> + <ok to="end"/> + <error to="fail"/> + </action> + <kill name="fail"> + <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message> + </kill> + <end name="end"/> +</workflow-app> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/app/pig/hcat-wordcount.pig ---------------------------------------------------------------------- diff --git a/examples/app/pig/hcat-wordcount.pig b/examples/app/pig/hcat-wordcount.pig new file mode 100644 index 0000000..3df93c0 --- /dev/null +++ b/examples/app/pig/hcat-wordcount.pig @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +indata = LOAD '$falcon_inparts_table' USING org.apache.hive.hcatalog.pig.HCatLoader(); +filterdata = FILTER indata BY $falcon_inparts_filter; +grpdata = GROUP filterdata BY (word); +finaldata = FOREACH grpdata GENERATE FLATTEN(group) as word, (int)SUM(filterdata.cnt) as cnt; +STORE finaldata INTO '$falcon_outpart_table' USING org.apache.hive.hcatalog.pig.HCatStorer('$falcon_outpart_partitions'); http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/app/pig/wordcount.pig ---------------------------------------------------------------------- diff --git a/examples/app/pig/wordcount.pig b/examples/app/pig/wordcount.pig new file mode 100644 index 0000000..190d2d4 --- /dev/null +++ b/examples/app/pig/wordcount.pig @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +indata = LOAD '$inpaths' USING PigStorage() AS (words:chararray); +tokdata = FOREACH indata GENERATE FLATTEN(TOKENIZE(words)) as word; +grpdata = GROUP tokdata BY (word); +finaldata = FOREACH grpdata GENERATE FLATTEN(group), COUNT(tokdata); +STORE finaldata INTO '$outpath' USING PigStorage('\t'); http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/data/generate.sh ---------------------------------------------------------------------- diff --git a/examples/data/generate.sh b/examples/data/generate.sh new file mode 100644 index 0000000..54db3d7 --- /dev/null +++ b/examples/data/generate.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +PRG="${0}" + +while [ -h "${PRG}" ]; do + ls=`ls -ld "${PRG}"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "${PRG}"`/"$link" + fi +done + +BASEDIR=`dirname ${PRG}` +BASEDIR=`cd ${BASEDIR};pwd` + +rm -rf generated-data +YEAR=`date +%Y` +MONTH=`date +m` +DAY=`date +%d` +HOUR=`date +%H` + +DELIM='\t' +input=(first second third fourth fifth) +for MINUTE in `seq -w 00 59` +do + mkdir -p generated-data/00/$MINUTE/ + word=${input[$RANDOM % 5]} + cnt=`expr $RANDOM % 10` + echo -e "$word$DELIM$cnt" > generated-data/00/$MINUTE/data +done + +hadoop fs -rmr /data/in/2013/11/15/ +hadoop fs -mkdir -p /data/in/2013/11/15/ +hadoop fs -put generated-data/00 /data/in/2013/11/15/ +rm -rf generated-data http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/data/hcat-generate.sh ---------------------------------------------------------------------- diff --git a/examples/data/hcat-generate.sh b/examples/data/hcat-generate.sh new file mode 100644 index 0000000..5c4b40a --- /dev/null +++ b/examples/data/hcat-generate.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +PRG="${0}" + +while [ -h "${PRG}" ]; do + ls=`ls -ld "${PRG}"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "${PRG}"`/"$link" + fi +done + +BASEDIR=`dirname ${PRG}` +BASEDIR=`cd ${BASEDIR};pwd` + +${BASEDIR}/generate.sh + +hcat -e "DROP TABLE IF EXISTS in_table" +hcat -e "DROP TABLE IF EXISTS repl_in_table" +hcat -e "DROP TABLE IF EXISTS out_table" +hcat -e "CREATE TABLE in_table (word STRING, cnt INT) PARTITIONED BY (ds STRING);" +hcat -e "CREATE TABLE repl_in_table (word STRING, cnt INT) PARTITIONED BY (ds STRING);" +hcat -e "CREATE TABLE out_table (word STRING, cnt INT) PARTITIONED BY (ds STRING);" +for MINUTE in `seq -w 00 59` +do + hcat -e "ALTER TABLE in_table ADD PARTITION (ds='2013-11-15-00-$MINUTE') LOCATION '/data/in/2013/11/15/00/$MINUTE';" +done http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/filesystem/embedded-cluster.xml ---------------------------------------------------------------------- diff --git a/examples/entity/filesystem/embedded-cluster.xml b/examples/entity/filesystem/embedded-cluster.xml new file mode 100644 index 0000000..c505066 --- /dev/null +++ b/examples/entity/filesystem/embedded-cluster.xml @@ -0,0 +1,51 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<!-- + Example cluster definition that can be used with the embedded + version of the falcon system started through mvn jetty:run + + How to bring up the test falcon instance: + + export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=256m" && mvn clean install + rm -rf target/store/* + rm -rf webapp/target/webapps/oozie/data/* + cd webapp + mvn jetty:run +--> +<cluster colo="local" description="" name="local" xmlns="uri:falcon:cluster:0.1"> + <interfaces> + <interface type="readonly" endpoint="hftp://localhost:41110"; version="1.1.2"/> + + <interface type="write" endpoint="hdfs://localhost:41020" version="1.1.2"/> + + <interface type="execute" endpoint="localhost:41021" version="1.1.2"/> + + <interface type="workflow" endpoint="http://localhost:41000/oozie/"; version="4.0.0"/> + + <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.4.3"/> + </interfaces> + <locations> + <location name="staging" path="/projects/falcon/staging"/> <!--mandatory--> + <location name="temp" path="/projects/falcon/tmp"/> <!--optional--> + <location name="working" path="/projects/falcon/working"/> <!--optional--> + </locations> + <properties> + </properties> +</cluster> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/filesystem/in-feed.xml ---------------------------------------------------------------------- diff --git a/examples/entity/filesystem/in-feed.xml b/examples/entity/filesystem/in-feed.xml new file mode 100644 index 0000000..798de9c --- /dev/null +++ b/examples/entity/filesystem/in-feed.xml @@ -0,0 +1,39 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> +<feed description="input" name="in" xmlns="uri:falcon:feed:0.1"> + <groups>input</groups> + + <frequency>minutes(1)</frequency> + <timezone>UTC</timezone> + <late-arrival cut-off="hours(1)"/> + + <clusters> + <cluster name="local"> + <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/> + <retention limit="hours(2)" action="delete"/> + </cluster> + </clusters> + + <locations> + <location type="data" path="/data/in/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}"/> + </locations> + + <ACL owner="testuser-ut-user" group="group" permission="0x644"/> + <schema location="/schema/log/log.format.csv" provider="csv"/> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/filesystem/oozie-mr-process.xml ---------------------------------------------------------------------- diff --git a/examples/entity/filesystem/oozie-mr-process.xml b/examples/entity/filesystem/oozie-mr-process.xml new file mode 100644 index 0000000..c080436 --- /dev/null +++ b/examples/entity/filesystem/oozie-mr-process.xml @@ -0,0 +1,50 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<process name="oozie-mr-process" xmlns="uri:falcon:process:0.1"> + <clusters> + <cluster name="local"> + <validity start="2013-11-15T00:05Z" end="2013-11-15T01:05Z"/> + </cluster> + </clusters> + + <parallel>1</parallel> + <order>FIFO</order> + <frequency>minutes(5)</frequency> + <timezone>UTC</timezone> + + <inputs> + <!-- In the workflow, the input paths will be available in a variable 'inpaths' --> + <input name="inpaths" feed="in" start="now(0,-5)" end="now(0,-1)"/> + </inputs> + + <outputs> + <!-- In the workflow, the output path will be available in a variable 'outpath' --> + <output name="outpath" feed="out" instance="now(0,0)"/> + </outputs> + + <properties> + <!-- In the workflow, these properties will be available with variable - key --> + <property name="queueName" value="default"/> + <!-- The schedule time available as a property in workflow --> + <property name="time" value="${instanceTime()}"/> + </properties> + + <workflow engine="oozie" path="/app/oozie-mr"/> +</process> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/filesystem/out-feed.xml ---------------------------------------------------------------------- diff --git a/examples/entity/filesystem/out-feed.xml b/examples/entity/filesystem/out-feed.xml new file mode 100644 index 0000000..f537bbc --- /dev/null +++ b/examples/entity/filesystem/out-feed.xml @@ -0,0 +1,39 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> +<feed description="output" name="out" xmlns="uri:falcon:feed:0.1"> + <groups>output</groups> + + <frequency>minutes(5)</frequency> + <timezone>UTC</timezone> + <late-arrival cut-off="hours(1)"/> + + <clusters> + <cluster name="local"> + <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/> + <retention limit="hours(2)" action="delete"/> + </cluster> + </clusters> + + <locations> + <location type="data" path="/data/out/${YEAR}/${MONTH}/${DAY}/${HOUR}"/> + </locations> + + <ACL owner="testuser-ut-user" group="group" permission="0x644"/> + <schema location="/schema/out/out.format.csv" provider="csv"/> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/filesystem/pig-process.xml ---------------------------------------------------------------------- diff --git a/examples/entity/filesystem/pig-process.xml b/examples/entity/filesystem/pig-process.xml new file mode 100644 index 0000000..99e03bd --- /dev/null +++ b/examples/entity/filesystem/pig-process.xml @@ -0,0 +1,43 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<process name="pig-process" xmlns="uri:falcon:process:0.1"> + <clusters> + <cluster name="local"> + <validity start="2013-11-15T00:05Z" end="2013-11-15T01:05Z"/> + </cluster> + </clusters> + + <parallel>1</parallel> + <order>FIFO</order> + <frequency>minutes(5)</frequency> + <timezone>UTC</timezone> + + <inputs> + <!-- In the pig script, the input paths will be available in a variable 'inpaths' --> + <input name="inpaths" feed="in" start="now(0,-5)" end="now(0,-1)"/> + </inputs> + + <outputs> + <!-- In the pig script, the output path will be available in a variable 'outpath' --> + <output name="outpath" feed="out" instance="now(0,0)"/> + </outputs> + + <workflow engine="pig" path="/app/pig/wordcount.pig"/> +</process> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/filesystem/replication-feed.xml ---------------------------------------------------------------------- diff --git a/examples/entity/filesystem/replication-feed.xml b/examples/entity/filesystem/replication-feed.xml new file mode 100644 index 0000000..7387d45 --- /dev/null +++ b/examples/entity/filesystem/replication-feed.xml @@ -0,0 +1,46 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> +<feed description="replication feed" name="repl-feed" xmlns="uri:falcon:feed:0.1"> + <groups>input</groups> + + <frequency>minutes(1)</frequency> + <timezone>UTC</timezone> + <late-arrival cut-off="hours(1)"/> + + <clusters> + <cluster name="local" type="source"> + <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/> + <retention limit="hours(2)" action="delete"/> + </cluster> + <cluster name="local-target" type="target"> + <validity start="2013-11-15T00:00Z" end="2030-01-01T00:00Z"/> + <retention limit="hours(2)" action="delete"/> + <locations> + <location type="data" path="/data/repl-in/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}"/> + </locations> + </cluster> + </clusters> + + <locations> + <location type="data" path="/data/in/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}"/> + </locations> + + <ACL owner="testuser-ut-user" group="group" permission="0x644"/> + <schema location="/schema/log/log.format.csv" provider="csv"/> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/filesystem/standalone-cluster.xml ---------------------------------------------------------------------- diff --git a/examples/entity/filesystem/standalone-cluster.xml b/examples/entity/filesystem/standalone-cluster.xml new file mode 100644 index 0000000..4f8a5fc --- /dev/null +++ b/examples/entity/filesystem/standalone-cluster.xml @@ -0,0 +1,43 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<!-- + Example cluster definition that can be used with falcon and standalone + hadoop and oozie instances +--> +<cluster colo="local" description="Standalone cluster" name="local" xmlns="uri:falcon:cluster:0.1"> + <interfaces> + <interface type="readonly" endpoint="hdfs://localhost:8020" version="1.1.2"/> + + <interface type="write" endpoint="hdfs://localhost:8020" version="1.1.2"/> + + <interface type="execute" endpoint="localhost:8021" version="1.1.2"/> + + <interface type="workflow" endpoint="http://localhost:11000/oozie/"; version="4.0.0"/> + + <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.4.3"/> + </interfaces> + <locations> + <location name="staging" path="/projects/falcon/staging"/> <!--mandatory--> + <location name="temp" path="/projects/falcon/tmp"/> <!--optional--> + <location name="working" path="/projects/falcon/working"/> <!--optional--> + </locations> + <properties> + </properties> +</cluster> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/filesystem/standalone-target-cluster.xml ---------------------------------------------------------------------- diff --git a/examples/entity/filesystem/standalone-target-cluster.xml b/examples/entity/filesystem/standalone-target-cluster.xml new file mode 100644 index 0000000..cb5c4c6 --- /dev/null +++ b/examples/entity/filesystem/standalone-target-cluster.xml @@ -0,0 +1,43 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<!-- + Example cluster definition that can be used with falcon and standalone + hadoop and oozie instances +--> +<cluster colo="local" description="Standalone cluster" name="local-target" xmlns="uri:falcon:cluster:0.1"> + <interfaces> + <interface type="readonly" endpoint="hdfs://localhost:8020" version="1.1.2"/> + + <interface type="write" endpoint="hdfs://localhost:8020" version="1.1.2"/> + + <interface type="execute" endpoint="localhost:8021" version="1.1.2"/> + + <interface type="workflow" endpoint="http://localhost:11000/oozie/"; version="4.0.0"/> + + <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.4.3"/> + </interfaces> + <locations> + <location name="staging" path="/projects/falcon/staging-target"/> <!--mandatory--> + <location name="temp" path="/projects/falcon/tmp-target"/> <!--optional--> + <location name="working" path="/projects/falcon/working-target"/> <!--optional--> + </locations> + <properties> + </properties> +</cluster> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/hcat/hcat-in-feed.xml ---------------------------------------------------------------------- diff --git a/examples/entity/hcat/hcat-in-feed.xml b/examples/entity/hcat/hcat-in-feed.xml new file mode 100644 index 0000000..f978e8c --- /dev/null +++ b/examples/entity/hcat/hcat-in-feed.xml @@ -0,0 +1,37 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> +<feed description="input" name="hcat-in" xmlns="uri:falcon:feed:0.1"> + <groups>input</groups> + + <frequency>minutes(1)</frequency> + <timezone>UTC</timezone> + <late-arrival cut-off="hours(1)"/> + + <clusters> + <cluster name="hcat-local"> + <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/> + <retention limit="hours(2)" action="delete"/> + </cluster> + </clusters> + + <table uri="catalog:default:in_table#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}" /> + + <ACL owner="testuser-ut-user" group="group" permission="0x644"/> + <schema location="/schema/log/log.format.csv" provider="csv"/> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/hcat/hcat-out-feed.xml ---------------------------------------------------------------------- diff --git a/examples/entity/hcat/hcat-out-feed.xml b/examples/entity/hcat/hcat-out-feed.xml new file mode 100644 index 0000000..23b22b9 --- /dev/null +++ b/examples/entity/hcat/hcat-out-feed.xml @@ -0,0 +1,35 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> +<feed description="output" name="hcat-out" xmlns="uri:falcon:feed:0.1"> + <frequency>minutes(5)</frequency> + <timezone>UTC</timezone> + <late-arrival cut-off="hours(1)"/> + + <clusters> + <cluster name="hcat-local"> + <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/> + <retention limit="hours(2)" action="delete"/> + </cluster> + </clusters> + + <table uri="catalog:default:out_table#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}" /> + + <ACL owner="testuser-ut-user" group="group" permission="0x644"/> + <schema location="/schema/out/out.format.csv" provider="csv"/> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/hcat/hcat-pig-process.xml ---------------------------------------------------------------------- diff --git a/examples/entity/hcat/hcat-pig-process.xml b/examples/entity/hcat/hcat-pig-process.xml new file mode 100644 index 0000000..60836c4 --- /dev/null +++ b/examples/entity/hcat/hcat-pig-process.xml @@ -0,0 +1,43 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<process name="hcat-pig-process" xmlns="uri:falcon:process:0.1"> + <clusters> + <cluster name="hcat-local"> + <validity start="2013-11-15T00:05Z" end="2013-11-15T01:05Z"/> + </cluster> + </clusters> + + <parallel>1</parallel> + <order>FIFO</order> + <frequency>minutes(5)</frequency> + <timezone>UTC</timezone> + + <inputs> + <!-- In the pig script, the input paths will be available in a variable 'inparts' --> + <input name="inparts" feed="hcat-in" start="now(0,-5)" end="now(0,-1)"/> + </inputs> + + <outputs> + <!-- In the pig script, the output path will be available in a variable 'outparts' --> + <output name="outpart" feed="hcat-out" instance="now(0,0)"/> + </outputs> + + <workflow engine="pig" path="/app/pig/hcat-wordcount.pig"/> +</process> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/hcat/hcat-replication-feed.xml ---------------------------------------------------------------------- diff --git a/examples/entity/hcat/hcat-replication-feed.xml b/examples/entity/hcat/hcat-replication-feed.xml new file mode 100644 index 0000000..2b8f026 --- /dev/null +++ b/examples/entity/hcat/hcat-replication-feed.xml @@ -0,0 +1,42 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> +<feed description="input" name="hcat-repl-feed" xmlns="uri:falcon:feed:0.1"> + <groups>input</groups> + + <frequency>minutes(1)</frequency> + <timezone>UTC</timezone> + <late-arrival cut-off="hours(1)"/> + + <clusters> + <cluster name="hcat-local" type="source"> + <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/> + <retention limit="hours(2)" action="delete"/> + </cluster> + <cluster name="hcat-local-target" type="target"> + <validity start="2013-11-15T00:00Z" end="2030-01-01T00:00Z"/> + <retention limit="hours(2)" action="delete"/> + <table uri="catalog:default:repl_in_table#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}" /> + </cluster> + </clusters> + + <table uri="catalog:default:in_table#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}" /> + + <ACL owner="testuser-ut-user" group="group" permission="0x644"/> + <schema location="/schema/log/log.format.csv" provider="csv"/> +</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/hcat/hcat-standalone-cluster.xml ---------------------------------------------------------------------- diff --git a/examples/entity/hcat/hcat-standalone-cluster.xml b/examples/entity/hcat/hcat-standalone-cluster.xml new file mode 100644 index 0000000..7962b1f --- /dev/null +++ b/examples/entity/hcat/hcat-standalone-cluster.xml @@ -0,0 +1,45 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<!-- + Example cluster definition that can be used with falcon and standalone + hadoop and oozie instances +--> +<cluster colo="local" description="Standalone cluster with hacatalog" name="hcat-local" xmlns="uri:falcon:cluster:0.1"> + <interfaces> + <interface type="readonly" endpoint="hftp://localhost:50010"; version="1.1.2"/> + + <interface type="write" endpoint="hdfs://localhost:8020" version="1.1.2"/> + + <interface type="execute" endpoint="localhost:8021" version="1.1.2"/> + + <interface type="workflow" endpoint="http://localhost:11000/oozie/"; version="4.0.0"/> + + <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.4.3"/> + + <interface type="registry" endpoint="thrift://localhost:12000" version="0.11.0"/> + </interfaces> + <locations> + <location name="staging" path="/projects/falcon/hcat-staging"/> + <location name="temp" path="/projects/falcon/hcat-tmp"/> + <location name="working" path="/projects/falcon/hcat-working"/> + </locations> + <properties> + </properties> +</cluster> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/hcat/hcat-standalone-target-cluster.xml ---------------------------------------------------------------------- diff --git a/examples/entity/hcat/hcat-standalone-target-cluster.xml b/examples/entity/hcat/hcat-standalone-target-cluster.xml new file mode 100644 index 0000000..8de78cf --- /dev/null +++ b/examples/entity/hcat/hcat-standalone-target-cluster.xml @@ -0,0 +1,45 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<!-- + Example cluster definition that can be used with falcon and standalone + hadoop and oozie instances +--> +<cluster colo="hcat-local-target" description="Standalone cluster with hacatalog" name="hcat-local-target" xmlns="uri:falcon:cluster:0.1"> + <interfaces> + <interface type="readonly" endpoint="hftp://localhost:50010"; version="1.1.2"/> + + <interface type="write" endpoint="hdfs://localhost:8020" version="1.1.2"/> + + <interface type="execute" endpoint="localhost:8021" version="1.1.2"/> + + <interface type="workflow" endpoint="http://localhost:11000/oozie/"; version="4.0.0"/> + + <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.4.3"/> + + <interface type="registry" endpoint="thrift://localhost:12000" version="0.11.0"/> + </interfaces> + <locations> + <location name="staging" path="/projects/falcon/hcat-staging-target"/> + <location name="temp" path="/projects/falcon/hcat-tmp-target"/> + <location name="working" path="/projects/falcon/hcat-working-target"/> + </locations> + <properties> + </properties> +</cluster> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/entity/hcat/hive-process.xml ---------------------------------------------------------------------- diff --git a/examples/entity/hcat/hive-process.xml b/examples/entity/hcat/hive-process.xml new file mode 100644 index 0000000..ad8b219 --- /dev/null +++ b/examples/entity/hcat/hive-process.xml @@ -0,0 +1,43 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<process name="hive-process" xmlns="uri:falcon:process:0.1"> + <clusters> + <cluster name="hcat-local"> + <validity start="2013-11-15T00:05Z" end="2013-11-15T01:05Z"/> + </cluster> + </clusters> + + <parallel>1</parallel> + <order>FIFO</order> + <frequency>minutes(5)</frequency> + <timezone>UTC</timezone> + + <inputs> + <!-- In the pig script, the input paths will be available in a variable 'inparts' --> + <input name="inparts" feed="hcat-in" start="now(0,-5)" end="now(0,-1)"/> + </inputs> + + <outputs> + <!-- In the pig script, the output path will be available in a variable 'outpart' --> + <output name="outpart" feed="hcat-out" instance="now(0,0)"/> + </outputs> + + <workflow engine="hive" path="/app/hive/wordcount.hql"/> +</process> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/examples/pom.xml ---------------------------------------------------------------------- diff --git a/examples/pom.xml b/examples/pom.xml new file mode 100644 index 0000000..f8f606c --- /dev/null +++ b/examples/pom.xml @@ -0,0 +1,47 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.falcon</groupId> + <artifactId>falcon-main</artifactId> + <version>0.10-SNAPSHOT</version> + </parent> + <artifactId>falcon-examples</artifactId> + <description>Apache Falcon Examples</description> + <name>Apache Falcon Examples</name> + + <profiles> + <profile> + <id>hadoop-2</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + </dependency> + </dependencies> + </profile> + </profiles> +</project> + http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 8f4561c..a72889e 100644 --- a/pom.xml +++ b/pom.xml @@ -469,6 +469,7 @@ <module>prism</module> <module>unit</module> <module>lifecycle</module> + <module>examples</module> <module>webapp</module> <module>docs</module> <module>distro</module> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/assemblies/assembly-standalone.xml ---------------------------------------------------------------------- diff --git a/src/main/assemblies/assembly-standalone.xml b/src/main/assemblies/assembly-standalone.xml index d3111b7..b9d705e 100644 --- a/src/main/assemblies/assembly-standalone.xml +++ b/src/main/assemblies/assembly-standalone.xml @@ -99,7 +99,7 @@ </fileSet> <fileSet> - <directory>src/main/examples</directory> + <directory>examples</directory> <outputDirectory>examples</outputDirectory> </fileSet> @@ -292,4 +292,4 @@ <fileMode>0755</fileMode> </file> </files> -</assembly> \ No newline at end of file +</assembly> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/assemblies/distributed-package.xml ---------------------------------------------------------------------- diff --git a/src/main/assemblies/distributed-package.xml b/src/main/assemblies/distributed-package.xml index eb45c6f..a5171c4 100644 --- a/src/main/assemblies/distributed-package.xml +++ b/src/main/assemblies/distributed-package.xml @@ -218,10 +218,23 @@ </fileSet> <fileSet> - <directory>../src/main/examples</directory> - <outputDirectory>examples</outputDirectory> + <directory>../examples/app</directory> + <outputDirectory>examples/app</outputDirectory> <directoryMode>0755</directoryMode> </fileSet> + + <fileSet> + <directory>../examples/data</directory> + <outputDirectory>examples/data</outputDirectory> + <directoryMode>0755</directoryMode> + </fileSet> + + <fileSet> + <directory>../examples/entity</directory> + <outputDirectory>examples/entity</outputDirectory> + <directoryMode>0755</directoryMode> + </fileSet> + </fileSets> <files> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/assemblies/standalone-package.xml ---------------------------------------------------------------------- diff --git a/src/main/assemblies/standalone-package.xml b/src/main/assemblies/standalone-package.xml index 0b5c69a..b5db01c 100644 --- a/src/main/assemblies/standalone-package.xml +++ b/src/main/assemblies/standalone-package.xml @@ -115,8 +115,20 @@ </fileSet> <fileSet> - <directory>../src/main/examples</directory> - <outputDirectory>examples</outputDirectory> + <directory>../examples/app</directory> + <outputDirectory>examples/app</outputDirectory> + <directoryMode>0755</directoryMode> + </fileSet> + + <fileSet> + <directory>../examples/data</directory> + <outputDirectory>examples/data</outputDirectory> + <directoryMode>0755</directoryMode> + </fileSet> + + <fileSet> + <directory>../examples/entity</directory> + <outputDirectory>examples/entity</outputDirectory> <directoryMode>0755</directoryMode> </fileSet> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/app/hive/wordcount.hql ---------------------------------------------------------------------- diff --git a/src/main/examples/app/hive/wordcount.hql b/src/main/examples/app/hive/wordcount.hql deleted file mode 100644 index 4c101c5..0000000 --- a/src/main/examples/app/hive/wordcount.hql +++ /dev/null @@ -1,19 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"); you may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -FROM ${falcon_inparts_table} INSERT OVERWRITE TABLE ${falcon_outpart_table} PARTITION(${falcon_outpart_partitions}) SELECT word, SUM(cnt) as cnt WHERE ${falcon_inparts_filter} GROUP BY word; http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/app/oozie-mr/workflow.xml ---------------------------------------------------------------------- diff --git a/src/main/examples/app/oozie-mr/workflow.xml b/src/main/examples/app/oozie-mr/workflow.xml deleted file mode 100644 index c715c91..0000000 --- a/src/main/examples/app/oozie-mr/workflow.xml +++ /dev/null @@ -1,63 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> - -<workflow-app xmlns="uri:oozie:workflow:0.2" name="map-reduce-wf"> - <start to="mr-node"/> - <action name="mr-node"> - <map-reduce> - <job-tracker>${jobTracker}</job-tracker> - <name-node>${nameNode}</name-node> - <prepare> - <delete path="${outpath}"/> - </prepare> - <configuration> - <property> - <name>mapred.job.queue.name</name> - <value>${queueName}</value> - </property> - <property> - <name>mapred.mapper.class</name> - <value>org.apache.hadoop.mapred.lib.IdentityMapper</value> - </property> - <property> - <name>mapred.reducer.class</name> - <value>org.apache.hadoop.mapred.lib.IdentityReducer</value> - </property> - <property> - <name>mapred.map.tasks</name> - <value>1</value> - </property> - <property> - <name>mapred.input.dir</name> - <value>${inpaths}</value> - </property> - <property> - <name>mapred.output.dir</name> - <value>${outpath}</value> - </property> - </configuration> - </map-reduce> - <ok to="end"/> - <error to="fail"/> - </action> - <kill name="fail"> - <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message> - </kill> - <end name="end"/> -</workflow-app> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/app/pig/hcat-wordcount.pig ---------------------------------------------------------------------- diff --git a/src/main/examples/app/pig/hcat-wordcount.pig b/src/main/examples/app/pig/hcat-wordcount.pig deleted file mode 100644 index 3df93c0..0000000 --- a/src/main/examples/app/pig/hcat-wordcount.pig +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -indata = LOAD '$falcon_inparts_table' USING org.apache.hive.hcatalog.pig.HCatLoader(); -filterdata = FILTER indata BY $falcon_inparts_filter; -grpdata = GROUP filterdata BY (word); -finaldata = FOREACH grpdata GENERATE FLATTEN(group) as word, (int)SUM(filterdata.cnt) as cnt; -STORE finaldata INTO '$falcon_outpart_table' USING org.apache.hive.hcatalog.pig.HCatStorer('$falcon_outpart_partitions'); http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/app/pig/wordcount.pig ---------------------------------------------------------------------- diff --git a/src/main/examples/app/pig/wordcount.pig b/src/main/examples/app/pig/wordcount.pig deleted file mode 100644 index 190d2d4..0000000 --- a/src/main/examples/app/pig/wordcount.pig +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -indata = LOAD '$inpaths' USING PigStorage() AS (words:chararray); -tokdata = FOREACH indata GENERATE FLATTEN(TOKENIZE(words)) as word; -grpdata = GROUP tokdata BY (word); -finaldata = FOREACH grpdata GENERATE FLATTEN(group), COUNT(tokdata); -STORE finaldata INTO '$outpath' USING PigStorage('\t'); http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/data/generate.sh ---------------------------------------------------------------------- diff --git a/src/main/examples/data/generate.sh b/src/main/examples/data/generate.sh deleted file mode 100755 index 54db3d7..0000000 --- a/src/main/examples/data/generate.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -PRG="${0}" - -while [ -h "${PRG}" ]; do - ls=`ls -ld "${PRG}"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "${PRG}"`/"$link" - fi -done - -BASEDIR=`dirname ${PRG}` -BASEDIR=`cd ${BASEDIR};pwd` - -rm -rf generated-data -YEAR=`date +%Y` -MONTH=`date +m` -DAY=`date +%d` -HOUR=`date +%H` - -DELIM='\t' -input=(first second third fourth fifth) -for MINUTE in `seq -w 00 59` -do - mkdir -p generated-data/00/$MINUTE/ - word=${input[$RANDOM % 5]} - cnt=`expr $RANDOM % 10` - echo -e "$word$DELIM$cnt" > generated-data/00/$MINUTE/data -done - -hadoop fs -rmr /data/in/2013/11/15/ -hadoop fs -mkdir -p /data/in/2013/11/15/ -hadoop fs -put generated-data/00 /data/in/2013/11/15/ -rm -rf generated-data http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/data/hcat-generate.sh ---------------------------------------------------------------------- diff --git a/src/main/examples/data/hcat-generate.sh b/src/main/examples/data/hcat-generate.sh deleted file mode 100644 index 5c4b40a..0000000 --- a/src/main/examples/data/hcat-generate.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/sh - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -PRG="${0}" - -while [ -h "${PRG}" ]; do - ls=`ls -ld "${PRG}"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "${PRG}"`/"$link" - fi -done - -BASEDIR=`dirname ${PRG}` -BASEDIR=`cd ${BASEDIR};pwd` - -${BASEDIR}/generate.sh - -hcat -e "DROP TABLE IF EXISTS in_table" -hcat -e "DROP TABLE IF EXISTS repl_in_table" -hcat -e "DROP TABLE IF EXISTS out_table" -hcat -e "CREATE TABLE in_table (word STRING, cnt INT) PARTITIONED BY (ds STRING);" -hcat -e "CREATE TABLE repl_in_table (word STRING, cnt INT) PARTITIONED BY (ds STRING);" -hcat -e "CREATE TABLE out_table (word STRING, cnt INT) PARTITIONED BY (ds STRING);" -for MINUTE in `seq -w 00 59` -do - hcat -e "ALTER TABLE in_table ADD PARTITION (ds='2013-11-15-00-$MINUTE') LOCATION '/data/in/2013/11/15/00/$MINUTE';" -done http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/filesystem/embedded-cluster.xml ---------------------------------------------------------------------- diff --git a/src/main/examples/entity/filesystem/embedded-cluster.xml b/src/main/examples/entity/filesystem/embedded-cluster.xml deleted file mode 100644 index c505066..0000000 --- a/src/main/examples/entity/filesystem/embedded-cluster.xml +++ /dev/null @@ -1,51 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> - -<!-- - Example cluster definition that can be used with the embedded - version of the falcon system started through mvn jetty:run - - How to bring up the test falcon instance: - - export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=256m" && mvn clean install - rm -rf target/store/* - rm -rf webapp/target/webapps/oozie/data/* - cd webapp - mvn jetty:run ---> -<cluster colo="local" description="" name="local" xmlns="uri:falcon:cluster:0.1"> - <interfaces> - <interface type="readonly" endpoint="hftp://localhost:41110"; version="1.1.2"/> - - <interface type="write" endpoint="hdfs://localhost:41020" version="1.1.2"/> - - <interface type="execute" endpoint="localhost:41021" version="1.1.2"/> - - <interface type="workflow" endpoint="http://localhost:41000/oozie/"; version="4.0.0"/> - - <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.4.3"/> - </interfaces> - <locations> - <location name="staging" path="/projects/falcon/staging"/> <!--mandatory--> - <location name="temp" path="/projects/falcon/tmp"/> <!--optional--> - <location name="working" path="/projects/falcon/working"/> <!--optional--> - </locations> - <properties> - </properties> -</cluster> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/filesystem/in-feed.xml ---------------------------------------------------------------------- diff --git a/src/main/examples/entity/filesystem/in-feed.xml b/src/main/examples/entity/filesystem/in-feed.xml deleted file mode 100644 index 798de9c..0000000 --- a/src/main/examples/entity/filesystem/in-feed.xml +++ /dev/null @@ -1,39 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> -<feed description="input" name="in" xmlns="uri:falcon:feed:0.1"> - <groups>input</groups> - - <frequency>minutes(1)</frequency> - <timezone>UTC</timezone> - <late-arrival cut-off="hours(1)"/> - - <clusters> - <cluster name="local"> - <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/> - <retention limit="hours(2)" action="delete"/> - </cluster> - </clusters> - - <locations> - <location type="data" path="/data/in/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}"/> - </locations> - - <ACL owner="testuser-ut-user" group="group" permission="0x644"/> - <schema location="/schema/log/log.format.csv" provider="csv"/> -</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/filesystem/oozie-mr-process.xml ---------------------------------------------------------------------- diff --git a/src/main/examples/entity/filesystem/oozie-mr-process.xml b/src/main/examples/entity/filesystem/oozie-mr-process.xml deleted file mode 100644 index c080436..0000000 --- a/src/main/examples/entity/filesystem/oozie-mr-process.xml +++ /dev/null @@ -1,50 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> - -<process name="oozie-mr-process" xmlns="uri:falcon:process:0.1"> - <clusters> - <cluster name="local"> - <validity start="2013-11-15T00:05Z" end="2013-11-15T01:05Z"/> - </cluster> - </clusters> - - <parallel>1</parallel> - <order>FIFO</order> - <frequency>minutes(5)</frequency> - <timezone>UTC</timezone> - - <inputs> - <!-- In the workflow, the input paths will be available in a variable 'inpaths' --> - <input name="inpaths" feed="in" start="now(0,-5)" end="now(0,-1)"/> - </inputs> - - <outputs> - <!-- In the workflow, the output path will be available in a variable 'outpath' --> - <output name="outpath" feed="out" instance="now(0,0)"/> - </outputs> - - <properties> - <!-- In the workflow, these properties will be available with variable - key --> - <property name="queueName" value="default"/> - <!-- The schedule time available as a property in workflow --> - <property name="time" value="${instanceTime()}"/> - </properties> - - <workflow engine="oozie" path="/app/oozie-mr"/> -</process> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/filesystem/out-feed.xml ---------------------------------------------------------------------- diff --git a/src/main/examples/entity/filesystem/out-feed.xml b/src/main/examples/entity/filesystem/out-feed.xml deleted file mode 100644 index f537bbc..0000000 --- a/src/main/examples/entity/filesystem/out-feed.xml +++ /dev/null @@ -1,39 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> -<feed description="output" name="out" xmlns="uri:falcon:feed:0.1"> - <groups>output</groups> - - <frequency>minutes(5)</frequency> - <timezone>UTC</timezone> - <late-arrival cut-off="hours(1)"/> - - <clusters> - <cluster name="local"> - <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/> - <retention limit="hours(2)" action="delete"/> - </cluster> - </clusters> - - <locations> - <location type="data" path="/data/out/${YEAR}/${MONTH}/${DAY}/${HOUR}"/> - </locations> - - <ACL owner="testuser-ut-user" group="group" permission="0x644"/> - <schema location="/schema/out/out.format.csv" provider="csv"/> -</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/filesystem/pig-process.xml ---------------------------------------------------------------------- diff --git a/src/main/examples/entity/filesystem/pig-process.xml b/src/main/examples/entity/filesystem/pig-process.xml deleted file mode 100644 index 99e03bd..0000000 --- a/src/main/examples/entity/filesystem/pig-process.xml +++ /dev/null @@ -1,43 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> - -<process name="pig-process" xmlns="uri:falcon:process:0.1"> - <clusters> - <cluster name="local"> - <validity start="2013-11-15T00:05Z" end="2013-11-15T01:05Z"/> - </cluster> - </clusters> - - <parallel>1</parallel> - <order>FIFO</order> - <frequency>minutes(5)</frequency> - <timezone>UTC</timezone> - - <inputs> - <!-- In the pig script, the input paths will be available in a variable 'inpaths' --> - <input name="inpaths" feed="in" start="now(0,-5)" end="now(0,-1)"/> - </inputs> - - <outputs> - <!-- In the pig script, the output path will be available in a variable 'outpath' --> - <output name="outpath" feed="out" instance="now(0,0)"/> - </outputs> - - <workflow engine="pig" path="/app/pig/wordcount.pig"/> -</process> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/filesystem/replication-feed.xml ---------------------------------------------------------------------- diff --git a/src/main/examples/entity/filesystem/replication-feed.xml b/src/main/examples/entity/filesystem/replication-feed.xml deleted file mode 100644 index 7387d45..0000000 --- a/src/main/examples/entity/filesystem/replication-feed.xml +++ /dev/null @@ -1,46 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> -<feed description="replication feed" name="repl-feed" xmlns="uri:falcon:feed:0.1"> - <groups>input</groups> - - <frequency>minutes(1)</frequency> - <timezone>UTC</timezone> - <late-arrival cut-off="hours(1)"/> - - <clusters> - <cluster name="local" type="source"> - <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/> - <retention limit="hours(2)" action="delete"/> - </cluster> - <cluster name="local-target" type="target"> - <validity start="2013-11-15T00:00Z" end="2030-01-01T00:00Z"/> - <retention limit="hours(2)" action="delete"/> - <locations> - <location type="data" path="/data/repl-in/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}"/> - </locations> - </cluster> - </clusters> - - <locations> - <location type="data" path="/data/in/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}"/> - </locations> - - <ACL owner="testuser-ut-user" group="group" permission="0x644"/> - <schema location="/schema/log/log.format.csv" provider="csv"/> -</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/filesystem/standalone-cluster.xml ---------------------------------------------------------------------- diff --git a/src/main/examples/entity/filesystem/standalone-cluster.xml b/src/main/examples/entity/filesystem/standalone-cluster.xml deleted file mode 100644 index 4f8a5fc..0000000 --- a/src/main/examples/entity/filesystem/standalone-cluster.xml +++ /dev/null @@ -1,43 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> - -<!-- - Example cluster definition that can be used with falcon and standalone - hadoop and oozie instances ---> -<cluster colo="local" description="Standalone cluster" name="local" xmlns="uri:falcon:cluster:0.1"> - <interfaces> - <interface type="readonly" endpoint="hdfs://localhost:8020" version="1.1.2"/> - - <interface type="write" endpoint="hdfs://localhost:8020" version="1.1.2"/> - - <interface type="execute" endpoint="localhost:8021" version="1.1.2"/> - - <interface type="workflow" endpoint="http://localhost:11000/oozie/"; version="4.0.0"/> - - <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.4.3"/> - </interfaces> - <locations> - <location name="staging" path="/projects/falcon/staging"/> <!--mandatory--> - <location name="temp" path="/projects/falcon/tmp"/> <!--optional--> - <location name="working" path="/projects/falcon/working"/> <!--optional--> - </locations> - <properties> - </properties> -</cluster> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/filesystem/standalone-target-cluster.xml ---------------------------------------------------------------------- diff --git a/src/main/examples/entity/filesystem/standalone-target-cluster.xml b/src/main/examples/entity/filesystem/standalone-target-cluster.xml deleted file mode 100644 index cb5c4c6..0000000 --- a/src/main/examples/entity/filesystem/standalone-target-cluster.xml +++ /dev/null @@ -1,43 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> - -<!-- - Example cluster definition that can be used with falcon and standalone - hadoop and oozie instances ---> -<cluster colo="local" description="Standalone cluster" name="local-target" xmlns="uri:falcon:cluster:0.1"> - <interfaces> - <interface type="readonly" endpoint="hdfs://localhost:8020" version="1.1.2"/> - - <interface type="write" endpoint="hdfs://localhost:8020" version="1.1.2"/> - - <interface type="execute" endpoint="localhost:8021" version="1.1.2"/> - - <interface type="workflow" endpoint="http://localhost:11000/oozie/"; version="4.0.0"/> - - <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.4.3"/> - </interfaces> - <locations> - <location name="staging" path="/projects/falcon/staging-target"/> <!--mandatory--> - <location name="temp" path="/projects/falcon/tmp-target"/> <!--optional--> - <location name="working" path="/projects/falcon/working-target"/> <!--optional--> - </locations> - <properties> - </properties> -</cluster> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/hcat/hcat-in-feed.xml ---------------------------------------------------------------------- diff --git a/src/main/examples/entity/hcat/hcat-in-feed.xml b/src/main/examples/entity/hcat/hcat-in-feed.xml deleted file mode 100644 index f978e8c..0000000 --- a/src/main/examples/entity/hcat/hcat-in-feed.xml +++ /dev/null @@ -1,37 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> -<feed description="input" name="hcat-in" xmlns="uri:falcon:feed:0.1"> - <groups>input</groups> - - <frequency>minutes(1)</frequency> - <timezone>UTC</timezone> - <late-arrival cut-off="hours(1)"/> - - <clusters> - <cluster name="hcat-local"> - <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/> - <retention limit="hours(2)" action="delete"/> - </cluster> - </clusters> - - <table uri="catalog:default:in_table#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}" /> - - <ACL owner="testuser-ut-user" group="group" permission="0x644"/> - <schema location="/schema/log/log.format.csv" provider="csv"/> -</feed> http://git-wip-us.apache.org/repos/asf/falcon/blob/2945fa80/src/main/examples/entity/hcat/hcat-out-feed.xml ---------------------------------------------------------------------- diff --git a/src/main/examples/entity/hcat/hcat-out-feed.xml b/src/main/examples/entity/hcat/hcat-out-feed.xml deleted file mode 100644 index 23b22b9..0000000 --- a/src/main/examples/entity/hcat/hcat-out-feed.xml +++ /dev/null @@ -1,35 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> -<feed description="output" name="hcat-out" xmlns="uri:falcon:feed:0.1"> - <frequency>minutes(5)</frequency> - <timezone>UTC</timezone> - <late-arrival cut-off="hours(1)"/> - - <clusters> - <cluster name="hcat-local"> - <validity start="2013-01-01T00:00Z" end="2030-01-01T00:00Z"/> - <retention limit="hours(2)" action="delete"/> - </cluster> - </clusters> - - <table uri="catalog:default:out_table#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}" /> - - <ACL owner="testuser-ut-user" group="group" permission="0x644"/> - <schema location="/schema/out/out.format.csv" provider="csv"/> -</feed>
