Repository: oozie Updated Branches: refs/heads/master 1fd6a6b41 -> 3c8c2016e
OOZIE-1963 Create a Hive Server 2 example (qwertymaniac via shwethags) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/3c8c2016 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/3c8c2016 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/3c8c2016 Branch: refs/heads/master Commit: 3c8c2016ece188e1a68209c6fb2640f747756c2c Parents: 1fd6a6b Author: Shwetha GS <[email protected]> Authored: Fri May 22 11:12:18 2015 +0530 Committer: Shwetha GS <[email protected]> Committed: Fri May 22 11:12:18 2015 +0530 ---------------------------------------------------------------------- examples/src/main/apps/hive2/README | 14 +++++ examples/src/main/apps/hive2/job.properties | 26 +++++++++ .../src/main/apps/hive2/job.properties.security | 27 +++++++++ examples/src/main/apps/hive2/script.q | 19 ++++++ examples/src/main/apps/hive2/workflow.xml | 49 ++++++++++++++++ .../src/main/apps/hive2/workflow.xml.security | 61 ++++++++++++++++++++ release-log.txt | 1 + 7 files changed, 197 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/3c8c2016/examples/src/main/apps/hive2/README ---------------------------------------------------------------------- diff --git a/examples/src/main/apps/hive2/README b/examples/src/main/apps/hive2/README new file mode 100644 index 0000000..4558e17 --- /dev/null +++ b/examples/src/main/apps/hive2/README @@ -0,0 +1,14 @@ +Running the HiveServer2 example + +1. Setup and run a HiveServer2 (hive --service hiveserver2) to run on localhost, port 10000, + prior to running the example. +2. When running in a secure cluster, follow below + a. The files job.properties.security and workflow.xml.security can be + used and modified as needed. Please refer to those two files and change the jdbcURL + and jdbcPrincipal as necessary. + b. Make sure oozie-site.xml under /etc/oozie/conf has the following property + specified. + <property> + <name>oozie.credentials.credentialclasses</name> + <value>hive2=org.apache.oozie.action.hadoop.Hive2Credentials</value> + </property> http://git-wip-us.apache.org/repos/asf/oozie/blob/3c8c2016/examples/src/main/apps/hive2/job.properties ---------------------------------------------------------------------- diff --git a/examples/src/main/apps/hive2/job.properties b/examples/src/main/apps/hive2/job.properties new file mode 100644 index 0000000..2640402 --- /dev/null +++ b/examples/src/main/apps/hive2/job.properties @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +nameNode=hdfs://localhost:8020 +jobTracker=localhost:8021 +queueName=default +jdbcURL=jdbc:hive2://localhost:10000/default +examplesRoot=examples + +oozie.use.system.libpath=true + +oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/hive2 http://git-wip-us.apache.org/repos/asf/oozie/blob/3c8c2016/examples/src/main/apps/hive2/job.properties.security ---------------------------------------------------------------------- diff --git a/examples/src/main/apps/hive2/job.properties.security b/examples/src/main/apps/hive2/job.properties.security new file mode 100644 index 0000000..ac980cd --- /dev/null +++ b/examples/src/main/apps/hive2/job.properties.security @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +nameNode=hdfs://localhost:8020 +jobTracker=localhost:8021 +queueName=default +jdbcURL=jdbc:hive2://localhost:10000/default +jdbcPrincipal=hive/[email protected] +examplesRoot=examples + +oozie.use.system.libpath=true + +oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/hive2 http://git-wip-us.apache.org/repos/asf/oozie/blob/3c8c2016/examples/src/main/apps/hive2/script.q ---------------------------------------------------------------------- diff --git a/examples/src/main/apps/hive2/script.q b/examples/src/main/apps/hive2/script.q new file mode 100644 index 0000000..3abc757 --- /dev/null +++ b/examples/src/main/apps/hive2/script.q @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +CREATE EXTERNAL TABLE test (a INT) STORED AS TEXTFILE LOCATION '${INPUT}'; +INSERT OVERWRITE DIRECTORY '${OUTPUT}' SELECT * FROM test; http://git-wip-us.apache.org/repos/asf/oozie/blob/3c8c2016/examples/src/main/apps/hive2/workflow.xml ---------------------------------------------------------------------- diff --git a/examples/src/main/apps/hive2/workflow.xml b/examples/src/main/apps/hive2/workflow.xml new file mode 100644 index 0000000..5f1ccc4 --- /dev/null +++ b/examples/src/main/apps/hive2/workflow.xml @@ -0,0 +1,49 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<workflow-app xmlns="uri:oozie:workflow:0.5" name="hive2-wf"> + <start to="hive2-node"/> + + <action name="hive2-node"> + <hive2 xmlns="uri:oozie:hive2-action:0.1"> + <job-tracker>${jobTracker}</job-tracker> + <name-node>${nameNode}</name-node> + <prepare> + <delete path="${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/hive2"/> + <mkdir path="${nameNode}/user/${wf:user()}/${examplesRoot}/output-data"/> + </prepare> + <configuration> + <property> + <name>mapred.job.queue.name</name> + <value>${queueName}</value> + </property> + </configuration> + <jdbc-url>${jdbcURL}</jdbc-url> + <script>script.q</script> + <param>INPUT=/user/${wf:user()}/${examplesRoot}/input-data/table</param> + <param>OUTPUT=/user/${wf:user()}/${examplesRoot}/output-data/hive2</param> + </hive2> + <ok to="end"/> + <error to="fail"/> + </action> + + <kill name="fail"> + <message>Hive2 (Beeline) action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message> + </kill> + <end name="end"/> +</workflow-app> http://git-wip-us.apache.org/repos/asf/oozie/blob/3c8c2016/examples/src/main/apps/hive2/workflow.xml.security ---------------------------------------------------------------------- diff --git a/examples/src/main/apps/hive2/workflow.xml.security b/examples/src/main/apps/hive2/workflow.xml.security new file mode 100644 index 0000000..42ea4cf --- /dev/null +++ b/examples/src/main/apps/hive2/workflow.xml.security @@ -0,0 +1,61 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<workflow-app xmlns="uri:oozie:workflow:0.5" name="hive2-wf"> + <credentials> + <credential name="hs2-creds" type="hive2"> + <property> + <name>hive2.server.principal</name> + <value>${jdbcPrincipal}</value> + </property> + <property> + <name>hive2.jdbc.url</name> + <value>${jdbcURL}</value> + </property> + </credential> + </credentials> + <start to="hive2-node"/> + + <action name="hive2-node"> + <hive2 xmlns="uri:oozie:hive2-action:0.1" cred="hs2-creds"> + <job-tracker>${jobTracker}</job-tracker> + <name-node>${nameNode}</name-node> + <prepare> + <delete path="${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/hive2"/> + <mkdir path="${nameNode}/user/${wf:user()}/${examplesRoot}/output-data"/> + </prepare> + <configuration> + <property> + <name>mapred.job.queue.name</name> + <value>${queueName}</value> + </property> + </configuration> + <jdbc-url>${jdbcURL}</jdbc-url> + <script>script.q</script> + <param>INPUT=/user/${wf:user()}/${examplesRoot}/input-data/table</param> + <param>OUTPUT=/user/${wf:user()}/${examplesRoot}/output-data/hive2</param> + </hive2> + <ok to="end"/> + <error to="fail"/> + </action> + + <kill name="fail"> + <message>Hive2 (Beeline) action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message> + </kill> + <end name="end"/> +</workflow-app> http://git-wip-us.apache.org/repos/asf/oozie/blob/3c8c2016/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 23a8601..32eea5c 100644 --- a/release-log.txt +++ b/release-log.txt @@ -2,6 +2,7 @@ -- Oozie 4.2.0 release (unreleased) +OOZIE-1963 Create a Hive Server 2 example (qwertymaniac via shwethags) OOZIE-1993 Rerun fails during join in certain condition (shwethags) OOZIE-2236 Need to package hive-hcatalog-server-extensions.jar in the hcatalog sharelib (venkatnrangan via bzhang) OOZIE-2232 Oozie should invalidate bulk write command when "-filter" is missing (venkatnrangan via bzhang)
