BIGTOP-3075: Drop pig packaging Drop pig packaging as it doesn't work with Hive-2.0+. See BIGTOP-2975
This closes #396 Change-Id: I7e6ca27979864f04e36595afbd5475e628e21b07 Signed-off-by: Yuqi Gu <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/fb7a80fd Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/fb7a80fd Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/fb7a80fd Branch: refs/heads/branch-1.3 Commit: fb7a80fd75dd3403cd2b45b0546956bb0d3a6a78 Parents: a745f03 Author: Yuqi Gu <[email protected]> Authored: Wed Sep 12 08:10:47 2018 +0000 Committer: Jun He <[email protected]> Committed: Tue Sep 18 18:25:59 2018 +0000 ---------------------------------------------------------------------- bigtop-deploy/puppet/hieradata/site.yaml | 1 - bigtop-deploy/puppet/manifests/cluster.pp | 4 - .../puppet/modules/hadoop_pig/manifests/init.pp | 40 - .../modules/hadoop_pig/templates/pig.properties | 70 - .../puppet/modules/hadoop_pig/tests/init.pp | 17 - .../src/charm/pig/layer-pig/README.md | 136 - .../src/charm/pig/layer-pig/actions.yaml | 6 - .../src/charm/pig/layer-pig/actions/smoke-test | 68 - .../src/charm/pig/layer-pig/copyright | 16 - .../src/charm/pig/layer-pig/icon.svg | 1431 ------- .../src/charm/pig/layer-pig/layer.yaml | 13 - .../layer-pig/lib/charms/layer/bigtop_pig.py | 80 - .../src/charm/pig/layer-pig/metadata.yaml | 10 - .../src/charm/pig/layer-pig/reactive/pig.py | 37 - .../src/charm/pig/layer-pig/tests/01-deploy.py | 49 - .../src/charm/pig/layer-pig/tests/tests.yaml | 3 - .../src/common/phoenix/do-component-build | 5 - .../src/common/pig/do-component-build | 82 - bigtop-packages/src/common/pig/install_pig.sh | 158 - ...IG-5304-Fix-build-issue-with-Hadoop-2.8.diff | 30 - bigtop-packages/src/common/pig/pig.1 | 130 - bigtop-packages/src/deb/pig/changelog | 1 - bigtop-packages/src/deb/pig/compat | 1 - bigtop-packages/src/deb/pig/control | 47 - bigtop-packages/src/deb/pig/copyright | 16 - bigtop-packages/src/deb/pig/pig.dirs | 2 - bigtop-packages/src/deb/pig/pig.postinst | 35 - bigtop-packages/src/deb/pig/pig.preinst | 53 - bigtop-packages/src/deb/pig/pig.prerm | 57 - bigtop-packages/src/deb/pig/rules | 58 - bigtop-packages/src/deb/pig/source/format | 1 - bigtop-packages/src/rpm/pig/RPMS/.gitignore | 1 - bigtop-packages/src/rpm/pig/SPECS/.gitignore | 3 - bigtop-packages/src/rpm/pig/SPECS/pig.spec | 171 - .../smoke-tests/pig/TestPigSmoke.groovy | 58 - bigtop-tests/smoke-tests/pig/build.gradle | 38 - bigtop-tests/smoke-tests/pig/log4j.properties | 24 - bigtop-tests/smoke-tests/pig/wordcount.pig | 24 - .../package/src/main/resources/apt/pig.xml | 3168 -------------- .../package/src/main/resources/package_data.xml | 38 - .../src/main/resources/urpmi/package_data.xml | 3861 ------------------ .../src/main/resources/yum/package_data.xml | 7 - .../package/src/main/resources/yum/pig.xml | 3152 -------------- .../src/main/resources/zypper/package_data.xml | 7 - .../package/src/main/resources/zypper/pig.xml | 3152 -------------- bigtop-tests/test-execution/smokes/pom.xml | 7 - bigtop.bom | 25 +- pom.xml | 12 - 48 files changed, 2 insertions(+), 16403 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb7a80fd/bigtop-deploy/puppet/hieradata/site.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/hieradata/site.yaml b/bigtop-deploy/puppet/hieradata/site.yaml index 9c2dfc8..f6a6c8b 100644 --- a/bigtop-deploy/puppet/hieradata/site.yaml +++ b/bigtop-deploy/puppet/hieradata/site.yaml @@ -28,7 +28,6 @@ hadoop::hadoop_storage_dirs: # - mahout # - mapred-app # - oozie -# - pig # - qfs # - solrcloud # - spark http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb7a80fd/bigtop-deploy/puppet/manifests/cluster.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp index 9dbde21..121b030 100644 --- a/bigtop-deploy/puppet/manifests/cluster.pp +++ b/bigtop-deploy/puppet/manifests/cluster.pp @@ -111,9 +111,6 @@ $roles_map = { crunch => { client => ["crunch-client"], }, - pig => { - client => ["pig-client"], - }, hive => { master => ["hive-server2", "hive-metastore"], client => ["hive-client"], @@ -202,7 +199,6 @@ class node_with_roles ($roles = hiera("bigtop::roles")) inherits hadoop_cluster_ "ignite_hadoop", "hadoop_flume", "hadoop_hive", - "hadoop_pig", "sqoop2", "hadoop_zookeeper", "hcatalog", http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb7a80fd/bigtop-deploy/puppet/modules/hadoop_pig/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_pig/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop_pig/manifests/init.pp deleted file mode 100644 index 49f871e..0000000 --- a/bigtop-deploy/puppet/modules/hadoop_pig/manifests/init.pp +++ /dev/null @@ -1,40 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -class hadoop_pig { - - class deploy ($roles) { - if ("pig-client" in $roles) { - include hadoop_pig::client - } - } - - class client { - include hadoop::common - - package { "pig": - ensure => latest, - require => Package["hadoop"], - } - - file { "/etc/pig/conf/pig.properties": - content => template('hadoop_pig/pig.properties'), - require => Package["pig"], - owner => "root", /* FIXME: I'm really no sure about these -- we might end */ - mode => "755", /* up deploying/testing a different thing compared */ - /* to a straight rpm/deb deployment */ - } - } -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb7a80fd/bigtop-deploy/puppet/modules/hadoop_pig/templates/pig.properties ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_pig/templates/pig.properties b/bigtop-deploy/puppet/modules/hadoop_pig/templates/pig.properties deleted file mode 100644 index 1e1c297..0000000 --- a/bigtop-deploy/puppet/modules/hadoop_pig/templates/pig.properties +++ /dev/null @@ -1,70 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Pig configuration file. All values can be overwritten by command line arguments. -# see bin/pig -help - -# log4jconf log4j configuration file -# log4jconf=./conf/log4j.properties - -# brief logging (no timestamps) -brief=false - -# clustername, name of the hadoop jobtracker. If no port is defined port 50020 will be used. -#cluster - -#debug level, INFO is default -debug=INFO - -# a file that contains pig script -#file= - -# load jarfile, colon separated -#jar= - -#verbose print all log messages to screen (default to print only INFO and above to screen) -verbose=false - -#exectype local|mapreduce, mapreduce is default -#exectype=mapreduce -# hod realted properties -#ssh.gateway -#hod.expect.root -#hod.expect.uselatest -#hod.command -#hod.config.dir -#hod.param - - -#Do not spill temp files smaller than this size (bytes) -pig.spill.size.threshold=5000000 -#EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes) -#This should help reduce the number of files being spilled. -pig.spill.gc.activation.size=40000000 - - -###################### -# Everything below this line is Yahoo specific. Note that I've made -# (almost) no changes to the lines above to make merging in from Apache -# easier. Any values I don't want from above I override below. -# -# This file is configured for use with HOD on the production clusters. If you -# want to run pig with a static cluster you will need to remove everything -# below this line and set the cluster value (above) to the -# hostname and port of your job tracker. - -exectype=mapreduce -log.file= http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb7a80fd/bigtop-deploy/puppet/modules/hadoop_pig/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_pig/tests/init.pp b/bigtop-deploy/puppet/modules/hadoop_pig/tests/init.pp deleted file mode 100644 index 9692630..0000000 --- a/bigtop-deploy/puppet/modules/hadoop_pig/tests/init.pp +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -include hadoop_pig -hadoop_pig::client { "test-pig": } http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb7a80fd/bigtop-packages/src/charm/pig/layer-pig/README.md ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/pig/layer-pig/README.md b/bigtop-packages/src/charm/pig/layer-pig/README.md deleted file mode 100644 index f38763d..0000000 --- a/bigtop-packages/src/charm/pig/layer-pig/README.md +++ /dev/null @@ -1,136 +0,0 @@ -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -# Overview - -Apache Pig is a platform for creating MapReduce programs used with Hadoop. -It consists of a high-level language (Pig Latin) for expressing data analysis -programs, coupled with infrastructure for evaluating these programs. Learn more -at [pig.apache.org][]. - -This charm deploys version 0.15.0 of the Pig component from [Apache Bigtop][]. - -[pig.apache.org]: http://pig.apache.org/ -[Apache Bigtop]: http://bigtop.apache.org/ - - -# Deploying - -This charm requires Juju 2.0 or greater. If Juju is not yet set up, please -follow the [getting-started][] instructions prior to deploying this charm. - -This charm is intended to be deployed via one of the [apache bigtop bundles][]. -For example: - - juju deploy hadoop-processing - -This will deploy an Apache Bigtop Hadoop cluster. More information about this -deployment can be found in the [bundle readme](https://jujucharms.com/hadoop-processing/). - -Now add Pig and relate it to the cluster via the hadoop-plugin: - - juju deploy pig - juju add-relation pig plugin - -## Network-Restricted Environments -Charms can be deployed in environments with limited network access. To deploy -in this environment, configure a Juju model with appropriate proxy and/or -mirror options. See [Configuring Models][] for more information. - -[getting-started]: https://jujucharms.com/docs/stable/getting-started -[apache bigtop bundles]: https://jujucharms.com/u/bigdata-charmers/#bundles -[Configuring Models]: https://jujucharms.com/docs/stable/models-config - - -# Verifying - -## Status -Apache Bigtop charms provide extended status reporting to indicate when they -are ready: - - juju status - -This is particularly useful when combined with `watch` to track the on-going -progress of the deployment: - - watch -n 2 juju status - -The message column will provide information about a given unit's state. -This charm is ready for use once the status message indicates that it is -ready. - -## Smoke Test -This charm provides a `smoke-test` action that can be used to verify the -application is functioning as expected. Run the action as follows: - - juju run-action pig/0 smoke-test - -Watch the progress of the smoke test actions with: - - watch -n 2 juju show-action-status - -Eventually, the action should settle to `status: completed`. If it -reports `status: failed`, the application is not working as expected. Get -more information about a specific smoke test with: - - juju show-action-output <action-id> - - -# Using - -Once the deployment has been verified, Apache Pig will be available to execute -Pig Latin jobs on your data. You can run Pig in a variety of modes: - -## Local Mode -Run Pig in local mode on the Pig unit with the following: - - juju ssh pig/0 - pig -x local - -## MapReduce Mode -MapReduce mode is the default for Pig. To run in this mode, ssh to the Pig unit -and run pig as follows: - - juju ssh pig/0 - pig - - -# Issues - -Apache Bigtop tracks issues using JIRA (Apache account required). File an -issue for this charm at: - -https://issues.apache.org/jira/secure/CreateIssue!default.jspa - -Ensure `Bigtop` is selected as the project. Typically, charm issues are filed -in the `deployment` component with the latest stable release selected as the -affected version. Any uncertain fields may be left blank. - - -# Contact Information - -- <[email protected]> - - -# Resources - -- [Apache Pig home page](http://pig.apache.org/) -- [Apache Bigtop home page](http://bigtop.apache.org/) -- [Apache Bigtop issue tracking](http://bigtop.apache.org/issue-tracking.html) -- [Apache Bigtop mailing lists](http://bigtop.apache.org/mail-lists.html) -- [Juju Big Data](https://jujucharms.com/big-data) -- [Juju Bigtop charms](https://jujucharms.com/q/bigtop) -- [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb7a80fd/bigtop-packages/src/charm/pig/layer-pig/actions.yaml ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/pig/layer-pig/actions.yaml b/bigtop-packages/src/charm/pig/layer-pig/actions.yaml deleted file mode 100644 index f6e503e..0000000 --- a/bigtop-packages/src/charm/pig/layer-pig/actions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -smoke-test: - description: | - Verify the Bigtop Pig service is working by executing a smoke test. If - pig is in 'local' mode, the smoke test will execute a local /etc/passwd - analysis job. If pig is in 'mapreduce' mode, the Bigtop smoke test will - be executed, which exercises Pig with HDFS and MapReduce. http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb7a80fd/bigtop-packages/src/charm/pig/layer-pig/actions/smoke-test ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/pig/layer-pig/actions/smoke-test b/bigtop-packages/src/charm/pig/layer-pig/actions/smoke-test deleted file mode 100755 index fc21d4b..0000000 --- a/bigtop-packages/src/charm/pig/layer-pig/actions/smoke-test +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/local/sbin/charm-env python3 - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys - -from charmhelpers.core import hookenv -from charms.layer.apache_bigtop_base import Bigtop -from charms.reactive import is_state -from jujubigdata.utils import run_as -from path import Path - - -def fail(msg): - hookenv.action_set({'outcome': 'failure'}) - hookenv.action_fail(msg) - sys.exit() - - -if not is_state('pig.installed'): - fail('Pig is not yet ready') - - -if is_state('hadoop.ready'): - hookenv.log('Testing Pig in mapreduce mode using the Bigtop smoke test') - # Remove any output from previous runs - run_as('ubuntu', - 'hdfs', 'dfs', '-rm', '-f', '-R', '/user/ubuntu/pig-output-wordcount') - - bigtop = Bigtop() - bigtop.run_smoke_tests(['pig']) - output = run_as('ubuntu', - 'hdfs', 'dfs', '-cat', 'pig-output-wordcount/part-r-00000', - capture_output=True) -else: - hookenv.log('Testing Pig in local mode (ignoring bigtop component smoke test)') - # Remove any output from previous runs - run_as('ubuntu', 'rm', '-R', '-f', '/tmp/test.pig.out') - - script = ("A = load '/etc/passwd' using PigStorage(':'); " - "B = foreach A generate $0 as id; store B into '/tmp/test.pig.out';") - Path('/tmp/test.pig').write_text(script) - run_as('ubuntu', 'pig', '-x', 'local', '-l', '/tmp/test.pig.log', '/tmp/test.pig') - - output = run_as('ubuntu', - 'cat', '/tmp/test.pig.out/part-m-00000', - capture_output=True) - -# The smoke tests analyze /etc/passwd. Successful runs will have an 'ubuntu' -# passwd entry in the output. -hookenv.action_set({'raw': output}) -if 'ubuntu' not in output: - fail('Unable to analyze passwd file with Pig') -else: - hookenv.action_set({'outcome': 'success'}) http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb7a80fd/bigtop-packages/src/charm/pig/layer-pig/copyright ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/pig/layer-pig/copyright b/bigtop-packages/src/charm/pig/layer-pig/copyright deleted file mode 100644 index e900b97..0000000 --- a/bigtop-packages/src/charm/pig/layer-pig/copyright +++ /dev/null @@ -1,16 +0,0 @@ -Format: http://dep.debian.net/deps/dep5/ - -Files: * -Copyright: Copyright 2015, Canonical Ltd., All Rights Reserved. -License: Apache License 2.0 - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - . - http://www.apache.org/licenses/LICENSE-2.0 - . - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License.
