AMBARI-6972. Pig downloaded configs do not contain file : pig.properties at 1.3 stack (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/533d109c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/533d109c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/533d109c Branch: refs/heads/branch-alerts-dev Commit: 533d109c8595ec3b625be422a76c5e7d1ec7b1eb Parents: ce87ca7 Author: Andrew Onishuk <[email protected]> Authored: Thu Aug 21 17:38:47 2014 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Thu Aug 21 17:38:47 2014 +0300 ---------------------------------------------------------------------- .../PIG/configuration/pig-properties.xml | 83 ++++++++++++++++++++ .../services/PIG/configuration/pig.properties | 52 ------------ .../stacks/HDP/1.3.2/services/PIG/metainfo.xml | 8 +- .../services/PIG/package/scripts/params.py | 2 + .../1.3.2/services/PIG/package/scripts/pig.py | 24 ++---- .../PIG/package/templates/pig.properties.j2 | 74 ----------------- .../services/PIG/package/scripts/params.py | 16 +--- .../2.0.6/services/PIG/package/scripts/pig.py | 12 --- .../python/stacks/1.3.2/PIG/test_pig_client.py | 14 +++- .../python/stacks/1.3.2/configs/default.json | 3 + .../python/stacks/1.3.2/configs/secured.json | 3 + .../python/stacks/2.0.6/configs/default.json | 2 +- .../python/stacks/2.0.6/configs/secured.json | 2 +- .../PIG/configuration/pig-properties.xml | 2 +- ambari-web/app/data/site_properties.js | 15 ++++ 15 files changed, 137 insertions(+), 175 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/533d109c/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/configuration/pig-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/configuration/pig-properties.xml b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/configuration/pig-properties.xml new file mode 100644 index 0000000..2277a40 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/configuration/pig-properties.xml @@ -0,0 +1,83 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration supports_final="false"> + + <property> + <name>content</name> + <description>Describe all the Pig agent configurations</description> + <value> +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +# Pig configuration file. All values can be overwritten by command line arguments. + +# log4jconf log4j configuration file +# log4jconf=./conf/log4j.properties + +# a file that contains pig script +#file= + +# load jarfile, colon separated +#jar= + +#verbose print all log messages to screen (default to print only INFO and above to screen) +#verbose=true + +#exectype local|mapreduce, mapreduce is default +#exectype=local + +#pig.logfile= + +#Do not spill temp files smaller than this size (bytes) +#pig.spill.size.threshold=5000000 +#EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes) +#This should help reduce the number of files being spilled. +#pig.spill.gc.activation.size=40000000 + +#the following two parameters are to help estimate the reducer number +#pig.exec.reducers.bytes.per.reducer=1000000000 +#pig.exec.reducers.max=999 + +#Use this option only when your Pig job will otherwise die because of +#using more counter than hadoop configured limit +#pig.disable.counter=true +hcat.bin=/usr/bin/hcat + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/533d109c/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/configuration/pig.properties ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/configuration/pig.properties b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/configuration/pig.properties deleted file mode 100644 index 01000b5..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/configuration/pig.properties +++ /dev/null @@ -1,52 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Pig default configuration file. All values can be overwritten by pig.properties and command line arguments. -# see bin/pig -help - -# brief logging (no timestamps) -brief=false - -#debug level, INFO is default -debug=INFO - -#verbose print all log messages to screen (default to print only INFO and above to screen) -verbose=false - -#exectype local|mapreduce, mapreduce is default -exectype=mapreduce - -#Enable insertion of information about script into hadoop job conf -pig.script.info.enabled=true - -#Do not spill temp files smaller than this size (bytes) -pig.spill.size.threshold=5000000 -#EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes) -#This should help reduce the number of files being spilled. -pig.spill.gc.activation.size=40000000 - -#the following two parameters are to help estimate the reducer number -pig.exec.reducers.bytes.per.reducer=1000000000 -pig.exec.reducers.max=999 - -#Temporary location to store the intermediate data. -pig.temp.dir=/tmp/ - -#Threshold for merging FRJoin fragment files -pig.files.concatenation.threshold=100 -pig.optimistic.files.concatenation=false; - -pig.disable.counter=false http://git-wip-us.apache.org/repos/asf/ambari/blob/533d109c/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/metainfo.xml index ee745e8..bbc078f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/metainfo.xml @@ -42,7 +42,12 @@ <type>env</type> <fileName>pig-log4j.properties</fileName> <dictionaryName>pig-log4j</dictionaryName> - </configFile> + </configFile> + <configFile> + <type>env</type> + <fileName>pig.properties</fileName> + <dictionaryName>pig-properties</dictionaryName> + </configFile> </configFiles> </component> </components> @@ -70,6 +75,7 @@ <configuration-dependencies> <config-type>pig-env</config-type> <config-type>pig-log4j</config-type> + <config-type>pig-properties</config-type> </configuration-dependencies> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/533d109c/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/params.py index c1058e5..e84070c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/params.py @@ -45,3 +45,5 @@ if (('pig-log4j' in config['configurations']) and ('content' in config['configur log4j_props = config['configurations']['pig-log4j']['content'] else: log4j_props = None + +pig_properties = config['configurations']['pig-properties']['content'] http://git-wip-us.apache.org/repos/asf/ambari/blob/533d109c/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/pig.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/pig.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/pig.py index e73a0d0..0b61dd3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/pig.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/pig.py @@ -34,9 +34,14 @@ def pig(): owner=params.hdfs_user, content=InlineTemplate(params.pig_env_sh_template) ) - - pig_TemplateConfig( ['pig.properties']) - + + File(format("{params.pig_conf_dir}/pig.properties"), + mode=0644, + group=params.user_group, + owner=params.hdfs_user, + content=params.pig_properties + ) + if (params.log4j_props != None): File(format("{params.pig_conf_dir}/log4j.properties"), mode=0644, @@ -50,16 +55,3 @@ def pig(): group=params.user_group, owner=params.hdfs_user ) - - -def pig_TemplateConfig(name): - import params - - if not isinstance(name, list): - name = [name] - - for x in name: - TemplateConfig( format("{pig_conf_dir}/{x}"), - owner = params.hdfs_user - ) - http://git-wip-us.apache.org/repos/asf/ambari/blob/533d109c/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/templates/pig.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/templates/pig.properties.j2 b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/templates/pig.properties.j2 deleted file mode 100644 index 0c5fa47..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/templates/pig.properties.j2 +++ /dev/null @@ -1,74 +0,0 @@ -{# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - - -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# -# - -# Pig configuration file. All values can be overwritten by command line arguments. - -# log4jconf log4j configuration file -# log4jconf=./conf/log4j.properties - -# a file that contains pig script -#file= - -# load jarfile, colon separated -#jar= - -#verbose print all log messages to screen (default to print only INFO and above to screen) -#verbose=true - -#exectype local|mapreduce, mapreduce is default -#exectype=local - -#pig.logfile= - -#Do not spill temp files smaller than this size (bytes) -#pig.spill.size.threshold=5000000 -#EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes) -#This should help reduce the number of files being spilled. -#pig.spill.gc.activation.size=40000000 - -#the following two parameters are to help estimate the reducer number -#pig.exec.reducers.bytes.per.reducer=1000000000 -#pig.exec.reducers.max=999 - -#Use this option only when your Pig job will otherwise die because of -#using more counter than hadoop configured limit -#pig.disable.counter=true -hcat.bin=/usr/bin/hcat http://git-wip-us.apache.org/repos/asf/ambari/blob/533d109c/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py index e676f13..941860b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py @@ -40,16 +40,6 @@ pig_env_sh_template = config['configurations']['pig-env']['content'] java64_home = config['hostLevelParams']['java_home'] hadoop_home = "/usr" -# pig.properties - if not in the JSON command, then we need to esnure some -# basic properties are set; this is a safety mechanism -if (('pig-properties' in config['configurations']) and ('pig-content' in config['configurations']['pig-properties'])): - pig_properties = config['configurations']['pig-properties']['pig-content'] -else: - pig_properties = """hcat.bin=/usr/bin/hcat -pig.location.check.strict=false""" - -# log4j.properties -if (('pig-log4j' in config['configurations']) and ('content' in config['configurations']['pig-log4j'])): - log4j_props = config['configurations']['pig-log4j']['content'] -else: - log4j_props = None +pig_properties = config['configurations']['pig-properties']['content'] + +log4j_props = config['configurations']['pig-log4j']['content'] http://git-wip-us.apache.org/repos/asf/ambari/blob/533d109c/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig.py index 8a8d50d..8326262 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig.py @@ -56,15 +56,3 @@ def pig(): group=params.user_group, owner=params.hdfs_user ) - -def pig_TemplateConfig(name): - import params - - if not isinstance(name, list): - name = [name] - - for x in name: - TemplateConfig( format("{pig_conf_dir}/{x}"), - owner = params.hdfs_user - ) - http://git-wip-us.apache.org/repos/asf/ambari/blob/533d109c/ambari-server/src/test/python/stacks/1.3.2/PIG/test_pig_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/1.3.2/PIG/test_pig_client.py b/ambari-server/src/test/python/stacks/1.3.2/PIG/test_pig_client.py index f5f985c..d97a797 100644 --- a/ambari-server/src/test/python/stacks/1.3.2/PIG/test_pig_client.py +++ b/ambari-server/src/test/python/stacks/1.3.2/PIG/test_pig_client.py @@ -37,8 +37,11 @@ class TestPigClient(RMFTestCase): owner = 'hdfs', content = InlineTemplate(self.getConfig()['configurations']['pig-env']['content']) ) - self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig.properties', - owner = 'hdfs', + self.assertResourceCalled('File', '/etc/pig/conf/pig.properties', + owner = 'hdfs', + group = 'hadoop', + mode = 0644, + content = 'pigproperties\nline2' ) self.assertResourceCalled('File', '/etc/pig/conf/log4j.properties', owner = 'hdfs', @@ -63,8 +66,11 @@ class TestPigClient(RMFTestCase): owner = 'hdfs', content = InlineTemplate(self.getConfig()['configurations']['pig-env']['content']), ) - self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig.properties', - owner = 'hdfs', + self.assertResourceCalled('File', '/etc/pig/conf/pig.properties', + owner = 'hdfs', + group = 'hadoop', + mode = 0644, + content = 'pigproperties\nline2' ) self.assertResourceCalled('File', '/etc/pig/conf/log4j.properties', owner = 'hdfs', http://git-wip-us.apache.org/repos/asf/ambari/blob/533d109c/ambari-server/src/test/python/stacks/1.3.2/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/1.3.2/configs/default.json b/ambari-server/src/test/python/stacks/1.3.2/configs/default.json index 96a7682..40e79c2 100644 --- a/ambari-server/src/test/python/stacks/1.3.2/configs/default.json +++ b/ambari-server/src/test/python/stacks/1.3.2/configs/default.json @@ -421,6 +421,9 @@ "pig-log4j": { "content": "log4jproperties\nline2" }, + "pig-properties": { + "content": "pigproperties\nline2" + }, "oozie-log4j": { "content": "log4jproperties\nline2" } http://git-wip-us.apache.org/repos/asf/ambari/blob/533d109c/ambari-server/src/test/python/stacks/1.3.2/configs/secured.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/1.3.2/configs/secured.json b/ambari-server/src/test/python/stacks/1.3.2/configs/secured.json index 75c53fc..03447b3 100644 --- a/ambari-server/src/test/python/stacks/1.3.2/configs/secured.json +++ b/ambari-server/src/test/python/stacks/1.3.2/configs/secured.json @@ -606,6 +606,9 @@ "pig-log4j": { "content": "log4jproperties\nline2" }, + "pig-properties": { + "content": "pigproperties\nline2" + }, "oozie-log4j": { "content": "log4jproperties\nline2" } http://git-wip-us.apache.org/repos/asf/ambari/blob/533d109c/ambari-server/src/test/python/stacks/2.0.6/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json index bae564f..f32f4bb 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json @@ -567,7 +567,7 @@ "content": "log4jproperties\nline2" }, "pig-properties": { - "pig-content": "pigproperties\nline2" + "content": "pigproperties\nline2" }, "oozie-log4j": { "content": "log4jproperties\nline2" http://git-wip-us.apache.org/repos/asf/ambari/blob/533d109c/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json index 88de472..ece713b 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json @@ -596,7 +596,7 @@ "content": "log4jproperties\nline2" }, "pig-properties": { - "pig-content": "pigproperties\nline2" + "content": "pigproperties\nline2" }, "oozie-log4j": { "content": "log4jproperties\nline2" http://git-wip-us.apache.org/repos/asf/ambari/blob/533d109c/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/PIG/configuration/pig-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/PIG/configuration/pig-properties.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/PIG/configuration/pig-properties.xml index 63b99e2..480be75 100644 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/PIG/configuration/pig-properties.xml +++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/PIG/configuration/pig-properties.xml @@ -23,7 +23,7 @@ <configuration supports_final="false"> <property> - <name>pig-content</name> + <name>content</name> <value> # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file http://git-wip-us.apache.org/repos/asf/ambari/blob/533d109c/ambari-web/app/data/site_properties.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/data/site_properties.js b/ambari-web/app/data/site_properties.js index 5fa4b37..6f82134 100644 --- a/ambari-web/app/data/site_properties.js +++ b/ambari-web/app/data/site_properties.js @@ -784,6 +784,21 @@ module.exports = "serviceName": "WEBHCAT", "category": "Advanced webhcat-site" }, + /**********************************************pig.properties*****************************************/ + { + "id": "site property", + "name": "content", + "displayName": "content", + "value": "", + "defaultValue": "", + "description": "pig properties", + "displayType": "content", + "isRequired": false, + "showLabel": false, + "serviceName": "PIG", + "filename": "pig-properties.xml", + "category": "Advanced pig-properties" + }, //***************************************** GLUSTERFS stack******************************************** { "id": "site property",
