Repository: ambari Updated Branches: refs/heads/trunk 1fbb933ef -> fe02e3865
AMBARI-21265 - parquet-logging.properties is missing in hive conf when HDP is upgraded (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fe02e386 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fe02e386 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fe02e386 Branch: refs/heads/trunk Commit: fe02e386526adfa19ea6a4abad34043d40822a6f Parents: 1fbb933 Author: Jonathan Hurley <[email protected]> Authored: Thu Jun 15 14:59:20 2017 -0400 Committer: Jonathan Hurley <[email protected]> Committed: Fri Jun 16 08:52:12 2017 -0400 ---------------------------------------------------------------------- .../HIVE/0.12.0.2.0/metainfo.xml | 19 ++-- .../HIVE/0.12.0.2.0/package/scripts/hive.py | 8 +- .../0.12.0.2.0/package/scripts/params_linux.py | 5 + .../2.1.0.3.0/configuration/parquet-logging.xml | 106 +++++++++++++++++++ .../HIVE/2.1.0.3.0/package/scripts/hive.py | 40 ++++--- .../2.1.0.3.0/package/scripts/params_linux.py | 5 + .../HIVE/configuration/parquet-logging.xml | 106 +++++++++++++++++++ 7 files changed, 265 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/fe02e386/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/metainfo.xml index 4ba3cf1..e3300e3 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/metainfo.xml @@ -185,12 +185,12 @@ <customCommands> <customCommand> <name>CLEAN</name> - <commandScript> - <script>scripts/mysql_server.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </customCommand> + <commandScript> + <script>scripts/mysql_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </customCommand> </customCommands> </component> @@ -225,11 +225,17 @@ <fileName>hive-exec-log4j.properties</fileName> <dictionaryName>hive-exec-log4j</dictionaryName> </configFile> + <configFile> + <type>env</type> + <fileName>parquet-logging.properties</fileName> + <dictionaryName>parquet-logging</dictionaryName> + </configFile> </configFiles> <configuration-dependencies> <config-type>hive-site</config-type> </configuration-dependencies> </component> + <component> <name>HCAT</name> <displayName>HCat Client</displayName> @@ -341,6 +347,7 @@ <config-type>hivemetastore-site.xml</config-type> <config-type>webhcat-site</config-type> <config-type>webhcat-env</config-type> + <config-type>parquet-logging</config-type> <config-type>ranger-hive-plugin-properties</config-type> <config-type>ranger-hive-audit</config-type> <config-type>ranger-hive-policymgr-ssl</config-type> http://git-wip-us.apache.org/repos/asf/ambari/blob/fe02e386/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py index 959e111..36725c3 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py @@ -409,7 +409,13 @@ def fill_conf_dir(component_conf_dir): owner=params.hive_user, content=StaticFile(format("{component_conf_dir}/{log4j_filename}.template")) ) - pass # if params.log4j_version == '1' + + if params.parquet_logging_properties is not None: + File(format("{component_conf_dir}/parquet-logging.properties"), + mode = mode_identified_for_file, + group = params.user_group, + owner = params.hive_user, + content = params.parquet_logging_properties) def jdbc_connector(target, hive_previous_jdbc_jar): http://git-wip-us.apache.org/repos/asf/ambari/blob/fe02e386/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index f680799..078076a 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -494,6 +494,11 @@ if (('hive-exec-log4j' in config['configurations']) and ('content' in config['co else: log4j_exec_props = None +# parquet-logging.properties +parquet_logging_properties = None +if 'parquet-logging' in config['configurations']: + parquet_logging_properties = config['configurations']['parquet-logging']['content'] + daemon_name = status_params.daemon_name process_name = status_params.process_name hive_env_sh_template = config['configurations']['hive-env']['content'] http://git-wip-us.apache.org/repos/asf/ambari/blob/fe02e386/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/parquet-logging.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/parquet-logging.xml b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/parquet-logging.xml new file mode 100644 index 0000000..c80f65b --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/parquet-logging.xml @@ -0,0 +1,106 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="false"> + <property> + <name>content</name> + <display-name>parquet-logging</display-name> + <description>Custom Parquet Logging</description> + <value> +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Properties file which configures the operation of the JDK +# logging facility. + +# The system will look for this config file, first using +# a System property specified at startup: +# +# >java -Djava.util.logging.config.file=myLoggingConfigFilePath +# +# If this property is not specified, then the config file is +# retrieved from its default location at: +# +# JDK_HOME/jre/lib/logging.properties + +# Global logging properties. +# ------------------------------------------ +# The set of handlers to be loaded upon startup. +# Comma-separated list of class names. +# (? LogManager docs say no comma here, but JDK example has comma.) +# handlers=java.util.logging.ConsoleHandler +org.apache.parquet.handlers= java.util.logging.FileHandler + +# Default global logging level. +# Loggers and Handlers may override this level +.level=INFO + +# Handlers +# ----------------------------------------- + +# --- ConsoleHandler --- +# Override of global logging level +java.util.logging.ConsoleHandler.level=INFO +java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format=[%1$tc] %4$s: %2$s - %5$s %6$s%n + +# --- FileHandler --- +# Override of global logging level +java.util.logging.FileHandler.level=ALL + +# Naming style for the output file: +# (The output file is placed in the system temporary directory. +# %u is used to provide unique identifier for the file. +# For more information refer +# https://docs.oracle.com/javase/7/docs/api/java/util/logging/FileHandler.html) +java.util.logging.FileHandler.pattern=%t/parquet-%u.log + +# Limiting size of output file in bytes: +java.util.logging.FileHandler.limit=50000000 + +# Number of output files to cycle through, by appending an +# integer to the base file name: +java.util.logging.FileHandler.count=1 + +# Style of output (Simple or XML): +java.util.logging.FileHandler.formatter=java.util.logging.SimpleFormatter + + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/fe02e386/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py index b7b04a2..e7e92ec 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py @@ -83,7 +83,7 @@ def hive(name=None): if params.enable_atlas_hook: atlas_hook_filepath = os.path.join(params.hive_config_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group) - + File(format("{hive_config_dir}/hive-env.sh"), owner=params.hive_user, group=params.user_group, @@ -125,7 +125,7 @@ def hive(name=None): def setup_hiveserver2(): import params - + File(params.start_hiveserver2_path, mode=0755, content=Template(format('{start_hiveserver2_script}')) @@ -144,16 +144,16 @@ def setup_hiveserver2(): owner=params.hive_user, group=params.user_group, mode=0600) - + # copy tarball to HDFS feature not supported - if not (params.stack_version_formatted_major and check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted_major)): + if not (params.stack_version_formatted_major and check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted_major)): params.HdfsResource(params.webhcat_apps_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=0755 ) - + # Create webhcat dirs. if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir: params.HdfsResource(params.hcat_hdfs_user_dir, @@ -216,7 +216,7 @@ def setup_hiveserver2(): skip=params.sysprep_skip_copy_tarballs_hdfs) # ******* End Copy Tarballs ******* # ********************************* - + # if warehouse directory is in DFS if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse(params.default_fs).scheme: # Create Hive Metastore Warehouse Dir @@ -236,7 +236,7 @@ def setup_hiveserver2(): owner=params.hive_user, mode=params.hive_hdfs_user_mode ) - + if not is_empty(params.hive_exec_scratchdir) and not urlparse(params.hive_exec_scratchdir).path.startswith("/tmp"): params.HdfsResource(params.hive_exec_scratchdir, type="directory", @@ -244,12 +244,12 @@ def setup_hiveserver2(): owner=params.hive_user, group=params.hdfs_user, mode=0777) # Hive expects this dir to be writeable by everyone as it is used as a temp dir - + params.HdfsResource(None, action="execute") - + def setup_non_client(): import params - + Directory(params.hive_pid_dir, create_parents = True, cd_access='a', @@ -273,10 +273,10 @@ def setup_non_client(): jdbc_connector(params.hive_jdbc_target, params.hive_previous_jdbc_jar) if params.hive2_jdbc_target is not None and not os.path.exists(params.hive2_jdbc_target): jdbc_connector(params.hive2_jdbc_target, params.hive2_previous_jdbc_jar) - + def setup_metastore(): import params - + if params.hive_metastore_site_supported: hivemetastore_site_config = get_config("hivemetastore-site") if hivemetastore_site_config: @@ -287,7 +287,7 @@ def setup_metastore(): owner=params.hive_user, group=params.user_group, mode=0600) - + File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hivemetastore.properties"), owner=params.hive_user, group=params.user_group, @@ -328,7 +328,7 @@ def create_metastore_schema(): not_if = check_schema_created_cmd, user = params.hive_user ) - + """ Writes configuration files required by Hive. """ @@ -400,7 +400,13 @@ def fill_conf_dir(component_conf_dir): owner=params.hive_user, content=StaticFile(format("{component_conf_dir}/{log4j_filename}.template")) ) - pass # if params.log4j_version == '1' + + if params.parquet_logging_properties is not None: + File(format("{component_conf_dir}/parquet-logging.properties"), + mode = mode_identified_for_file, + group = params.user_group, + owner = params.hive_user, + content = params.parquet_logging_properties) def jdbc_connector(target, hive_previous_jdbc_jar): @@ -426,7 +432,7 @@ def jdbc_connector(target, hive_previous_jdbc_jar): Execute(('rm', '-f', params.prepackaged_ojdbc_symlink), path=["/bin", "/usr/bin/"], sudo = True) - + File(params.downloaded_custom_connector, content = DownloadSource(params.driver_curl_source)) @@ -463,7 +469,7 @@ def jdbc_connector(target, hive_previous_jdbc_jar): File(target, mode = 0644, ) - + @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) def hive(name=None): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/fe02e386/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py index 91f10d8..77206e7 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py @@ -494,6 +494,11 @@ if (('hive-exec-log4j' in config['configurations']) and ('content' in config['co else: log4j_exec_props = None +# parquet-logging.properties +parquet_logging_properties = None +if 'parquet-logging' in config['configurations']: + parquet_logging_properties = config['configurations']['parquet-logging']['content'] + daemon_name = status_params.daemon_name process_name = status_params.process_name hive_env_sh_template = config['configurations']['hive-env']['content'] http://git-wip-us.apache.org/repos/asf/ambari/blob/fe02e386/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/parquet-logging.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/parquet-logging.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/parquet-logging.xml new file mode 100644 index 0000000..c80f65b --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/parquet-logging.xml @@ -0,0 +1,106 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="false"> + <property> + <name>content</name> + <display-name>parquet-logging</display-name> + <description>Custom Parquet Logging</description> + <value> +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Properties file which configures the operation of the JDK +# logging facility. + +# The system will look for this config file, first using +# a System property specified at startup: +# +# >java -Djava.util.logging.config.file=myLoggingConfigFilePath +# +# If this property is not specified, then the config file is +# retrieved from its default location at: +# +# JDK_HOME/jre/lib/logging.properties + +# Global logging properties. +# ------------------------------------------ +# The set of handlers to be loaded upon startup. +# Comma-separated list of class names. +# (? LogManager docs say no comma here, but JDK example has comma.) +# handlers=java.util.logging.ConsoleHandler +org.apache.parquet.handlers= java.util.logging.FileHandler + +# Default global logging level. +# Loggers and Handlers may override this level +.level=INFO + +# Handlers +# ----------------------------------------- + +# --- ConsoleHandler --- +# Override of global logging level +java.util.logging.ConsoleHandler.level=INFO +java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format=[%1$tc] %4$s: %2$s - %5$s %6$s%n + +# --- FileHandler --- +# Override of global logging level +java.util.logging.FileHandler.level=ALL + +# Naming style for the output file: +# (The output file is placed in the system temporary directory. +# %u is used to provide unique identifier for the file. +# For more information refer +# https://docs.oracle.com/javase/7/docs/api/java/util/logging/FileHandler.html) +java.util.logging.FileHandler.pattern=%t/parquet-%u.log + +# Limiting size of output file in bytes: +java.util.logging.FileHandler.limit=50000000 + +# Number of output files to cycle through, by appending an +# integer to the base file name: +java.util.logging.FileHandler.count=1 + +# Style of output (Simple or XML): +java.util.logging.FileHandler.formatter=java.util.logging.SimpleFormatter + + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration>
