Repository: metron Updated Branches: refs/heads/master 38b7d5e33 -> 31f3aa8cb
METRON-1447 Heap Size Not Set Correctly by MPack for ES 5.x (nickwallen) closes apache/metron#927 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/31f3aa8c Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/31f3aa8c Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/31f3aa8c Branch: refs/heads/master Commit: 31f3aa8cb0f4c3821c264e375aa14eda207adcb0 Parents: 38b7d5e Author: nickwallen <[email protected]> Authored: Wed Feb 7 14:50:06 2018 -0500 Committer: nickallen <[email protected]> Committed: Wed Feb 7 14:50:06 2018 -0500 ---------------------------------------------------------------------- .../5.6.2/configuration/elastic-jvm-options.xml | 144 +++++++++++++++++++ .../5.6.2/configuration/elastic-sysconfig.xml | 12 +- .../ELASTICSEARCH/5.6.2/metainfo.xml | 1 + .../5.6.2/package/scripts/elastic_commands.py | 13 ++ .../5.6.2/package/scripts/params.py | 3 + 5 files changed, 163 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/31f3aa8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml new file mode 100644 index 0000000..5c6aaca --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml @@ -0,0 +1,144 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration> + <property> + <name>heap_size</name> + <value>512m</value> + <description>JVM heap size</description> + </property> + <property> + <name>content</name> + <description>The jinja template for the Elasticsearch JVM options file.</description> + <value> +## JVM configuration + +################################################################ +## IMPORTANT: JVM heap size +################################################################ +## +## You should always set the min and max JVM heap +## size to the same value. For example, to set +## the heap to 4 GB, set: +## +## -Xms4g +## -Xmx4g +## +## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html +## for more information +## +################################################################ + +# Xms represents the initial size of total heap space +# Xmx represents the maximum size of total heap space + +-Xms{{heap_size}} +-Xmx{{heap_size}} + +################################################################ +## Expert settings +################################################################ +## +## All settings below this section are considered +## expert settings. Don't tamper with them unless +## you understand what you are doing +## +################################################################ + +## GC configuration +-XX:+UseConcMarkSweepGC +-XX:CMSInitiatingOccupancyFraction=75 +-XX:+UseCMSInitiatingOccupancyOnly + +## optimizations + +# pre-touch memory pages used by the JVM during initialization +-XX:+AlwaysPreTouch + +## basic + +# force the server VM (remove on 32-bit client JVMs) +-server + +# explicitly set the stack size (reduce to 320k on 32-bit client JVMs) +-Xss1m + +# set to headless, just in case +-Djava.awt.headless=true + +# ensure UTF-8 encoding by default (e.g. filenames) +-Dfile.encoding=UTF-8 + +# use our provided JNA always versus the system one +-Djna.nosys=true + +# use old-style file permissions on JDK9 +-Djdk.io.permissionsUseCanonicalPath=true + +# flags to configure Netty +-Dio.netty.noUnsafe=true +-Dio.netty.noKeySetOptimization=true +-Dio.netty.recycler.maxCapacityPerThread=0 + +# log4j 2 +-Dlog4j.shutdownHookEnabled=false +-Dlog4j2.disable.jmx=true +-Dlog4j.skipJansi=true + +## heap dumps + +# generate a heap dump when an allocation from the Java heap fails +# heap dumps are created in the working directory of the JVM +-XX:+HeapDumpOnOutOfMemoryError + +# specify an alternative path for heap dumps +# ensure the directory exists and has sufficient space +#-XX:HeapDumpPath=${heap.dump.path} + +## GC logging + +#-XX:+PrintGCDetails +#-XX:+PrintGCTimeStamps +#-XX:+PrintGCDateStamps +#-XX:+PrintClassHistogram +#-XX:+PrintTenuringDistribution +#-XX:+PrintGCApplicationStoppedTime + +# log GC status to a file with time stamps +# ensure the directory exists +#-Xloggc:${loggc} + +# By default, the GC log file will not rotate. +# By uncommenting the lines below, the GC log file +# will be rotated every 128MB at most 32 times. +#-XX:+UseGCLogFileRotation +#-XX:NumberOfGCLogFiles=32 +#-XX:GCLogFileSize=128M + +# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON. +# If documents were already indexed with unquoted fields in a previous version +# of Elasticsearch, some operations may throw errors. +# +# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided +# only for migration purposes. +#-Delasticsearch.json.allow_unquoted_field_names=true + </value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/31f3aa8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml index cb069b8..ea6ca38 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml @@ -40,11 +40,6 @@ <description>Elasticsearch Configuration Directory</description> </property> <property> - <name>heap_size</name> - <value>512m</value> - <description>Heap size</description> - </property> - <property> <name>max_open_files</name> <value>65536</value> <description>Maximum number of open files</description> @@ -92,11 +87,8 @@ PID_DIR={{pid_dir}} # JAVA_HOME must be provided here for OS that use systemd service launch JAVA_HOME={{java64_home}} -# Additional Java OPTS -ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}/elasticsearch_gc.log -XX:-CMSConcurrentMTEnabled \ --XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCTimeStamps \ --XX:ErrorFile={{log_dir}}/elasticsearch_err.log -XX:ParallelGCThreads=8 \ --Xms{{heap_size}} -Xmx{{heap_size}}" +# Additional Java options - now preferential to use 'jvm.options' file instead +ES_JAVA_OPTS="" # https://www.elastic.co/guide/en/elasticsearch/reference/5.6/_memory_lock_check.html MAX_LOCKED_MEMORY=unlimited http://git-wip-us.apache.org/repos/asf/metron/blob/31f3aa8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml index 0943eec..47abb45 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml @@ -83,6 +83,7 @@ <config-type>elastic-site</config-type> <config-type>elastic-sysconfig</config-type> <config-type>elastic-systemd</config-type> + <config-type>elastic-jvm-options</config-type> </configuration-dependencies> <restartRequiredAfterChange>true</restartRequiredAfterChange> <quickLinksConfigurations> http://git-wip-us.apache.org/repos/asf/metron/blob/31f3aa8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py index afbaff2..618d10a 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py @@ -190,6 +190,17 @@ def create_elastic_pam_limits(params): owner="root", group="root") +def create_elastic_jvm_options(params): + """ + Defines the jvm.options file used to specify JVM options. + """ + path = "{0}/jvm.options".format(params.conf_dir) + Logger.info("Creating Elasticsearch JVM Options; file={0}".format(path)) + File(path, + content=InlineTemplate(params.jvm_options_template), + owner=params.elastic_user, + group=params.elastic_group) + def get_data_directories(params): """ Returns the directories to use for storing Elasticsearch data. @@ -225,6 +236,7 @@ def configure_master(): create_elastic_site(params, "elasticsearch.master.yaml.j2") create_elastic_config(params) create_elastic_pam_limits(params) + create_elastic_jvm_options(params) if is_systemd_running(): configure_systemd(params) @@ -249,5 +261,6 @@ def configure_slave(): create_elastic_site(params, "elasticsearch.slave.yaml.j2") create_elastic_config(params) create_elastic_pam_limits(params) + create_elastic_jvm_options(params) if is_systemd_running(): configure_systemd(params) http://git-wip-us.apache.org/repos/asf/metron/blob/31f3aa8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py index 0399c60..24f2306 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py @@ -103,3 +103,6 @@ systemd_parent_dir = '/etc/systemd/system/' systemd_elasticsearch_dir = systemd_parent_dir + 'elasticsearch.service.d/' systemd_override_file = systemd_elasticsearch_dir + 'override.conf' systemd_override_template = config['configurations']['elastic-systemd']['content'] + +heap_size = config['configurations']['elastic-jvm-options']['heap_size'] +jvm_options_template = config['configurations']['elastic-jvm-options']['content']
