METRON-1447 Heap Size Not Set Correctly by MPack for ES 5.x (nickwallen) closes 
apache/metron#927


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/31f3aa8c
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/31f3aa8c
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/31f3aa8c

Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 31f3aa8cb0f4c3821c264e375aa14eda207adcb0
Parents: 38b7d5e
Author: nickwallen <n...@nickallen.org>
Authored: Wed Feb 7 14:50:06 2018 -0500
Committer: nickallen <nickal...@apache.org>
Committed: Wed Feb 7 14:50:06 2018 -0500

----------------------------------------------------------------------
 .../5.6.2/configuration/elastic-jvm-options.xml | 144 +++++++++++++++++++
 .../5.6.2/configuration/elastic-sysconfig.xml   |  12 +-
 .../ELASTICSEARCH/5.6.2/metainfo.xml            |   1 +
 .../5.6.2/package/scripts/elastic_commands.py   |  13 ++
 .../5.6.2/package/scripts/params.py             |   3 +
 5 files changed, 163 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/31f3aa8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
----------------------------------------------------------------------
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
new file mode 100644
index 0000000..5c6aaca
--- /dev/null
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
@@ -0,0 +1,144 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<configuration>
+    <property>
+        <name>heap_size</name>
+        <value>512m</value>
+        <description>JVM heap size</description>
+    </property>
+    <property>
+        <name>content</name>
+        <description>The jinja template for the Elasticsearch JVM options 
file.</description>
+        <value>
+## JVM configuration
+
+################################################################
+## IMPORTANT: JVM heap size
+################################################################
+##
+## You should always set the min and max JVM heap
+## size to the same value. For example, to set
+## the heap to 4 GB, set:
+##
+## -Xms4g
+## -Xmx4g
+##
+## See 
https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html
+## for more information
+##
+################################################################
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+
+-Xms{{heap_size}}
+-Xmx{{heap_size}}
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below this section are considered
+## expert settings. Don't tamper with them unless
+## you understand what you are doing
+##
+################################################################
+
+## GC configuration
+-XX:+UseConcMarkSweepGC
+-XX:CMSInitiatingOccupancyFraction=75
+-XX:+UseCMSInitiatingOccupancyOnly
+
+## optimizations
+
+# pre-touch memory pages used by the JVM during initialization
+-XX:+AlwaysPreTouch
+
+## basic
+
+# force the server VM (remove on 32-bit client JVMs)
+-server
+
+# explicitly set the stack size (reduce to 320k on 32-bit client JVMs)
+-Xss1m
+
+# set to headless, just in case
+-Djava.awt.headless=true
+
+# ensure UTF-8 encoding by default (e.g. filenames)
+-Dfile.encoding=UTF-8
+
+# use our provided JNA always versus the system one
+-Djna.nosys=true
+
+# use old-style file permissions on JDK9
+-Djdk.io.permissionsUseCanonicalPath=true
+
+# flags to configure Netty
+-Dio.netty.noUnsafe=true
+-Dio.netty.noKeySetOptimization=true
+-Dio.netty.recycler.maxCapacityPerThread=0
+
+# log4j 2
+-Dlog4j.shutdownHookEnabled=false
+-Dlog4j2.disable.jmx=true
+-Dlog4j.skipJansi=true
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps
+# ensure the directory exists and has sufficient space
+#-XX:HeapDumpPath=${heap.dump.path}
+
+## GC logging
+
+#-XX:+PrintGCDetails
+#-XX:+PrintGCTimeStamps
+#-XX:+PrintGCDateStamps
+#-XX:+PrintClassHistogram
+#-XX:+PrintTenuringDistribution
+#-XX:+PrintGCApplicationStoppedTime
+
+# log GC status to a file with time stamps
+# ensure the directory exists
+#-Xloggc:${loggc}
+
+# By default, the GC log file will not rotate.
+# By uncommenting the lines below, the GC log file
+# will be rotated every 128MB at most 32 times.
+#-XX:+UseGCLogFileRotation
+#-XX:NumberOfGCLogFiles=32
+#-XX:GCLogFileSize=128M
+
+# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON.
+# If documents were already indexed with unquoted fields in a previous version
+# of Elasticsearch, some operations may throw errors.
+#
+# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided
+# only for migration purposes.
+#-Delasticsearch.json.allow_unquoted_field_names=true
+        </value>
+    </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/metron/blob/31f3aa8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
----------------------------------------------------------------------
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
index cb069b8..ea6ca38 100755
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
@@ -40,11 +40,6 @@
         <description>Elasticsearch Configuration Directory</description>
     </property>
     <property>
-        <name>heap_size</name>
-        <value>512m</value>
-        <description>Heap size</description>
-    </property>
-    <property>
         <name>max_open_files</name>
         <value>65536</value>
         <description>Maximum number of open files</description>
@@ -92,11 +87,8 @@ PID_DIR={{pid_dir}}
 # JAVA_HOME must be provided here for OS that use systemd service launch
 JAVA_HOME={{java64_home}}
 
-# Additional Java OPTS
-ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}/elasticsearch_gc.log 
-XX:-CMSConcurrentMTEnabled \
--XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCTimeStamps \
--XX:ErrorFile={{log_dir}}/elasticsearch_err.log -XX:ParallelGCThreads=8 \
--Xms{{heap_size}} -Xmx{{heap_size}}"
+# Additional Java options - now preferential to use 'jvm.options' file instead
+ES_JAVA_OPTS=""
 
 # 
https://www.elastic.co/guide/en/elasticsearch/reference/5.6/_memory_lock_check.html
 MAX_LOCKED_MEMORY=unlimited

http://git-wip-us.apache.org/repos/asf/metron/blob/31f3aa8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
----------------------------------------------------------------------
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
index 0943eec..47abb45 100755
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
@@ -83,6 +83,7 @@
                 <config-type>elastic-site</config-type>
                 <config-type>elastic-sysconfig</config-type>
                 <config-type>elastic-systemd</config-type>
+                <config-type>elastic-jvm-options</config-type>
             </configuration-dependencies>
             <restartRequiredAfterChange>true</restartRequiredAfterChange>
             <quickLinksConfigurations>

http://git-wip-us.apache.org/repos/asf/metron/blob/31f3aa8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
----------------------------------------------------------------------
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
index afbaff2..618d10a 100644
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
@@ -190,6 +190,17 @@ def create_elastic_pam_limits(params):
          owner="root",
          group="root")
 
+def create_elastic_jvm_options(params):
+    """
+    Defines the jvm.options file used to specify JVM options.
+    """
+    path = "{0}/jvm.options".format(params.conf_dir)
+    Logger.info("Creating Elasticsearch JVM Options; file={0}".format(path))
+    File(path,
+         content=InlineTemplate(params.jvm_options_template),
+         owner=params.elastic_user,
+         group=params.elastic_group)
+
 def get_data_directories(params):
     """
     Returns the directories to use for storing Elasticsearch data.
@@ -225,6 +236,7 @@ def configure_master():
     create_elastic_site(params,  "elasticsearch.master.yaml.j2")
     create_elastic_config(params)
     create_elastic_pam_limits(params)
+    create_elastic_jvm_options(params)
     if is_systemd_running():
         configure_systemd(params)
 
@@ -249,5 +261,6 @@ def configure_slave():
     create_elastic_site(params, "elasticsearch.slave.yaml.j2")
     create_elastic_config(params)
     create_elastic_pam_limits(params)
+    create_elastic_jvm_options(params)
     if is_systemd_running():
         configure_systemd(params)

http://git-wip-us.apache.org/repos/asf/metron/blob/31f3aa8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
index 0399c60..24f2306 100755
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
@@ -103,3 +103,6 @@ systemd_parent_dir = '/etc/systemd/system/'
 systemd_elasticsearch_dir = systemd_parent_dir + 'elasticsearch.service.d/'
 systemd_override_file = systemd_elasticsearch_dir + 'override.conf'
 systemd_override_template = 
config['configurations']['elastic-systemd']['content']
+
+heap_size = config['configurations']['elastic-jvm-options']['heap_size']
+jvm_options_template = 
config['configurations']['elastic-jvm-options']['content']

Reply via email to