Repository: ambari Updated Branches: refs/heads/branch-2.4 7b775aa9f -> 6a6281937
AMBARI-17473: Add additional parameters to hawq-site.xml (bhuvnesh2703, mithmatt) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6a628193 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6a628193 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6a628193 Branch: refs/heads/branch-2.4 Commit: 6a62819375776255a8510452a9240ca366542e67 Parents: 7b775aa Author: Matt <[email protected]> Authored: Thu Jun 30 00:46:25 2016 -0700 Committer: Matt <[email protected]> Committed: Thu Jun 30 00:46:25 2016 -0700 ---------------------------------------------------------------------- .../HAWQ/2.0.0/configuration/hawq-site.xml | 238 +++++++++++++++++++ .../2.0.0/configuration/hawq-sysctl-env.xml | 9 + .../HAWQ/2.0.0/service_advisor.py | 4 + .../HAWQ/2.0.0/themes/theme.json | 86 ++++++- .../stacks/2.3/HAWQ/test_service_advisor.py | 17 ++ 5 files changed, 341 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6a628193/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml index 6e21d84..150b2c6 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml @@ -352,4 +352,242 @@ </value-attributes> <on-ambari-upgrade add="true"/> </property> + <property> + <name>gp_workfile_compress_algorithm</name> + <display-name>Spill Files Compression Algorithm</display-name> + <value>none</value> + <description> + Compression algorithm to use on the spill files when a hash aggregation or hash join operation spills to disk during query processing. + If using zlib, it must be in your $PATH on all segments. The default is none. + </description> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>none</value> + <label>None</label> + </entry> + <entry> + <value>zlib</value> + <label>zlib</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>gp_workfile_limit_files_per_query</name> + <display-name>Max. No. of Temporary Spill Files allowed per Query per Segment</display-name> + <value>3000000</value> + <description> + Sets the maximum number of temporary spill files (also known as workfiles) allowed per query per segment. + Spill files are created when executing a query that requires more memory than it is allocated. + The current query is terminated when the limit is exceeded. The default is 3000000. + </description> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>gp_workfile_limit_per_query</name> + <display-name>Max. Disk Size Allowed for Individual Query for creating Temporary Spill Files per Segment</display-name> + <value>0</value> + <description> + Sets the maximum disk size an individual query is allowed to use for creating temporary spill files at each segment. + The default value is 0, which means a limit is not enforced. + </description> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + <unit>kB</unit> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>gp_workfile_limit_per_segment</name> + <display-name>Max. Disk Size Allowed for All Queries for creating Temporary Spill Files per Segment</display-name> + <value>0</value> + <description> + Sets the maximum total disk size that all running queries are allowed to use for creating temporary spill files at each segment. + The default value is 0, which means a limit is not enforced. + </description> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + <unit>kB</unit> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>gp_autostats_mode</name> + <value>on_no_stats</value> + <description> + Specifies the mode for triggering automatic statistics collection with ANALYZE. + The on_no_stats option triggers statistics collection for CREATE TABLE AS SELECT, INSERT, or COPY operations on any table that has no existing statistics. + The on_change option triggers statistics collection only when the number of rows affected exceeds the threshold defined by gp_autostats_on_change_threshold. + Operations that can trigger automatic statistics collection with on_change are CREATE TABLE AS SELECT, UPDATE, DELETE, INSERT and COPY. + The default is on_no_stats. + </description> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>none</value> + </entry> + <entry> + <value>on_change</value> + </entry> + <entry> + <value>on_no_stats</value> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>gp_autostats_on_change_threshold</name> + <value>2147483647</value> + <description> + Specifies the threshold for automatic statistics collection when gp_autostats_mode is set to on_change. + When a triggering table operation affects a number of rows exceeding this threshold, ANALYZE is added and statistics are collected for the table. + The default is 2147483647. + </description> + <value-attributes> + <type>int</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>gp_external_max_segs</name> + <value>64</value> + <description> + Sets the number of segments that will scan external table data during an external table operation, the purpose being not to overload the system with scanning data and take away resources from other concurrent operations. + This only applies to external tables that use the gpfdist://protocol to access external table data. The default is 64. + </description> + <value-attributes> + <type>int</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>hawq_rm_resource_idle_timeout</name> + <value>300</value> + <description> + Amount of time, in seconds, before idle resources are returned to YARN. The default is 300. + </description> + <value-attributes> + <type>int</type> + <minimum>-1</minimum> + <maximum>65535</maximum> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>log_duration</name> + <value>off</value> + <description> + Causes the duration of every completed statement which satisfies log_statement to be logged. The default is off. + </description> + <value-attributes> + <type>string</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>log_rotation_age</name> + <value>1d</value> + <description> + Determines the maximum lifetime of an individual log file. After this time has elapsed, a new log file will be created. + Set to zero to disable time-based creation of new log files. The default is 1d. + </description> + <value-attributes> + <type>string</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>log_statement</name> + <value>ALL</value> + <description> + Controls which SQL statements are logged. DDL logs all data definition commands like CREATE, ALTER, and DROP commands. + MOD logs all DDL statements, plus INSERT, UPDATE, DELETE, TRUNCATE, and COPY FROM. + PREPARE and EXPLAIN ANALYZE statements are also logged if their contained command is of an appropriate type. + The default is ALL. + </description> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>NONE</value> + </entry> + <entry> + <value>DDL</value> + </entry> + <entry> + <value>MOD</value> + </entry> + <entry> + <value>ALL</value> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>log_statement_stats</name> + <value>off</value> + <description> + For each query, write total performance statistics of the query parser, planner, and executor to the server log. + This is a crude profiling instrument. The default is off. + </description> + <value-attributes> + <type>string</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>log_truncate_on_rotation</name> + <value>off</value> + <description> + Truncates (overwrites), rather than appends to, any existing log file of the same name. + Truncation will occur only when a new file is being opened due to time-based rotation. + For example, using this setting in combination with a log_filename such as hawq-%H.log would result in generating twenty-four hourly log files and then cyclically overwriting them. + When off, pre-existing files will be appended to in all cases. The default is off. + </description> + <value-attributes> + <type>string</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>runaway_detector_activation_percent</name> + <value>95</value> + <description> + Sets the percentage of the virtual memory quota that will trigger the termination of queries. + If the amount of virtual memory utilized by a physical segment exceeds the calculated threshold, then HAWQ begins terminating queries based on memory usage, starting with the query that is consuming the largest amount of memory. + Queries are terminated until the percentage of utilized virtual memory is below the specified percentage. The default is 95. + </description> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + <maximum>100</maximum> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>statement_timeout</name> + <value>0</value> + <description> + Abort any statement that takes over the specified number of milliseconds. 0 turns off the limitation. The default is 0. + </description> + <value-attributes> + <type>int</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/6a628193/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml index c132dcd..6350344 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml @@ -342,8 +342,17 @@ When overcommit_memory is set to 2, the committed address space is not permitted to exceed swap plus this percentage of physical RAM. Default is 50. </description> + <depends-on> + <property> + <type>hawq-sysctl-env</type> + <name>vm.overcommit_memory</name> + </property> + </depends-on> <value-attributes> <type>int</type> + <minimum>0</minimum> + <maximum>200</maximum> + <increment-step>25</increment-step> </value-attributes> <on-ambari-upgrade add="true"/> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/6a628193/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py index f0e8b33..e254094 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py @@ -146,6 +146,7 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): putHawqSitePropertyAttribute = self.putPropertyAttribute(configurations, "hawq-site") hawq_sysctl_env = services["configurations"]["hawq-sysctl-env"]["properties"] putHawqSysctlEnvProperty = self.putProperty(configurations, "hawq-sysctl-env", services) + putHawqSysctlEnvPropertyAttribute = self.putPropertyAttribute(configurations, "hawq-sysctl-env") # remove master port when master is colocated with Ambari server if self.isHawqMasterComponentOnAmbariServer(services) and "hawq_master_address_port" in hawq_site: @@ -188,6 +189,9 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor): # set vm.overcommit_memory to 2 if the minimum memory among all hawqHosts is greater than 32GB vm_overcommit_mem_value = int(hawq_sysctl_env["vm.overcommit_memory"]) putHawqSysctlEnvProperty("vm.overcommit_ratio", vm_overcommit_ratio) + # Show vm.overcommit_ratio on theme only if vm.overcommit_memory is set to 2 + overcommit_ratio_visibility = "true" if vm_overcommit_mem_value == 2 else "false" + putHawqSysctlEnvPropertyAttribute("vm.overcommit_ratio", "visible", overcommit_ratio_visibility) putHawqSysctlEnvProperty("vm.overcommit_memory", vm_overcommit_mem_value) host_ram_kb = minHawqHostsMemory * vm_overcommit_ratio / 100 if vm_overcommit_mem_value == 2 else minHawqHostsMemory host_ram_gb = float(host_ram_kb) / (1024 * 1024) http://git-wip-us.apache.org/repos/asf/ambari/blob/6a628193/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/themes/theme.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/themes/theme.json b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/themes/theme.json index d7940a1..3c8a897 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/themes/theme.json +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/themes/theme.json @@ -11,7 +11,7 @@ "display-name":"Settings", "layout":{ "tab-columns":"1", - "tab-rows":"2", + "tab-rows":"3", "sections":[ { "name":"general", @@ -66,6 +66,32 @@ "column-span":"1" } ] + }, + { + "name":"workfile-limits", + "display-name":"Workfile Limits", + "row-index":"2", + "column-index":"0", + "row-span":"1", + "column-span":"1", + "section-columns":"2", + "section-rows":"1", + "subsections":[ + { + "name":"workfile-limits-col1", + "row-index":"0", + "column-index":"0", + "row-span":"1", + "column-span":"1" + }, + { + "name":"workfile-limits-col2", + "row-index":"0", + "column-index":"1", + "row-span":"1", + "column-span":"1" + } + ] } ] } @@ -80,60 +106,65 @@ "config":"hawq-site/hawq_master_directory", "subsection-name":"general-col1" }, - { "config":"hawq-site/hawq_segment_directory", "subsection-name":"general-col2" }, - { "config":"hawq-site/hawq_master_temp_directory", "subsection-name":"general-col1" }, - { "config":"hawq-site/hawq_segment_temp_directory", "subsection-name":"general-col2" }, - { "config":"hawq-site/default_hash_table_bucket_number", "subsection-name":"resource-management-col2" }, - { "config":"hawq-site/hawq_rm_nvseg_perquery_perseg_limit", "subsection-name":"resource-management-col2" }, - { "config":"hawq-site/hawq_rm_nvseg_perquery_limit", "subsection-name":"resource-management-col2" }, - { "config":"hawq-site/hawq_global_rm_type", "subsection-name":"resource-management-col1" }, - { "config":"hawq-sysctl-env/vm.overcommit_memory", "subsection-name":"resource-management-col1" }, - { "config":"hawq-sysctl-env/vm.overcommit_ratio", "subsection-name":"resource-management-col1" }, - { "config":"hawq-site/hawq_rm_memory_limit_perseg", "subsection-name":"resource-management-col1" }, - { "config":"hawq-site/hawq_rm_nvcore_limit_perseg", "subsection-name":"resource-management-col2" + }, + { + "config":"hawq-site/gp_workfile_compress_algorithm", + "subsection-name":"workfile-limits-col1" + }, + { + "config":"hawq-site/gp_workfile_limit_files_per_query", + "subsection-name":"workfile-limits-col1" + }, + { + "config":"hawq-site/gp_workfile_limit_per_query", + "subsection-name":"workfile-limits-col2" + }, + { + "config":"hawq-site/gp_workfile_limit_per_segment", + "subsection-name":"workfile-limits-col2" } ] }, @@ -198,7 +229,12 @@ { "config":"hawq-sysctl-env/vm.overcommit_ratio", "widget":{ - "type":"text-field" + "type":"slider", + "units": [ + { + "unit-name":"int" + } + ] } }, { @@ -234,6 +270,30 @@ } ] } + }, + { + "config":"hawq-site/gp_workfile_compress_algorithm", + "widget":{ + "type":"combo" + } + }, + { + "config":"hawq-site/gp_workfile_limit_files_per_query", + "widget":{ + "type":"text-field" + } + }, + { + "config":"hawq-site/gp_workfile_limit_per_query", + "widget":{ + "type":"text-field" + } + }, + { + "config":"hawq-site/gp_workfile_limit_per_segment", + "widget":{ + "type":"text-field" + } } ] } http://git-wip-us.apache.org/repos/asf/ambari/blob/6a628193/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py index 12f4fa1..8d97baa 100644 --- a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py @@ -299,3 +299,20 @@ class TestHAWQ200ServiceAdvisor(TestCase): self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) for property, status in properties_visible_status.iteritems(): self.assertEqual(configurations["hawq-site"]["property_attributes"][property]["visible"], status) + + ## Test if vm.overcommit_ratio is set to visible / invisible based on the value of vm.overcommit_memory + + # Case 1: vm.overcommit_ratio should be invisible when overcommit_memory is set as 0 + services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"] = 0 + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) + self.assertEqual(configurations["hawq-sysctl-env"]["property_attributes"]["vm.overcommit_ratio"]["visible"], "false") + + # Case 2: vm.overcommit_ratio should be invisible when overcommit_memory is set as 1 + services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"] = 1 + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) + self.assertEqual(configurations["hawq-sysctl-env"]["property_attributes"]["vm.overcommit_ratio"]["visible"], "false") + + # Case 3: vm.overcommit_ratio should be visible when overcommit_memory is set as 2 + services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"] = 2 + self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts) + self.assertEqual(configurations["hawq-sysctl-env"]["property_attributes"]["vm.overcommit_ratio"]["visible"], "true") \ No newline at end of file
