[
https://issues.apache.org/jira/browse/HIVE-26727?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
László Bodor updated HIVE-26727:
--------------------------------
Summary: Log all mr -> tez config overrides, fallbacks (was: Log all mr ->
tez config overrides)
> Log all mr -> tez config overrides, fallbacks
> ---------------------------------------------
>
> Key: HIVE-26727
> URL: https://issues.apache.org/jira/browse/HIVE-26727
> Project: Hive
> Issue Type: Improvement
> Reporter: László Bodor
> Priority: Major
>
> in TezSessionState.setupTezParamsBasedOnMR
> {code}
> private void setupTezParamsBasedOnMR(TezConfiguration conf) {
> String env = conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV);
> if (conf.get(MRJobConfig.MR_AM_ENV) != null) {
> env = (env == null) ? conf.get(MRJobConfig.MR_AM_ENV) : env + "," +
> conf.get(MRJobConfig.MR_AM_ENV);
> }
> if (env != null) {
> conf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_ENV, env);
> }
> conf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
> org.apache.tez.mapreduce.hadoop.MRHelpers.getJavaOptsForMRAM(conf));
> String queueName = conf.get(JobContext.QUEUE_NAME,
> YarnConfiguration.DEFAULT_QUEUE_NAME);
> conf.setIfUnset(TezConfiguration.TEZ_QUEUE_NAME, queueName); <---- NOT
> LOGGED AT ALL
> int amMemMB = conf.getInt(MRJobConfig.MR_AM_VMEM_MB,
> MRJobConfig.DEFAULT_MR_AM_VMEM_MB);
> conf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, "" +
> amMemMB);<---- NOT LOGGED AT ALL
> int amCores = conf.getInt(MRJobConfig.MR_AM_CPU_VCORES,
> MRJobConfig.DEFAULT_MR_AM_CPU_VCORES);
> conf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, "" +
> amCores);<---- NOT LOGGED AT ALL
> conf.setIfUnset(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, ""<---- NOT
> LOGGED AT ALL
> + conf.getInt(MRJobConfig.MR_AM_MAX_ATTEMPTS,
> MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS));
> conf.setIfUnset(TezConfiguration.TEZ_AM_VIEW_ACLS,<---- NOT LOGGED AT ALL
> conf.get(MRJobConfig.JOB_ACL_VIEW_JOB,
> MRJobConfig.DEFAULT_JOB_ACL_VIEW_JOB));
> conf.setIfUnset(TezConfiguration.TEZ_AM_MODIFY_ACLS,<---- NOT LOGGED AT
> ALL
> conf.get(MRJobConfig.JOB_ACL_MODIFY_JOB,
> MRJobConfig.DEFAULT_JOB_ACL_MODIFY_JOB));
> // Refer to
> org.apache.tez.mapreduce.hadoop.MRHelpers.processDirectConversion.
> ArrayList<Map<String, String>> maps = new ArrayList<Map<String,
> String>>(2);
> maps.add(DeprecatedKeys.getMRToTezRuntimeParamMap());
> maps.add(DeprecatedKeys.getMRToDAGParamMap());
> boolean preferTez = true; // Can make this configurable.
> for (Map<String, String> map : maps) {
> for (Map.Entry<String, String> dep : map.entrySet()) {
> if (conf.get(dep.getKey()) != null) {
> // TODO Deprecation reason does not seem to reflect in the config ?
> // The ordering is important in case of keys which are also
> deprecated.
> // Unset will unset the deprecated keys and all its variants.
> final String mrValue = conf.get(dep.getKey());
> final String tezValue = conf.get(dep.getValue());
> conf.unset(dep.getKey());
> if (tezValue == null) {
> conf.set(dep.getValue(), mrValue, "TRANSLATED_TO_TEZ");
> } else if (!preferTez) {
> conf.set(dep.getValue(), mrValue,
> "TRANSLATED_TO_TEZ_AND_MR_OVERRIDE");
> }
> LOG.info("Config: mr(unset):" + dep.getKey() + ", mr initial
> value=" <---- LOGGED PROPERLY
> + mrValue
> + ", tez(original):" + dep.getValue() + "=" + tezValue
> + ", tez(final):" + dep.getValue() + "=" +
> conf.get(dep.getValue()));
> }
> }
> }
> }
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
