Author: khorgath Date: Thu May 8 23:24:17 2014 New Revision: 1593453 URL: http://svn.apache.org/r1593453 Log: HIVE-4576 : templeton.hive.properties does not allow values with commas (Eugene Koifman via Ashutosh Chauhan) ((from https://issues.apache.org/jira/secure/attachment/12643464/HIVE-4576.0.13.patch)
Modified: hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java hive/branches/branch-0.13/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java Modified: hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml?rev=1593453&r1=1593452&r2=1593453&view=diff ============================================================================== --- hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml (original) +++ hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml Thu May 8 23:24:17 2014 @@ -136,8 +136,9 @@ <property> <name>templeton.hive.properties</name> - <value>hive.metastore.local=false,hive.metastore.uris=thrift://localhost:9933,hive.metastore.sasl.enabled=false</value> - <description>Properties to set when running hive.</description> + <value>hive.metastore.uris=thrift://localhost:9933,hive.metastore.sasl.enabled=false</value> + <description>Properties to set when running hive. This is expected to be a comma-separated prop=value list. + If some value is itself a comma-separated list the escape character is '\'</description> </property> <property> Modified: hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java?rev=1593453&r1=1593452&r2=1593453&view=diff ============================================================================== --- hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java (original) +++ hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java Thu May 8 23:24:17 2014 @@ -21,6 +21,8 @@ package org.apache.hive.hcatalog.templet import java.io.File; import java.net.URL; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Iterator; @@ -31,8 +33,10 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.VersionInfo; import org.apache.hive.hcatalog.templeton.tool.JobState; +import org.apache.hive.hcatalog.templeton.tool.TempletonUtils; import org.apache.hive.hcatalog.templeton.tool.ZooKeeperCleanup; import org.apache.hive.hcatalog.templeton.tool.ZooKeeperStorage; @@ -104,6 +108,11 @@ public class AppConfig extends Configura * see webhcat-default.xml */ public static final String HCAT_HOME_PATH = "templeton.hcat.home"; + /** + * is a comma separated list of name=value pairs; + * In case some value is itself a comma-separated list, the comma needs to + * be escaped with {@link org.apache.hadoop.util.StringUtils#ESCAPE_CHAR} + */ public static final String HIVE_PROPS_NAME = "templeton.hive.properties"; public static final String LIB_JARS_NAME = "templeton.libjars"; public static final String PIG_ARCHIVE_NAME = "templeton.pig.archive"; @@ -247,8 +256,18 @@ public class AppConfig extends Configura public String controllerMRChildOpts() { return get(TEMPLETON_CONTROLLER_MR_CHILD_OPTS); } - - + /** + * @see #HIVE_PROPS_NAME + */ + public Collection<String> hiveProps() { + String[] props= StringUtils.split(get(HIVE_PROPS_NAME)); + //since raw data was (possibly) escaped to make split work, + //now need to remove escape chars so they don't interfere with downstream processing + for(int i = 0; i < props.length; i++) { + props[i] = TempletonUtils.unEscape(props[i]); + } + return Arrays.asList(props); + } public String[] overrideJars() { if (getBoolean(OVERRIDE_JARS_ENABLED, true)) Modified: hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java?rev=1593453&r1=1593452&r2=1593453&view=diff ============================================================================== --- hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java (original) +++ hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java Thu May 8 23:24:17 2014 @@ -82,7 +82,7 @@ public class HiveDelegator extends Launc args.add("--hiveconf"); args.add(JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER_TEZ); - for (String prop : appConf.getStrings(AppConfig.HIVE_PROPS_NAME)) { + for (String prop : appConf.hiveProps()) { args.add("--hiveconf"); args.add(TempletonUtils.quoteForWindows(prop)); } Modified: hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java?rev=1593453&r1=1593452&r2=1593453&view=diff ============================================================================== --- hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java (original) +++ hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java Thu May 8 23:24:17 2014 @@ -45,6 +45,7 @@ public interface JobSubmissionConstants "__TEZ_CREDENTIALS_OPTION=WEBHCAT_TOKEN_FILE_LOCATION_TEZ__"; /** * constants needed for Pig job submission + * The string values here are what Pig expects to see in it's environment */ public static interface PigConstants { public static final String HIVE_HOME = "HIVE_HOME"; Modified: hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java?rev=1593453&r1=1593452&r2=1593453&view=diff ============================================================================== --- hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java (original) +++ hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java Thu May 8 23:24:17 2014 @@ -29,6 +29,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.util.Shell; +import org.apache.hadoop.util.StringUtils; import org.apache.hive.hcatalog.templeton.BadParam; import org.apache.hive.hcatalog.templeton.LauncherDelegator; @@ -79,8 +80,8 @@ public class LaunchMapper extends Mapper } if(conf.get(PigConstants.PIG_OPTS) != null) { StringBuilder pigOpts = new StringBuilder(); - for(String prop : conf.get(PigConstants.PIG_OPTS).split(",")) { - pigOpts.append("-D").append(prop).append(" "); + for(String prop : StringUtils.split(conf.get(PigConstants.PIG_OPTS))) { + pigOpts.append("-D").append(TempletonUtils.unEscape(prop)).append(" "); } env.put(PigConstants.PIG_OPTS, pigOpts.toString()); } Modified: hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java?rev=1593453&r1=1593452&r2=1593453&view=diff ============================================================================== --- hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java (original) +++ hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java Thu May 8 23:24:17 2014 @@ -36,6 +36,7 @@ import java.util.regex.Pattern; import javax.ws.rs.core.UriBuilder; +import org.apache.commons.lang.StringEscapeUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -354,4 +355,8 @@ public class TempletonUtils { args.add("call"); } } + + public static String unEscape(String input) { + return StringEscapeUtils.unescapeJava(input); + } } Modified: hive/branches/branch-0.13/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java?rev=1593453&r1=1593452&r2=1593453&view=diff ============================================================================== --- hive/branches/branch-0.13/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java (original) +++ hive/branches/branch-0.13/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java Thu May 8 23:24:17 2014 @@ -290,5 +290,23 @@ public class TestTempletonUtils { } catch (URISyntaxException ex) { } } - + + @Test + public void testPropertiesParsing() throws Exception { + String[] props = {"hive.metastore.uris=thrift://localhost:9933\\,thrift://127.0.0.1:9933", + "hive.metastore.sasl.enabled=false", + "hive.some.fake.path=C:\\foo\\bar.txt\\"}; + StringBuilder input = new StringBuilder(); + for(String prop : props) { + if(input.length() > 0) { + input.append(','); + } + input.append(prop); + } + String[] newProps = StringUtils.split(input.toString()); + for(int i = 0; i < newProps.length; i++) { + Assert.assertEquals("Pre/post split values don't match", + TempletonUtils.unEscape(props[i]), TempletonUtils.unEscape(newProps[i])); + } + } }