[
https://issues.apache.org/jira/browse/HIVE-14801?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15507845#comment-15507845
]
Thejas M Nair commented on HIVE-14801:
--------------------------------------
Overall around 5 seconds saved -
cc [~sseth]
Before -
{code}
<?xml version="1.0" encoding="UTF-8"?>
<testsuite xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="https://maven.apache.org/surefire/maven-surefire-plugin/xsd/surefire-test-report.xsd";
name="org.apache.hadoop.hive.metastore.TestPartitionNameWhitelistValidation"
time="14.252" tests="6" errors="0" skipped="0" failures="0">
<properties>
<property name="java.runtime.name" value="Java(TM) SE Runtime Environment"/>
<property name="sun.boot.library.path"
value="/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/lib"/>
<property name="java.vm.version" value="24.71-b01"/>
<property name="gopherProxySet" value="false"/>
<property name="qfile_regex" value=".*acid_mapjoin.*"/>
<property name="java.vm.vendor" value="Oracle Corporation"/>
<property name="maven.multiModuleProjectDirectory"
value="/Users/thejas/hive-thejas-tez2/hive/itests"/>
<property name="java.vendor.url" value="http://java.oracle.com/"/>
<property name="path.separator" value=":"/>
<property name="guice.disable.misplaced.annotation.check" value="true"/>
<property name="java.vm.name" value="Java HotSpot(TM) 64-Bit Server VM"/>
<property name="file.encoding.pkg" value="sun.io"/>
<property name="user.country" value="US"/>
<property name="sun.java.launcher" value="SUN_STANDARD"/>
<property name="sun.os.patch.level" value="unknown"/>
<property name="test" value="TestPartitionNameWhitelistValidation"/>
<property name="java.vm.specification.name" value="Java Virtual Machine
Specification"/>
<property name="user.dir"
value="/Users/thejas/hive-thejas-tez2/hive/itests"/>
<property name="java.runtime.version" value="1.7.0_71-b14"/>
<property name="java.awt.graphicsenv" value="sun.awt.CGraphicsEnvironment"/>
<property name="java.endorsed.dirs"
value="/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/lib/endorsed"/>
<property name="os.arch" value="x86_64"/>
<property name="java.io.tmpdir"
value="/var/folders/gw/s6wmsdd92j94s9xpqx781xb00000gp/T/"/>
<property name="line.separator" value=" "/>
<property name="java.vm.specification.vendor" value="Oracle Corporation"/>
<property name="os.name" value="Mac OS X"/>
<property name="classworlds.conf"
value="/Users/thejas/bin/apache-maven-3.3.9/bin/m2.conf"/>
<property name="test.output.overwrite" value="true"/>
<property name="sun.jnu.encoding" value="UTF-8"/>
<property name="java.library.path"
value="/Users/thejas/Library/Java/Extensions:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java:."/>
<property name="java.specification.name" value="Java Platform API
Specification"/>
<property name="java.class.version" value="51.0"/>
<property name="sun.management.compiler" value="HotSpot 64-Bit Tiered
Compilers"/>
<property name="os.version" value="10.11.6"/>
<property name="http.nonProxyHosts"
value="local|*.local|169.254/16|*.169.254/16"/>
<property name="user.home" value="/Users/thejas"/>
<property name="user.timezone" value="America/Los_Angeles"/>
<property name="java.awt.printerjob" value="sun.lwawt.macosx.CPrinterJob"/>
<property name="java.specification.version" value="1.7"/>
<property name="file.encoding" value="UTF-8"/>
<property name="user.name" value="thejas"/>
<property name="java.class.path"
value="/Users/thejas/bin/apache-maven-3.3.9/boot/plexus-classworlds-2.5.2.jar"/>
<property name="java.vm.specification.version" value="1.7"/>
<property name="sun.arch.data.model" value="64"/>
<property name="java.home"
value="/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre"/>
<property name="sun.java.command"
value="org.codehaus.plexus.classworlds.launcher.Launcher test
-Dtest=TestPartitionNameWhitelistValidation -Dqfile_regex=.*acid_mapjoin.*
-Phadoop-2 -Dtest.output.overwrite=true"/>
<property name="java.specification.vendor" value="Oracle Corporation"/>
<property name="user.language" value="en"/>
<property name="awt.toolkit" value="sun.lwawt.macosx.LWCToolkit"/>
<property name="java.vm.info" value="mixed mode"/>
<property name="java.version" value="1.7.0_71"/>
<property name="java.ext.dirs"
value="/Users/thejas/Library/Java/Extensions:/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/lib/ext:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java"/>
<property name="sun.boot.class.path"
value="/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/lib/resources.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/lib/rt.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/lib/sunrsasign.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/lib/jsse.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/lib/jce.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/lib/charsets.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/lib/jfr.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/classes"/>
<property name="java.vendor" value="Oracle Corporation"/>
<property name="java.awt.headless" value="true"/>
<property name="maven.home" value="/Users/thejas/bin/apache-maven-3.3.9"/>
<property name="file.separator" value="/"/>
<property name="java.vendor.url.bug"
value="http://bugreport.sun.com/bugreport/"/>
<property name="sun.cpu.endian" value="little"/>
<property name="sun.io.unicode.encoding" value="UnicodeBig"/>
<property name="socksNonProxyHosts"
value="local|*.local|169.254/16|*.169.254/16"/>
<property name="ftp.nonProxyHosts"
value="local|*.local|169.254/16|*.169.254/16"/>
<property name="sun.cpu.isalist" value=""/>
</properties>
<testcase name="testAppendPartitionWithValidCharacters"
classname="org.apache.hadoop.hive.metastore.TestPartitionNameWhitelistValidation"
time="10.883"/>
<testcase name="testAppendPartitionWithCommas"
classname="org.apache.hadoop.hive.metastore.TestPartitionNameWhitelistValidation"
time="1.243"/>
<testcase name="testAppendPartitionWithUnicode"
classname="org.apache.hadoop.hive.metastore.TestPartitionNameWhitelistValidation"
time="0.286"/>
<testcase name="testAddPartitionWithValidPartVal"
classname="org.apache.hadoop.hive.metastore.TestPartitionNameWhitelistValidation"
time="0.281"/>
<testcase name="testAddPartitionWithUnicode"
classname="org.apache.hadoop.hive.metastore.TestPartitionNameWhitelistValidation"
time="0.246"/>
<testcase name="testAddPartitionWithCommas"
classname="org.apache.hadoop.hive.metastore.TestPartitionNameWhitelistValidation"
time="1.239"/>
</testsuite>
{code}
After -
{code}
<?xml version="1.0" encoding="UTF-8"?>
<testsuite xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="https://maven.apache.org/surefire/maven-surefire-plugin/xsd/surefire-test-report.xsd";
name="org.apache.hadoop.hive.metastore.TestPartitionNameWhitelistValidation"
time="9.412" tests="6" errors="0" skipped="0" failures="0">
<properties>
<property name="java.runtime.name" value="Java(TM) SE Runtime Environment"/>
<property name="sun.boot.library.path"
value="/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/lib"/>
<property name="java.vm.version" value="24.71-b01"/>
<property name="gopherProxySet" value="false"/>
<property name="qfile_regex" value=".*acid_mapjoin.*"/>
<property name="java.vm.vendor" value="Oracle Corporation"/>
<property name="maven.multiModuleProjectDirectory"
value="/Users/thejas/hive-thejas-tez2/hive/itests"/>
<property name="java.vendor.url" value="http://java.oracle.com/"/>
<property name="path.separator" value=":"/>
<property name="guice.disable.misplaced.annotation.check" value="true"/>
<property name="java.vm.name" value="Java HotSpot(TM) 64-Bit Server VM"/>
<property name="file.encoding.pkg" value="sun.io"/>
<property name="user.country" value="US"/>
<property name="sun.java.launcher" value="SUN_STANDARD"/>
<property name="sun.os.patch.level" value="unknown"/>
<property name="test" value="TestPartitionNameWhitelistValidation"/>
<property name="java.vm.specification.name" value="Java Virtual Machine
Specification"/>
<property name="user.dir"
value="/Users/thejas/hive-thejas-tez2/hive/itests"/>
<property name="java.runtime.version" value="1.7.0_71-b14"/>
<property name="java.awt.graphicsenv" value="sun.awt.CGraphicsEnvironment"/>
<property name="java.endorsed.dirs"
value="/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/lib/endorsed"/>
<property name="os.arch" value="x86_64"/>
<property name="java.io.tmpdir"
value="/var/folders/gw/s6wmsdd92j94s9xpqx781xb00000gp/T/"/>
<property name="line.separator" value=" "/>
<property name="java.vm.specification.vendor" value="Oracle Corporation"/>
<property name="os.name" value="Mac OS X"/>
<property name="classworlds.conf"
value="/Users/thejas/bin/apache-maven-3.3.9/bin/m2.conf"/>
<property name="test.output.overwrite" value="true"/>
<property name="sun.jnu.encoding" value="UTF-8"/>
<property name="java.library.path"
value="/Users/thejas/Library/Java/Extensions:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java:."/>
<property name="java.specification.name" value="Java Platform API
Specification"/>
<property name="java.class.version" value="51.0"/>
<property name="sun.management.compiler" value="HotSpot 64-Bit Tiered
Compilers"/>
<property name="os.version" value="10.11.6"/>
<property name="http.nonProxyHosts"
value="local|*.local|169.254/16|*.169.254/16"/>
<property name="user.home" value="/Users/thejas"/>
<property name="user.timezone" value="America/Los_Angeles"/>
<property name="java.awt.printerjob" value="sun.lwawt.macosx.CPrinterJob"/>
<property name="java.specification.version" value="1.7"/>
<property name="file.encoding" value="UTF-8"/>
<property name="user.name" value="thejas"/>
<property name="java.class.path"
value="/Users/thejas/bin/apache-maven-3.3.9/boot/plexus-classworlds-2.5.2.jar"/>
<property name="java.vm.specification.version" value="1.7"/>
<property name="sun.arch.data.model" value="64"/>
<property name="java.home"
value="/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre"/>
<property name="sun.java.command"
value="org.codehaus.plexus.classworlds.launcher.Launcher test
-Dtest=TestPartitionNameWhitelistValidation -Dqfile_regex=.*acid_mapjoin.*
-Phadoop-2 -Dtest.output.overwrite=true"/>
<property name="java.specification.vendor" value="Oracle Corporation"/>
<property name="user.language" value="en"/>
<property name="awt.toolkit" value="sun.lwawt.macosx.LWCToolkit"/>
<property name="java.vm.info" value="mixed mode"/>
<property name="java.version" value="1.7.0_71"/>
<property name="java.ext.dirs"
value="/Users/thejas/Library/Java/Extensions:/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/lib/ext:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java"/>
<property name="sun.boot.class.path"
value="/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/lib/resources.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/lib/rt.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/lib/sunrsasign.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/lib/jsse.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/lib/jce.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/lib/charsets.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/lib/jfr.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home/jre/classes"/>
<property name="java.vendor" value="Oracle Corporation"/>
<property name="java.awt.headless" value="true"/>
<property name="maven.home" value="/Users/thejas/bin/apache-maven-3.3.9"/>
<property name="file.separator" value="/"/>
<property name="java.vendor.url.bug"
value="http://bugreport.sun.com/bugreport/"/>
<property name="sun.cpu.endian" value="little"/>
<property name="sun.io.unicode.encoding" value="UnicodeBig"/>
<property name="socksNonProxyHosts"
value="local|*.local|169.254/16|*.169.254/16"/>
<property name="ftp.nonProxyHosts"
value="local|*.local|169.254/16|*.169.254/16"/>
<property name="sun.cpu.isalist" value=""/>
</properties>
<testcase name="testAppendPartitionWithValidCharacters"
classname="org.apache.hadoop.hive.metastore.TestPartitionNameWhitelistValidation"
time="8.676"/>
<testcase name="testAppendPartitionWithCommas"
classname="org.apache.hadoop.hive.metastore.TestPartitionNameWhitelistValidation"
time="0.145"/>
<testcase name="testAppendPartitionWithUnicode"
classname="org.apache.hadoop.hive.metastore.TestPartitionNameWhitelistValidation"
time="0.136"/>
<testcase name="testAddPartitionWithValidPartVal"
classname="org.apache.hadoop.hive.metastore.TestPartitionNameWhitelistValidation"
time="0.132"/>
<testcase name="testAddPartitionWithUnicode"
classname="org.apache.hadoop.hive.metastore.TestPartitionNameWhitelistValidation"
time="0.135"/>
<testcase name="testAddPartitionWithCommas"
classname="org.apache.hadoop.hive.metastore.TestPartitionNameWhitelistValidation"
time="0.14"/>
</testsuite>
{code}
> improve TestPartitionNameWhitelistValidation stability
> ------------------------------------------------------
>
> Key: HIVE-14801
> URL: https://issues.apache.org/jira/browse/HIVE-14801
> Project: Hive
> Issue Type: Bug
> Reporter: Thejas M Nair
> Assignee: Thejas M Nair
> Attachments: HIVE-14801.1.patch, HIVE-14801.2.patch
>
>
> TestPartitionNameWhitelistValidation uses remote metastore. However, there
> can be multiple issues around startup of remote metastore, including race
> conditions in finding available port. In addition, all the initialization
> done at startup of remote metastore is likely to make the test case take more
> time.
> This test case doesn't need remote metastore, so it should be moved to using
> embedded metastore.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
