Propchange: hive/branches/llap/ ------------------------------------------------------------------------------ Merged /hive/branches/cbo:r1605012-1627125 Merged /hive/trunk:r1625337-1631832
Modified: hive/branches/llap/accumulo-handler/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/llap/accumulo-handler/pom.xml?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/accumulo-handler/pom.xml (original) +++ hive/branches/llap/accumulo-handler/pom.xml Tue Oct 14 19:06:45 2014 @@ -112,6 +112,12 @@ <dependencies> <dependency> <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop-20S.version}</version> + <optional>true</optional> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-core</artifactId> <version>${hadoop-20S.version}</version> <optional>true</optional> @@ -123,6 +129,12 @@ <dependencies> <dependency> <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop-23.version}</version> + <optional>true</optional> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop-23.version}</version> <optional>true</optional> Modified: hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/columns/ColumnMapper.java URL: http://svn.apache.org/viewvc/hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/columns/ColumnMapper.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/columns/ColumnMapper.java (original) +++ hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/columns/ColumnMapper.java Tue Oct 14 19:06:45 2014 @@ -20,11 +20,11 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.accumulo.AccumuloHiveConstants; import org.apache.hadoop.hive.accumulo.serde.TooManyAccumuloColumnsException; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Logger; import com.google.common.base.Preconditions; Modified: hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/HiveAccumuloTableOutputFormat.java URL: http://svn.apache.org/viewvc/hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/HiveAccumuloTableOutputFormat.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/HiveAccumuloTableOutputFormat.java (original) +++ hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/HiveAccumuloTableOutputFormat.java Tue Oct 14 19:06:45 2014 @@ -22,10 +22,14 @@ import org.apache.accumulo.core.client.A import org.apache.accumulo.core.client.mapred.AccumuloOutputFormat; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.data.Mutation; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hive.accumulo.AccumuloConnectionParameters; import org.apache.hadoop.hive.accumulo.serde.AccumuloSerDeParameters; +import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordWriter; +import org.apache.hadoop.util.Progressable; import com.google.common.base.Preconditions; @@ -41,6 +45,13 @@ public class HiveAccumuloTableOutputForm super.checkOutputSpecs(ignored, job); } + @Override + public RecordWriter<Text,Mutation> getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException { + configureAccumuloOutputFormat(job); + + return super.getRecordWriter(ignored, job, name, progress); + } + protected void configureAccumuloOutputFormat(JobConf job) throws IOException { AccumuloConnectionParameters cnxnParams = new AccumuloConnectionParameters(job); @@ -76,16 +87,32 @@ public class HiveAccumuloTableOutputForm protected void setAccumuloConnectorInfo(JobConf conf, String username, AuthenticationToken token) throws AccumuloSecurityException { - AccumuloOutputFormat.setConnectorInfo(conf, username, token); + try { + AccumuloOutputFormat.setConnectorInfo(conf, username, token); + } catch (IllegalStateException e) { + // AccumuloOutputFormat complains if you re-set an already set value. We just don't care. + log.debug("Ignoring exception setting Accumulo Connector instance for user " + username, e); + } } @SuppressWarnings("deprecation") protected void setAccumuloZooKeeperInstance(JobConf conf, String instanceName, String zookeepers) { - AccumuloOutputFormat.setZooKeeperInstance(conf, instanceName, zookeepers); + try { + AccumuloOutputFormat.setZooKeeperInstance(conf, instanceName, zookeepers); + } catch (IllegalStateException ise) { + // AccumuloOutputFormat complains if you re-set an already set value. We just don't care. + log.debug("Ignoring exception setting ZooKeeper instance of " + instanceName + " at " + + zookeepers, ise); + } } protected void setAccumuloMockInstance(JobConf conf, String instanceName) { - AccumuloOutputFormat.setMockInstance(conf, instanceName); + try { + AccumuloOutputFormat.setMockInstance(conf, instanceName); + } catch (IllegalStateException e) { + // AccumuloOutputFormat complains if you re-set an already set value. We just don't care. + log.debug("Ignoring exception setting mock instance of " + instanceName, e); + } } protected void setDefaultAccumuloTableName(JobConf conf, String tableName) { Modified: hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/AccumuloRowSerializer.java URL: http://svn.apache.org/viewvc/hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/AccumuloRowSerializer.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/AccumuloRowSerializer.java (original) +++ hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/AccumuloRowSerializer.java Tue Oct 14 19:06:45 2014 @@ -99,9 +99,6 @@ public class AccumuloRowSerializer { // The ObjectInspector for the row ID ObjectInspector fieldObjectInspector = field.getFieldObjectInspector(); - log.info("Serializing rowId with " + value + " in " + field + " using " - + rowIdFactory.getClass()); - // Serialize the row component using the RowIdFactory. In the normal case, this will just // delegate back to the "local" serializeRowId method byte[] data = rowIdFactory.serializeRowId(value, field, output); Modified: hive/branches/llap/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/columns/TestColumnMapper.java URL: http://svn.apache.org/viewvc/hive/branches/llap/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/columns/TestColumnMapper.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/columns/TestColumnMapper.java (original) +++ hive/branches/llap/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/columns/TestColumnMapper.java Tue Oct 14 19:06:45 2014 @@ -20,12 +20,12 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.accumulo.AccumuloHiveConstants; import org.apache.hadoop.hive.accumulo.serde.TooManyAccumuloColumnsException; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import org.apache.hadoop.util.StringUtils; import org.junit.Assert; import org.junit.Test; Modified: hive/branches/llap/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/FirstCharAccumuloCompositeRowId.java URL: http://svn.apache.org/viewvc/hive/branches/llap/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/FirstCharAccumuloCompositeRowId.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/FirstCharAccumuloCompositeRowId.java (original) +++ hive/branches/llap/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/FirstCharAccumuloCompositeRowId.java Tue Oct 14 19:06:45 2014 @@ -19,10 +19,10 @@ package org.apache.hadoop.hive.accumulo. import java.util.Arrays; import java.util.Properties; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector; -import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Logger; /** Modified: hive/branches/llap/beeline/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/llap/beeline/pom.xml?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/beeline/pom.xml (original) +++ hive/branches/llap/beeline/pom.xml Tue Oct 14 19:06:45 2014 @@ -49,6 +49,11 @@ <artifactId>hive-shims</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-jdbc</artifactId> + <version>${project.version}</version> + </dependency> <!-- inter-project --> <dependency> <groupId>commons-cli</groupId> @@ -88,12 +93,6 @@ <!-- test intra-project --> <dependency> <groupId>org.apache.hive</groupId> - <artifactId>hive-jdbc</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.hive</groupId> <artifactId>hive-exec</artifactId> <version>${project.version}</version> <classifier>tests</classifier> Modified: hive/branches/llap/beeline/src/java/org/apache/hive/beeline/BeeLine.java URL: http://svn.apache.org/viewvc/hive/branches/llap/beeline/src/java/org/apache/hive/beeline/BeeLine.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/beeline/src/java/org/apache/hive/beeline/BeeLine.java (original) +++ hive/branches/llap/beeline/src/java/org/apache/hive/beeline/BeeLine.java Tue Oct 14 19:06:45 2014 @@ -692,10 +692,6 @@ public class BeeLine implements Closeabl int code = 0; if (!commands.isEmpty()) { - // for single command execute, disable color - getOpts().setColor(false); - getOpts().setHeaderInterval(-1); - for (Iterator<String> i = commands.iterator(); i.hasNext();) { String command = i.next().toString(); debug(loc("executing-command", command)); Modified: hive/branches/llap/beeline/src/java/org/apache/hive/beeline/Commands.java URL: http://svn.apache.org/viewvc/hive/branches/llap/beeline/src/java/org/apache/hive/beeline/Commands.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/beeline/src/java/org/apache/hive/beeline/Commands.java (original) +++ hive/branches/llap/beeline/src/java/org/apache/hive/beeline/Commands.java Tue Oct 14 19:06:45 2014 @@ -38,6 +38,7 @@ import java.sql.Driver; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.sql.SQLWarning; import java.util.Arrays; import java.util.Iterator; import java.util.LinkedList; @@ -47,10 +48,13 @@ import java.util.Set; import java.util.TreeSet; import org.apache.hadoop.hive.common.cli.ShellCmdExecutor; +import org.apache.hive.jdbc.HiveStatement; public class Commands { private final BeeLine beeLine; + private static final int DEFAULT_QUERY_PROGRESS_INTERVAL = 1000; + private static final int DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT = 10 * 1000; /** * @param beeLine @@ -728,7 +732,7 @@ public class Commands { beeLine.handleException(e); } - + line = line.trim(); if (line.endsWith(";")) { line = line.substring(0, line.length() - 1); } @@ -758,6 +762,7 @@ public class Commands { try { Statement stmnt = null; boolean hasResults; + Thread logThread = null; try { long start = System.currentTimeMillis(); @@ -767,7 +772,15 @@ public class Commands { hasResults = ((CallableStatement) stmnt).execute(); } else { stmnt = beeLine.createStatement(); - hasResults = stmnt.execute(sql); + if (beeLine.getOpts().isSilent()) { + hasResults = stmnt.execute(sql); + } else { + logThread = new Thread(createLogRunnable(stmnt)); + logThread.setDaemon(true); + logThread.start(); + hasResults = stmnt.execute(sql); + logThread.interrupt(); + } } beeLine.showWarnings(); @@ -782,6 +795,11 @@ public class Commands { beeLine.info(beeLine.loc("rows-selected", count) + " " + beeLine.locElapsedTime(end - start)); } finally { + if (logThread != null) { + logThread.join(DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT); + showRemainingLogsIfAny(stmnt); + logThread = null; + } rs.close(); } } while (BeeLine.getMoreResults(stmnt)); @@ -792,6 +810,13 @@ public class Commands { + " " + beeLine.locElapsedTime(end - start)); } } finally { + if (logThread != null) { + if (!logThread.isInterrupted()) { + logThread.interrupt(); + } + logThread.join(DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT); + showRemainingLogsIfAny(stmnt); + } if (stmnt != null) { stmnt.close(); } @@ -803,6 +828,61 @@ public class Commands { return true; } + private Runnable createLogRunnable(Statement statement) { + if (statement instanceof HiveStatement) { + final HiveStatement hiveStatement = (HiveStatement) statement; + + Runnable runnable = new Runnable() { + @Override + public void run() { + while (hiveStatement.hasMoreLogs()) { + try { + // fetch the log periodically and output to beeline console + for (String log : hiveStatement.getQueryLog()) { + beeLine.info(log); + } + Thread.sleep(DEFAULT_QUERY_PROGRESS_INTERVAL); + } catch (SQLException e) { + beeLine.error(new SQLWarning(e)); + return; + } catch (InterruptedException e) { + beeLine.debug("Getting log thread is interrupted, since query is done!"); + return; + } + } + } + }; + return runnable; + } else { + beeLine.debug("The statement instance is not HiveStatement type: " + statement.getClass()); + return new Runnable() { + @Override + public void run() { + // do nothing. + } + }; + } + } + + private void showRemainingLogsIfAny(Statement statement) { + if (statement instanceof HiveStatement) { + HiveStatement hiveStatement = (HiveStatement) statement; + List<String> logs; + do { + try { + logs = hiveStatement.getQueryLog(); + } catch (SQLException e) { + beeLine.error(new SQLWarning(e)); + return; + } + for (String log : logs) { + beeLine.info(log); + } + } while (logs.size() > 0); + } else { + beeLine.debug("The statement instance is not HiveStatement type: " + statement.getClass()); + } + } public boolean quit(String line) { beeLine.setExit(true); Modified: hive/branches/llap/bin/beeline.cmd URL: http://svn.apache.org/viewvc/hive/branches/llap/bin/beeline.cmd?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/bin/beeline.cmd (original) +++ hive/branches/llap/bin/beeline.cmd Tue Oct 14 19:06:45 2014 @@ -43,7 +43,22 @@ if not exist %HADOOP_HOME%\libexec\hadoo @rem supress the HADOOP_HOME warnings in 1.x.x set HADOOP_HOME_WARN_SUPPRESS=true call %HADOOP_HOME%\libexec\hadoop-config.cmd -set CLASSPATH=%CLASSPATH%;%HIVE_HOME%\lib\*; + +@rem include only the beeline client jar and its dependencies +pushd %HIVE_HOME%\lib +for /f %%a IN ('dir /b hive-beeline-**.jar') do ( + set CLASSPATH=%CLASSPATH%;%HIVE_HOME%\lib\%%a +) +for /f %%a IN ('dir /b super-csv-**.jar') do ( + set CLASSPATH=%CLASSPATH%;%HIVE_HOME%\lib\%%a +) +for /f %%a IN ('dir /b jline-**.jar') do ( + set CLASSPATH=%CLASSPATH%;%HIVE_HOME%\lib\%%a +) +for /f %%a IN ('dir /b hive-jdbc-**-standalone.jar') do ( + set CLASSPATH=%CLASSPATH%;%HIVE_HOME%\lib\%%a +) +popd call %JAVA_HOME%\bin\java %JAVA_HEAP_MAX% %HADOOP_OPTS% -classpath %CLASSPATH% org.apache.hive.beeline.BeeLine %* Modified: hive/branches/llap/bin/ext/beeline.sh URL: http://svn.apache.org/viewvc/hive/branches/llap/bin/ext/beeline.sh?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/bin/ext/beeline.sh (original) +++ hive/branches/llap/bin/ext/beeline.sh Tue Oct 14 19:06:45 2014 @@ -19,11 +19,17 @@ export SERVICE_LIST="${SERVICE_LIST}${TH beeline () { CLASS=org.apache.hive.beeline.BeeLine; - execHiveCmd $CLASS "$@" + + # include only the beeline client jar and its dependencies + beelineJarPath=`ls ${HIVE_LIB}/hive-beeline-*.jar` + superCsvJarPath=`ls ${HIVE_LIB}/super-csv-*.jar` + jlineJarPath=`ls ${HIVE_LIB}/jline-*.jar` + jdbcStandaloneJarPath=`ls ${HIVE_LIB}/hive-jdbc-*-standalone.jar` + export HADOOP_CLASSPATH=${beelineJarPath}:${superCsvJarPath}:${jlineJarPath}:${jdbcStandaloneJarPath} + + exec $HADOOP jar ${beelineJarPath} $CLASS $HIVE_OPTS "$@" } beeline_help () { - CLASS=org.apache.hive.beeline.BeeLine; - execHiveCmd $CLASS "--help" + beeline "--help" } - Modified: hive/branches/llap/bin/ext/hiveserver2.cmd URL: http://svn.apache.org/viewvc/hive/branches/llap/bin/ext/hiveserver2.cmd?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/bin/ext/hiveserver2.cmd (original) +++ hive/branches/llap/bin/ext/hiveserver2.cmd Tue Oct 14 19:06:45 2014 @@ -59,10 +59,7 @@ if [%1]==[hiveserver2_help] goto :hivese if [%1]==[hiveserver2_catservice] goto :hiveserver2_catservice -if [%1]==[hiveserver2_catcmd] goto :hiveserver2_catcmd - :hiveserver2 - echo "Starting Hive Thrift Server" @rem hadoop 20 or newer - skip the aux_jars option and hiveconf call %HIVE_BIN_PATH%\ext\util\execHiveCmd.cmd %CLASS% @@ -78,21 +75,11 @@ goto :EOF @echo ^<id^>HiveServer2^</id^> @echo ^<name^>HiveServer2^</name^> @echo ^<description^>Hadoop HiveServer2 Service^</description^> -@echo ^<executable^>%SystemRoot%\system32\cmd.exe^</executable^> -@echo ^<arguments^>/c %HIVE_BIN_PATH%\ext\hs2service.cmd ^</arguments^> +@echo ^<executable^>%JAVA_HOME%\bin\java^</executable^> +@echo ^<arguments^>%JAVA_HEAP_MAX% %HADOOP_OPTS% -classpath %CLASSPATH%;%HIVE_HBASE_PATH% %CLASS% -hiveconf hive.hadoop.classpath=%HIVE_LIB%\* -hiveconf hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory -hiveconf hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator -hiveconf hive.metastore.uris=" " %HIVE_OPTS%^</arguments^> @echo ^</service^> goto :EOF - -:hiveserver2_catcmd -if not defined HADOOP_CLASSPATH ( - @echo set HADOOP_CLASSPATH=%HIVE_LIB%\* - ) else ( - @echo set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%HIVE_LIB%\* - ) -@echo %JAVA_HOME%\bin\java %JAVA_HEAP_MAX% %HADOOP_OPTS% -classpath %CLASSPATH%;%HIVE_HBASE_PATH% %CLASS% -hiveconf hive.metastore.uris=" " -hiveconf hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory -hiveconf hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator %HIVE_OPTS% -goto :EOF - :AddToHiveHbasePath if not defined HIVE_HBASE_PATH ( set HIVE_HBASE_PATH=%1 Modified: hive/branches/llap/bin/ext/hiveserver2.sh URL: http://svn.apache.org/viewvc/hive/branches/llap/bin/ext/hiveserver2.sh?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/bin/ext/hiveserver2.sh (original) +++ hive/branches/llap/bin/ext/hiveserver2.sh Tue Oct 14 19:06:45 2014 @@ -17,7 +17,6 @@ THISSERVICE=hiveserver2 export SERVICE_LIST="${SERVICE_LIST}${THISSERVICE} " hiveserver2() { - echo "Starting HiveServer2" CLASS=org.apache.hive.service.server.HiveServer2 if $cygwin; then HIVE_LIB=`cygpath -w "$HIVE_LIB"` Modified: hive/branches/llap/bin/hive.cmd URL: http://svn.apache.org/viewvc/hive/branches/llap/bin/hive.cmd?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/bin/hive.cmd (original) +++ hive/branches/llap/bin/hive.cmd Tue Oct 14 19:06:45 2014 @@ -284,9 +284,6 @@ if defined CATSERVICE ( ) else ( call %HADOOP_HOME%\libexec\hadoop-config.cmd ) - if %TORUN% == hiveserver2 ( - call %HIVE_BIN_PATH%\ext\hiveserver2.cmd hiveserver2_catcmd > %HIVE_BIN_PATH%\ext\hs2service.cmd - ) call %HIVE_BIN_PATH%\ext\%TORUN%.cmd %TORUN%%CATSERVICE% %* goto :EOF ) Modified: hive/branches/llap/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java URL: http://svn.apache.org/viewvc/hive/branches/llap/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java (original) +++ hive/branches/llap/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java Tue Oct 14 19:06:45 2014 @@ -60,6 +60,7 @@ import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.mr.HadoopJobExecHelper; +import org.apache.hadoop.hive.ql.exec.tez.TezJobExecHelper; import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.hadoop.hive.ql.parse.VariableSubstitution; import org.apache.hadoop.hive.ql.processors.CommandProcessor; @@ -384,6 +385,7 @@ public class CliDriver { // First, kill any running MR jobs HadoopJobExecHelper.killRunningJobs(); + TezJobExecHelper.killRunningJobs(); HiveInterruptUtils.interrupt(); } }); Modified: hive/branches/llap/common/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/llap/common/pom.xml?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/common/pom.xml (original) +++ hive/branches/llap/common/pom.xml Tue Oct 14 19:06:45 2014 @@ -72,6 +72,12 @@ </dependency> <!-- test inter-project --> <dependency> + <groupId>com.google.code.tempus-fugit</groupId> + <artifactId>tempus-fugit</artifactId> + <version>${tempus-fugit.version}</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>${junit.version}</version> Modified: hive/branches/llap/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java URL: http://svn.apache.org/viewvc/hive/branches/llap/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original) +++ hive/branches/llap/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Tue Oct 14 19:06:45 2014 @@ -205,8 +205,8 @@ public class HiveConf extends Configurat PLAN_SERIALIZATION("hive.plan.serialization.format", "kryo", "Query plan format serialization between client and task nodes. \n" + "Two supported values are : kryo and javaXML. Kryo is default."), - SCRATCHDIR("hive.exec.scratchdir", "/tmp/hive", - "HDFS root scratch dir for Hive jobs which gets created with 777 permission. " + + SCRATCHDIR("hive.exec.scratchdir", "/tmp/hive", + "HDFS root scratch dir for Hive jobs which gets created with write all (733) permission. " + "For each connecting user, an HDFS scratch dir: ${hive.exec.scratchdir}/<username> is created, " + "with ${hive.scratch.dir.permission}."), LOCALSCRATCHDIR("hive.exec.local.scratchdir", @@ -215,7 +215,7 @@ public class HiveConf extends Configurat DOWNLOADED_RESOURCES_DIR("hive.downloaded.resources.dir", "${system:java.io.tmpdir}" + File.separator + "${hive.session.id}_resources", "Temporary local directory for added resources in the remote file system."), - SCRATCHDIRPERMISSION("hive.scratch.dir.permission", "700", + SCRATCHDIRPERMISSION("hive.scratch.dir.permission", "700", "The permission for the user specific scratch directories that get created."), SUBMITVIACHILD("hive.exec.submitviachild", false, ""), SUBMITLOCALTASKVIACHILD("hive.exec.submit.local.task.via.child", true, @@ -240,9 +240,9 @@ public class HiveConf extends Configurat "The compression codec and other options are determined from Hadoop config variables mapred.output.compress*"), COMPRESSINTERMEDIATECODEC("hive.intermediate.compression.codec", "", ""), COMPRESSINTERMEDIATETYPE("hive.intermediate.compression.type", "", ""), - BYTESPERREDUCER("hive.exec.reducers.bytes.per.reducer", (long) (1000 * 1000 * 1000), - "size per reducer.The default is 1G, i.e if the input size is 10G, it will use 10 reducers."), - MAXREDUCERS("hive.exec.reducers.max", 999, + BYTESPERREDUCER("hive.exec.reducers.bytes.per.reducer", (long) (256 * 1000 * 1000), + "size per reducer.The default is 256Mb, i.e if the input size is 1G, it will use 4 reducers."), + MAXREDUCERS("hive.exec.reducers.max", 1009, "max number of reducers will be used. If the one specified in the configuration parameter mapred.reduce.tasks is\n" + "negative, Hive will use this one as the max number of reducers when automatically determine number of reducers."), PREEXECHOOKS("hive.exec.pre.hooks", "", @@ -640,6 +640,9 @@ public class HiveConf extends Configurat HIVEJOINCACHESIZE("hive.join.cache.size", 25000, "How many rows in the joining tables (except the streaming table) should be cached in memory."), + // CBO related + HIVE_CBO_ENABLED("hive.cbo.enable", false, "Flag to control enabling Cost Based Optimizations using Calcite framework."), + // hive.mapjoin.bucket.cache.size has been replaced by hive.smbjoin.cache.row, // need to remove by hive .13. Also, do not change default (see SMB operator) HIVEMAPJOINBUCKETCACHESIZE("hive.mapjoin.bucket.cache.size", 100, ""), @@ -1041,7 +1044,7 @@ public class HiveConf extends Configurat "That means if reducer-num of the child RS is fixed (order by or forced bucketing) and small, it can make very slow, single MR.\n" + "The optimization will be automatically disabled if number of reducers would be less than specified value."), - HIVEOPTSORTDYNAMICPARTITION("hive.optimize.sort.dynamic.partition", true, + HIVEOPTSORTDYNAMICPARTITION("hive.optimize.sort.dynamic.partition", false, "When enabled dynamic partitioning column will be globally sorted.\n" + "This way we can keep only one record writer open for each partition value\n" + "in the reducer thereby reducing the memory pressure on reducers."), @@ -1186,13 +1189,6 @@ public class HiveConf extends Configurat "Average row size is computed from average column size of all columns in the row. In the absence\n" + "of column statistics and for variable length complex columns like map, the average number of\n" + "entries/values can be specified using this config."), - // to accurately compute statistics for GROUPBY map side parallelism needs to be known - HIVE_STATS_MAP_SIDE_PARALLELISM("hive.stats.map.parallelism", 1, - "Hive/Tez optimizer estimates the data size flowing through each of the operators.\n" + - "For GROUPBY operator, to accurately compute the data size map-side parallelism needs to\n" + - "be known. By default, this value is set to 1 since optimizer is not aware of the number of\n" + - "mappers during compile-time. This Hive config can be used to specify the number of mappers\n" + - "to be used for data size computation of GROUPBY operator."), // statistics annotation fetches stats for each partition, which can be expensive. turning // this off will result in basic sizes being fetched from namenode instead HIVE_STATS_FETCH_PARTITION_STATS("hive.stats.fetch.partition.stats", true, @@ -1243,10 +1239,16 @@ public class HiveConf extends Configurat "This param is to control whether or not only do lock on queries\n" + "that need to execute at least one mapred job."), + // Zookeeper related configs HIVE_ZOOKEEPER_QUORUM("hive.zookeeper.quorum", "", - "The list of ZooKeeper servers to talk to. This is only needed for read/write locks."), + "List of ZooKeeper servers to talk to. This is needed for: " + + "1. Read/write locks - when hive.lock.manager is set to " + + "org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager, " + + "2. When HiveServer2 supports service discovery via Zookeeper."), HIVE_ZOOKEEPER_CLIENT_PORT("hive.zookeeper.client.port", "2181", - "The port of ZooKeeper servers to talk to. This is only needed for read/write locks."), + "The port of ZooKeeper servers to talk to. " + + "If the list of Zookeeper servers specified in hive.zookeeper.quorum," + + "does not contain port numbers, this value is used."), HIVE_ZOOKEEPER_SESSION_TIMEOUT("hive.zookeeper.session.timeout", 600*1000, "ZooKeeper client's session timeout. The client is disconnected, and as a result, all locks released, \n" + "if a heartbeat is not sent in the timeout."), @@ -1294,6 +1296,9 @@ public class HiveConf extends Configurat "Number of aborted transactions involving a particular table or partition before major\n" + "compaction is initiated."), + HIVE_COMPACTOR_CLEANER_RUN_INTERVAL("hive.compactor.cleaner.run.interval", "5000ms", + new TimeValidator(TimeUnit.MILLISECONDS), "Time between runs of the cleaner thread"), + // For HBase storage handler HIVE_HBASE_WAL_ENABLED("hive.hbase.wal.enabled", true, "Whether writes to HBase should be forced to the write-ahead log. \n" + @@ -1368,6 +1373,8 @@ public class HiveConf extends Configurat "authorization manager class name to be used in the metastore for authorization.\n" + "The user defined authorization class should implement interface \n" + "org.apache.hadoop.hive.ql.security.authorization.HiveMetastoreAuthorizationProvider. "), + HIVE_METASTORE_AUTHORIZATION_AUTH_READS("hive.security.metastore.authorization.auth.reads", true, + "If this is true, metastore authorizer authorizes read actions on database, table"), HIVE_METASTORE_AUTHENTICATOR_MANAGER("hive.security.metastore.authenticator.manager", "org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator", "authenticator manager class name to be used in the metastore for authentication. \n" + @@ -1446,11 +1453,6 @@ public class HiveConf extends Configurat "If the property is set, the value must be a valid URI (java.net.URI, e.g. \"file:///tmp/my-logging.properties\"), \n" + "which you can then extract a URL from and pass to PropertyConfigurator.configure(URL)."), - // Hive global init file location - HIVE_GLOBAL_INIT_FILE_LOCATION("hive.server2.global.init.file.location", "${env:HIVE_CONF_DIR}", - "The location of HS2 global init file (.hiverc).\n" + - "If the property is reset, the value must be a valid path where the init file is located."), - // prefix used to auto generated column aliases (this should be started with '_') HIVE_AUTOGEN_COLUMNALIAS_PREFIX_LABEL("hive.autogen.columnalias.prefix.label", "_c", "String used as a prefix when auto generating column alias.\n" + @@ -1468,10 +1470,10 @@ public class HiveConf extends Configurat HIVE_INSERT_INTO_MULTILEVEL_DIRS("hive.insert.into.multilevel.dirs", false, "Where to insert into multilevel directories like\n" + "\"insert directory '/HIVEFT25686/chinna/' from table\""), - HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS("hive.warehouse.subdir.inherit.perms", false, - "Set this to true if the the table directories should inherit the\n" + - "permission of the warehouse or database directory instead of being created\n" + - "with the permissions derived from dfs umask"), + HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS("hive.warehouse.subdir.inherit.perms", true, + "Set this to false if the table directories should be created\n" + + "with the permissions derived from dfs umask instead of\n" + + "inheriting the permission of the warehouse or database directory."), HIVE_INSERT_INTO_EXTERNAL_TABLES("hive.insert.into.external.tables", true, "whether insert into external tables is allowed"), @@ -1489,16 +1491,29 @@ public class HiveConf extends Configurat "table. From 0.12 onwards, they are displayed separately. This flag will let you\n" + "get old behavior, if desired. See, test-case in patch for HIVE-6689."), + // HiveServer2 specific configs HIVE_SERVER2_MAX_START_ATTEMPTS("hive.server2.max.start.attempts", 30L, new RangeValidator(0L, null), - "This number of times HiveServer2 will attempt to start before exiting, sleeping 60 seconds between retries. \n" + - "The default of 30 will keep trying for 30 minutes."), - + "Number of times HiveServer2 will attempt to start before exiting, sleeping 60 seconds " + + "between retries. \n The default of 30 will keep trying for 30 minutes."), + HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY("hive.server2.support.dynamic.service.discovery", false, + "Whether HiveServer2 supports dynamic service discovery for its clients. " + + "To support this, each instance of HiveServer2 currently uses ZooKeeper to register itself, " + + "when it is brought up. JDBC/ODBC clients should use the ZooKeeper ensemble: " + + "hive.zookeeper.quorum in their connection string."), + HIVE_SERVER2_ZOOKEEPER_NAMESPACE("hive.server2.zookeeper.namespace", "hiveserver2", + "The parent node in ZooKeeper used by HiveServer2 when supporting dynamic service discovery."), + // HiveServer2 global init file location + HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION("hive.server2.global.init.file.location", "${env:HIVE_CONF_DIR}", + "Either the location of a HS2 global init file or a directory containing a .hiverc file. If the \n" + + "property is set, the value must be a valid path to an init file or directory where the init file is located."), HIVE_SERVER2_TRANSPORT_MODE("hive.server2.transport.mode", "binary", new StringSet("binary", "http"), "Transport mode of HiveServer2."), + HIVE_SERVER2_THRIFT_BIND_HOST("hive.server2.thrift.bind.host", "", + "Bind host on which to run the HiveServer2 Thrift service."), // http (over thrift) transport settings HIVE_SERVER2_THRIFT_HTTP_PORT("hive.server2.thrift.http.port", 10001, - "Port number when in HTTP mode."), + "Port number of HiveServer2 Thrift interface when hive.server2.transport.mode is 'http'."), HIVE_SERVER2_THRIFT_HTTP_PATH("hive.server2.thrift.http.path", "cliservice", "Path component of URL endpoint when in HTTP mode."), HIVE_SERVER2_THRIFT_HTTP_MIN_WORKER_THREADS("hive.server2.thrift.http.min.worker.threads", 5, @@ -1515,11 +1530,7 @@ public class HiveConf extends Configurat // binary transport settings HIVE_SERVER2_THRIFT_PORT("hive.server2.thrift.port", 10000, - "Port number of HiveServer2 Thrift interface.\n" + - "Can be overridden by setting $HIVE_SERVER2_THRIFT_PORT"), - HIVE_SERVER2_THRIFT_BIND_HOST("hive.server2.thrift.bind.host", "", - "Bind host on which to run the HiveServer2 Thrift interface.\n" + - "Can be overridden by setting $HIVE_SERVER2_THRIFT_BIND_HOST"), + "Port number of HiveServer2 Thrift interface when hive.server2.transport.mode is 'binary'."), // hadoop.rpc.protection being set to a higher level than HiveServer2 // does not make sense in most situations. // HiveServer2 ignores hadoop.rpc.protection in favor of hive.server2.thrift.sasl.qop. @@ -1702,6 +1713,9 @@ public class HiveConf extends Configurat HIVE_VECTORIZATION_REDUCE_ENABLED("hive.vectorized.execution.reduce.enabled", true, "This flag should be set to true to enable vectorized mode of the reduce-side of query execution.\n" + "The default value is true."), + HIVE_VECTORIZATION_REDUCE_GROUPBY_ENABLED("hive.vectorized.execution.reduce.groupby.enabled", true, + "This flag should be set to true to enable vectorized mode of the reduce-side GROUP BY query execution.\n" + + "The default value is true."), HIVE_VECTORIZATION_GROUPBY_CHECKINTERVAL("hive.vectorized.groupby.checkinterval", 100000, "Number of entries added to the group by aggregation hash before a recomputation of average entry size is performed."), HIVE_VECTORIZATION_GROUPBY_MAXENTRIES("hive.vectorized.groupby.maxentries", 1000000, @@ -1710,8 +1724,10 @@ public class HiveConf extends Configurat HIVE_VECTORIZATION_GROUPBY_FLUSH_PERCENT("hive.vectorized.groupby.flush.percent", (float) 0.1, "Percent of entries in the group by aggregation hash flushed when the memory threshold is exceeded."), - HIVE_TYPE_CHECK_ON_INSERT("hive.typecheck.on.insert", true, ""), + HIVE_HADOOP_CLASSPATH("hive.hadoop.classpath", null, + "For Windows OS, we need to pass HIVE_HADOOP_CLASSPATH Java parameter while starting HiveServer2 \n" + + "using \"-hiveconf hive.hadoop.classpath=%HIVE_LIB%\"."), HIVE_RPC_QUERY_PLAN("hive.rpc.query.plan", false, "Whether to send the query plan via local resource or RPC"), Modified: hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveChar.java URL: http://svn.apache.org/viewvc/hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveChar.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveChar.java (original) +++ hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveChar.java Tue Oct 14 19:06:45 2014 @@ -18,10 +18,19 @@ package org.apache.hadoop.hive.common.type; -import junit.framework.TestCase; - -public class TestHiveChar extends TestCase { - +import com.google.code.tempusfugit.concurrency.annotations.*; +import com.google.code.tempusfugit.concurrency.*; +import org.junit.*; +import static org.junit.Assert.*; + +public class TestHiveChar { + + @Rule public ConcurrentRule concurrentRule = new ConcurrentRule(); + @Rule public RepeatingRule repeatingRule = new RepeatingRule(); + + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testBasic() { HiveChar hc = new HiveChar("abc", 10); assertEquals("abc ", hc.toString()); @@ -47,6 +56,9 @@ public class TestHiveChar extends TestCa assertEquals(3, hc.getCharacterLength()); } + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testStringLength() { HiveChar hc = new HiveChar(); @@ -60,6 +72,9 @@ public class TestHiveChar extends TestCa assertEquals("0123456789 ", hc.toString()); } + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testComparison() { HiveChar hc1 = new HiveChar(); HiveChar hc2 = new HiveChar(); Modified: hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java URL: http://svn.apache.org/viewvc/hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java (original) +++ hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java Tue Oct 14 19:06:45 2014 @@ -20,12 +20,19 @@ package org.apache.hadoop.hive.common.ty import java.math.BigDecimal; import java.math.BigInteger; -import org.junit.Assert; -import org.junit.Test; +import com.google.code.tempusfugit.concurrency.annotations.*; +import com.google.code.tempusfugit.concurrency.*; +import org.junit.*; +import static org.junit.Assert.*; public class TestHiveDecimal { + @Rule public ConcurrentRule concurrentRule = new ConcurrentRule(); + @Rule public RepeatingRule repeatingRule = new RepeatingRule(); + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testPrecisionScaleEnforcement() { String decStr = "1786135888657847525803324040144343378.09799306448796128931113691624"; HiveDecimal dec = HiveDecimal.create(decStr); @@ -82,6 +89,8 @@ public class TestHiveDecimal { } @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testMultiply() { HiveDecimal dec1 = HiveDecimal.create("0.00001786135888657847525803"); HiveDecimal dec2 = HiveDecimal.create("3.0000123456789"); @@ -105,6 +114,8 @@ public class TestHiveDecimal { } @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testPow() { HiveDecimal dec = HiveDecimal.create("3.00001415926"); Assert.assertEquals(dec.pow(2), dec.multiply(dec)); @@ -118,6 +129,8 @@ public class TestHiveDecimal { } @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testDivide() { HiveDecimal dec1 = HiveDecimal.create("3.14"); HiveDecimal dec2 = HiveDecimal.create("3"); @@ -133,6 +146,8 @@ public class TestHiveDecimal { } @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testPlus() { HiveDecimal dec1 = HiveDecimal.create("99999999999999999999999999999999999"); HiveDecimal dec2 = HiveDecimal.create("1"); @@ -145,6 +160,8 @@ public class TestHiveDecimal { @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testSubtract() { HiveDecimal dec1 = HiveDecimal.create("3.140"); HiveDecimal dec2 = HiveDecimal.create("1.00"); @@ -152,6 +169,8 @@ public class TestHiveDecimal { } @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testPosMod() { HiveDecimal hd1 = HiveDecimal.create("-100.91"); HiveDecimal hd2 = HiveDecimal.create("9.8"); @@ -160,12 +179,16 @@ public class TestHiveDecimal { } @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testHashCode() { Assert.assertEquals(HiveDecimal.create("9").hashCode(), HiveDecimal.create("9.00").hashCode()); Assert.assertEquals(HiveDecimal.create("0").hashCode(), HiveDecimal.create("0.00").hashCode()); } @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testException() { HiveDecimal dec = HiveDecimal.create("3.1415.926"); Assert.assertNull(dec); @@ -174,6 +197,8 @@ public class TestHiveDecimal { } @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testBinaryConversion() { testBinaryConversion("0.00"); testBinaryConversion("-12.25"); Modified: hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveVarchar.java URL: http://svn.apache.org/viewvc/hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveVarchar.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveVarchar.java (original) +++ hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveVarchar.java Tue Oct 14 19:06:45 2014 @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hive.common.type; -import junit.framework.TestCase; import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.common.LogUtils; @@ -28,8 +27,15 @@ import java.io.IOException; import java.io.InputStreamReader; import java.util.Random; +import com.google.code.tempusfugit.concurrency.annotations.*; +import com.google.code.tempusfugit.concurrency.*; +import org.junit.*; +import static org.junit.Assert.*; + +public class TestHiveVarchar { + @Rule public ConcurrentRule concurrentRule = new ConcurrentRule(); + @Rule public RepeatingRule repeatingRule = new RepeatingRule(); -public class TestHiveVarchar extends TestCase { public TestHiveVarchar() { super(); } @@ -65,6 +71,9 @@ public class TestHiveVarchar extends Tes } } + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testStringLength() throws Exception { int strLen = 20; int[] lengths = { 15, 20, 25 }; @@ -124,6 +133,9 @@ public class TestHiveVarchar extends Tes assertEquals(5, vc1.getCharacterLength()); } + @Test + @Concurrent(count=4) + @Repeating(repetition=100) public void testComparison() throws Exception { HiveVarchar hc1 = new HiveVarchar("abcd", 20); HiveVarchar hc2 = new HiveVarchar("abcd", 20); Modified: hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleAvg.java URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleAvg.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleAvg.java (original) +++ hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleAvg.java Tue Oct 14 19:06:45 2014 @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.contrib.udaf.example; +import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDAF; import org.apache.hadoop.hive.ql.exec.UDAFEvaluator; @@ -32,6 +33,8 @@ import org.apache.hadoop.hive.ql.exec.UD * more efficient. * */ +@Description(name = "example_avg", +value = "_FUNC_(col) - Example UDAF to compute average") public final class UDAFExampleAvg extends UDAF { /** Modified: hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleGroupConcat.java URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleGroupConcat.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleGroupConcat.java (original) +++ hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleGroupConcat.java Tue Oct 14 19:06:45 2014 @@ -21,6 +21,7 @@ package org.apache.hadoop.hive.contrib.u import java.util.ArrayList; import java.util.Collections; +import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDAF; import org.apache.hadoop.hive.ql.exec.UDAFEvaluator; @@ -35,6 +36,8 @@ import org.apache.hadoop.hive.ql.exec.UD * implement built-in aggregation functions, which are harder to program but * more efficient. */ +@Description(name = "example_group_concat", +value = "_FUNC_(col) - Example UDAF that concatenates all arguments from different rows into a single string") public class UDAFExampleGroupConcat extends UDAF { /** Modified: hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMaxN.java URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMaxN.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMaxN.java (original) +++ hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMaxN.java Tue Oct 14 19:06:45 2014 @@ -19,11 +19,13 @@ package org.apache.hadoop.hive.contrib.udaf.example; +import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDAF; /** * Returns the max N double values. */ +@Description(name = "example_max_n", value = "_FUNC_(expr) - Example UDAF that returns the max N double values") public class UDAFExampleMaxN extends UDAF { /** Modified: hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMinN.java URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMinN.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMinN.java (original) +++ hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMinN.java Tue Oct 14 19:06:45 2014 @@ -19,11 +19,13 @@ package org.apache.hadoop.hive.contrib.udaf.example; +import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDAF; /** * Returns the min N double values. */ +@Description(name = "example_min_n", value = "_FUNC_(expr) - Example UDAF that returns the min N double values") public class UDAFExampleMinN extends UDAF{ /** Modified: hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleAdd.java URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleAdd.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleAdd.java (original) +++ hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleAdd.java Tue Oct 14 19:06:45 2014 @@ -17,12 +17,14 @@ */ package org.apache.hadoop.hive.contrib.udf.example; +import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; /** * UDFExampleAdd. * */ +@Description(name = "example_add", value = "_FUNC_(expr) - Example UDAF that returns the sum") public class UDFExampleAdd extends UDF { public Integer evaluate(Integer... a) { Modified: hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleArraySum.java URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleArraySum.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleArraySum.java (original) +++ hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleArraySum.java Tue Oct 14 19:06:45 2014 @@ -19,12 +19,14 @@ package org.apache.hadoop.hive.contrib.u import java.util.List; +import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; /** * UDFExampleArraySum. * */ +@Description(name = "example_arraysum", value = "_FUNC_(expr) - Example UDAF that returns the sum") public class UDFExampleArraySum extends UDF { public Double evaluate(List<Double> a) { Modified: hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleFormat.java URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleFormat.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleFormat.java (original) +++ hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleFormat.java Tue Oct 14 19:06:45 2014 @@ -17,12 +17,14 @@ */ package org.apache.hadoop.hive.contrib.udf.example; +import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; /** * UDFExampleFormat. * */ +@Description(name = "example_format", value = "_FUNC_(expr) - Example UDAF that returns formated String") public class UDFExampleFormat extends UDF { public String evaluate(String format, Object... args) { Modified: hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleMapConcat.java URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleMapConcat.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleMapConcat.java (original) +++ hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleMapConcat.java Tue Oct 14 19:06:45 2014 @@ -21,12 +21,15 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Map; +import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; /** * UDFExampleMapConcat. * */ +@Description(name = "example_mapconcat", +value = "_FUNC_(expr) - Example UDAF that returns contents of Map as a formated String") public class UDFExampleMapConcat extends UDF { public String evaluate(Map<String, String> a) { Modified: hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleStructPrint.java URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleStructPrint.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleStructPrint.java (original) +++ hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleStructPrint.java Tue Oct 14 19:06:45 2014 @@ -19,12 +19,15 @@ package org.apache.hadoop.hive.contrib.u import java.util.List; +import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; /** * UDFExampleStructPrint. * */ +@Description(name = "example_structprint", +value = "_FUNC_(obj) - Example UDAF that returns contents of an object") public class UDFExampleStructPrint extends UDF { public String evaluate(Object a) { Modified: hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFCount2.java URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFCount2.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFCount2.java (original) +++ hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFCount2.java Tue Oct 14 19:06:45 2014 @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.contrib.u import java.util.ArrayList; +import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF; @@ -34,6 +35,8 @@ import org.apache.hadoop.hive.serde2.obj * to test outputting of rows on close with lateral view. * */ +@Description(name = "udtfCount2", +value = "_FUNC_(col) - UDF outputs the number of rows seen, twice.") public class GenericUDTFCount2 extends GenericUDTF { private transient Integer count = Integer.valueOf(0); Modified: hive/branches/llap/contrib/src/test/results/clientpositive/udf_example_add.q.out URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/test/results/clientpositive/udf_example_add.q.out?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/contrib/src/test/results/clientpositive/udf_example_add.q.out (original) +++ hive/branches/llap/contrib/src/test/results/clientpositive/udf_example_add.q.out Tue Oct 14 19:06:45 2014 @@ -25,36 +25,24 @@ SELECT example_add(1, 2), FROM src LIMIT 1 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 3 (type: int), 6 (type: int), 10 (type: int), 3.3000000000000003 (type: double), 6.6 (type: double), 11.0 (type: double), 10.4 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 500 Data size: 22000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator limit: 1 Processor Tree: - ListSink + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 3 (type: int), 6 (type: int), 10 (type: int), 3.3000000000000003 (type: double), 6.6 (type: double), 11.0 (type: double), 10.4 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 500 Data size: 22000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE + ListSink PREHOOK: query: SELECT example_add(1, 2), example_add(1, 2, 3), Modified: hive/branches/llap/contrib/src/test/results/clientpositive/udf_example_format.q.out URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/test/results/clientpositive/udf_example_format.q.out?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/contrib/src/test/results/clientpositive/udf_example_format.q.out (original) +++ hive/branches/llap/contrib/src/test/results/clientpositive/udf_example_format.q.out Tue Oct 14 19:06:45 2014 @@ -19,36 +19,24 @@ SELECT example_format("abc"), FROM src LIMIT 1 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'abc' (type: string), '1.1' (type: string), '1.1 1.200000e+00' (type: string), 'a 12 10' (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 182500 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 365 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 365 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator limit: 1 Processor Tree: - ListSink + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'abc' (type: string), '1.1' (type: string), '1.1 1.200000e+00' (type: string), 'a 12 10' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 182500 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 365 Basic stats: COMPLETE Column stats: COMPLETE + ListSink PREHOOK: query: SELECT example_format("abc"), example_format("%1$s", 1.1), Modified: hive/branches/llap/data/files/parquet_types.txt URL: http://svn.apache.org/viewvc/hive/branches/llap/data/files/parquet_types.txt?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/data/files/parquet_types.txt (original) +++ hive/branches/llap/data/files/parquet_types.txt Tue Oct 14 19:06:45 2014 @@ -1,21 +1,21 @@ -100|1|1|1.0|0.0|abc|2011-01-01 01:01:01.111111111|a |a -101|2|2|1.1|0.3|def|2012-02-02 02:02:02.222222222|ab |ab -102|3|3|1.2|0.6|ghi|2013-03-03 03:03:03.333333333|abc|abc -103|1|4|1.3|0.9|jkl|2014-04-04 04:04:04.444444444|abcd|abcd -104|2|5|1.4|1.2|mno|2015-05-05 05:05:05.555555555|abcde|abcde -105|3|1|1.0|1.5|pqr|2016-06-06 06:06:06.666666666|abcdef|abcdef -106|1|2|1.1|1.8|stu|2017-07-07 07:07:07.777777777|abcdefg|abcdefg -107|2|3|1.2|2.1|vwx|2018-08-08 08:08:08.888888888|bcdefg|abcdefgh -108|3|4|1.3|2.4|yza|2019-09-09 09:09:09.999999999|cdefg|abcdefghijklmnop -109|1|5|1.4|2.7|bcd|2020-10-10 10:10:10.101010101|klmno|abcdedef -110|2|1|1.0|3.0|efg|2021-11-11 11:11:11.111111111|pqrst|abcdede -111|3|2|1.1|3.3|hij|2022-12-12 12:12:12.121212121|nopqr|abcded -112|1|3|1.2|3.6|klm|2023-01-02 13:13:13.131313131|opqrs|abcdd -113|2|4|1.3|3.9|nop|2024-02-02 14:14:14.141414141|pqrst|abc -114|3|5|1.4|4.2|qrs|2025-03-03 15:15:15.151515151|qrstu|b -115|1|1|1.0|4.5|tuv|2026-04-04 16:16:16.161616161|rstuv|abcded -116|2|2|1.1|4.8|wxy|2027-05-05 17:17:17.171717171|stuvw|abcded -117|3|3|1.2|5.1|zab|2028-06-06 18:18:18.181818181|tuvwx|abcded -118|1|4|1.3|5.4|cde|2029-07-07 19:19:19.191919191|uvwzy|abcdede -119|2|5|1.4|5.7|fgh|2030-08-08 20:20:20.202020202|vwxyz|abcdede -120|3|1|1.0|6.0|ijk|2031-09-09 21:21:21.212121212|wxyza|abcde +100|1|1|1.0|0.0|abc|2011-01-01 01:01:01.111111111|a |a |k1:v1|101,200|10,abc +101|2|2|1.1|0.3|def|2012-02-02 02:02:02.222222222|ab |ab |k2:v2|102,200|10,def +102|3|3|1.2|0.6|ghi|2013-03-03 03:03:03.333333333|abc|abc|k3:v3|103,200|10,ghi +103|1|4|1.3|0.9|jkl|2014-04-04 04:04:04.444444444|abcd|abcd|k4:v4|104,200|10,jkl +104|2|5|1.4|1.2|mno|2015-05-05 05:05:05.555555555|abcde|abcde|k5:v5|105,200|10,mno +105|3|1|1.0|1.5|pqr|2016-06-06 06:06:06.666666666|abcdef|abcdef|k6:v6|106,200|10,pqr +106|1|2|1.1|1.8|stu|2017-07-07 07:07:07.777777777|abcdefg|abcdefg|k7:v7|107,200|10,stu +107|2|3|1.2|2.1|vwx|2018-08-08 08:08:08.888888888|bcdefg|abcdefgh|k8:v8|108,200|10,vwx +108|3|4|1.3|2.4|yza|2019-09-09 09:09:09.999999999|cdefg|abcdefghijklmnop|k9:v9|109,200|10,yza +109|1|5|1.4|2.7|bcd|2020-10-10 10:10:10.101010101|klmno|abcdedef|k10:v10|110,200|10,bcd +110|2|1|1.0|3.0|efg|2021-11-11 11:11:11.111111111|pqrst|abcdede|k11:v11|111,200|10,efg +111|3|2|1.1|3.3|hij|2022-12-12 12:12:12.121212121|nopqr|abcded|k12:v12|112,200|10,hij +112|1|3|1.2|3.6|klm|2023-01-02 13:13:13.131313131|opqrs|abcdd|k13:v13|113,200|10,klm +113|2|4|1.3|3.9|nop|2024-02-02 14:14:14.141414141|pqrst|abc|k14:v14|114,200|10,nop +114|3|5|1.4|4.2|qrs|2025-03-03 15:15:15.151515151|qrstu|b|k15:v15|115,200|10,qrs +115|1|1|1.0|4.5|qrs|2026-04-04 16:16:16.161616161|rstuv|abcded|k16:v16|116,200|10,qrs +116|2|2|1.1|4.8|wxy|2027-05-05 17:17:17.171717171|stuvw|abcded|k17:v17|117,200|10,wxy +117|3|3|1.2|5.1|zab|2028-06-06 18:18:18.181818181|tuvwx|abcded|k18:v18|118,200|10,zab +118|1|4|1.3|5.4|cde|2029-07-07 19:19:19.191919191|uvwzy|abcdede|k19:v19|119,200|10,cde +119|2|5|1.4|5.7|fgh|2030-08-08 20:20:20.202020202|vwxyz|abcdede|k20:v20|120,200|10,fgh +120|3|1|1.0|6.0|ijk|2031-09-09 21:21:21.212121212|wxyza|abcde|k21:v21|121,200|10,ijk Propchange: hive/branches/llap/hbase-handler/pom.xml ------------------------------------------------------------------------------ Merged /hive/branches/cbo/hbase-handler/pom.xml:r1605012-1627125 Merged /hive/trunk/hbase-handler/pom.xml:r1625337-1631832 Modified: hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/DefaultHBaseKeyFactory.java URL: http://svn.apache.org/viewvc/hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/DefaultHBaseKeyFactory.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/DefaultHBaseKeyFactory.java (original) +++ hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/DefaultHBaseKeyFactory.java Tue Oct 14 19:06:45 2014 @@ -18,6 +18,9 @@ package org.apache.hadoop.hive.hbase; +import java.io.IOException; +import java.util.Properties; + import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.lazy.LazyFactory; import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase; @@ -26,9 +29,6 @@ import org.apache.hadoop.hive.serde2.obj import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import java.io.IOException; -import java.util.Properties; - public class DefaultHBaseKeyFactory extends AbstractHBaseKeyFactory implements HBaseKeyFactory { protected LazySimpleSerDe.SerDeParameters serdeParams; Modified: hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java URL: http://svn.apache.org/viewvc/hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java (original) +++ hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java Tue Oct 14 19:06:45 2014 @@ -53,6 +53,7 @@ public class HBaseSerDe extends Abstract public static final String HBASE_COMPOSITE_KEY_CLASS = "hbase.composite.key.class"; public static final String HBASE_COMPOSITE_KEY_TYPES = "hbase.composite.key.types"; public static final String HBASE_COMPOSITE_KEY_FACTORY = "hbase.composite.key.factory"; + public static final String HBASE_STRUCT_SERIALIZER_CLASS = "hbase.struct.serialization.class"; public static final String HBASE_SCAN_CACHE = "hbase.scan.cache"; public static final String HBASE_SCAN_CACHEBLOCKS = "hbase.scan.cacheblock"; public static final String HBASE_SCAN_BATCH = "hbase.scan.batch"; @@ -98,7 +99,7 @@ public class HBaseSerDe extends Abstract cachedHBaseRow = new LazyHBaseRow( (LazySimpleStructObjectInspector) cachedObjectInspector, - serdeParams.getKeyIndex(), serdeParams.getKeyFactory()); + serdeParams.getKeyIndex(), serdeParams.getKeyFactory(), serdeParams.getValueFactories()); serializer = new HBaseRowSerializer(serdeParams); Modified: hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeHelper.java URL: http://svn.apache.org/viewvc/hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeHelper.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeHelper.java (original) +++ hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeHelper.java Tue Oct 14 19:06:45 2014 @@ -41,6 +41,10 @@ import org.apache.hadoop.hive.serde.serd import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.avro.AvroObjectInspectorGenerator; import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils; +import org.apache.hadoop.hive.serde2.lazy.LazyFactory; +import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.util.StringUtils; @@ -371,6 +375,19 @@ public class HBaseSerDeHelper { } /** + * Create the {@link LazyObjectBase lazy field} + * */ + public static LazyObjectBase createLazyField(ColumnMapping[] columnMappings, int fieldID, + ObjectInspector inspector) { + ColumnMapping colMap = columnMappings[fieldID]; + if (colMap.getQualifierName() == null && !colMap.isHbaseRowKey()) { + // a column family + return new LazyHBaseCellMap((LazyMapObjectInspector) inspector); + } + return LazyFactory.createLazyObject(inspector, colMap.getBinaryStorage().get(0)); + } + + /** * Auto-generates the key struct for composite keys * * @param compositeKeyParts map of composite key part name to its type. Usually this would be Modified: hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeParameters.java URL: http://svn.apache.org/viewvc/hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeParameters.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeParameters.java (original) +++ hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeParameters.java Tue Oct 14 19:06:45 2014 @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.hbase.Colu import org.apache.hadoop.hive.hbase.struct.AvroHBaseValueFactory; import org.apache.hadoop.hive.hbase.struct.DefaultHBaseValueFactory; import org.apache.hadoop.hive.hbase.struct.HBaseValueFactory; +import org.apache.hadoop.hive.hbase.struct.StructHBaseValueFactory; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils; @@ -204,11 +205,21 @@ public class HBaseSerDeParameters { for (int i = 0; i < columnMappings.size(); i++) { String serType = getSerializationType(conf, tbl, columnMappings.getColumnsMapping()[i]); - if (serType != null && serType.equals(AVRO_SERIALIZATION_TYPE)) { + if (AVRO_SERIALIZATION_TYPE.equals(serType)) { Schema schema = getSchema(conf, tbl, columnMappings.getColumnsMapping()[i]); - valueFactories.add(new AvroHBaseValueFactory(schema)); + valueFactories.add(new AvroHBaseValueFactory(i, schema)); + } else if (STRUCT_SERIALIZATION_TYPE.equals(serType)) { + String structValueClassName = tbl.getProperty(HBaseSerDe.HBASE_STRUCT_SERIALIZER_CLASS); + + if (structValueClassName == null) { + throw new IllegalArgumentException(HBaseSerDe.HBASE_STRUCT_SERIALIZER_CLASS + + " must be set for hbase columns of type [" + STRUCT_SERIALIZATION_TYPE + "]"); + } + + Class<?> structValueClass = job.getClassByName(structValueClassName); + valueFactories.add(new StructHBaseValueFactory(i, structValueClass)); } else { - valueFactories.add(new DefaultHBaseValueFactory()); + valueFactories.add(new DefaultHBaseValueFactory(i)); } } } catch (Exception e) { Modified: hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java URL: http://svn.apache.org/viewvc/hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java (original) +++ hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java Tue Oct 14 19:06:45 2014 @@ -20,15 +20,15 @@ package org.apache.hadoop.hive.hbase; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping; +import org.apache.hadoop.hive.hbase.struct.HBaseValueFactory; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; -import org.apache.hadoop.hive.serde2.lazy.LazyFactory; import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase; import org.apache.hadoop.hive.serde2.lazy.LazyStruct; -import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -47,18 +47,21 @@ public class LazyHBaseRow extends LazySt private final int iKey; private final HBaseKeyFactory keyFactory; + private final List<HBaseValueFactory> valueFactories; public LazyHBaseRow(LazySimpleStructObjectInspector oi) { - this(oi, -1, null); + this(oi, -1, null, null); } /** * Construct a LazyHBaseRow object with the ObjectInspector. */ - public LazyHBaseRow(LazySimpleStructObjectInspector oi, int iKey, HBaseKeyFactory keyFactory) { + public LazyHBaseRow(LazySimpleStructObjectInspector oi, int iKey, HBaseKeyFactory keyFactory, + List<HBaseValueFactory> valueFactories) { super(oi); this.iKey = iKey; this.keyFactory = keyFactory; + this.valueFactories = valueFactories; } /** @@ -76,13 +79,14 @@ public class LazyHBaseRow extends LazySt if (fieldID == iKey) { return keyFactory.createKey(fieldRef.getFieldObjectInspector()); } - ColumnMapping colMap = columnsMapping[fieldID]; - if (colMap.qualifierName == null && !colMap.hbaseRowKey) { - // a column family - return new LazyHBaseCellMap((LazyMapObjectInspector) fieldRef.getFieldObjectInspector()); + + if (valueFactories != null) { + return valueFactories.get(fieldID).createValueObject(fieldRef.getFieldObjectInspector()); } - return LazyFactory.createLazyObject(fieldRef.getFieldObjectInspector(), - colMap.binaryStorage.get(0)); + + // fallback to default + return HBaseSerDeHelper.createLazyField(columnsMapping, fieldID, + fieldRef.getFieldObjectInspector()); } /** Modified: hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/AvroHBaseValueFactory.java URL: http://svn.apache.org/viewvc/hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/AvroHBaseValueFactory.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/AvroHBaseValueFactory.java (original) +++ hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/AvroHBaseValueFactory.java Tue Oct 14 19:06:45 2014 @@ -48,7 +48,8 @@ public class AvroHBaseValueFactory exten * * @param schema the associated {@link Schema schema} * */ - public AvroHBaseValueFactory(Schema schema) { + public AvroHBaseValueFactory(int fieldID, Schema schema) { + super(fieldID); this.schema = schema; } Modified: hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/DefaultHBaseValueFactory.java URL: http://svn.apache.org/viewvc/hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/DefaultHBaseValueFactory.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/DefaultHBaseValueFactory.java (original) +++ hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/DefaultHBaseValueFactory.java Tue Oct 14 19:06:45 2014 @@ -21,9 +21,12 @@ import java.io.IOException; import java.util.Properties; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.hbase.ColumnMappings; +import org.apache.hadoop.hive.hbase.HBaseSerDeHelper; import org.apache.hadoop.hive.hbase.HBaseSerDeParameters; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.lazy.LazyFactory; +import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -32,33 +35,46 @@ import org.apache.hadoop.hive.serde2.typ /** * Default implementation of the {@link HBaseValueFactory} * */ -public class DefaultHBaseValueFactory implements HBaseValueFactory{ +public class DefaultHBaseValueFactory implements HBaseValueFactory { protected LazySimpleSerDe.SerDeParameters serdeParams; + protected ColumnMappings columnMappings; protected HBaseSerDeParameters hbaseParams; protected Properties properties; protected Configuration conf; - @Override + private int fieldID; + + public DefaultHBaseValueFactory(int fieldID) { + this.fieldID = fieldID; + } + + @Override public void init(HBaseSerDeParameters hbaseParams, Configuration conf, Properties properties) - throws SerDeException { + throws SerDeException { this.hbaseParams = hbaseParams; this.serdeParams = hbaseParams.getSerdeParams(); + this.columnMappings = hbaseParams.getColumnMappings(); this.properties = properties; this.conf = conf; - } + } - @Override - public ObjectInspector createValueObjectInspector(TypeInfo type) - throws SerDeException { + @Override + public ObjectInspector createValueObjectInspector(TypeInfo type) + throws SerDeException { return LazyFactory.createLazyObjectInspector(type, serdeParams.getSeparators(), 1, serdeParams.getNullSequence(), serdeParams.isEscaped(), serdeParams.getEscapeChar()); - } + } - @Override - public byte[] serializeValue(Object object, StructField field) - throws IOException { + @Override + public LazyObjectBase createValueObject(ObjectInspector inspector) throws SerDeException { + return HBaseSerDeHelper.createLazyField(columnMappings.getColumnsMapping(), fieldID, inspector); + } + + @Override + public byte[] serializeValue(Object object, StructField field) + throws IOException { // TODO Add support for serialization of values here - return null; - } -} \ No newline at end of file + return null; + } +} Modified: hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/HBaseValueFactory.java URL: http://svn.apache.org/viewvc/hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/HBaseValueFactory.java?rev=1631841&r1=1631840&r2=1631841&view=diff ============================================================================== --- hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/HBaseValueFactory.java (original) +++ hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/HBaseValueFactory.java Tue Oct 14 19:06:45 2014 @@ -22,8 +22,10 @@ import java.io.IOException; import java.util.Properties; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.hbase.HBaseKeyFactory; import org.apache.hadoop.hive.hbase.HBaseSerDeParameters; import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -53,6 +55,13 @@ public interface HBaseValueFactory { ObjectInspector createValueObjectInspector(TypeInfo type) throws SerDeException; /** + * create custom object for hbase value + * + * @param inspector OI create by {@link HBaseKeyFactory#createKeyObjectInspector} + */ + LazyObjectBase createValueObject(ObjectInspector inspector) throws SerDeException; + + /** * Serialize the given hive object * * @param object the object to be serialized
