Author: gunther
Date: Tue Sep 23 19:30:11 2014
New Revision: 1627125
URL: http://svn.apache.org/r1627125
Log:
Merge latest trunk into cbo branch. (Gunther Hagleitner)
Added:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAppMasterEventOperator.java
- copied unchanged from r1627116,
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAppMasterEventOperator.java
hive/branches/cbo/ql/src/test/queries/clientpositive/constantPropagateForSubQuery.q
- copied unchanged from r1627116,
hive/trunk/ql/src/test/queries/clientpositive/constantPropagateForSubQuery.q
hive/branches/cbo/ql/src/test/queries/clientpositive/drop_table_purge.q
- copied unchanged from r1627116,
hive/trunk/ql/src/test/queries/clientpositive/drop_table_purge.q
hive/branches/cbo/ql/src/test/queries/clientpositive/vectorized_dynamic_partition_pruning.q
- copied unchanged from r1627116,
hive/trunk/ql/src/test/queries/clientpositive/vectorized_dynamic_partition_pruning.q
hive/branches/cbo/ql/src/test/results/clientpositive/constantPropagateForSubQuery.q.out
- copied unchanged from r1627116,
hive/trunk/ql/src/test/results/clientpositive/constantPropagateForSubQuery.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/drop_table_purge.q.out
- copied unchanged from r1627116,
hive/trunk/ql/src/test/results/clientpositive/drop_table_purge.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/vectorization_0.q.out
- copied unchanged from r1627116,
hive/trunk/ql/src/test/results/clientpositive/tez/vectorization_0.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out
- copied unchanged from r1627116,
hive/trunk/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out
Modified:
hive/branches/cbo/ (props changed)
hive/branches/cbo/beeline/src/java/org/apache/hive/beeline/BeeLine.java
hive/branches/cbo/bin/beeline.cmd
hive/branches/cbo/bin/ext/beeline.sh
hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/branches/cbo/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java
hive/branches/cbo/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
hive/branches/cbo/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestExtendedAcls.java
hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java
hive/branches/cbo/itests/src/test/resources/testconfiguration.properties
hive/branches/cbo/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
hive/branches/cbo/jdbc/pom.xml
hive/branches/cbo/metastore/scripts/upgrade/oracle/upgrade-0.13.0-to-0.14.0.oracle.sql
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriterV2.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SerializationUtils.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java
hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java
hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java
hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestSerializationUtils.java
hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java
hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/parse/TestIUD.java
hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java
hive/branches/cbo/ql/src/test/queries/clientpositive/acid_vectorization.q
hive/branches/cbo/ql/src/test/queries/clientpositive/delete_all_non_partitioned.q
hive/branches/cbo/ql/src/test/queries/clientpositive/delete_all_partitioned.q
hive/branches/cbo/ql/src/test/queries/clientpositive/delete_where_partitioned.q
hive/branches/cbo/ql/src/test/queries/clientpositive/delete_whole_partition.q
hive/branches/cbo/ql/src/test/queries/clientpositive/insert_update_delete.q
hive/branches/cbo/ql/src/test/queries/clientpositive/insert_values_dynamic_partitioned.q
hive/branches/cbo/ql/src/test/queries/clientpositive/insert_values_non_partitioned.q
hive/branches/cbo/ql/src/test/queries/clientpositive/insert_values_partitioned.q
hive/branches/cbo/ql/src/test/queries/clientpositive/insert_values_tmp_table.q
hive/branches/cbo/ql/src/test/queries/clientpositive/update_after_multiple_inserts.q
hive/branches/cbo/ql/src/test/queries/clientpositive/update_all_partitioned.q
hive/branches/cbo/ql/src/test/queries/clientpositive/update_all_types.q
hive/branches/cbo/ql/src/test/queries/clientpositive/update_where_partitioned.q
hive/branches/cbo/ql/src/test/queries/clientpositive/vectorization_0.q
hive/branches/cbo/ql/src/test/queries/clientpositive/vectorized_date_funcs.q
hive/branches/cbo/ql/src/test/queries/clientpositive/vectorized_timestamp_funcs.q
hive/branches/cbo/ql/src/test/resources/orc-file-dump-dictionary-threshold.out
hive/branches/cbo/ql/src/test/resources/orc-file-dump.out
hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_part.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_select.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/auto_join8.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/insert_values_non_partitioned.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/insert_values_tmp_table.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/join8.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/join_view.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/lateral_view_ppd.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/orc_analyze.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/orc_merge1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/orc_merge5.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/orc_merge6.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/orc_merge7.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/orc_merge_incompat1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/orc_merge_incompat2.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/ppd_repeated_alias.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/select_dummy_source.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/subquery_multiinsert.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/subquery_notin.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/subquery_notin_having.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/subquery_views.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/insert_values_non_partitioned.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/insert_values_tmp_table.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/orc_analyze.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/orc_merge1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/orc_merge5.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/orc_merge6.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/orc_merge7.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/orc_merge_incompat1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/orc_merge_incompat2.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/update_all_types.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/vectorized_timestamp_funcs.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/udf4.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/udf7.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/udf_case.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/udf_elt.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/udf_when.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/update_all_types.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/vector_elt.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/vectorization_0.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/vectorized_date_funcs.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/vectorized_timestamp_funcs.q.out
hive/branches/cbo/ql/src/test/results/compiler/plan/cast1.q.xml
hive/branches/cbo/ql/src/test/results/compiler/plan/groupby1.q.xml
hive/branches/cbo/ql/src/test/results/compiler/plan/groupby2.q.xml
hive/branches/cbo/ql/src/test/results/compiler/plan/groupby3.q.xml
hive/branches/cbo/ql/src/test/results/compiler/plan/groupby4.q.xml
hive/branches/cbo/ql/src/test/results/compiler/plan/groupby5.q.xml
hive/branches/cbo/ql/src/test/results/compiler/plan/groupby6.q.xml
hive/branches/cbo/ql/src/test/results/compiler/plan/join8.q.xml
hive/branches/cbo/ql/src/test/results/compiler/plan/udf1.q.xml
hive/branches/cbo/ql/src/test/results/compiler/plan/udf4.q.xml
hive/branches/cbo/ql/src/test/results/compiler/plan/udf6.q.xml
hive/branches/cbo/ql/src/test/results/compiler/plan/udf_case.q.xml
hive/branches/cbo/ql/src/test/results/compiler/plan/udf_when.q.xml
hive/branches/cbo/ql/src/test/results/compiler/plan/union.q.xml
hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
hive/branches/cbo/service/src/test/org/apache/hive/service/cli/session/TestSessionGlobalInitFile.java
Propchange: hive/branches/cbo/
------------------------------------------------------------------------------
Merged /hive/trunk:r1626363-1627116
Modified:
hive/branches/cbo/beeline/src/java/org/apache/hive/beeline/BeeLine.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/beeline/src/java/org/apache/hive/beeline/BeeLine.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
--- hive/branches/cbo/beeline/src/java/org/apache/hive/beeline/BeeLine.java
(original)
+++ hive/branches/cbo/beeline/src/java/org/apache/hive/beeline/BeeLine.java Tue
Sep 23 19:30:11 2014
@@ -692,10 +692,6 @@ public class BeeLine implements Closeabl
int code = 0;
if (!commands.isEmpty()) {
- // for single command execute, disable color
- getOpts().setColor(false);
- getOpts().setHeaderInterval(-1);
-
for (Iterator<String> i = commands.iterator(); i.hasNext();) {
String command = i.next().toString();
debug(loc("executing-command", command));
Modified: hive/branches/cbo/bin/beeline.cmd
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/bin/beeline.cmd?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
--- hive/branches/cbo/bin/beeline.cmd (original)
+++ hive/branches/cbo/bin/beeline.cmd Tue Sep 23 19:30:11 2014
@@ -43,7 +43,22 @@ if not exist %HADOOP_HOME%\libexec\hadoo
@rem supress the HADOOP_HOME warnings in 1.x.x
set HADOOP_HOME_WARN_SUPPRESS=true
call %HADOOP_HOME%\libexec\hadoop-config.cmd
-set CLASSPATH=%CLASSPATH%;%HIVE_HOME%\lib\*;
+
+@rem include only the beeline client jar and its dependencies
+pushd %HIVE_HOME%\lib
+for /f %%a IN ('dir /b hive-beeline-**.jar') do (
+ set CLASSPATH=%CLASSPATH%;%HIVE_HOME%\lib\%%a
+)
+for /f %%a IN ('dir /b super-csv-**.jar') do (
+ set CLASSPATH=%CLASSPATH%;%HIVE_HOME%\lib\%%a
+)
+for /f %%a IN ('dir /b jline-**.jar') do (
+ set CLASSPATH=%CLASSPATH%;%HIVE_HOME%\lib\%%a
+)
+for /f %%a IN ('dir /b hive-jdbc-**-standalone.jar') do (
+ set CLASSPATH=%CLASSPATH%;%HIVE_HOME%\lib\%%a
+)
+popd
call %JAVA_HOME%\bin\java %JAVA_HEAP_MAX% %HADOOP_OPTS% -classpath %CLASSPATH%
org.apache.hive.beeline.BeeLine %*
Modified: hive/branches/cbo/bin/ext/beeline.sh
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/bin/ext/beeline.sh?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
--- hive/branches/cbo/bin/ext/beeline.sh (original)
+++ hive/branches/cbo/bin/ext/beeline.sh Tue Sep 23 19:30:11 2014
@@ -19,11 +19,17 @@ export SERVICE_LIST="${SERVICE_LIST}${TH
beeline () {
CLASS=org.apache.hive.beeline.BeeLine;
- execHiveCmd $CLASS "$@"
+
+ # include only the beeline client jar and its dependencies
+ beelineJarPath=`ls ${HIVE_LIB}/hive-beeline-*.jar`
+ superCsvJarPath=`ls ${HIVE_LIB}/super-csv-*.jar`
+ jlineJarPath=`ls ${HIVE_LIB}/jline-*.jar`
+ jdbcStandaloneJarPath=`ls ${HIVE_LIB}/hive-jdbc-*-standalone.jar`
+ export
HADOOP_CLASSPATH=${beelineJarPath}:${superCsvJarPath}:${jlineJarPath}:${jdbcStandaloneJarPath}
+
+ exec $HADOOP jar ${beelineJarPath} $CLASS $HIVE_OPTS "$@"
}
beeline_help () {
- CLASS=org.apache.hive.beeline.BeeLine;
- execHiveCmd $CLASS "--help"
+ beeline "--help"
}
-
Modified:
hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
--- hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
(original)
+++ hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
Tue Sep 23 19:30:11 2014
@@ -1506,8 +1506,8 @@ public class HiveConf extends Configurat
"The parent node in ZooKeeper used by HiveServer2 when supporting
dynamic service discovery."),
// HiveServer2 global init file location
HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION("hive.server2.global.init.file.location",
"${env:HIVE_CONF_DIR}",
- "The location of HS2 global init file (.hiverc).\n" +
- "If the property is reset, the value must be a valid path where the
init file is located."),
+ "Either the location of a HS2 global init file or a directory
containing a .hiverc file. If the \n" +
+ "property is set, the value must be a valid path to an init file or
directory where the init file is located."),
HIVE_SERVER2_TRANSPORT_MODE("hive.server2.transport.mode", "binary", new
StringSet("binary", "http"),
"Transport mode of HiveServer2."),
HIVE_SERVER2_THRIFT_BIND_HOST("hive.server2.thrift.bind.host", "",
@@ -1715,6 +1715,9 @@ public class HiveConf extends Configurat
HIVE_VECTORIZATION_REDUCE_ENABLED("hive.vectorized.execution.reduce.enabled",
true,
"This flag should be set to true to enable vectorized mode of the
reduce-side of query execution.\n" +
"The default value is true."),
+
HIVE_VECTORIZATION_REDUCE_GROUPBY_ENABLED("hive.vectorized.execution.reduce.groupby.enabled",
true,
+ "This flag should be set to true to enable vectorized mode of the
reduce-side GROUP BY query execution.\n" +
+ "The default value is true."),
HIVE_VECTORIZATION_GROUPBY_CHECKINTERVAL("hive.vectorized.groupby.checkinterval",
100000,
"Number of entries added to the group by aggregation hash before a
recomputation of average entry size is performed."),
HIVE_VECTORIZATION_GROUPBY_MAXENTRIES("hive.vectorized.groupby.maxentries",
1000000,
Modified:
hive/branches/cbo/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java
(original)
+++
hive/branches/cbo/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java
Tue Sep 23 19:30:11 2014
@@ -90,6 +90,7 @@ public class TestHCatPartitionPublish {
File workDir = handleWorkDir();
conf.set("yarn.scheduler.capacity.root.queues", "default");
conf.set("yarn.scheduler.capacity.root.default.capacity", "100");
+ conf.set("fs.pfile.impl", "org.apache.hadoop.fs.ProxyLocalFileSystem");
fs = FileSystem.get(conf);
System.setProperty("hadoop.log.dir", new File(workDir,
"/logs").getAbsolutePath());
Modified:
hive/branches/cbo/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
(original)
+++
hive/branches/cbo/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
Tue Sep 23 19:30:11 2014
@@ -320,6 +320,6 @@ public class TestTempletonUtils {
result = TempletonUtils.findContainingJar(FileSystem.class,
".*hadoop.*\\.jar.*");
Assert.assertNotNull(result);
result = TempletonUtils.findContainingJar(HadoopShimsSecure.class,
".*unknownjar.*");
- Assert.assertNull(result);
+ Assert.assertNull("unexpectedly found jar for HadoopShimsSecure class: " +
result, result);
}
}
Modified:
hive/branches/cbo/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestExtendedAcls.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestExtendedAcls.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestExtendedAcls.java
(original)
+++
hive/branches/cbo/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestExtendedAcls.java
Tue Sep 23 19:30:11 2014
@@ -34,6 +34,7 @@ import org.apache.hadoop.hive.conf.HiveC
import org.junit.Assert;
import org.junit.BeforeClass;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
public class TestExtendedAcls extends FolderPermissionBase {
@@ -46,7 +47,7 @@ public class TestExtendedAcls extends Fo
baseSetup();
}
- List<AclEntry> aclSpec1 = Lists.newArrayList(
+ private final ImmutableList<AclEntry> aclSpec1 = ImmutableList.of(
aclEntry(ACCESS, USER, FsAction.ALL),
aclEntry(ACCESS, GROUP, FsAction.ALL),
aclEntry(ACCESS, OTHER, FsAction.ALL),
@@ -55,7 +56,7 @@ public class TestExtendedAcls extends Fo
aclEntry(ACCESS, GROUP, "bar", FsAction.READ_WRITE),
aclEntry(ACCESS, GROUP, "foo", FsAction.READ_EXECUTE));
- List<AclEntry> aclSpec2 = Lists.newArrayList(
+ private final ImmutableList<AclEntry> aclSpec2 = ImmutableList.of(
aclEntry(ACCESS, USER, FsAction.ALL),
aclEntry(ACCESS, GROUP, FsAction.ALL),
aclEntry(ACCESS, OTHER, FsAction.READ_EXECUTE),
@@ -83,20 +84,20 @@ public class TestExtendedAcls extends Fo
switch (permIndex) {
case 0:
FsPermission perm = fs.getFileStatus(new Path(locn)).getPermission();
- Assert.assertEquals(perm.toString(), "rwxrwxrwx");
+ Assert.assertEquals("Location: " + locn, "rwxrwxrwx",
String.valueOf(perm));
List<AclEntry> actual = getAcl(locn);
verifyAcls(aclSpec1, actual);
break;
case 1:
perm = fs.getFileStatus(new Path(locn)).getPermission();
- Assert.assertEquals(perm.toString(), "rwxrwxr-x");
+ Assert.assertEquals("Location: " + locn, "rwxrwxr-x",
String.valueOf(perm));
List<AclEntry> acls = getAcl(locn);
verifyAcls(aclSpec2, acls);
break;
default:
- throw new RuntimeException("Only 2 permissions by this test");
+ throw new RuntimeException("Only 2 permissions by this test: " +
permIndex);
}
}
Modified:
hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java
(original)
+++
hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java
Tue Sep 23 19:30:11 2014
@@ -52,7 +52,7 @@ public abstract class FolderPermissionBa
protected static Path warehouseDir;
protected static Path baseDfsDir;
- public static final PathFilter hiddenFileFilter = new PathFilter(){
+ protected static final PathFilter hiddenFileFilter = new PathFilter(){
public boolean accept(Path p){
String name = p.getName();
return !name.startsWith("_") && !name.startsWith(".");
@@ -591,7 +591,7 @@ public abstract class FolderPermissionBa
private List<String> listStatus(String locn) throws Exception {
List<String> results = new ArrayList<String>();
- FileStatus[] listStatus = fs.listStatus(new Path(locn));
+ FileStatus[] listStatus = fs.listStatus(new Path(locn), hiddenFileFilter);
for (FileStatus status : listStatus) {
results.add(status.getPath().toString());
}
Modified:
hive/branches/cbo/itests/src/test/resources/testconfiguration.properties
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/itests/src/test/resources/testconfiguration.properties?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
--- hive/branches/cbo/itests/src/test/resources/testconfiguration.properties
(original)
+++ hive/branches/cbo/itests/src/test/resources/testconfiguration.properties
Tue Sep 23 19:30:11 2014
@@ -159,6 +159,7 @@ minitez.query.files.shared=alter_merge_2
vector_left_outer_join.q,\
vector_mapjoin_reduce.q,\
vector_string_concat.q,\
+ vectorization_0.q,\
vectorization_12.q,\
vectorization_13.q,\
vectorization_14.q,\
@@ -187,7 +188,8 @@ minitez.query.files=bucket_map_join_tez1
tez_joins_explain.q,\
tez_schema_evolution.q,\
tez_union.q,\
- tez_union_decimal.q
+ tez_union_decimal.q,\
+ vectorized_dynamic_partition_pruning.q
beeline.positive.exclude=add_part_exist.q,\
alter1.q,\
Modified:
hive/branches/cbo/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
(original)
+++
hive/branches/cbo/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
Tue Sep 23 19:30:11 2014
@@ -103,6 +103,7 @@ public class QTestUtil {
public static final String UTF_8 = "UTF-8";
private static final Log LOG = LogFactory.getLog("QTestUtil");
+ private static final String QTEST_LEAVE_FILES = "QTEST_LEAVE_FILES";
private final String defaultInitScript = "q_test_init.sql";
private final String defaultCleanupScript = "q_test_cleanup.sql";
@@ -537,6 +538,9 @@ public class QTestUtil {
* Clear out any side effects of running tests
*/
public void clearTestSideEffects() throws Exception {
+ if (System.getenv(QTEST_LEAVE_FILES) != null) {
+ return;
+ }
// Delete any tables other than the source tables
// and any databases other than the default database.
for (String dbName : db.getAllDatabases()) {
@@ -598,6 +602,9 @@ public class QTestUtil {
if(!isSessionStateStarted) {
startSessionState();
}
+ if (System.getenv(QTEST_LEAVE_FILES) != null) {
+ return;
+ }
SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", true);
Modified: hive/branches/cbo/jdbc/pom.xml
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/jdbc/pom.xml?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
--- hive/branches/cbo/jdbc/pom.xml (original)
+++ hive/branches/cbo/jdbc/pom.xml Tue Sep 23 19:30:11 2014
@@ -152,6 +152,16 @@
<exclude>org.apache.velocity:*</exclude>
</excludes>
</artifactSet>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
</configuration>
</execution>
</executions>
Modified:
hive/branches/cbo/metastore/scripts/upgrade/oracle/upgrade-0.13.0-to-0.14.0.oracle.sql
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/metastore/scripts/upgrade/oracle/upgrade-0.13.0-to-0.14.0.oracle.sql?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/metastore/scripts/upgrade/oracle/upgrade-0.13.0-to-0.14.0.oracle.sql
(original)
+++
hive/branches/cbo/metastore/scripts/upgrade/oracle/upgrade-0.13.0-to-0.14.0.oracle.sql
Tue Sep 23 19:30:11 2014
@@ -1,5 +1,6 @@
SELECT 'Upgrading MetaStore schema from 0.13.0 to 0.14.0' AS Status from dual;
[email protected];
@020-HIVE-7784.oracle.sql;
UPDATE VERSION SET SCHEMA_VERSION='0.14.0', VERSION_COMMENT='Hive release
version 0.14.0' where VER_ID=1;
Modified:
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
(original)
+++
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
Tue Sep 23 19:30:11 2014
@@ -1424,10 +1424,14 @@ public class HiveMetaStore extends Thrif
if (!success) {
ms.rollbackTransaction();
} else if (deleteData && !isExternal) {
+ boolean ifPurge = false;
+ if (envContext != null){
+ ifPurge =
Boolean.parseBoolean(envContext.getProperties().get("ifPurge"));
+ }
// Delete the data in the partitions which have other locations
- deletePartitionData(partPaths);
+ deletePartitionData(partPaths, ifPurge);
// Delete the data in the table
- deleteTableData(tblPath);
+ deleteTableData(tblPath, ifPurge);
// ok even if the data is not deleted
}
for (MetaStoreEventListener listener : listeners) {
@@ -1444,9 +1448,21 @@ public class HiveMetaStore extends Thrif
* @param tablePath
*/
private void deleteTableData(Path tablePath) {
+ deleteTableData(tablePath, false);
+ }
+
+ /**
+ * Deletes the data in a table's location, if it fails logs an error
+ *
+ * @param tablePath
+ * @param ifPurge completely purge the table (skipping trash) while
removing
+ * data from warehouse
+ */
+ private void deleteTableData(Path tablePath, boolean ifPurge) {
+
if (tablePath != null) {
try {
- wh.deleteDir(tablePath, true);
+ wh.deleteDir(tablePath, true, ifPurge);
} catch (Exception e) {
LOG.error("Failed to delete table directory: " + tablePath +
" " + e.getMessage());
@@ -1461,10 +1477,22 @@ public class HiveMetaStore extends Thrif
* @param partPaths
*/
private void deletePartitionData(List<Path> partPaths) {
+ deletePartitionData(partPaths, false);
+ }
+
+ /**
+ * Give a list of partitions' locations, tries to delete each one
+ * and for each that fails logs an error.
+ *
+ * @param partPaths
+ * @param ifPurge completely purge the partition (skipping trash) while
+ * removing data from warehouse
+ */
+ private void deletePartitionData(List<Path> partPaths, boolean ifPurge) {
if (partPaths != null && !partPaths.isEmpty()) {
for (Path partPath : partPaths) {
try {
- wh.deleteDir(partPath, true);
+ wh.deleteDir(partPath, true, ifPurge);
} catch (Exception e) {
LOG.error("Failed to delete partition directory: " + partPath +
" " + e.getMessage());
Modified:
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
(original)
+++
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
Tue Sep 23 19:30:11 2014
@@ -763,18 +763,34 @@ public class HiveMetaStoreClient impleme
}
/**
- * @param name
- * @param dbname
- * @throws NoSuchObjectException
- * @throws MetaException
- * @throws TException
- * @see
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#drop_table(java.lang.String,
- * java.lang.String, boolean)
+ * {@inheritDoc}
+ * @see #dropTable(String, String, boolean, boolean, EnvironmentContext)
*/
@Override
- public void dropTable(String dbname, String name)
- throws NoSuchObjectException, MetaException, TException {
- dropTable(dbname, name, true, true, null);
+ public void dropTable(String dbname, String name, boolean deleteData,
+ boolean ignoreUnknownTab) throws MetaException, TException,
+ NoSuchObjectException, UnsupportedOperationException {
+ dropTable(dbname, name, deleteData, ignoreUnknownTab, null);
+ }
+
+ /**
+ * Drop the table and choose whether to save the data in the trash.
+ * @param ifPurge completely purge the table (skipping trash) while removing
+ * data from warehouse
+ * @see #dropTable(String, String, boolean, boolean, EnvironmentContext)
+ */
+ public void dropTable(String dbname, String name, boolean deleteData,
+ boolean ignoreUnknownTab, boolean ifPurge)
+ throws MetaException, TException, NoSuchObjectException,
UnsupportedOperationException {
+ //build new environmentContext with ifPurge;
+ EnvironmentContext envContext = null;
+ if(ifPurge){
+ Map<String, String> warehouseOptions = null;
+ warehouseOptions = new HashMap<String, String>();
+ warehouseOptions.put("ifPurge", "TRUE");
+ envContext = new EnvironmentContext(warehouseOptions);
+ }
+ dropTable(dbname, name, deleteData, ignoreUnknownTab, envContext);
}
/** {@inheritDoc} */
@@ -786,23 +802,37 @@ public class HiveMetaStoreClient impleme
}
/**
+ * @see #dropTable(String, String, boolean, boolean, EnvironmentContext)
+ */
+ @Override
+ public void dropTable(String dbname, String name)
+ throws NoSuchObjectException, MetaException, TException {
+ dropTable(dbname, name, true, true, null);
+ }
+
+ /**
+ * Drop the table and choose whether to: delete the underlying table data;
+ * throw if the table doesn't exist; save the data in the trash.
+ *
* @param dbname
* @param name
* @param deleteData
* delete the underlying data or just delete the table in metadata
- * @throws NoSuchObjectException
+ * @param ignoreUnknownTab
+ * don't throw if the requested table doesn't exist
+ * @param envContext
+ * for communicating with thrift
* @throws MetaException
+ * could not drop table properly
+ * @throws NoSuchObjectException
+ * the table wasn't found
* @throws TException
+ * a thrift communication error occurred
+ * @throws UnsupportedOperationException
+ * dropping an index table is not allowed
* @see
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#drop_table(java.lang.String,
* java.lang.String, boolean)
*/
- @Override
- public void dropTable(String dbname, String name, boolean deleteData,
- boolean ignoreUnknownTab) throws MetaException, TException,
- NoSuchObjectException, UnsupportedOperationException {
- dropTable(dbname, name, deleteData, ignoreUnknownTab, null);
- }
-
public void dropTable(String dbname, String name, boolean deleteData,
boolean ignoreUnknownTab, EnvironmentContext envContext) throws
MetaException, TException,
NoSuchObjectException, UnsupportedOperationException {
Modified:
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
(original)
+++
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
Tue Sep 23 19:30:11 2014
@@ -37,12 +37,14 @@ public class HiveMetaStoreFsImpl impleme
@Override
public boolean deleteDir(FileSystem fs, Path f, boolean recursive,
- Configuration conf) throws MetaException {
+ boolean ifPurge, Configuration conf) throws MetaException {
LOG.info("deleting " + f);
HadoopShims hadoopShim = ShimLoader.getHadoopShims();
try {
- if (hadoopShim.moveToAppropriateTrash(fs, f, conf)) {
+ if (ifPurge) {
+ LOG.info("Not moving "+ f +" to trash");
+ } else if (hadoopShim.moveToAppropriateTrash(fs, f, conf)) {
LOG.info("Moved to trash: " + f);
return true;
}
Modified:
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
(original)
+++
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
Tue Sep 23 19:30:11 2014
@@ -192,6 +192,10 @@ public interface IMetaStoreClient {
* The database for this table
* @param tableName
* The table to drop
+ * @param deleteData
+ * Should we delete the underlying data
+ * @param ignoreUnknownTab
+ * don't throw if the requested table doesn't exist
* @throws MetaException
* Could not drop table properly.
* @throws NoSuchObjectException
@@ -200,7 +204,16 @@ public interface IMetaStoreClient {
* A thrift communication error occurred
*/
void dropTable(String dbname, String tableName, boolean deleteData,
- boolean ignoreUknownTab) throws MetaException, TException,
+ boolean ignoreUnknownTab) throws MetaException, TException,
+ NoSuchObjectException;
+
+ /**
+ * @param ifPurge
+ * completely purge the table (skipping trash) while removing data
from warehouse
+ * @see #dropTable(String, String, boolean, boolean)
+ */
+ public void dropTable(String dbname, String tableName, boolean deleteData,
+ boolean ignoreUnknownTab, boolean ifPurge) throws MetaException,
TException,
NoSuchObjectException;
/**
@@ -226,6 +239,9 @@ public interface IMetaStoreClient {
void dropTable(String tableName, boolean deleteData)
throws MetaException, UnknownTableException, TException,
NoSuchObjectException;
+ /**
+ * @see #dropTable(String, String, boolean, boolean)
+ */
void dropTable(String dbname, String tableName)
throws MetaException, TException, NoSuchObjectException;
Modified:
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java
(original)
+++
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java
Tue Sep 23 19:30:11 2014
@@ -32,11 +32,12 @@ public interface MetaStoreFS {
* delete a directory
*
* @param f
+ * @param ifPurge
* @param recursive
* @return true on success
* @throws MetaException
*/
public boolean deleteDir(FileSystem fs, Path f, boolean recursive,
- Configuration conf) throws MetaException;
+ boolean ifPurge, Configuration conf) throws MetaException;
}
Modified:
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
(original)
+++
hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
Tue Sep 23 19:30:11 2014
@@ -224,8 +224,12 @@ public class Warehouse {
}
public boolean deleteDir(Path f, boolean recursive) throws MetaException {
+ return deleteDir(f, recursive, false);
+ }
+
+ public boolean deleteDir(Path f, boolean recursive, boolean ifPurge) throws
MetaException {
FileSystem fs = getFs(f);
- return fsHandler.deleteDir(fs, f, recursive, conf);
+ return fsHandler.deleteDir(fs, f, recursive, ifPurge, conf);
}
public boolean isEmpty(Path path) throws IOException, MetaException {
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java
(original)
+++
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java
Tue Sep 23 19:30:11 2014
@@ -44,10 +44,10 @@ import org.apache.tez.runtime.api.events
@SuppressWarnings({ "deprecation", "serial" })
public class AppMasterEventOperator extends Operator<AppMasterEventDesc> {
- private transient Serializer serializer;
- private transient DataOutputBuffer buffer;
- private transient boolean hasReachedMaxSize = false;
- private transient long MAX_SIZE;
+ protected transient Serializer serializer;
+ protected transient DataOutputBuffer buffer;
+ protected transient boolean hasReachedMaxSize = false;
+ protected transient long MAX_SIZE;
@Override
public void initializeOp(Configuration hconf) throws HiveException {
@@ -57,7 +57,7 @@ public class AppMasterEventOperator exte
initDataBuffer(false);
}
- private void initDataBuffer(boolean skipPruning) throws HiveException {
+ protected void initDataBuffer(boolean skipPruning) throws HiveException {
buffer = new DataOutputBuffer();
try {
// where does this go to?
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
(original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
Tue Sep 23 19:30:11 2014
@@ -3689,7 +3689,7 @@ public class DDLTask extends Task<DDLWor
}
// drop the table
- db.dropTable(dropTbl.getTableName());
+ db.dropTable(dropTbl.getTableName(), dropTbl.getIfPurge());
if (tbl != null) {
// We have already locked the table in DDLSemanticAnalyzer, don't do it
again here
work.getOutputs().add(new WriteEntity(tbl,
WriteEntity.WriteType.DDL_NO_LOCK));
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
(original)
+++
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
Tue Sep 23 19:30:11 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.exec;
+import org.apache.hadoop.hive.ql.exec.vector.VectorAppMasterEventOperator;
import org.apache.hadoop.hive.ql.exec.vector.VectorExtractOperator;
import org.apache.hadoop.hive.ql.exec.vector.VectorFileSinkOperator;
import org.apache.hadoop.hive.ql.exec.vector.VectorFilterOperator;
@@ -118,6 +119,10 @@ public final class OperatorFactory {
static {
vectorOpvec = new ArrayList<OpTuple>();
+ vectorOpvec.add(new OpTuple<AppMasterEventDesc>(AppMasterEventDesc.class,
+ VectorAppMasterEventOperator.class));
+ vectorOpvec.add(new
OpTuple<DynamicPruningEventDesc>(DynamicPruningEventDesc.class,
+ VectorAppMasterEventOperator.class));
vectorOpvec.add(new OpTuple<SelectDesc>(SelectDesc.class,
VectorSelectOperator.class));
vectorOpvec.add(new OpTuple<GroupByDesc>(GroupByDesc.class,
VectorGroupByOperator.class));
vectorOpvec.add(new OpTuple<MapJoinDesc>(MapJoinDesc.class,
VectorMapJoinOperator.class));
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
(original)
+++
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
Tue Sep 23 19:30:11 2014
@@ -437,6 +437,7 @@ public class ReduceRecordProcessor exte
private boolean processVectors(Iterable<Object> values, byte tag) throws
HiveException {
VectorizedRowBatch batch = batches[tag];
batch.reset();
+ buffer.reset();
/* deserialize key into columns */
VectorizedBatchUtil.addRowToBatchFrom(keyObject, keyStructInspector,
@@ -459,6 +460,7 @@ public class ReduceRecordProcessor exte
VectorizedBatchUtil.setBatchSize(batch, rowIdx);
reducer.processOp(batch, tag);
rowIdx = 0;
+ buffer.reset();
if (isLogInfoEnabled) {
logProgress();
}
@@ -467,6 +469,7 @@ public class ReduceRecordProcessor exte
if (rowIdx > 0) {
VectorizedBatchUtil.setBatchSize(batch, rowIdx);
reducer.processOp(batch, tag);
+ buffer.reset();
}
if (isLogInfoEnabled) {
logProgress();
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
(original)
+++
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
Tue Sep 23 19:30:11 2014
@@ -270,11 +270,16 @@ public class TezSessionPoolManager {
public void closeAndOpen(TezSessionState sessionState, HiveConf conf)
throws Exception {
+ closeAndOpen(sessionState, conf, null);
+ }
+
+ public void closeAndOpen(TezSessionState sessionState, HiveConf conf,
+ String[] additionalFiles) throws Exception {
HiveConf sessionConf = sessionState.getConf();
if (sessionConf != null && sessionConf.get("tez.queue.name") != null) {
conf.set("tez.queue.name", sessionConf.get("tez.queue.name"));
}
close(sessionState);
- sessionState.open(conf);
+ sessionState.open(conf, additionalFiles);
}
}
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
(original)
+++
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
Tue Sep 23 19:30:11 2014
@@ -55,6 +55,7 @@ import org.apache.tez.dag.api.DAG;
import org.apache.tez.dag.api.Edge;
import org.apache.tez.dag.api.GroupInputEdge;
import org.apache.tez.dag.api.SessionNotRunning;
+import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.Vertex;
import org.apache.tez.dag.api.VertexGroup;
import org.apache.tez.dag.api.client.DAGClient;
@@ -124,14 +125,11 @@ public class TezTask extends Task<TezWor
// create the tez tmp dir
scratchDir = utils.createTezDir(scratchDir, conf);
- if (!session.isOpen()) {
- // can happen if the user sets the tez flag after the session was
- // established
- LOG.info("Tez session hasn't been created yet. Opening session");
- session.open(conf, inputOutputJars);
- } else {
- session.refreshLocalResourcesFromConf(conf);
- }
+ Map<String,LocalResource> inputOutputLocalResources =
+ getExtraLocalResources(jobConf, scratchDir, inputOutputJars);
+
+ // Ensure the session is open and has the necessary local resources
+ updateSession(session, jobConf, scratchDir, inputOutputJars,
inputOutputLocalResources);
List<LocalResource> additionalLr = session.getLocalizedResources();
@@ -153,8 +151,12 @@ public class TezTask extends Task<TezWor
// next we translate the TezWork to a Tez DAG
DAG dag = build(jobConf, work, scratchDir, appJarLr, additionalLr, ctx);
+ // Add the extra resources to the dag
+ addExtraResourcesToDag(session, dag, inputOutputJars,
inputOutputLocalResources);
+
// submit will send the job to the cluster and start executing
- client = submit(jobConf, dag, scratchDir, appJarLr, session,
additionalLr);
+ client = submit(jobConf, dag, scratchDir, appJarLr, session,
+ additionalLr, inputOutputJars, inputOutputLocalResources);
// finally monitor will print progress until the job is done
TezJobMonitor monitor = new TezJobMonitor();
@@ -195,6 +197,63 @@ public class TezTask extends Task<TezWor
return rc;
}
+ /**
+ * Converted the list of jars into local resources
+ */
+ Map<String,LocalResource> getExtraLocalResources(JobConf jobConf, Path
scratchDir,
+ String[] inputOutputJars) throws Exception {
+ final Map<String,LocalResource> resources = new
HashMap<String,LocalResource>();
+ final List<LocalResource> localResources = utils.localizeTempFiles(
+ scratchDir.toString(), jobConf, inputOutputJars);
+ if (null != localResources) {
+ for (LocalResource lr : localResources) {
+ resources.put(utils.getBaseName(lr), lr);
+ }
+ }
+ return resources;
+ }
+
+ /**
+ * Ensures that the Tez Session is open and the AM has all necessary jars
configured.
+ */
+ void updateSession(TezSessionState session,
+ JobConf jobConf, Path scratchDir, String[] inputOutputJars,
+ Map<String,LocalResource> extraResources) throws Exception {
+ final boolean missingLocalResources = !session
+ .hasResources(inputOutputJars);
+
+ if (!session.isOpen()) {
+ // can happen if the user sets the tez flag after the session was
+ // established
+ LOG.info("Tez session hasn't been created yet. Opening session");
+ session.open(conf, inputOutputJars);
+ } else {
+ LOG.info("Session is already open");
+
+ // Ensure the open session has the necessary resources (StorageHandler)
+ if (missingLocalResources) {
+ LOG.info("Tez session missing resources," +
+ " adding additional necessary resources");
+ session.getSession().addAppMasterLocalFiles(extraResources);
+ }
+
+ session.refreshLocalResourcesFromConf(conf);
+ }
+ }
+
+ /**
+ * Adds any necessary resources that must be localized in each vertex to the
DAG.
+ */
+ void addExtraResourcesToDag(TezSessionState session, DAG dag,
+ String[] inputOutputJars,
+ Map<String,LocalResource> inputOutputLocalResources) throws Exception {
+ if (!session.hasResources(inputOutputJars)) {
+ if (null != inputOutputLocalResources) {
+ dag.addTaskLocalFiles(inputOutputLocalResources);
+ }
+ }
+ }
+
DAG build(JobConf conf, TezWork work, Path scratchDir,
LocalResource appJarLr, List<LocalResource> additionalLr, Context ctx)
throws Exception {
@@ -287,7 +346,8 @@ public class TezTask extends Task<TezWor
DAGClient submit(JobConf conf, DAG dag, Path scratchDir,
LocalResource appJarLr, TezSessionState sessionState,
- List<LocalResource> additionalLr)
+ List<LocalResource> additionalLr, String[] inputOutputJars,
+ Map<String,LocalResource> inputOutputLocalResources)
throws Exception {
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_SUBMIT_DAG);
@@ -308,7 +368,7 @@ public class TezTask extends Task<TezWor
console.printInfo("Tez session was closed. Reopening...");
// close the old one, but keep the tmp files around
- TezSessionPoolManager.getInstance().closeAndOpen(sessionState,
this.conf);
+ TezSessionPoolManager.getInstance().closeAndOpen(sessionState,
this.conf, inputOutputJars);
console.printInfo("Session re-established.");
dagClient = sessionState.getSession().submitDAG(dag);
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java
(original)
+++
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java
Tue Sep 23 19:30:11 2014
@@ -67,6 +67,7 @@ public class VectorExpressionDescriptor
DATE (0x040),
TIMESTAMP (0x080),
DATETIME_FAMILY (DATE.value | TIMESTAMP.value),
+ INT_TIMESTAMP_FAMILY (INT_FAMILY.value | TIMESTAMP.value),
INT_DATETIME_FAMILY (INT_FAMILY.value | DATETIME_FAMILY.value),
STRING_DATETIME_FAMILY (STRING_FAMILY.value | DATETIME_FAMILY.value),
ALL_FAMILY (0xFFF);
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
(original)
+++
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
Tue Sep 23 19:30:11 2014
@@ -1889,47 +1889,47 @@ public class VectorizationContext {
// TODO: And, investigate if different reduce-side versions are needed for
var* and std*, or if map-side aggregate can be used.. Right now they are
conservatively
// marked map-side (HASH).
static ArrayList<AggregateDefinition> aggregatesDefinition = new
ArrayList<AggregateDefinition>() {{
- add(new AggregateDefinition("min",
VectorExpressionDescriptor.ArgumentType.INT_FAMILY, null,
VectorUDAFMinLong.class));
- add(new AggregateDefinition("min",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, null,
VectorUDAFMinDouble.class));
- add(new AggregateDefinition("min",
VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, null,
VectorUDAFMinString.class));
- add(new AggregateDefinition("min",
VectorExpressionDescriptor.ArgumentType.DECIMAL, null,
VectorUDAFMinDecimal.class));
- add(new AggregateDefinition("max",
VectorExpressionDescriptor.ArgumentType.INT_FAMILY, null,
VectorUDAFMaxLong.class));
- add(new AggregateDefinition("max",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, null,
VectorUDAFMaxDouble.class));
- add(new AggregateDefinition("max",
VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, null,
VectorUDAFMaxString.class));
- add(new AggregateDefinition("max",
VectorExpressionDescriptor.ArgumentType.DECIMAL, null,
VectorUDAFMaxDecimal.class));
- add(new AggregateDefinition("count",
VectorExpressionDescriptor.ArgumentType.NONE, GroupByDesc.Mode.HASH,
VectorUDAFCountStar.class));
- add(new AggregateDefinition("count",
VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFCount.class));
- add(new AggregateDefinition("count",
VectorExpressionDescriptor.ArgumentType.INT_FAMILY,
GroupByDesc.Mode.MERGEPARTIAL, VectorUDAFCountMerge.class));
- add(new AggregateDefinition("count",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFCount.class));
- add(new AggregateDefinition("count",
VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFCount.class));
- add(new AggregateDefinition("count",
VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH,
VectorUDAFCount.class));
- add(new AggregateDefinition("sum",
VectorExpressionDescriptor.ArgumentType.INT_FAMILY, null,
VectorUDAFSumLong.class));
- add(new AggregateDefinition("sum",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, null,
VectorUDAFSumDouble.class));
- add(new AggregateDefinition("sum",
VectorExpressionDescriptor.ArgumentType.DECIMAL, null,
VectorUDAFSumDecimal.class));
- add(new AggregateDefinition("avg",
VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFAvgLong.class));
- add(new AggregateDefinition("avg",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFAvgDouble.class));
- add(new AggregateDefinition("avg",
VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH,
VectorUDAFAvgDecimal.class));
- add(new AggregateDefinition("variance",
VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFVarPopLong.class));
- add(new AggregateDefinition("var_pop",
VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFVarPopLong.class));
- add(new AggregateDefinition("variance",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFVarPopDouble.class));
- add(new AggregateDefinition("var_pop",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFVarPopDouble.class));
- add(new AggregateDefinition("variance",
VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH,
VectorUDAFVarPopDecimal.class));
- add(new AggregateDefinition("var_pop",
VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH,
VectorUDAFVarPopDecimal.class));
- add(new AggregateDefinition("var_samp",
VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFVarSampLong.class));
- add(new AggregateDefinition("var_samp" ,
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFVarSampDouble.class));
- add(new AggregateDefinition("var_samp" ,
VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH,
VectorUDAFVarSampDecimal.class));
- add(new AggregateDefinition("std",
VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFStdPopLong.class));
- add(new AggregateDefinition("stddev",
VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFStdPopLong.class));
- add(new AggregateDefinition("stddev_pop",
VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFStdPopLong.class));
- add(new AggregateDefinition("std",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFStdPopDouble.class));
- add(new AggregateDefinition("stddev",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFStdPopDouble.class));
- add(new AggregateDefinition("stddev_pop",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFStdPopDouble.class));
- add(new AggregateDefinition("std",
VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH,
VectorUDAFStdPopDecimal.class));
- add(new AggregateDefinition("stddev",
VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH,
VectorUDAFStdPopDecimal.class));
- add(new AggregateDefinition("stddev_pop",
VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH,
VectorUDAFStdPopDecimal.class));
- add(new AggregateDefinition("stddev_samp",
VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFStdSampLong.class));
- add(new AggregateDefinition("stddev_samp",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH,
VectorUDAFStdSampDouble.class));
- add(new AggregateDefinition("stddev_samp",
VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH,
VectorUDAFStdSampDecimal.class));
+ add(new AggregateDefinition("min",
VectorExpressionDescriptor.ArgumentType.INT_DATETIME_FAMILY, null,
VectorUDAFMinLong.class));
+ add(new AggregateDefinition("min",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, null,
VectorUDAFMinDouble.class));
+ add(new AggregateDefinition("min",
VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, null,
VectorUDAFMinString.class));
+ add(new AggregateDefinition("min",
VectorExpressionDescriptor.ArgumentType.DECIMAL, null,
VectorUDAFMinDecimal.class));
+ add(new AggregateDefinition("max",
VectorExpressionDescriptor.ArgumentType.INT_DATETIME_FAMILY, null,
VectorUDAFMaxLong.class));
+ add(new AggregateDefinition("max",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, null,
VectorUDAFMaxDouble.class));
+ add(new AggregateDefinition("max",
VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, null,
VectorUDAFMaxString.class));
+ add(new AggregateDefinition("max",
VectorExpressionDescriptor.ArgumentType.DECIMAL, null,
VectorUDAFMaxDecimal.class));
+ add(new AggregateDefinition("count",
VectorExpressionDescriptor.ArgumentType.NONE,
GroupByDesc.Mode.HASH, VectorUDAFCountStar.class));
+ add(new AggregateDefinition("count",
VectorExpressionDescriptor.ArgumentType.INT_DATETIME_FAMILY,
GroupByDesc.Mode.HASH, VectorUDAFCount.class));
+ add(new AggregateDefinition("count",
VectorExpressionDescriptor.ArgumentType.INT_FAMILY,
GroupByDesc.Mode.MERGEPARTIAL, VectorUDAFCountMerge.class));
+ add(new AggregateDefinition("count",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY,
GroupByDesc.Mode.HASH, VectorUDAFCount.class));
+ add(new AggregateDefinition("count",
VectorExpressionDescriptor.ArgumentType.STRING_FAMILY,
GroupByDesc.Mode.HASH, VectorUDAFCount.class));
+ add(new AggregateDefinition("count",
VectorExpressionDescriptor.ArgumentType.DECIMAL,
GroupByDesc.Mode.HASH, VectorUDAFCount.class));
+ add(new AggregateDefinition("sum",
VectorExpressionDescriptor.ArgumentType.INT_FAMILY, null,
VectorUDAFSumLong.class));
+ add(new AggregateDefinition("sum",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, null,
VectorUDAFSumDouble.class));
+ add(new AggregateDefinition("sum",
VectorExpressionDescriptor.ArgumentType.DECIMAL, null,
VectorUDAFSumDecimal.class));
+ add(new AggregateDefinition("avg",
VectorExpressionDescriptor.ArgumentType.INT_TIMESTAMP_FAMILY,
GroupByDesc.Mode.HASH, VectorUDAFAvgLong.class));
+ add(new AggregateDefinition("avg",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY,
GroupByDesc.Mode.HASH, VectorUDAFAvgDouble.class));
+ add(new AggregateDefinition("avg",
VectorExpressionDescriptor.ArgumentType.DECIMAL,
GroupByDesc.Mode.HASH, VectorUDAFAvgDecimal.class));
+ add(new AggregateDefinition("variance",
VectorExpressionDescriptor.ArgumentType.INT_TIMESTAMP_FAMILY,
GroupByDesc.Mode.HASH, VectorUDAFVarPopLong.class));
+ add(new AggregateDefinition("var_pop",
VectorExpressionDescriptor.ArgumentType.INT_TIMESTAMP_FAMILY,
GroupByDesc.Mode.HASH, VectorUDAFVarPopLong.class));
+ add(new AggregateDefinition("variance",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY,
GroupByDesc.Mode.HASH, VectorUDAFVarPopDouble.class));
+ add(new AggregateDefinition("var_pop",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY,
GroupByDesc.Mode.HASH, VectorUDAFVarPopDouble.class));
+ add(new AggregateDefinition("variance",
VectorExpressionDescriptor.ArgumentType.DECIMAL,
GroupByDesc.Mode.HASH, VectorUDAFVarPopDecimal.class));
+ add(new AggregateDefinition("var_pop",
VectorExpressionDescriptor.ArgumentType.DECIMAL,
GroupByDesc.Mode.HASH, VectorUDAFVarPopDecimal.class));
+ add(new AggregateDefinition("var_samp",
VectorExpressionDescriptor.ArgumentType.INT_TIMESTAMP_FAMILY,
GroupByDesc.Mode.HASH, VectorUDAFVarSampLong.class));
+ add(new AggregateDefinition("var_samp" ,
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY,
GroupByDesc.Mode.HASH, VectorUDAFVarSampDouble.class));
+ add(new AggregateDefinition("var_samp" ,
VectorExpressionDescriptor.ArgumentType.DECIMAL,
GroupByDesc.Mode.HASH, VectorUDAFVarSampDecimal.class));
+ add(new AggregateDefinition("std",
VectorExpressionDescriptor.ArgumentType.INT_TIMESTAMP_FAMILY,
GroupByDesc.Mode.HASH, VectorUDAFStdPopLong.class));
+ add(new AggregateDefinition("stddev",
VectorExpressionDescriptor.ArgumentType.INT_TIMESTAMP_FAMILY,
GroupByDesc.Mode.HASH, VectorUDAFStdPopLong.class));
+ add(new AggregateDefinition("stddev_pop",
VectorExpressionDescriptor.ArgumentType.INT_TIMESTAMP_FAMILY,
GroupByDesc.Mode.HASH, VectorUDAFStdPopLong.class));
+ add(new AggregateDefinition("std",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY,
GroupByDesc.Mode.HASH, VectorUDAFStdPopDouble.class));
+ add(new AggregateDefinition("stddev",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY,
GroupByDesc.Mode.HASH, VectorUDAFStdPopDouble.class));
+ add(new AggregateDefinition("stddev_pop",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY,
GroupByDesc.Mode.HASH, VectorUDAFStdPopDouble.class));
+ add(new AggregateDefinition("std",
VectorExpressionDescriptor.ArgumentType.DECIMAL,
GroupByDesc.Mode.HASH, VectorUDAFStdPopDecimal.class));
+ add(new AggregateDefinition("stddev",
VectorExpressionDescriptor.ArgumentType.DECIMAL,
GroupByDesc.Mode.HASH, VectorUDAFStdPopDecimal.class));
+ add(new AggregateDefinition("stddev_pop",
VectorExpressionDescriptor.ArgumentType.DECIMAL,
GroupByDesc.Mode.HASH, VectorUDAFStdPopDecimal.class));
+ add(new AggregateDefinition("stddev_samp",
VectorExpressionDescriptor.ArgumentType.INT_TIMESTAMP_FAMILY,
GroupByDesc.Mode.HASH, VectorUDAFStdSampLong.class));
+ add(new AggregateDefinition("stddev_samp",
VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY,
GroupByDesc.Mode.HASH, VectorUDAFStdSampDouble.class));
+ add(new AggregateDefinition("stddev_samp",
VectorExpressionDescriptor.ArgumentType.DECIMAL,
GroupByDesc.Mode.HASH, VectorUDAFStdSampDecimal.class));
}};
public VectorAggregateExpression getAggregatorExpression(AggregationDesc
desc, boolean isReduce)
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriterV2.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriterV2.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriterV2.java
(original)
+++
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriterV2.java
Tue Sep 23 19:30:11 2014
@@ -418,138 +418,120 @@ class RunLengthIntegerWriterV2 implement
private void determineEncoding() {
- int idx = 0;
+ // we need to compute zigzag values for DIRECT encoding if we decide to
+ // break early for delta overflows or for shorter runs
+ computeZigZagLiterals();
- // for identifying monotonic sequences
- boolean isIncreasing = false;
- int increasingCount = 1;
- boolean isDecreasing = false;
- int decreasingCount = 1;
+ zzBits100p = utils.percentileBits(zigzagLiterals, 0, numLiterals, 1.0);
- // for identifying type of delta encoding
- min = literals[0];
- long max = literals[0];
- isFixedDelta = true;
- long currDelta = 0;
+ // not a big win for shorter runs to determine encoding
+ if (numLiterals <= MIN_REPEAT) {
+ encoding = EncodingType.DIRECT;
+ return;
+ }
- min = literals[0];
- long deltaMax = 0;
+ // DELTA encoding check
- // populate all variables to identify the encoding type
- if (numLiterals >= 1) {
- currDelta = literals[1] - literals[0];
- for(int i = 0; i < numLiterals; i++) {
- if (i > 0 && literals[i] >= max) {
- max = literals[i];
- increasingCount++;
- }
+ // for identifying monotonic sequences
+ boolean isIncreasing = true;
+ boolean isDecreasing = true;
+ this.isFixedDelta = true;
- if (i > 0 && literals[i] <= min) {
- min = literals[i];
- decreasingCount++;
- }
+ this.min = literals[0];
+ long max = literals[0];
+ final long initialDelta = literals[1] - literals[0];
+ long currDelta = initialDelta;
+ long deltaMax = initialDelta;
+ this.adjDeltas[0] = initialDelta;
+
+ for (int i = 1; i < numLiterals; i++) {
+ final long l1 = literals[i];
+ final long l0 = literals[i - 1];
+ currDelta = l1 - l0;
+ min = Math.min(min, l1);
+ max = Math.max(max, l1);
+
+ isIncreasing &= (l0 <= l1);
+ isDecreasing &= (l0 >= l1);
+
+ isFixedDelta &= (currDelta == initialDelta);
+ if (i > 1) {
+ adjDeltas[i - 1] = Math.abs(currDelta);
+ deltaMax = Math.max(deltaMax, adjDeltas[i - 1]);
+ }
+ }
- // if delta doesn't changes then mark it as fixed delta
- if (i > 0 && isFixedDelta) {
- if (literals[i] - literals[i - 1] != currDelta) {
- isFixedDelta = false;
- }
+ // its faster to exit under delta overflow condition without checking for
+ // PATCHED_BASE condition as encoding using DIRECT is faster and has less
+ // overhead than PATCHED_BASE
+ if (!utils.isSafeSubtract(max, min)) {
+ encoding = EncodingType.DIRECT;
+ return;
+ }
- fixedDelta = currDelta;
- }
+ // invariant - subtracting any number from any other in the literals after
+ // this point won't overflow
- // populate zigzag encoded literals
- long zzEncVal = 0;
- if (signed) {
- zzEncVal = utils.zigzagEncode(literals[i]);
- } else {
- zzEncVal = literals[i];
- }
- zigzagLiterals[idx] = zzEncVal;
- idx++;
+ // if initialDelta is 0 then we cannot delta encode as we cannot identify
+ // the sign of deltas (increasing or decreasing)
+ if (initialDelta != 0) {
+
+ // if min is equal to max then the delta is 0, this condition happens for
+ // fixed values run >10 which cannot be encoded with SHORT_REPEAT
+ if (min == max) {
+ assert isFixedDelta : min + "==" + max +
+ ", isFixedDelta cannot be false";
+ assert currDelta == 0 : min + "==" + max + ", currDelta should be
zero";
+ fixedDelta = 0;
+ encoding = EncodingType.DELTA;
+ return;
+ }
- // max delta value is required for computing the fixed bits
- // required for delta blob in delta encoding
- if (i > 0) {
- if (i == 1) {
- // first value preserve the sign
- adjDeltas[i - 1] = literals[i] - literals[i - 1];
- } else {
- adjDeltas[i - 1] = Math.abs(literals[i] - literals[i - 1]);
- if (adjDeltas[i - 1] > deltaMax) {
- deltaMax = adjDeltas[i - 1];
- }
- }
- }
+ if (isFixedDelta) {
+ assert currDelta == initialDelta
+ : "currDelta should be equal to initialDelta for fixed delta
encoding";
+ encoding = EncodingType.DELTA;
+ fixedDelta = currDelta;
+ return;
}
// stores the number of bits required for packing delta blob in
// delta encoding
bitsDeltaMax = utils.findClosestNumBits(deltaMax);
- // if decreasing count equals total number of literals then the
- // sequence is monotonically decreasing
- if (increasingCount == 1 && decreasingCount == numLiterals) {
- isDecreasing = true;
- }
-
- // if increasing count equals total number of literals then the
- // sequence is monotonically increasing
- if (decreasingCount == 1 && increasingCount == numLiterals) {
- isIncreasing = true;
+ // monotonic condition
+ if (isIncreasing || isDecreasing) {
+ encoding = EncodingType.DELTA;
+ return;
}
}
- // if the sequence is both increasing and decreasing then it is not
- // monotonic
- if (isDecreasing && isIncreasing) {
- isDecreasing = false;
- isIncreasing = false;
- }
-
- // fixed delta condition
- if (isIncreasing == false && isDecreasing == false && isFixedDelta ==
true) {
- encoding = EncodingType.DELTA;
- return;
- }
-
- // monotonic condition
- if (isIncreasing || isDecreasing) {
- encoding = EncodingType.DELTA;
- return;
- }
+ // PATCHED_BASE encoding check
// percentile values are computed for the zigzag encoded values. if the
// number of bit requirement between 90th and 100th percentile varies
// beyond a threshold then we need to patch the values. if the variation
- // is not significant then we can use direct or delta encoding
-
- double p = 0.9;
- zzBits90p = utils.percentileBits(zigzagLiterals, 0, numLiterals, p);
-
- p = 1.0;
- zzBits100p = utils.percentileBits(zigzagLiterals, 0, numLiterals, p);
+ // is not significant then we can use direct encoding
+ zzBits90p = utils.percentileBits(zigzagLiterals, 0, numLiterals, 0.9);
int diffBitsLH = zzBits100p - zzBits90p;
// if the difference between 90th percentile and 100th percentile fixed
// bits is > 1 then we need patch the values
- if (isIncreasing == false && isDecreasing == false && diffBitsLH > 1
- && isFixedDelta == false) {
+ if (diffBitsLH > 1) {
+
// patching is done only on base reduced values.
// remove base from literals
- for(int i = 0; i < numLiterals; i++) {
+ for (int i = 0; i < numLiterals; i++) {
baseRedLiterals[i] = literals[i] - min;
}
// 95th percentile width is used to determine max allowed value
// after which patching will be done
- p = 0.95;
- brBits95p = utils.percentileBits(baseRedLiterals, 0, numLiterals, p);
+ brBits95p = utils.percentileBits(baseRedLiterals, 0, numLiterals, 0.95);
// 100th percentile is used to compute the max patch width
- p = 1.0;
- brBits100p = utils.percentileBits(baseRedLiterals, 0, numLiterals, p);
+ brBits100p = utils.percentileBits(baseRedLiterals, 0, numLiterals, 1.0);
// after base reducing the values, if the difference in bits between
// 95th percentile and 100th percentile value is zero then there
@@ -565,19 +547,24 @@ class RunLengthIntegerWriterV2 implement
encoding = EncodingType.DIRECT;
return;
}
- }
-
- // if difference in bits between 95th percentile and 100th percentile is
- // 0, then patch length will become 0. Hence we will fallback to direct
- if (isIncreasing == false && isDecreasing == false && diffBitsLH <= 1
- && isFixedDelta == false) {
+ } else {
+ // if difference in bits between 95th percentile and 100th percentile is
+ // 0, then patch length will become 0. Hence we will fallback to direct
encoding = EncodingType.DIRECT;
return;
}
+ }
- // this should not happen
- if (encoding == null) {
- throw new RuntimeException("Integer encoding cannot be determined.");
+ private void computeZigZagLiterals() {
+ // populate zigzag encoded literals
+ long zzEncVal = 0;
+ for (int i = 0; i < numLiterals; i++) {
+ if (signed) {
+ zzEncVal = utils.zigzagEncode(literals[i]);
+ } else {
+ zzEncVal = literals[i];
+ }
+ zigzagLiterals[i] = zzEncVal;
}
}
@@ -700,7 +687,7 @@ class RunLengthIntegerWriterV2 implement
patchWidth = 0;
gapVsPatchList = null;
min = 0;
- isFixedDelta = false;
+ isFixedDelta = true;
}
@Override
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SerializationUtils.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SerializationUtils.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SerializationUtils.java
(original)
+++
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SerializationUtils.java
Tue Sep 23 19:30:11 2014
@@ -1283,4 +1283,9 @@ final class SerializationUtils {
+ ((readBuffer[rbOffset + 7] & 255) << 0));
}
+ // Do not want to use Guava LongMath.checkedSubtract() here as it will throw
+ // ArithmeticException in case of overflow
+ public boolean isSafeSubtract(long left, long right) {
+ return (left ^ right) >= 0 | (left ^ (left - right)) >= 0;
+ }
}
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
(original)
+++
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
Tue Sep 23 19:30:11 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.lockmg
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.*;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.thrift.TException;
@@ -42,10 +43,10 @@ public class DbLockManager implements Hi
private static final long MAX_SLEEP = 15000;
private HiveLockManagerCtx context;
private Set<DbHiveLock> locks;
- private HiveMetaStoreClient client;
+ private IMetaStoreClient client;
private long nextSleep = 50;
- DbLockManager(HiveMetaStoreClient client) {
+ DbLockManager(IMetaStoreClient client) {
locks = new HashSet<DbHiveLock>();
this.client = client;
}
@@ -210,8 +211,8 @@ public class DbLockManager implements Hi
/**
* Clear the memory of the locks in this object. This won't clear the locks
from the database.
* It is for use with
- * {@link
#DbLockManager(org.apache.hadoop.hive.metastore.HiveMetaStoreClient).commitTxn}
and
- * {@link
#DbLockManager(org.apache.hadoop.hive.metastore.HiveMetaStoreClient).rollbackTxn}.
+ * {@link
#DbLockManager(org.apache.hadoop.hive.metastore.IMetaStoreClient).commitTxn} and
+ * {@link
#DbLockManager(org.apache.hadoop.hive.metastore.IMetaStoreClient).rollbackTxn}.
*/
void clearLocalLockRecords() {
locks.clear();
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
---
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
(original)
+++
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
Tue Sep 23 19:30:11 2014
@@ -31,6 +31,8 @@ import org.apache.hadoop.hive.ql.QueryPl
import org.apache.hadoop.hive.ql.hooks.Entity;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.thrift.TException;
@@ -46,7 +48,7 @@ public class DbTxnManager extends HiveTx
static final private Log LOG = LogFactory.getLog(CLASS_NAME);
private DbLockManager lockMgr = null;
- private HiveMetaStoreClient client = null;
+ private IMetaStoreClient client = null;
private long txnId = 0;
DbTxnManager() {
@@ -311,7 +313,6 @@ public class DbTxnManager extends HiveTx
try {
if (txnId > 0) rollbackTxn();
if (lockMgr != null) lockMgr.close();
- if (client != null) client.close();
} catch (Exception e) {
LOG.error("Caught exception " + e.getClass().getName() + " with message
<" + e.getMessage()
+ ">, swallowing as there is nothing we can do with it.");
@@ -326,10 +327,12 @@ public class DbTxnManager extends HiveTx
"methods.");
}
try {
- client = new HiveMetaStoreClient(conf);
+ Hive db = Hive.get(conf);
+ client = db.getMSC();
} catch (MetaException e) {
- throw new LockException(ErrorMsg.METASTORE_COULD_NOT_INITIATE.getMsg(),
- e);
+ throw new
LockException(ErrorMsg.METASTORE_COULD_NOT_INITIATE.getMsg(), e);
+ } catch (HiveException e) {
+ throw new
LockException(ErrorMsg.METASTORE_COULD_NOT_INITIATE.getMsg(), e);
}
}
}
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1627125&r1=1627124&r2=1627125&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
(original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
Tue Sep 23 19:30:11 2014
@@ -878,6 +878,23 @@ public class Hive {
/**
* Drops table along with the data in it. If the table doesn't exist then it
+ * is a no-op. If ifPurge option is specified it is passed to the
+ * hdfs command that removes table data from warehouse to make it skip trash.
+ *
+ * @param tableName
+ * table to drop
+ * @param ifPurge
+ * completely purge the table (skipping trash) while removing data
from warehouse
+ * @throws HiveException
+ * thrown if the drop fails
+ */
+ public void dropTable(String tableName, boolean ifPurge) throws
HiveException {
+ String[] names = Utilities.getDbTableName(tableName);
+ dropTable(names[0], names[1], true, true, ifPurge);
+ }
+
+ /**
+ * Drops table along with the data in it. If the table doesn't exist then it
* is a no-op
*
* @param tableName
@@ -886,8 +903,7 @@ public class Hive {
* thrown if the drop fails
*/
public void dropTable(String tableName) throws HiveException {
- String[] names = Utilities.getDbTableName(tableName);
- dropTable(names[0], names[1], true, true);
+ dropTable(tableName, false);
}
/**
@@ -902,7 +918,7 @@ public class Hive {
* thrown if the drop fails
*/
public void dropTable(String dbName, String tableName) throws HiveException {
- dropTable(dbName, tableName, true, true);
+ dropTable(dbName, tableName, true, true, false);
}
/**
@@ -913,14 +929,31 @@ public class Hive {
* @param deleteData
* deletes the underlying data along with metadata
* @param ignoreUnknownTab
- * an exception if thrown if this is falser and table doesn't exist
+ * an exception is thrown if this is false and the table doesn't
exist
* @throws HiveException
*/
public void dropTable(String dbName, String tableName, boolean deleteData,
boolean ignoreUnknownTab) throws HiveException {
+ dropTable(dbName, tableName, deleteData, ignoreUnknownTab, false);
+ }
+ /**
+ * Drops the table.
+ *
+ * @param dbName
+ * @param tableName
+ * @param deleteData
+ * deletes the underlying data along with metadata
+ * @param ignoreUnknownTab
+ * an exception is thrown if this is false and the table doesn't
exist
+ * @param ifPurge
+ * completely purge the table skipping trash while removing data
from warehouse
+ * @throws HiveException
+ */
+ public void dropTable(String dbName, String tableName, boolean deleteData,
+ boolean ignoreUnknownTab, boolean ifPurge) throws HiveException {
try {
- getMSC().dropTable(dbName, tableName, deleteData, ignoreUnknownTab);
+ getMSC().dropTable(dbName, tableName, deleteData, ignoreUnknownTab,
ifPurge);
} catch (NoSuchObjectException e) {
if (!ignoreUnknownTab) {
throw new HiveException(e);
@@ -1736,6 +1769,7 @@ private void constructOneLBLocationMap(F
public List<Partition> dropPartitions(String dbName, String tblName,
List<DropTableDesc.PartSpec> partSpecs, boolean deleteData, boolean
ignoreProtection,
boolean ifExists) throws HiveException {
+ //TODO: add support for ifPurge
try {
Table tbl = getTable(dbName, tblName);
List<ObjectPair<Integer, byte[]>> partExprs =
