This is an automated email from the ASF dual-hosted git repository.
anishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 3c42435 HIVE-23960: Partition with no column statistics leads to
unbalanced calls to openTransaction/commitTransaction error during
get_partitions_by_names (Pravin Kumar Sinha, reviewed by Aasha Medhi)
3c42435 is described below
commit 3c42435545000de08aa99f2bf225b027128ea1ba
Author: Anishek Agarwal <anis...@gmail.com>
AuthorDate: Mon Aug 10 11:02:55 2020 +0530
HIVE-23960: Partition with no column statistics leads to unbalanced calls
to openTransaction/commitTransaction error during get_partitions_by_names
(Pravin Kumar Sinha, reviewed by Aasha Medhi)
---
.../queries/clientpositive/repl_4_exim_nocolstat.q | 33 +++++
.../llap/repl_4_exim_nocolstat.q.out | 149 +++++++++++++++++++++
.../apache/hadoop/hive/metastore/ObjectStore.java | 9 +-
3 files changed, 189 insertions(+), 2 deletions(-)
diff --git a/ql/src/test/queries/clientpositive/repl_4_exim_nocolstat.q
b/ql/src/test/queries/clientpositive/repl_4_exim_nocolstat.q
new file mode 100644
index 0000000..4b4ad07
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/repl_4_exim_nocolstat.q
@@ -0,0 +1,33 @@
+set hive.mapred.mode=nonstrict;
+set hive.test.mode=true;
+set hive.test.mode.prefix=;
+set hive.test.mode.nosamplelist=replsrc,repldst;
+set metastore.try.direct.sql=false;
+set hive.metastore.rawstore.impl=org.apache.hadoop.hive.metastore.ObjectStore;
+
+drop table if exists replsrc;
+drop table if exists repldst;
+
+create table replsrc (emp_id int comment "employee id")
+ partitioned by (emp_country string, emp_state string)
+ stored as textfile;
+load data local inpath "../../data/files/test.dat"
+ into table replsrc partition (emp_country="us",emp_state="ca");
+
+alter table replsrc add partition (emp_country="zx",emp_state="ka");
+
+dfs ${system:test.dfs.mkdir} target/tmp/ql/test/data/exports/repldst/temp;
+dfs -rmr target/tmp/ql/test/data/exports/repldst;
+
+export table replsrc to 'ql/test/data/exports/repldst' for
replication('repldst');
+
+drop table replsrc;
+
+import table repldst from 'ql/test/data/exports/repldst';
+describe extended repldst;
+show table extended like repldst;
+show create table repldst;
+select * from repldst;
+
+drop table repldst;
+
diff --git
a/ql/src/test/results/clientpositive/llap/repl_4_exim_nocolstat.q.out
b/ql/src/test/results/clientpositive/llap/repl_4_exim_nocolstat.q.out
new file mode 100644
index 0000000..4ff902d
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/repl_4_exim_nocolstat.q.out
@@ -0,0 +1,149 @@
+PREHOOK: query: drop table if exists replsrc
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists replsrc
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table if exists repldst
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists repldst
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table replsrc (emp_id int comment "employee id")
+ partitioned by (emp_country string, emp_state string)
+ stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@replsrc
+POSTHOOK: query: create table replsrc (emp_id int comment "employee id")
+ partitioned by (emp_country string, emp_state string)
+ stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@replsrc
+PREHOOK: query: load data local inpath "../../data/files/test.dat"
+ into table replsrc partition (emp_country="us",emp_state="ca")
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@replsrc
+POSTHOOK: query: load data local inpath "../../data/files/test.dat"
+ into table replsrc partition (emp_country="us",emp_state="ca")
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@replsrc
+POSTHOOK: Output: default@replsrc@emp_country=us/emp_state=ca
+PREHOOK: query: alter table replsrc add partition
(emp_country="zx",emp_state="ka")
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@replsrc
+POSTHOOK: query: alter table replsrc add partition
(emp_country="zx",emp_state="ka")
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@replsrc
+POSTHOOK: Output: default@replsrc@emp_country=zx/emp_state=ka
+#### A masked pattern was here ####
+PREHOOK: query: export table replsrc to 'ql/test/data/exports/repldst' for
replication('repldst')
+PREHOOK: type: EXPORT
+PREHOOK: Input: default@replsrc@emp_country=us/emp_state=ca
+PREHOOK: Input: default@replsrc@emp_country=zx/emp_state=ka
+#### A masked pattern was here ####
+POSTHOOK: query: export table replsrc to 'ql/test/data/exports/repldst' for
replication('repldst')
+POSTHOOK: type: EXPORT
+POSTHOOK: Input: default@replsrc@emp_country=us/emp_state=ca
+POSTHOOK: Input: default@replsrc@emp_country=zx/emp_state=ka
+#### A masked pattern was here ####
+PREHOOK: query: drop table replsrc
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@replsrc
+PREHOOK: Output: default@replsrc
+POSTHOOK: query: drop table replsrc
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@replsrc
+POSTHOOK: Output: default@replsrc
+PREHOOK: query: import table repldst from 'ql/test/data/exports/repldst'
+PREHOOK: type: IMPORT
+#### A masked pattern was here ####
+PREHOOK: Output: default@repldst
+POSTHOOK: query: import table repldst from 'ql/test/data/exports/repldst'
+POSTHOOK: type: IMPORT
+#### A masked pattern was here ####
+POSTHOOK: Output: default@repldst
+POSTHOOK: Output: default@repldst@emp_country=us/emp_state=ca
+POSTHOOK: Output: default@repldst@emp_country=zx/emp_state=ka
+PREHOOK: query: describe extended repldst
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@repldst
+POSTHOOK: query: describe extended repldst
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@repldst
+emp_id int employee id
+emp_country string
+emp_state string
+
+# Partition Information
+# col_name data_type comment
+emp_country string
+emp_state string
+
+#### A masked pattern was here ####
+PREHOOK: query: show table extended like repldst
+PREHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: query: show table extended like repldst
+POSTHOOK: type: SHOW_TABLESTATUS
+tableName:repldst
+#### A masked pattern was here ####
+inputformat:org.apache.hadoop.mapred.TextInputFormat
+outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+columns:struct columns { i32 emp_id}
+partitioned:true
+partitionColumns:struct partition_columns { string emp_country, string
emp_state}
+totalNumberFiles:1
+totalFileSize:11
+maxFileSize:11
+minFileSize:11
+#### A masked pattern was here ####
+
+PREHOOK: query: show create table repldst
+PREHOOK: type: SHOW_CREATETABLE
+PREHOOK: Input: default@repldst
+POSTHOOK: query: show create table repldst
+POSTHOOK: type: SHOW_CREATETABLE
+POSTHOOK: Input: default@repldst
+CREATE TABLE `repldst`(
+ `emp_id` int COMMENT 'employee id')
+PARTITIONED BY (
+ `emp_country` string,
+ `emp_state` string)
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
+LOCATION
+#### A masked pattern was here ####
+TBLPROPERTIES (
+ 'bucketing_version'='2',
+ 'repl.last.id'='0',
+#### A masked pattern was here ####
+PREHOOK: query: select * from repldst
+PREHOOK: type: QUERY
+PREHOOK: Input: default@repldst
+PREHOOK: Input: default@repldst@emp_country=us/emp_state=ca
+PREHOOK: Input: default@repldst@emp_country=zx/emp_state=ka
+#### A masked pattern was here ####
+POSTHOOK: query: select * from repldst
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@repldst
+POSTHOOK: Input: default@repldst@emp_country=us/emp_state=ca
+POSTHOOK: Input: default@repldst@emp_country=zx/emp_state=ka
+#### A masked pattern was here ####
+1 us ca
+2 us ca
+3 us ca
+4 us ca
+5 us ca
+6 us ca
+PREHOOK: query: drop table repldst
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@repldst
+PREHOOK: Output: default@repldst
+POSTHOOK: query: drop table repldst
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@repldst
+POSTHOOK: Output: default@repldst
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 1fc0be7..f866b94 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -9226,6 +9226,10 @@ public class ObjectStore implements RawStore,
Configurable {
public List<ColumnStatistics> getPartitionColumnStatistics(String catName,
String dbName, String tableName,
List<String> partNames, List<String> colNames, String engine) throws
MetaException, NoSuchObjectException {
// Note: this will get stats without verifying ACID.
+ if (CollectionUtils.isEmpty(partNames) ||
CollectionUtils.isEmpty(colNames)) {
+ LOG.debug("PartNames and/or ColNames are empty");
+ return Collections.emptyList();
+ }
return getPartitionColumnStatisticsInternal(
catName, dbName, tableName, partNames, colNames, engine, true, true);
}
@@ -9236,8 +9240,9 @@ public class ObjectStore implements RawStore,
Configurable {
List<String> partNames, List<String> colNames,
String engine, String writeIdList)
throws MetaException, NoSuchObjectException {
- if (partNames == null || partNames.isEmpty()) {
- return null;
+ if (CollectionUtils.isEmpty(partNames) ||
CollectionUtils.isEmpty(colNames)) {
+ LOG.debug("PartNames and/or ColNames are empty");
+ return Collections.emptyList();
}
List<ColumnStatistics> allStats = getPartitionColumnStatisticsInternal(
catName, dbName, tableName, partNames, colNames, engine, true, true);
