This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 125a87d HIVE-23358: MSCK REPAIR should remove insignificant 0's from
numeric partition values (Adesh Rao, reviewed by Sankar Hariappan)
125a87d is described below
commit 125a87d4b2698793dda97a731e1b30d77b957eea
Author: Adesh Kumar Rao <[email protected]>
AuthorDate: Mon Aug 17 19:11:56 2020 +0530
HIVE-23358: MSCK REPAIR should remove insignificant 0's from numeric
partition values (Adesh Rao, reviewed by Sankar Hariappan)
Signed-off-by: Sankar Hariappan <[email protected]>
Closes (#1050)
---
ql/src/test/queries/clientpositive/msck_repair_7.q | 71 ++++
.../clientpositive/llap/msck_repair_7.q.out | 392 +++++++++++++++++++++
.../hive/metastore/HiveMetaStoreChecker.java | 5 +-
.../hive/metastore/utils/MetaStoreServerUtils.java | 43 ++-
.../metastore/utils/TestMetaStoreServerUtils.java | 18 +
5 files changed, 525 insertions(+), 4 deletions(-)
diff --git a/ql/src/test/queries/clientpositive/msck_repair_7.q
b/ql/src/test/queries/clientpositive/msck_repair_7.q
new file mode 100644
index 0000000..19f8d8f
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/msck_repair_7.q
@@ -0,0 +1,71 @@
+DROP TABLE IF EXISTS repairtable_n7_1;
+DROP TABLE IF EXISTS repairtable_n7_2;
+DROP TABLE IF EXISTS repairtable_n7_3;
+DROP TABLE IF EXISTS repairtable_n7_4;
+DROP TABLE IF EXISTS repairtable_n7_5;
+DROP TABLE IF EXISTS repairtable_n7_6;
+DROP TABLE IF EXISTS repairtable_n7_7;
+DROP TABLE IF EXISTS repairtable_n7_8;
+
+CREATE EXTERNAL TABLE repairtable_n7_1(key INT) PARTITIONED BY (p1 TINYINT)
stored as ORC location
'${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable_n7_1/';
+CREATE EXTERNAL TABLE repairtable_n7_2(key INT) PARTITIONED BY (p1 SMALLINT)
stored as ORC location
'${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable_n7_2/';
+CREATE EXTERNAL TABLE repairtable_n7_3(key INT) PARTITIONED BY (p1 INT) stored
as ORC location
'${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable_n7_3/';
+CREATE EXTERNAL TABLE repairtable_n7_4(key INT) PARTITIONED BY (p1 BIGINT)
stored as ORC location
'${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable_n7_4/';
+CREATE EXTERNAL TABLE repairtable_n7_5(key INT) PARTITIONED BY (p1 FLOAT)
stored as ORC location
'${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable_n7_5/';
+CREATE EXTERNAL TABLE repairtable_n7_6(key INT) PARTITIONED BY (p1 DOUBLE)
stored as ORC location
'${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable_n7_6/';
+CREATE EXTERNAL TABLE repairtable_n7_7(key INT) PARTITIONED BY (p1
DECIMAL(10,10)) stored as ORC location
'${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable_n7_7/';
+CREATE EXTERNAL TABLE repairtable_n7_8(key INT) PARTITIONED BY (p1 string)
stored as ORC location
'${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable_n7_8/';
+
+MSCK REPAIR TABLE repairtable_n7_1;
+MSCK REPAIR TABLE repairtable_n7_2;
+MSCK REPAIR TABLE repairtable_n7_3;
+MSCK REPAIR TABLE repairtable_n7_4;
+MSCK REPAIR TABLE repairtable_n7_5;
+MSCK REPAIR TABLE repairtable_n7_6;
+MSCK REPAIR TABLE repairtable_n7_7;
+MSCK REPAIR TABLE repairtable_n7_8;
+
+show partitions repairtable_n7_1;
+show partitions repairtable_n7_2;
+show partitions repairtable_n7_3;
+show partitions repairtable_n7_4;
+show partitions repairtable_n7_5;
+show partitions repairtable_n7_6;
+show partitions repairtable_n7_7;
+show partitions repairtable_n7_8;
+
+dfs ${system:test.dfs.mkdir} -p
${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable_n7_1/p1=01;
+dfs ${system:test.dfs.mkdir} -p
${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable_n7_2/p1=01;
+dfs ${system:test.dfs.mkdir} -p
${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable_n7_3/p1=010;
+dfs ${system:test.dfs.mkdir} -p
${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable_n7_4/p1=-00100;
+dfs ${system:test.dfs.mkdir} -p
${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable_n7_5/p1=010.010;
+dfs ${system:test.dfs.mkdir} -p
${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable_n7_6/p1=-0100.00100;
+dfs ${system:test.dfs.mkdir} -p
${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable_n7_7/p1=01.00100;
+dfs ${system:test.dfs.mkdir} -p
${system:test.tmp.dir}/apps/hive/warehouse/test.db/repairtable_n7_8/p1=01;
+
+MSCK REPAIR TABLE repairtable_n7_1;
+MSCK REPAIR TABLE repairtable_n7_2;
+MSCK REPAIR TABLE repairtable_n7_3;
+MSCK REPAIR TABLE repairtable_n7_4;
+MSCK REPAIR TABLE repairtable_n7_5;
+MSCK REPAIR TABLE repairtable_n7_6;
+MSCK REPAIR TABLE repairtable_n7_7;
+MSCK REPAIR TABLE repairtable_n7_8;
+
+show partitions repairtable_n7_1;
+show partitions repairtable_n7_2;
+show partitions repairtable_n7_3;
+show partitions repairtable_n7_4;
+show partitions repairtable_n7_5;
+show partitions repairtable_n7_6;
+show partitions repairtable_n7_7;
+show partitions repairtable_n7_8;
+
+DROP TABLE repairtable_n7_1;
+DROP TABLE repairtable_n7_2;
+DROP TABLE repairtable_n7_3;
+DROP TABLE repairtable_n7_4;
+DROP TABLE repairtable_n7_5;
+DROP TABLE repairtable_n7_6;
+DROP TABLE repairtable_n7_7;
+DROP TABLE repairtable_n7_8;
diff --git a/ql/src/test/results/clientpositive/llap/msck_repair_7.q.out
b/ql/src/test/results/clientpositive/llap/msck_repair_7.q.out
new file mode 100644
index 0000000..bf164f6
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/msck_repair_7.q.out
@@ -0,0 +1,392 @@
+PREHOOK: query: DROP TABLE IF EXISTS repairtable_n7_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS repairtable_n7_1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE IF EXISTS repairtable_n7_2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS repairtable_n7_2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE IF EXISTS repairtable_n7_3
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS repairtable_n7_3
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE IF EXISTS repairtable_n7_4
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS repairtable_n7_4
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE IF EXISTS repairtable_n7_5
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS repairtable_n7_5
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE IF EXISTS repairtable_n7_6
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS repairtable_n7_6
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE IF EXISTS repairtable_n7_7
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS repairtable_n7_7
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE IF EXISTS repairtable_n7_8
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS repairtable_n7_8
+POSTHOOK: type: DROPTABLE
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repairtable_n7_1
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repairtable_n7_1
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repairtable_n7_2
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repairtable_n7_2
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repairtable_n7_3
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repairtable_n7_3
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repairtable_n7_4
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repairtable_n7_4
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repairtable_n7_5
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repairtable_n7_5
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repairtable_n7_6
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repairtable_n7_6
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repairtable_n7_7
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repairtable_n7_7
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repairtable_n7_8
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repairtable_n7_8
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n7_1
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n7_1
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n7_1
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n7_1
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n7_2
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n7_2
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n7_2
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n7_2
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n7_3
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n7_3
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n7_3
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n7_3
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n7_4
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n7_4
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n7_4
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n7_4
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n7_5
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n7_5
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n7_5
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n7_5
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n7_6
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n7_6
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n7_6
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n7_6
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n7_7
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n7_7
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n7_7
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n7_7
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n7_8
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n7_8
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n7_8
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n7_8
+PREHOOK: query: show partitions repairtable_n7_1
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n7_1
+POSTHOOK: query: show partitions repairtable_n7_1
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n7_1
+PREHOOK: query: show partitions repairtable_n7_2
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n7_2
+POSTHOOK: query: show partitions repairtable_n7_2
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n7_2
+PREHOOK: query: show partitions repairtable_n7_3
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n7_3
+POSTHOOK: query: show partitions repairtable_n7_3
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n7_3
+PREHOOK: query: show partitions repairtable_n7_4
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n7_4
+POSTHOOK: query: show partitions repairtable_n7_4
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n7_4
+PREHOOK: query: show partitions repairtable_n7_5
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n7_5
+POSTHOOK: query: show partitions repairtable_n7_5
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n7_5
+PREHOOK: query: show partitions repairtable_n7_6
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n7_6
+POSTHOOK: query: show partitions repairtable_n7_6
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n7_6
+PREHOOK: query: show partitions repairtable_n7_7
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n7_7
+POSTHOOK: query: show partitions repairtable_n7_7
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n7_7
+PREHOOK: query: show partitions repairtable_n7_8
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n7_8
+POSTHOOK: query: show partitions repairtable_n7_8
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n7_8
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n7_1
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n7_1
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n7_1
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n7_1
+Partitions not in metastore: repairtable_n7_1:p1=1
+#### A masked pattern was here ####
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n7_2
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n7_2
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n7_2
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n7_2
+Partitions not in metastore: repairtable_n7_2:p1=1
+#### A masked pattern was here ####
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n7_3
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n7_3
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n7_3
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n7_3
+Partitions not in metastore: repairtable_n7_3:p1=10
+#### A masked pattern was here ####
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n7_4
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n7_4
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n7_4
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n7_4
+Partitions not in metastore: repairtable_n7_4:p1=-100
+#### A masked pattern was here ####
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n7_5
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n7_5
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n7_5
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n7_5
+Partitions not in metastore: repairtable_n7_5:p1=10.01
+#### A masked pattern was here ####
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n7_6
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n7_6
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n7_6
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n7_6
+Partitions not in metastore: repairtable_n7_6:p1=-100.001
+#### A masked pattern was here ####
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n7_7
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n7_7
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n7_7
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n7_7
+Partitions not in metastore: repairtable_n7_7:p1=1.001
+#### A masked pattern was here ####
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n7_8
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n7_8
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n7_8
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n7_8
+Partitions not in metastore: repairtable_n7_8:p1=01
+#### A masked pattern was here ####
+PREHOOK: query: show partitions repairtable_n7_1
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n7_1
+POSTHOOK: query: show partitions repairtable_n7_1
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n7_1
+p1=1
+PREHOOK: query: show partitions repairtable_n7_2
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n7_2
+POSTHOOK: query: show partitions repairtable_n7_2
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n7_2
+p1=1
+PREHOOK: query: show partitions repairtable_n7_3
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n7_3
+POSTHOOK: query: show partitions repairtable_n7_3
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n7_3
+p1=10
+PREHOOK: query: show partitions repairtable_n7_4
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n7_4
+POSTHOOK: query: show partitions repairtable_n7_4
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n7_4
+p1=-100
+PREHOOK: query: show partitions repairtable_n7_5
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n7_5
+POSTHOOK: query: show partitions repairtable_n7_5
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n7_5
+p1=10.01
+PREHOOK: query: show partitions repairtable_n7_6
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n7_6
+POSTHOOK: query: show partitions repairtable_n7_6
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n7_6
+p1=-100.001
+PREHOOK: query: show partitions repairtable_n7_7
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n7_7
+POSTHOOK: query: show partitions repairtable_n7_7
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n7_7
+p1=1.001
+PREHOOK: query: show partitions repairtable_n7_8
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n7_8
+POSTHOOK: query: show partitions repairtable_n7_8
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n7_8
+p1=01
+PREHOOK: query: DROP TABLE repairtable_n7_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@repairtable_n7_1
+PREHOOK: Output: default@repairtable_n7_1
+POSTHOOK: query: DROP TABLE repairtable_n7_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@repairtable_n7_1
+POSTHOOK: Output: default@repairtable_n7_1
+PREHOOK: query: DROP TABLE repairtable_n7_2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@repairtable_n7_2
+PREHOOK: Output: default@repairtable_n7_2
+POSTHOOK: query: DROP TABLE repairtable_n7_2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@repairtable_n7_2
+POSTHOOK: Output: default@repairtable_n7_2
+PREHOOK: query: DROP TABLE repairtable_n7_3
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@repairtable_n7_3
+PREHOOK: Output: default@repairtable_n7_3
+POSTHOOK: query: DROP TABLE repairtable_n7_3
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@repairtable_n7_3
+POSTHOOK: Output: default@repairtable_n7_3
+PREHOOK: query: DROP TABLE repairtable_n7_4
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@repairtable_n7_4
+PREHOOK: Output: default@repairtable_n7_4
+POSTHOOK: query: DROP TABLE repairtable_n7_4
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@repairtable_n7_4
+POSTHOOK: Output: default@repairtable_n7_4
+PREHOOK: query: DROP TABLE repairtable_n7_5
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@repairtable_n7_5
+PREHOOK: Output: default@repairtable_n7_5
+POSTHOOK: query: DROP TABLE repairtable_n7_5
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@repairtable_n7_5
+POSTHOOK: Output: default@repairtable_n7_5
+PREHOOK: query: DROP TABLE repairtable_n7_6
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@repairtable_n7_6
+PREHOOK: Output: default@repairtable_n7_6
+POSTHOOK: query: DROP TABLE repairtable_n7_6
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@repairtable_n7_6
+POSTHOOK: Output: default@repairtable_n7_6
+PREHOOK: query: DROP TABLE repairtable_n7_7
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@repairtable_n7_7
+PREHOOK: Output: default@repairtable_n7_7
+POSTHOOK: query: DROP TABLE repairtable_n7_7
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@repairtable_n7_7
+POSTHOOK: Output: default@repairtable_n7_7
+PREHOOK: query: DROP TABLE repairtable_n7_8
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@repairtable_n7_8
+PREHOOK: Output: default@repairtable_n7_8
+POSTHOOK: query: DROP TABLE repairtable_n7_8
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@repairtable_n7_8
+POSTHOOK: Output: default@repairtable_n7_8
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java
index 69c22c3..2882336 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java
@@ -25,6 +25,7 @@ import static
org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.getPar
import static
org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.getPartitionListByFilterExp;
import static
org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.getPartitionName;
import static
org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.getPartitionSpec;
+import static
org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.getPartitionColtoTypeMap;
import static
org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.getPath;
import static
org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.isPartitioned;
@@ -35,6 +36,7 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.Callable;
@@ -430,11 +432,12 @@ public class HiveMetaStoreChecker {
partColNames.add(fSchema.getName());
}
+ Map<String, String> partitionColToTypeMap =
getPartitionColtoTypeMap(table.getPartitionKeys());
// we should now only have the unexpected folders left
for (Path partPath : allPartDirs) {
FileSystem fs = partPath.getFileSystem(conf);
String partitionName = getPartitionName(fs.makeQualified(tablePath),
- partPath, partColNames);
+ partPath, partColNames, partitionColToTypeMap);
LOG.debug("PartitionName: " + partitionName);
if (partitionName != null) {
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
index 1969ddd..6dad7f4 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
@@ -1459,7 +1459,8 @@ public class MetaStoreServerUtils {
* Set of partition columns from table definition
* @return Partition name, for example partitiondate=2008-01-01
*/
- public static String getPartitionName(Path tablePath, Path partitionPath,
Set<String> partCols) {
+ public static String getPartitionName(Path tablePath, Path partitionPath,
Set<String> partCols,
+ Map<String, String>
partitionColToTypeMap) {
String result = null;
Path currPath = partitionPath;
LOG.debug("tablePath:" + tablePath + ", partCols: " + partCols);
@@ -1481,9 +1482,12 @@ public class MetaStoreServerUtils {
String partitionValue = parts[1];
if (partCols.contains(partitionName)) {
if (result == null) {
- result = partitionName + "=" + partitionValue;
+ result = partitionName + "="
+ + getNormalisedPartitionValue(partitionValue,
partitionColToTypeMap.get(partitionName));
} else {
- result = partitionName + "=" + partitionValue + Path.SEPARATOR +
result;
+ result = partitionName + "="
+ + getNormalisedPartitionValue(partitionValue,
partitionColToTypeMap.get(partitionName))
+ + Path.SEPARATOR + result;
}
}
}
@@ -1493,6 +1497,39 @@ public class MetaStoreServerUtils {
return result;
}
+ public static String getNormalisedPartitionValue(String partitionValue,
String type) {
+
+ LOG.debug("Converting '" + partitionValue + "' to type: '" + type + "'.");
+
+ if (type.equalsIgnoreCase("tinyint")
+ || type.equalsIgnoreCase("smallint")
+ || type.equalsIgnoreCase("int")){
+ return Integer.toString(Integer.parseInt(partitionValue));
+ } else if (type.equalsIgnoreCase("bigint")){
+ return Long.toString(Long.parseLong(partitionValue));
+ } else if (type.equalsIgnoreCase("float")){
+ return Float.toString(Float.parseFloat(partitionValue));
+ } else if (type.equalsIgnoreCase("double")){
+ return Double.toString(Double.parseDouble(partitionValue));
+ } else if (type.startsWith("decimal")){
+ // Decimal datatypes are stored like decimal(10,10)
+ return new
BigDecimal(partitionValue).stripTrailingZeros().toPlainString();
+ }
+ return partitionValue;
+ }
+
+ public static Map<String, String> getPartitionColtoTypeMap(List<FieldSchema>
partitionCols) {
+ Map<String, String> typeMap = new HashMap<>();
+
+ if (partitionCols != null) {
+ for (FieldSchema fSchema : partitionCols) {
+ typeMap.put(fSchema.getName(), fSchema.getType());
+ }
+ }
+
+ return typeMap;
+ }
+
public static Partition createMetaPartitionObject(Table tbl, Map<String,
String> partSpec, Path location)
throws MetastoreException {
List<String> pvals = new ArrayList<String>();
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreServerUtils.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreServerUtils.java
index c6ae77c..0e74d39 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreServerUtils.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreServerUtils.java
@@ -844,5 +844,23 @@ public class TestMetaStoreServerUtils {
assertEquals(connectionURL, result);
}
+ @Test
+ public void testConversionToSignificantNumericTypes() {
+ assertEquals("1", MetaStoreServerUtils.getNormalisedPartitionValue("0001",
"tinyint"));
+ assertEquals("1", MetaStoreServerUtils.getNormalisedPartitionValue("0001",
"smallint"));
+ assertEquals("10",
MetaStoreServerUtils.getNormalisedPartitionValue("00010", "int"));
+ assertEquals("-10",
MetaStoreServerUtils.getNormalisedPartitionValue("-00010", "int"));
+
+ assertEquals("10",
MetaStoreServerUtils.getNormalisedPartitionValue("00010", "bigint"));
+ assertEquals("-10",
MetaStoreServerUtils.getNormalisedPartitionValue("-00010", "bigint"));
+
+ assertEquals("1.01",
MetaStoreServerUtils.getNormalisedPartitionValue("0001.0100", "float"));
+ assertEquals("-1.01",
MetaStoreServerUtils.getNormalisedPartitionValue("-0001.0100", "float"));
+ assertEquals("1.01",
MetaStoreServerUtils.getNormalisedPartitionValue("0001.010000", "double"));
+ assertEquals("-1.01",
MetaStoreServerUtils.getNormalisedPartitionValue("-0001.010000", "double"));
+ assertEquals("1.01",
MetaStoreServerUtils.getNormalisedPartitionValue("0001.0100", "decimal"));
+ assertEquals("-1.01",
MetaStoreServerUtils.getNormalisedPartitionValue("-0001.0100", "decimal"));
+ }
+
}
