Repository: hive Updated Branches: refs/heads/branch-3.1 c39b5d1bc -> f07c45975
HIVE-18767: Some alterPartitions invocations throw 'NumberFormatException: null' (Mass Dosage, reviewed by Peter Vary) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f07c4597 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f07c4597 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f07c4597 Branch: refs/heads/branch-3.1 Commit: f07c459754c30d1ac2439934888868a64964a6c2 Parents: c39b5d1 Author: Mass Dosage <[email protected]> Authored: Fri Oct 19 14:03:36 2018 +0200 Committer: Peter Vary <[email protected]> Committed: Fri Oct 19 14:03:36 2018 +0200 ---------------------------------------------------------------------- .../hive/metastore/utils/MetaStoreUtils.java | 11 ++- .../metastore/utils/TestMetaStoreUtils.java | 79 ++++++++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f07c4597/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java index 83952eb..aa5457d 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java @@ -629,14 +629,21 @@ public class MetaStoreUtils { return true; } + /** + * Determines whether the "fast stats" for the passed partitions are the same. + * + * @param oldPart Old partition to compare. + * @param newPart New partition to compare. + * @return true if the partitions are not null, contain all the "fast stats" and have the same values for these stats, otherwise false. + */ public static boolean isFastStatsSame(Partition oldPart, Partition newPart) { // requires to calculate stats if new and old have different fast stats if ((oldPart != null) && (oldPart.getParameters() != null)) { for (String stat : StatsSetupConst.fastStats) { if (oldPart.getParameters().containsKey(stat)) { Long oldStat = Long.parseLong(oldPart.getParameters().get(stat)); - Long newStat = Long.parseLong(newPart.getParameters().get(stat)); - if (!oldStat.equals(newStat)) { + String newStat = newPart.getParameters().get(stat); + if (newStat == null || !oldStat.equals(Long.parseLong(newStat))) { return false; } } else { http://git-wip-us.apache.org/repos/asf/hive/blob/f07c4597/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java index 55ff150..0977014 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; @@ -270,5 +271,83 @@ public class TestMetaStoreUtils { updateTableStatsSlow(db, tbl2, wh, false, false, null); verify(wh, never()).getFileStatusesForUnpartitionedTable(db, tbl2); } + + @Test + public void isFastStatsSameWithNullPartitions() { + Partition partition = new Partition(); + Assert.assertFalse(MetaStoreUtils.isFastStatsSame(null, null)); + Assert.assertFalse(MetaStoreUtils.isFastStatsSame(null, partition)); + Assert.assertFalse(MetaStoreUtils.isFastStatsSame(partition, null)); + } + + @Test + public void isFastStatsSameWithNoMatchingStats() { + Partition oldPartition = new Partition(); + Map<String, String> stats = new HashMap<>(); + oldPartition.setParameters(stats); + Assert.assertFalse(MetaStoreUtils.isFastStatsSame(oldPartition, null)); + stats.put("someKeyThatIsNotInFastStats","value"); + oldPartition.setParameters(stats); + Assert.assertFalse(MetaStoreUtils.isFastStatsSame(oldPartition, null)); + } + + @Test + public void isFastStatsSameMatchingButOnlyOneStat() { + Partition oldPartition = new Partition(); + Partition newPartition = new Partition(); + Map<String, String> stats = new HashMap<>(); + stats.put(StatsSetupConst.fastStats[0], "1"); + oldPartition.setParameters(stats); + newPartition.setParameters(stats); + Assert.assertFalse(MetaStoreUtils.isFastStatsSame(oldPartition, newPartition)); + } + + @Test + public void isFastStatsSameMatching() { + Partition oldPartition = new Partition(); + Partition newPartition = new Partition(); + Map<String, String> stats = new HashMap<>(); + for (int i=0; i<StatsSetupConst.fastStats.length; i++) { + stats.put(StatsSetupConst.fastStats[i], String.valueOf(i)); + } + oldPartition.setParameters(stats); + newPartition.setParameters(stats); + Assert.assertTrue(MetaStoreUtils.isFastStatsSame(oldPartition, newPartition)); + } + + @Test + public void isFastStatsSameDifferent() { + Partition oldPartition = new Partition(); + Partition newPartition = new Partition(); + Map<String, String> oldStats = new HashMap<>(); + for (int i=0; i<StatsSetupConst.fastStats.length; i++) { + oldStats.put(StatsSetupConst.fastStats[i], String.valueOf(i)); + } + oldPartition.setParameters(oldStats); + Map<String, String> newStats = new HashMap<>(); + for (int i=0; i<StatsSetupConst.fastStats.length; i++) { + //set the values to i+1 so they are different in the new stats + newStats.put(StatsSetupConst.fastStats[i], String.valueOf(i+1)); + } + newPartition.setParameters(newStats); + Assert.assertFalse(MetaStoreUtils.isFastStatsSame(oldPartition, newPartition)); + } + + @Test + public void isFastStatsSameNullStatsInNew() { + Partition oldPartition = new Partition(); + Partition newPartition = new Partition(); + Map<String, String> oldStats = new HashMap<>(); + for (int i=0; i<StatsSetupConst.fastStats.length; i++) { + oldStats.put(StatsSetupConst.fastStats[i], String.valueOf(i)); + } + oldPartition.setParameters(oldStats); + Map<String, String> newStats = new HashMap<>(); + for (int i=0; i<StatsSetupConst.fastStats.length; i++) { + newStats.put(StatsSetupConst.fastStats[i], null); + } + newPartition.setParameters(newStats); + Assert.assertFalse(MetaStoreUtils.isFastStatsSame(oldPartition, newPartition)); + } }
