Repository: hive Updated Branches: refs/heads/branch-2 854b9afce -> a45701220
HIVE-18767: Some alterPartitions invocations throw 'NumberFormatException: null' (Mass Dosage, reviewed by Peter Vary) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a4570122 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a4570122 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a4570122 Branch: refs/heads/branch-2 Commit: a45701220605a1075c28a9dad91b31982e2616d2 Parents: 854b9af Author: Mass Dosage <[email protected]> Authored: Tue Sep 11 14:26:45 2018 +0200 Committer: Peter Vary <[email protected]> Committed: Tue Sep 11 14:29:09 2018 +0200 ---------------------------------------------------------------------- .../hadoop/hive/metastore/MetaStoreUtils.java | 11 ++- .../hive/metastore/TestMetaStoreUtils.java | 79 ++++++++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/a4570122/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index 3b2da10..1b90170 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -301,14 +301,21 @@ public class MetaStoreUtils { return !isFastStatsSame(oldPart, newPart); } + /** + * Determines whether the "fast stats" for the passed partitions are the same. + * + * @param oldPart Old partition to compare. + * @param newPart New partition to compare. + * @return true if the partitions are not null, contain all the "fast stats" and have the same values for these stats, otherwise false. + */ static boolean isFastStatsSame(Partition oldPart, Partition newPart) { // requires to calculate stats if new and old have different fast stats if ((oldPart != null) && (oldPart.getParameters() != null)) { for (String stat : StatsSetupConst.fastStats) { if (oldPart.getParameters().containsKey(stat)) { Long oldStat = Long.parseLong(oldPart.getParameters().get(stat)); - Long newStat = Long.parseLong(newPart.getParameters().get(stat)); - if (!oldStat.equals(newStat)) { + String newStat = newPart.getParameters().get(stat); + if (newStat == null || !oldStat.equals(Long.parseLong(newStat))) { return false; } } else { http://git-wip-us.apache.org/repos/asf/hive/blob/a4570122/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java index 0ffe782..1771681 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.thrift.TException; @@ -252,4 +253,82 @@ public class TestMetaStoreUtils { verify(wh, never()).getFileStatusesForUnpartitionedTable(db, tbl2); } + @Test + public void isFastStatsSameWithNullPartitions() { + Partition partition = new Partition(); + Assert.assertFalse(MetaStoreUtils.isFastStatsSame(null, null)); + Assert.assertFalse(MetaStoreUtils.isFastStatsSame(null, partition)); + Assert.assertFalse(MetaStoreUtils.isFastStatsSame(partition, null)); + } + + @Test + public void isFastStatsSameWithNoMatchingStats() { + Partition oldPartition = new Partition(); + Map<String, String> stats = new HashMap<>(); + oldPartition.setParameters(stats); + Assert.assertFalse(MetaStoreUtils.isFastStatsSame(oldPartition, null)); + stats.put("someKeyThatIsNotInFastStats","value"); + oldPartition.setParameters(stats); + Assert.assertFalse(MetaStoreUtils.isFastStatsSame(oldPartition, null)); + } + + @Test + public void isFastStatsSameMatchingButOnlyOneStat() { + Partition oldPartition = new Partition(); + Partition newPartition = new Partition(); + Map<String, String> stats = new HashMap<>(); + stats.put(StatsSetupConst.fastStats[0], "1"); + oldPartition.setParameters(stats); + newPartition.setParameters(stats); + Assert.assertFalse(MetaStoreUtils.isFastStatsSame(oldPartition, newPartition)); + } + + @Test + public void isFastStatsSameMatching() { + Partition oldPartition = new Partition(); + Partition newPartition = new Partition(); + Map<String, String> stats = new HashMap<>(); + for (int i=0; i<StatsSetupConst.fastStats.length; i++) { + stats.put(StatsSetupConst.fastStats[i], String.valueOf(i)); + } + oldPartition.setParameters(stats); + newPartition.setParameters(stats); + Assert.assertTrue(MetaStoreUtils.isFastStatsSame(oldPartition, newPartition)); + } + + @Test + public void isFastStatsSameDifferent() { + Partition oldPartition = new Partition(); + Partition newPartition = new Partition(); + Map<String, String> oldStats = new HashMap<>(); + for (int i=0; i<StatsSetupConst.fastStats.length; i++) { + oldStats.put(StatsSetupConst.fastStats[i], String.valueOf(i)); + } + oldPartition.setParameters(oldStats); + Map<String, String> newStats = new HashMap<>(); + for (int i=0; i<StatsSetupConst.fastStats.length; i++) { + //set the values to i+1 so they are different in the new stats + newStats.put(StatsSetupConst.fastStats[i], String.valueOf(i+1)); + } + newPartition.setParameters(newStats); + Assert.assertFalse(MetaStoreUtils.isFastStatsSame(oldPartition, newPartition)); + } + + @Test + public void isFastStatsSameNullStatsInNew() { + Partition oldPartition = new Partition(); + Partition newPartition = new Partition(); + Map<String, String> oldStats = new HashMap<>(); + for (int i=0; i<StatsSetupConst.fastStats.length; i++) { + oldStats.put(StatsSetupConst.fastStats[i], String.valueOf(i)); + } + oldPartition.setParameters(oldStats); + Map<String, String> newStats = new HashMap<>(); + for (int i=0; i<StatsSetupConst.fastStats.length; i++) { + newStats.put(StatsSetupConst.fastStats[i], null); + } + newPartition.setParameters(newStats); + Assert.assertFalse(MetaStoreUtils.isFastStatsSame(oldPartition, newPartition)); + } + }
