Repository: incubator-impala Updated Branches: refs/heads/master f11181cbe -> d1594298e
IMPALA-6124: Fix alter table ddl updates and test Impala would previously update the ddl time of a table when dropping a partition but not when adding one. This change removes updates to the ddl time when partitions are added or removed to be consistent with Hive. Additionally the check in the ddl update test would fail if some operations took longer than 20 seconds. Instead, this change makes sure that the ddl time increases as intended. To test this change I ran test_last_ddl_time_update in exhaustive mode and also ran a private S3 build. Change-Id: I3126252e7709304d3e1fa4bb06a0b847180bd6cf Reviewed-on: http://gerrit.cloudera.org:8080/8411 Reviewed-by: Bharath Vissapragada <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/471285be Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/471285be Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/471285be Branch: refs/heads/master Commit: 471285bee3b8d4e18f97153fa0bd95641bba0d78 Parents: f11181c Author: Lars Volker <[email protected]> Authored: Sat Oct 28 09:47:39 2017 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Tue Oct 31 04:00:37 2017 +0000 ---------------------------------------------------------------------- .../org/apache/impala/service/CatalogOpExecutor.java | 3 --- tests/metadata/test_last_ddl_time_update.py | 12 ++++++------ 2 files changed, 6 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/471285be/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java index bb466dd..1683cc0 100644 --- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java +++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java @@ -2148,7 +2148,6 @@ public class CatalogOpExecutor { " ifExists is true.", e, tableName)); } } - updateLastDdlTime(msTbl, msClient); } catch (TException e) { throw new ImpalaRuntimeException( String.format(HMS_RPC_ERROR_FORMAT_STR, "dropPartition"), e); @@ -2628,7 +2627,6 @@ public class CatalogOpExecutor { MetastoreShim.alterPartitions(msClient.getHiveClient(), tableName.getDb(), tableName.getTbl(), hmsAddedPartitions); } - updateLastDdlTime(msTbl, msClient); } } catch (TException e) { throw new ImpalaRuntimeException( @@ -3264,7 +3262,6 @@ public class CatalogOpExecutor { } } } - updateLastDdlTime(msTbl, msClient); } } catch (AlreadyExistsException e) { throw new InternalException( http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/471285be/tests/metadata/test_last_ddl_time_update.py ---------------------------------------------------------------------- diff --git a/tests/metadata/test_last_ddl_time_update.py b/tests/metadata/test_last_ddl_time_update.py index 0ec57ee..b1bcfbb 100644 --- a/tests/metadata/test_last_ddl_time_update.py +++ b/tests/metadata/test_last_ddl_time_update.py @@ -49,12 +49,12 @@ class TestLastDdlTimeUpdate(ImpalaTestSuite): # add/drop partitions self.run_test("alter table %s add partition (j=1, s='2012')" % FQ_TBL_NAME, - unique_database, TBL_NAME, True) + unique_database, TBL_NAME, False) self.run_test("alter table %s add if not exists " "partition (j=1, s='2012')" % FQ_TBL_NAME, unique_database, TBL_NAME, False) self.run_test("alter table %s drop partition (j=1, s='2012')" % FQ_TBL_NAME, - unique_database, TBL_NAME, True) + unique_database, TBL_NAME, False) self.run_test("alter table %s drop if exists " "partition (j=2, s='2012')" % FQ_TBL_NAME, unique_database, TBL_NAME, False) @@ -78,10 +78,10 @@ class TestLastDdlTimeUpdate(ImpalaTestSuite): "partitioned by (j int, s string)" % FQ_TBL_NAME) # static partition insert self.run_test("insert into %s partition(j=1, s='2012') " - "select 10" % FQ_TBL_NAME, unique_database, TBL_NAME, True) + "select 10" % FQ_TBL_NAME, unique_database, TBL_NAME, False) # dynamic partition insert self.run_test("insert into %s partition(j, s) " - "select 10, 2, '2013'" % FQ_TBL_NAME, unique_database, TBL_NAME, True) + "select 10, 2, '2013'" % FQ_TBL_NAME, unique_database, TBL_NAME, False) # dynamic partition insert changing no partitions (empty input) self.run_test("insert into %s partition(j, s) " "select * from (select 10 as i, 2 as j, '2013' as s) as t " @@ -116,7 +116,7 @@ class TestLastDdlTimeUpdate(ImpalaTestSuite): afterDdlTime = table.parameters[HIVE_LAST_DDL_TIME_PARAM_KEY] if expect_changed: - # check that the time difference is within 20s - assert long(afterDdlTime) - long(beforeDdlTime) <= 20 + # check that the new ddlTime is strictly greater than the old one. + assert long(afterDdlTime) > long(beforeDdlTime) else: assert long(afterDdlTime) == long(beforeDdlTime)
