[
https://issues.apache.org/jira/browse/IMPALA-3040?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16554603#comment-16554603
]
ASF subversion and git services commented on IMPALA-3040:
---------------------------------------------------------
Commit ac4acf1b77ccad95528741c255834d8ccdb84518 in impala's branch
refs/heads/master from [~tianyiwang]
[ https://git-wip-us.apache.org/repos/asf?p=impala.git;h=ac4acf1 ]
IMPALA-3040: Remove cache directives if a partition is dropped externally
HdfsTable.dropPartition() doesn't uncache the partition right now. If
the partition is dropped from Hive and refreshed in Impala, the
partition will be removed from the catalog but the cache directive
remains. Because Impala directly uses HMS client to drop a
table/database, the cache directive won't be removed even if the table
is dropped in Impala, if the backgroud loading is run concurrenty with
the HMS client RPC call. This patch removes the cache directive in
dropPartition() if the partition is removed from HMS.
Change-Id: Id7701a499405e961456adea63f3592b43bd69170
Reviewed-on: http://gerrit.cloudera.org:8080/10792
Reviewed-by: Bharath Vissapragada <[email protected]>
Tested-by: Tianyi Wang <[email protected]>
> test_caching_ddl failed with unexpected get_num_cache_requests
> --------------------------------------------------------------
>
> Key: IMPALA-3040
> URL: https://issues.apache.org/jira/browse/IMPALA-3040
> Project: IMPALA
> Issue Type: Bug
> Components: Catalog
> Affects Versions: Impala 2.5.0
> Reporter: Tim Armstrong
> Assignee: Tianyi Wang
> Priority: Major
> Labels: broken-build, flaky
>
> This test failed on the integration job.
> http://sandbox.jenkins.cloudera.com/view/Impala/view/Builds%20-%202.5.0%20release/job/impala-cdh5.7.0-integration/3/testReport/junit/query_test.test_hdfs_caching/TestHdfsCachingDdl/test_caching_ddl_exec_option____disable_codegen___False___abort_on_error___1___exec_single_node_rows_threshold___0___batch_size___0___num_nodes___0____table_format__text_none_/
> {code}
> query_test.test_hdfs_caching.TestHdfsCachingDdl.test_caching_ddl[exec_option:
> {'disable_codegen': False, 'abort_on_error': 1,
> 'exec_single_node_rows_threshold': 0, 'batch_size': 0, 'num_nodes': 0} |
> table_format: text/none] (from pytest)
> Failing for the past 1 build (Since Failed#3 )
> Took 47 sec.
> add description
> Error Message
> assert 9 == 10 + where 10 = get_num_cache_requests()
> Stacktrace
> query_test/test_hdfs_caching.py:132: in test_caching_ddl
> assert num_entries_pre == get_num_cache_requests()
> E assert 9 == 10
> E + where 10 = get_num_cache_requests()
> Standard Error
> -- connecting to: localhost:21000
> -- executing against localhost:21000
> use default;
> SET sync_ddl=1;
> -- executing against localhost:21000
> drop database if exists `cachedb` cascade;
> -- executing against localhost:21000
> create database cachedb;
> -- executing against localhost:21000
> use functional;
> SET disable_codegen=False;
> SET abort_on_error=1;
> SET exec_single_node_rows_threshold=0;
> SET batch_size=0;
> SET num_nodes=0;
> -- executing against localhost:21000
> use cachedb;
> -- executing against localhost:21000
> create table cached_tbl_nopart (i int) cached in 'testPool';
> -- executing against localhost:21000
> insert into cached_tbl_nopart select 1;
> -- executing against localhost:21000
> select * from cached_tbl_nopart;
> -- executing against localhost:21000
> create table like_cached_tbl_nopart like cached_tbl_nopart;
> -- executing against localhost:21000
> show table stats like_cached_tbl_nopart;
> -- executing against localhost:21000
> show table stats cached_tbl_nopart;
> -- executing against localhost:21000
> alter table cached_tbl_nopart set uncached;
> -- executing against localhost:21000
> show table stats cached_tbl_nopart;
> -- executing against localhost:21000
> drop table if exists cached_tbl_part;
> -- executing against localhost:21000
> create table cached_tbl_part (i int) partitioned by (j int) cached in
> 'testPool' with replication = 9;
> -- executing against localhost:21000
> alter table cached_tbl_part add partition (j=0);
> -- executing against localhost:21000
> alter table cached_tbl_part add partition (j=1) uncached;
> -- executing against localhost:21000
> alter table cached_tbl_part add partition (j=2) cached in 'testPool';
> -- executing against localhost:21000
> show partitions cached_tbl_part;
> -- executing against localhost:21000
> drop table if exists cached_tbl_part;
> -- executing against localhost:21000
> create table cached_tbl_part (i int) partitioned by (j int) cached in
> 'testPool';
> -- executing against localhost:21000
> alter table cached_tbl_part add partition (j=0);
> -- executing against localhost:21000
> alter table cached_tbl_part add partition (j=1) uncached;
> -- executing against localhost:21000
> alter table cached_tbl_part add partition (j=2) cached in 'testPool';
> -- executing against localhost:21000
> show partitions cached_tbl_part;
> -- executing against localhost:21000
> alter table cached_tbl_part partition (j=2) set uncached;
> -- executing against localhost:21000
> show partitions cached_tbl_part;
> -- executing against localhost:21000
> alter table cached_tbl_part partition (j=2) set uncached;
> -- executing against localhost:21000
> show partitions cached_tbl_part;
> -- executing against localhost:21000
> alter table cached_tbl_part partition (j=1) set cached in 'testPool';
> -- executing against localhost:21000
> show partitions cached_tbl_part;
> -- executing against localhost:21000
> insert into cached_tbl_part partition(j) values(3, 3), (4, 4);
> -- executing against localhost:21000
> alter table cached_tbl_part partition (j=3) set cached in 'testPool' with
> replication = 4;
> -- executing against localhost:21000
> show partitions cached_tbl_part;
> -- executing against localhost:21000
> alter table cached_tbl_part set uncached;
> -- executing against localhost:21000
> show partitions cached_tbl_part;
> -- executing against localhost:21000
> alter table cached_tbl_part set uncached;
> -- executing against localhost:21000
> alter table cached_tbl_part set cached in 'testPool';
> -- executing against localhost:21000
> show partitions cached_tbl_part;
> -- executing against localhost:21000
> alter table cached_tbl_part set cached in 'testPool';
> -- executing against localhost:21000
> show partitions cached_tbl_part;
> -- executing against localhost:21000
> alter table cached_tbl_part partition (j=3) set cached in 'testPool' with
> replication = 4;
> -- executing against localhost:21000
> alter table cached_tbl_part partition (j=1) set uncached;
> -- executing against localhost:21000
> show partitions cached_tbl_part;
> -- executing against localhost:21000
> alter table cached_tbl_part set cached in 'testPool' with replication = 8;
> -- executing against localhost:21000
> show partitions cached_tbl_part;
> -- executing against localhost:21000
> alter table cached_tbl_part add partition(j=5) cached in 'testPool' with
> replication = 3;
> -- executing against localhost:21000
> show partitions cached_tbl_part;
> -- executing against localhost:21000
> alter table cached_tbl_part partition(j=2) set cached in 'testPool' with
> replication = 3;
> -- executing against localhost:21000
> show partitions cached_tbl_part;
> -- executing against localhost:21000
> alter table cached_tbl_part add partition(j=6) cached in 'testPool';
> -- executing against localhost:21000
> show partitions cached_tbl_part;
> -- executing against localhost:21000
> alter table cached_tbl_part add partition(j=7)
> location
> 'file:///data/2/jenkins/workspace/impala-cdh5.7.0-integration/repos/Impala/testdata/data/cached_tbl_part';
> -- executing against localhost:21000
> alter table cached_tbl_part add partition(j=7)
> location
> 'file:///data/2/jenkins/workspace/impala-cdh5.7.0-integration/repos/Impala/testdata/data/cache_tbl_part'
> uncached;
> -- executing against localhost:21000
> show partitions cached_tbl_part;
> -- executing against localhost:21000
> create table uncached_tbl_local(i int)
> location
> 'file:///data/2/jenkins/workspace/impala-cdh5.7.0-integration/repos/Impala/testdata/data/uncached_tbl_local';
> -- executing against localhost:21000
> alter table uncached_tbl_local set cached in 'testPool';
> -- executing against localhost:21000
> create table cached_tbl_local(i int) partitioned by (j int);
> -- executing against localhost:21000
> alter table cached_tbl_local add partition(j=0)
> location
> 'file:///data/2/jenkins/workspace/impala-cdh5.7.0-integration/repos/Impala/testdata/data/cached_local_part';
> -- executing against localhost:21000
> alter table cached_tbl_local partition(j=0) set cached in 'testPool';
> -- executing against localhost:21000
> alter table cached_tbl_local set cached in 'testPool';
> -- executing against localhost:21000
> alter table cached_tbl_local add partition(j=1) cached in 'testPool';
> -- executing against localhost:21000
> alter table cached_tbl_local set cached in 'testPool';
> -- executing against localhost:21000
> show partitions cached_tbl_local;
> -- executing against localhost:21000
> drop table cachedb.cached_tbl_part;
> -- executing against localhost:21000
> drop table cachedb.cached_tbl_nopart;
> -- executing against localhost:21000
> drop table cachedb.cached_tbl_local;
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
