This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new d45e3a5 IMPALA-9673: Add external warehouse dir variable in E2E test
d45e3a5 is described below
commit d45e3a50b003259e4ef1023333b47781a028eb19
Author: xiaomeng <xiaom...@cloudera.com>
AuthorDate: Fri May 22 11:13:55 2020 -0700
IMPALA-9673: Add external warehouse dir variable in E2E test
Updated CDP build to 7.2.1.0-57 to include new Hive features such as
HIVE-22995.
In minicluster, we have default values of hive.create.as.acid and
hive.create.as.insert.only which are false. So by default hive creates
external type table located in external warehouse directory.
Due to HIVE-22995, desc db returns external warehouse directory.
With above reasons, we need use external warehouse dir in some tests.
Also add a new test for "CREATE DATABASE ... LOCATION".
Tested:
Re-run failed test in minicluster.
Run exhaustive tests.
Change-Id: I57926babf4caebfd365e6be65a399f12ea68687f
Reviewed-on: http://gerrit.cloudera.org:8080/15990
Reviewed-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com>
---
bin/impala-config.sh | 16 +++++-----
.../functional/functional_schema_template.sql | 23 ++++++++------
.../queries/QueryTest/create-database.test | 36 ++++++++++++++++++++--
.../queries/QueryTest/describe-db.test | 6 ++--
.../queries/QueryTest/describe-hive-db.test | 6 ++--
tests/common/environ.py | 1 +
tests/common/impala_test_suite.py | 12 ++++++--
tests/query_test/test_compressed_formats.py | 12 +++++---
8 files changed, 78 insertions(+), 34 deletions(-)
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 481ea4e..c6387b7 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -172,16 +172,16 @@ export CDH_BUILD_NUMBER=1814051
export CDH_MAVEN_REPOSITORY=\
"https://${IMPALA_TOOLCHAIN_HOST}/build/cdh_components/${CDH_BUILD_NUMBER}/maven";
-export CDP_BUILD_NUMBER=2523282
+export CDP_BUILD_NUMBER=3192304
export CDP_MAVEN_REPOSITORY=\
"https://${IMPALA_TOOLCHAIN_HOST}/build/cdp_components/${CDP_BUILD_NUMBER}/maven";
-export CDP_HADOOP_VERSION=3.1.1.7.1.1.0-380
-export CDP_HBASE_VERSION=2.2.3.7.1.1.0-380
-export CDP_HIVE_VERSION=3.1.3000.7.1.1.0-380
-export CDP_KNOX_VERSION=1.3.0.7.1.1.0-380
-export CDP_OZONE_VERSION=0.4.0.7.1.1.0-380
-export CDP_RANGER_VERSION=2.0.0.7.1.1.0-380
-export CDP_TEZ_VERSION=0.9.1.7.1.1.0-380
+export CDP_HADOOP_VERSION=3.1.1.7.2.1.0-57
+export CDP_HBASE_VERSION=2.2.3.7.2.1.0-57
+export CDP_HIVE_VERSION=3.1.3000.7.2.1.0-57
+export CDP_KNOX_VERSION=1.3.0.7.2.1.0-57
+export CDP_OZONE_VERSION=0.6.0.7.2.1.0-57
+export CDP_RANGER_VERSION=2.0.0.7.2.1.0-57
+export CDP_TEZ_VERSION=0.9.1.7.2.1.0-57
export IMPALA_PARQUET_VERSION=1.10.99-cdh6.x-SNAPSHOT
export IMPALA_AVRO_JAVA_VERSION=1.8.2-cdh6.x-SNAPSHOT
diff --git a/testdata/datasets/functional/functional_schema_template.sql
b/testdata/datasets/functional/functional_schema_template.sql
index dc53371..c323666 100644
--- a/testdata/datasets/functional/functional_schema_template.sql
+++ b/testdata/datasets/functional/functional_schema_template.sql
@@ -2273,16 +2273,6 @@ TBLPROPERTIES('transactional'='true');
---- DATASET
functional
---- BASE_TABLE_NAME
-materialized_view
----- HIVE_MAJOR_VERSION
-3
----- CREATE_HIVE
-CREATE MATERIALIZED VIEW IF NOT EXISTS {db_name}{db_suffix}.{table_name}
- AS SELECT * FROM {db_name}{db_suffix}.insert_only_transactional_table;
-====
----- DATASET
-functional
----- BASE_TABLE_NAME
insert_only_transactional_bucketed_table
---- HIVE_MAJOR_VERSION
3
@@ -2323,6 +2313,19 @@ SELECT * from functional.{table_name};
---- DATASET
functional
---- BASE_TABLE_NAME
+materialized_view
+---- HIVE_MAJOR_VERSION
+3
+---- CREATE_HIVE
+-- The create materialized view command is moved down so that the database's
+-- managed directory has been created. Otherwise the command would fail. This
+-- is a bug in Hive.
+CREATE MATERIALIZED VIEW IF NOT EXISTS {db_name}{db_suffix}.{table_name}
+ AS SELECT * FROM {db_name}{db_suffix}.insert_only_transactional_table;
+=====
+---- DATASET
+functional
+---- BASE_TABLE_NAME
uncomp_src_alltypes
---- CREATE_HIVE
CREATE TABLE {db_name}{db_suffix}.{table_name} LIKE functional.alltypes STORED
AS ORC;
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/create-database.test
b/testdata/workloads/functional-query/queries/QueryTest/create-database.test
index 1b698b5..5cdaed3 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/create-database.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/create-database.test
@@ -16,7 +16,7 @@ STRING, STRING
# for a newly created database (regression test for IMPALA-7439)
describe database $DATABASE_2
---- RESULTS
-'$DATABASE_2','$NAMENODE/$MANAGED_WAREHOUSE_DIR/$DATABASE_2.db','For testing'
+'$DATABASE_2','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/$DATABASE_2.db','For testing'
---- TYPES
string, string, string
====
@@ -24,7 +24,7 @@ string, string, string
# Test that DESCRIBE EXTENDED also has all of the necessary info.
describe database extended $DATABASE_2
---- RESULTS
-'$DATABASE_2','$NAMENODE/$MANAGED_WAREHOUSE_DIR/$DATABASE_2.db','For testing'
+'$DATABASE_2','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/$DATABASE_2.db','For testing'
'Owner: ','',''
'','$USER','USER'
---- TYPES
@@ -56,6 +56,38 @@ drop database if exists $DATABASE_2
'Database does not exist.'
====
---- QUERY
+# Test CREATE DATABASE ... LOCATION
+create database if not exists $DATABASE_loc comment "For testing"
+ location '$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/specified_location'
+---- RESULTS
+'Database has been created.'
+====
+---- QUERY
+# Test that DESCRIBE shows the proper database location
+# for a newly created database with location
+describe database $DATABASE_loc
+---- RESULTS
+'$DATABASE_loc','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/specified_location','For
testing'
+---- TYPES
+string, string, string
+====
+---- QUERY
+# Test that DESCRIBE EXTENDED also has all of the necessary info.
+describe database extended $DATABASE_loc
+---- RESULTS
+'$DATABASE_loc','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/specified_location','For
testing'
+'Owner: ','',''
+'','$USER','USER'
+---- TYPES
+string, string, string
+====
+---- QUERY
+# Test dropping the database.
+drop database $DATABASE_loc
+---- RESULTS
+'Database has been dropped.'
+====
+---- QUERY
# Test DROP DATABASE ... CASCADE
create database if not exists $DATABASE_cascade
====
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/describe-db.test
b/testdata/workloads/functional-query/queries/QueryTest/describe-db.test
index 8e79083..d651e79 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/describe-db.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/describe-db.test
@@ -3,7 +3,7 @@
# Test printing of default database.
describe database default
---- RESULTS
-'default','$NAMENODE/$MANAGED_WAREHOUSE_DIR','Default Hive database'
+'default','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR','Default Hive database'
---- TYPES
string, string, string
====
@@ -12,7 +12,7 @@ describe database extended impala_test_desc_db1
---- RESULTS
'','$USER','USER'
'Owner: ','',''
-'impala_test_desc_db1','$NAMENODE/$MANAGED_WAREHOUSE_DIR/impala_test_desc_db1.db',''
+'impala_test_desc_db1','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/impala_test_desc_db1.db',''
---- TYPES
string, string, string
====
@@ -21,7 +21,7 @@ describe database extended impala_test_desc_db2
---- RESULTS
'','$USER','USER'
'Owner: ','',''
-'impala_test_desc_db2','$NAMENODE/$MANAGED_WAREHOUSE_DIR/impala_test_desc_db2.db','test
comment'
+'impala_test_desc_db2','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/impala_test_desc_db2.db','test
comment'
---- TYPES
string, string, string
====
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/describe-hive-db.test
b/testdata/workloads/functional-query/queries/QueryTest/describe-hive-db.test
index bae388f..ff7c8a3 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/describe-hive-db.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/describe-hive-db.test
@@ -3,7 +3,7 @@
# Test printing of default database.
describe database default
---- RESULTS
-'default','$NAMENODE/$MANAGED_WAREHOUSE_DIR','Default Hive database'
+'default','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR','Default Hive database'
---- TYPES
string, string, string
====
@@ -11,7 +11,7 @@ string, string, string
# Test printing of hive_test_desc_db database.
describe database hive_test_desc_db
---- RESULTS
-'hive_test_desc_db','$NAMENODE/$MANAGED_WAREHOUSE_DIR/hive_test_desc_db.db','test
comment'
+'hive_test_desc_db','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/hive_test_desc_db.db','test
comment'
---- TYPES
string, string, string
====
@@ -19,7 +19,7 @@ string, string, string
# Test printing of hive_test_desc_db database with extended information.
describe database extended hive_test_desc_db
---- RESULTS
-'hive_test_desc_db','$NAMENODE/$MANAGED_WAREHOUSE_DIR/hive_test_desc_db.db','test
comment'
+'hive_test_desc_db','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/hive_test_desc_db.db','test
comment'
'Parameter: ','',''
'','$USER','USER'
'Owner: ','',''
diff --git a/tests/common/environ.py b/tests/common/environ.py
index 0987b1b..3c8d130 100644
--- a/tests/common/environ.py
+++ b/tests/common/environ.py
@@ -77,6 +77,7 @@ if HIVE_MAJOR_VERSION > 2:
MANAGED_WAREHOUSE_DIR = 'test-warehouse/managed'
else:
MANAGED_WAREHOUSE_DIR = 'test-warehouse'
+EXTERNAL_WAREHOUSE_DIR = 'test-warehouse'
# Resolve any symlinks in the path.
impalad_basedir = \
diff --git a/tests/common/impala_test_suite.py
b/tests/common/impala_test_suite.py
index a243e4a..4ee54d5 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -37,7 +37,10 @@ from getpass import getuser
from random import choice
from subprocess import check_call
from tests.common.base_test_suite import BaseTestSuite
-from tests.common.environ import HIVE_MAJOR_VERSION, MANAGED_WAREHOUSE_DIR
+from tests.common.environ import (
+ HIVE_MAJOR_VERSION,
+ MANAGED_WAREHOUSE_DIR,
+ EXTERNAL_WAREHOUSE_DIR)
from tests.common.errors import Timeout
from tests.common.impala_connection import create_connection
from tests.common.impala_service import ImpaladService
@@ -435,7 +438,8 @@ class ImpalaTestSuite(BaseTestSuite):
"IMPALA_HOME",
"INTERNAL_LISTEN_HOST",
"INTERNAL_LISTEN_IP",
- "MANAGED_WAREHOUSE_DIR"])
+ "MANAGED_WAREHOUSE_DIR",
+ "EXTERNAL_WAREHOUSE_DIR"])
repl.update({
'$SECONDARY_FILESYSTEM': os.environ.get("SECONDARY_FILESYSTEM", ""),
'$USER': getuser()})
@@ -500,7 +504,9 @@ class ImpalaTestSuite(BaseTestSuite):
INTERNAL_LISTEN_HOST) \
.replace('$INTERNAL_LISTEN_IP',
INTERNAL_LISTEN_IP) \
.replace('$MANAGED_WAREHOUSE_DIR',
- MANAGED_WAREHOUSE_DIR)
+ MANAGED_WAREHOUSE_DIR) \
+ .replace('$EXTERNAL_WAREHOUSE_DIR',
+ EXTERNAL_WAREHOUSE_DIR)
if use_db:
test_section[section_name] =
test_section[section_name].replace('$DATABASE', use_db)
result_section, type_section = 'RESULTS', 'TYPES'
diff --git a/tests/query_test/test_compressed_formats.py
b/tests/query_test/test_compressed_formats.py
index db470d0..9f9177a 100644
--- a/tests/query_test/test_compressed_formats.py
+++ b/tests/query_test/test_compressed_formats.py
@@ -22,7 +22,7 @@ import struct
import subprocess
from os.path import join
-from tests.common.environ import MANAGED_WAREHOUSE_DIR
+from tests.common.environ import EXTERNAL_WAREHOUSE_DIR
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
SkipIfLocal
from tests.common.test_dimensions import create_single_exec_option_dimension
@@ -99,15 +99,16 @@ class TestCompressedFormats(ImpalaTestSuite):
# Make sure destination table uses suffix, even if use_suffix=False, so
# unique tables are created for each compression format
- # Since Hive makes tables managed by default (only matters on Hive 3+), it
- # will use the managed warehouse location.
- dest_base_dir = '/{0}'.format(MANAGED_WAREHOUSE_DIR)
+ # In Hive3+ create table like behavior is still in discussion, add location
+ # to avoid impact on Impala test.
+ dest_base_dir = '/{0}'.format(EXTERNAL_WAREHOUSE_DIR)
dest_table = '%s_%s_copy' % (table_name, compression_codec)
dest_table_dir = join(dest_base_dir, dest_table)
dest_file = join(dest_table_dir, file_name + extension)
drop_cmd = 'DROP TABLE IF EXISTS %s;' % (dest_table)
- hive_cmd = drop_cmd + 'CREATE TABLE %s LIKE %s;' % (dest_table, src_table)
+ hive_cmd = drop_cmd + 'CREATE TABLE %s LIKE %s LOCATION \'%s\';' % \
+ (dest_table, src_table, dest_table_dir)
# Create the table
self.run_stmt_in_hive(hive_cmd)
@@ -129,6 +130,7 @@ class TestCompressedFormats(ImpalaTestSuite):
raise
finally:
self.run_stmt_in_hive(drop_cmd)
+ self.filesystem_client.delete_file_dir(dest_file)
class TestUnsupportedTableWriters(ImpalaTestSuite):
@classmethod
