Repository: incubator-hawq Updated Branches: refs/heads/master 7661dec7c -> 8cc4a042e
HAWQ-991. Add feature test cases for the update hawqregister. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/af483766 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/af483766 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/af483766 Branch: refs/heads/master Commit: af483766e8b710661f89e170318b1e856aafb26e Parents: 2596be6 Author: xunzhang <[email protected]> Authored: Wed Aug 17 22:05:30 2016 +0800 Committer: rlei <[email protected]> Committed: Fri Aug 19 10:57:09 2016 +0800 ---------------------------------------------------------------------- .../ManagementTool/test_hawq_register.cpp | 126 +++++++++++++++++-- tools/bin/hawqregister | 13 +- 2 files changed, 128 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/af483766/src/test/feature/ManagementTool/test_hawq_register.cpp ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/test_hawq_register.cpp b/src/test/feature/ManagementTool/test_hawq_register.cpp index afc2cb4..e6fead8 100644 --- a/src/test/feature/ManagementTool/test_hawq_register.cpp +++ b/src/test/feature/ManagementTool/test_hawq_register.cpp @@ -16,7 +16,7 @@ class TestHawqRegister : public ::testing::Test { ~TestHawqRegister() {} }; -TEST_F(TestHawqRegister, TestSingleHawqFile) { +TEST_F(TestHawqRegister, TestUsage1SingleHawqFile) { SQLUtility util; string rootPath(util.getTestRootPath()); string relativePath("/ManagementTool/test_hawq_register_hawq.paq"); @@ -35,7 +35,7 @@ TEST_F(TestHawqRegister, TestSingleHawqFile) { util.execute("drop table hawqregister;"); } -TEST_F(TestHawqRegister, TestSingleHiveFile) { +TEST_F(TestHawqRegister, TestUsage1SingleHiveFile) { SQLUtility util; string rootPath(util.getTestRootPath()); string relativePath("/ManagementTool/test_hawq_register_hive.paq"); @@ -122,7 +122,7 @@ TEST_F(TestHawqRegister, TestFiles) { util.execute("drop table hawqregister;"); } -TEST_F(TestHawqRegister, TestHashDistributedTable) { +TEST_F(TestHawqRegister, TestUsage1HashDistributedTable) { SQLUtility util; string rootPath(util.getTestRootPath()); string relativePath("/ManagementTool/test_hawq_register_hawq.paq"); @@ -140,7 +140,7 @@ TEST_F(TestHawqRegister, TestHashDistributedTable) { util.execute("drop table hawqregister;"); } -TEST_F(TestHawqRegister, TestNotParquetFile) { +TEST_F(TestHawqRegister, TestUsage1NotParquetFile) { SQLUtility util; string rootPath(util.getTestRootPath()); string relativePath("/ManagementTool/test_hawq_register_not_paq"); @@ -158,7 +158,7 @@ TEST_F(TestHawqRegister, TestNotParquetFile) { util.execute("drop table hawqregister;"); } -TEST_F(TestHawqRegister, TestNotParquetTable) { +TEST_F(TestHawqRegister, TestUsage1NotParquetTable) { SQLUtility util; string rootPath(util.getTestRootPath()); string relativePath("/ManagementTool/test_hawq_register_hawq.paq"); @@ -176,7 +176,7 @@ TEST_F(TestHawqRegister, TestNotParquetTable) { util.execute("drop table hawqregister;"); } -TEST_F(TestHawqRegister, TestFileNotExist) { +TEST_F(TestHawqRegister, TestUsage1FileNotExist) { SQLUtility util; util.execute("create table hawqregister(i int);"); @@ -188,7 +188,7 @@ TEST_F(TestHawqRegister, TestFileNotExist) { util.execute("drop table hawqregister;"); } -TEST_F(TestHawqRegister, TestNotHDFSPath) { +TEST_F(TestHawqRegister, TestUsage1NotHDFSPath) { SQLUtility util; string rootPath(util.getTestRootPath()); string relativePath("/ManagementTool/test_hawq_register_hawq.paq"); @@ -205,3 +205,115 @@ TEST_F(TestHawqRegister, TestNotHDFSPath) { EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -rm hdfs://localhost:8020/hawq_register_hawq.paq")); util.execute("drop table hawqregister;"); } + +TEST_F(TestHawqRegister, TestUsage1ParquetRandomly) { + SQLUtility util; + string rootPath(util.getTestRootPath()); + string relativePath("/ManagementTool/test_hawq_register_hawq.paq"); + string filePath = rootPath + relativePath; + EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put -f " + filePath + " hdfs://localhost:8020/hawq_register_hawq.paq")); + util.execute("drop table if exists nt;"); + util.execute("create table nt(i int) with (appendonly=true, orientation=parquet);"); + EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -f hdfs://localhost:8020/hawq_register_hawq.paq nt")); + util.query("select * from nt;", 3); + util.execute("insert into nt values(1);"); + util.query("select * from nt;", 4); + util.execute("drop table nt;"); +} + +TEST_F(TestHawqRegister, TestUsage1ParquetRandomly2) { + SQLUtility util; + string rootPath(util.getTestRootPath()); + string relativePath("/ManagementTool/test_hawq_register_hawq.paq"); + string filePath = rootPath + relativePath; + EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put -f " + filePath + " hdfs://localhost:8020/hawq_register_hawq.paq")); + util.execute("drop table if exists nt;"); + util.execute("create table nt(i int) with (appendonly=true, orientation=parquet);"); + EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -f hdfs://localhost:8020/hawq_register_hawq.paq nt")); + util.query("select * from nt;", 3); + util.execute("insert into nt values(1);"); + util.query("select * from nt;", 4); + util.execute("drop table nt;"); +} + +TEST_F(TestHawqRegister, TestUsage2ParquetRandomly) { + SQLUtility util; + util.execute("drop table if exists t;"); + util.execute("create table t(i int) with (appendonly=true, orientation=parquet) distributed randomly;"); + util.execute("insert into t values(1), (2), (3);"); + util.query("select * from t;", 3); + EXPECT_EQ(0, Command::getCommandStatus("hawq extract -d " + (string) HAWQ_DB + " -o t.yml testhawqregister_testusage2parquetrandomly.t")); + EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c t.yml testhawqregister_testusage2parquetrandomly.nt")); + util.query("select * from nt;", 3); + EXPECT_EQ(0, Command::getCommandStatus("rm -rf t.yml")); + util.execute("drop table t;"); + util.execute("drop table nt;"); +} + +TEST_F(TestHawqRegister, TestUsage2ParquetHash1) { + SQLUtility util; + util.execute("drop table if exists t4;"); + util.execute("create table t4(i int) with (appendonly=true, orientation=parquet) distributed by (i);"); + util.execute("insert into t4 values(1), (2), (3);"); + EXPECT_EQ(0, Command::getCommandStatus("hawq extract -d " + (string) HAWQ_DB + " -o t4.yml testhawqregister_testusage2parquethash1.t4")); + EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c t4.yml testhawqregister_testusage2parquethash1.nt4")); + util.query("select * from nt4;", 3); + EXPECT_EQ(0, Command::getCommandStatus("rm -rf t4.yml")); + util.execute("drop table t4;"); + util.execute("drop table nt4;"); +} + + +TEST_F(TestHawqRegister, TestUsage2ParquetHash2) { + SQLUtility util; + util.execute("drop table if exists t5;"); + util.execute("create table t5(i int, j varchar, k text) with (appendonly=true, orientation=parquet) distributed by (i, k);"); + util.execute("insert into t5 values(1, 'x', 'ab'), (2, 'y', 'cd'), (3, 'z', 'ef');"); + EXPECT_EQ(0, Command::getCommandStatus("hawq extract -d " + (string) HAWQ_DB + " -o t5.yml testhawqregister_testusage2parquethash2.t5")); + EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c t5.yml testhawqregister_testusage2parquethash2.nt5")); + util.query("select * from nt5;", 3); + EXPECT_EQ(0, Command::getCommandStatus("rm -rf t5.yml")); + util.execute("drop table t5;"); + util.execute("drop table nt5;"); +} + + +TEST_F(TestHawqRegister, TestUsage2AORandom) { + SQLUtility util; + util.execute("drop table if exists t6;"); + util.execute("create table t6(i int) with (appendonly=true, orientation=row) distributed randomly;"); + util.execute("insert into t6 values(1), (2), (3);"); + EXPECT_EQ(0, Command::getCommandStatus("hawq extract -d " + (string) HAWQ_DB + " -o t6.yml testhawqregister_testusage2aorandom.t6")); + EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c t6.yml testhawqregister_testusage2aorandom.nt6")); + util.query("select * from nt6;", 3); + EXPECT_EQ(0, Command::getCommandStatus("rm -rf t6.yml")); + util.execute("drop table t6;"); + util.execute("drop table nt6;"); +} + +TEST_F(TestHawqRegister, TestUsage2AOHash1) { + SQLUtility util; + util.execute("drop table if exists t7;"); + util.execute("create table t7(i int) with (appendonly=true, orientation=row) distributed by (i);"); + util.execute("insert into t7 values(1), (2), (3);"); + EXPECT_EQ(0, Command::getCommandStatus("hawq extract -d " + (string) HAWQ_DB + " -o t7.yml testhawqregister_testusage2aohash1.t7")); + EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c t7.yml testhawqregister_testusage2aohash1.nt7")); + util.query("select * from nt7;", 3); + EXPECT_EQ(0, Command::getCommandStatus("rm -rf t7.yml")); + util.execute("drop table t7;"); + util.execute("drop table nt7;"); +} + + +TEST_F(TestHawqRegister, TestUsage2AOHash2) { + SQLUtility util; + util.execute("drop table if exists t8;"); + util.execute("create table t8(i int, j varchar, k text) with (appendonly=true, orientation=row) distributed by (i, k);"); + util.execute("insert into t8 values(1, 'x', 'ab'), (2, 'y', 'cd'), (3, 'z', 'ef');"); + EXPECT_EQ(0, Command::getCommandStatus("hawq extract -d " + (string) HAWQ_DB + " -o t8.yml testhawqregister_testusage2aohash2.t8")); + EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c t8.yml testhawqregister_testusage2aohash2.nt8")); + util.query("select * from nt8;", 3); + EXPECT_EQ(0, Command::getCommandStatus("rm -rf t8.yml")); + util.execute("drop table t8;"); + util.execute("drop table nt8;"); +} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/af483766/tools/bin/hawqregister ---------------------------------------------------------------------- diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister index 6700f54..2b492de 100755 --- a/tools/bin/hawqregister +++ b/tools/bin/hawqregister @@ -80,9 +80,10 @@ def create_table(dburl, tablename, schema_info, fmt, distrbution_policy): sys.exit(1) -def get_seg_name(dburl, tablename, database): +def get_seg_name(dburl, tablename, database, fmt): try: relname = '' + tablename = tablename.split('.')[-1] query = ("select pg_class2.relname from pg_class as pg_class1, pg_appendonly, pg_class as pg_class2 " "where pg_class1.relname ='%s' and pg_class1.oid = pg_appendonly.relid and pg_appendonly.segrelid = pg_class2.oid;") % tablename conn = dbconn.connect(dburl, True) @@ -97,6 +98,11 @@ def get_seg_name(dburl, tablename, database): except DatabaseError, ex: logger.error('Failed to run query "%s" with dbname "%s"' % (query, database)) sys.exit(1) + if fmt == 'Parquet': + if relname.find("paq") == -1: + logger.error("table '%s' is not parquet format" % tablename) + sys.exit(1) + return relname @@ -139,7 +145,7 @@ def get_metadata_from_database(dburl, tablename, seg_name): query = ("select location, gp_persistent_tablespace_node.tablespace_oid, database_oid, relfilenode from pg_class, gp_persistent_relation_node, " "gp_persistent_tablespace_node, gp_persistent_filespace_node where relname = '%s' and pg_class.relfilenode = " "gp_persistent_relation_node.relfilenode_oid and gp_persistent_relation_node.tablespace_oid = gp_persistent_tablespace_node.tablespace_oid " - "and gp_persistent_filespace_node.filespace_oid = gp_persistent_filespace_node.filespace_oid;") % tablename + "and gp_persistent_filespace_node.filespace_oid = gp_persistent_filespace_node.filespace_oid;") % tablename.split('.')[-1] conn = dbconn.connect(dburl, False) rows = dbconn.execSQL(conn, query) conn.commit() @@ -178,7 +184,6 @@ def get_files_in_hdfs(filepath): logger.error("Path '%s' does not exist in hdfs" % filepath) sys.exit(1) hdfscmd = "hadoop fs -ls -R %s" % filepath - print filepath result, out, err = local_ssh_output(hdfscmd) outlines = out.splitlines() # recursively search all the files under path 'filepath' @@ -276,7 +281,7 @@ if __name__ == '__main__': fileformat = 'Parquet' check_hash_type(dburl, tablename) # Usage1 only support randomly distributed table - seg_name = get_seg_name(dburl, tablename, database) + seg_name = get_seg_name(dburl, tablename, database, fileformat) firstsegno, tabledir = get_metadata_from_database(dburl, tablename, seg_name) check_files_and_table_in_same_hdfs_cluster(filepath, tabledir)
