Repository: incubator-hawq Updated Branches: refs/heads/master 56e21fce3 -> 7363ea75c
HAWQ-1128. Support HAWQ register tables with same file name in different schema. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/7363ea75 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/7363ea75 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/7363ea75 Branch: refs/heads/master Commit: 7363ea75cbb900e8aefd5470b94c81bf1714e659 Parents: 56e21fc Author: xunzhang <[email protected]> Authored: Mon Oct 31 21:34:11 2016 +0800 Committer: Wen Lin <[email protected]> Committed: Wed Nov 2 11:49:09 2016 +0800 ---------------------------------------------------------------------- .../test_hawq_register_usage1.cpp | 76 +++++++++++++------- .../test_hawq_register_usage2_case1.cpp | 20 +++--- .../test_hawq_register_usage2_case2.cpp | 4 +- tools/bin/hawqregister | 73 +++++++++---------- 4 files changed, 100 insertions(+), 73 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7363ea75/src/test/feature/ManagementTool/test_hawq_register_usage1.cpp ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/test_hawq_register_usage1.cpp b/src/test/feature/ManagementTool/test_hawq_register_usage1.cpp index a2c4a69..ce6ed08 100644 --- a/src/test/feature/ManagementTool/test_hawq_register_usage1.cpp +++ b/src/test/feature/ManagementTool/test_hawq_register_usage1.cpp @@ -14,6 +14,32 @@ using hawq::test::SQLUtility; using hawq::test::Command; using hawq::test::HdfsConfig; +TEST_F(TestHawqRegister, TestUsage1ExpectSuccessDifferentSchema) { + SQLUtility util; + string rootPath(util.getTestRootPath()); + string filePath = rootPath + "/ManagementTool/data/parquet200/dat.paq"; + + util.execute("drop table if exists xz.t;"); + util.execute("drop schema if exists xz;"); + util.execute("create schema xz;"); + util.execute("drop table if exists t;"); + auto cmd = hawq::test::stringFormat("hdfs dfs -put -f %s %s/usage1dat.paq", filePath.c_str(), getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + std::string policy = "DISTRIBUTED RANDOMLY";; + auto sql1 = hawq::test::stringFormat("CREATE TABLE t(i int) with (appendonly=true, orientation=parquet) %s;", policy.c_str()); + auto sql2 = hawq::test::stringFormat("CREATE TABLE xz.t(i int) with (appendonly=true, orientation=parquet) %s;", policy.c_str()); + util.execute(sql1); util.execute(sql2); + util.query("SELECT * from t", 0); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/usage1dat.paq testhawqregister_testusage1expectsuccessdifferentschema.t", HAWQ_DB, getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + util.query("select * from t;", 200); + util.execute("insert into t values(201);"); + util.query("select * from t;", 201); + util.execute("drop table xz.t;"); + util.execute("drop table t;"); + util.execute("drop schema xz;"); +} + TEST_F(TestHawqRegister, TestUsage1ExpectSuccess) { // Register file/folder into HAWQ by specific file/folder name @@ -40,7 +66,7 @@ TEST_F(TestHawqRegister, TestUsage1ExpectSuccess) { auto sql = hawq::test::stringFormat("CREATE TABLE t_%s(i int) with (appendonly=true, orientation=%s) %s;", std::to_string(suffix).c_str(), ddl.c_str(), policy.c_str()); util.execute(sql); util.query(hawq::test::stringFormat("SELECT * from t_%s", std::to_string(suffix).c_str()), 0); - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/usage1dat.paq t_%s", HAWQ_DB, getHdfsLocation().c_str(), std::to_string(suffix).c_str()); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/usage1dat.paq testhawqregister_testusage1expectsuccess.t_%s", HAWQ_DB, getHdfsLocation().c_str(), std::to_string(suffix).c_str()); EXPECT_EQ(0, Command::getCommandStatus(cmd)); util.query(hawq::test::stringFormat("select * from t_%s;", std::to_string(suffix).c_str()), 200); @@ -60,7 +86,7 @@ TEST_F(TestHawqRegister, TestUsage1ExpectSuccess) { auto sql = hawq::test::stringFormat("CREATE TABLE t_%s(i int) with (appendonly=true, orientation=%s) %s;", std::to_string(suffix).c_str(), ddl.c_str(), policy.c_str()); util.execute(sql); util.query(hawq::test::stringFormat("SELECT * from t_%s", std::to_string(suffix).c_str()), 0); - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/usage1dat.paq -e 596 t_%s", HAWQ_DB, getHdfsLocation().c_str(), std::to_string(suffix).c_str()); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/usage1dat.paq -e 596 testhawqregister_testusage1expectsuccess.t_%s", HAWQ_DB, getHdfsLocation().c_str(), std::to_string(suffix).c_str()); EXPECT_EQ(0, Command::getCommandStatus(cmd)); util.query(hawq::test::stringFormat("select * from t_%s;", std::to_string(suffix).c_str()), 100); @@ -84,7 +110,7 @@ TEST_F(TestHawqRegister, TestUsage1ExpectSuccess) { auto sql = hawq::test::stringFormat("CREATE TABLE t_%s(i int) with (appendonly=true, orientation=%s) %s;", std::to_string(suffix).c_str(), ddl.c_str(), policy.c_str()); util.execute(sql); util.query(hawq::test::stringFormat("SELECT * from t_%s", std::to_string(suffix).c_str()), 0); - cmd = hawq::test::stringFormat("hawq register -d %s -f %s%s t_%s", HAWQ_DB, getHdfsLocation().c_str(), folder.c_str(), std::to_string(suffix).c_str()); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s%s testhawqregister_testusage1expectsuccess.t_%s", HAWQ_DB, getHdfsLocation().c_str(), folder.c_str(), std::to_string(suffix).c_str()); EXPECT_EQ(0, Command::getCommandStatus(cmd)); util.query(hawq::test::stringFormat("select * from t_%s;", std::to_string(suffix).c_str()), 200); @@ -122,7 +148,7 @@ TEST_F(TestHawqRegister, TestUsage1SingleHawqFile) { util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); util.query("select * from hawqregister;", 0); - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq testhawqregister_testusage1singlehawqfile.hawqregister", HAWQ_DB, getHdfsLocation().c_str()); EXPECT_EQ(0, Command::getCommandStatus(cmd)); util.query("select * from hawqregister;", 3); @@ -143,7 +169,7 @@ TEST_F(TestHawqRegister, TestUsage1SingleHiveFile) { util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); util.query("select * from hawqregister;", 0); - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hive.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hive.paq testhawqregister_testusage1singlehivefile.hawqregister", HAWQ_DB, getHdfsLocation().c_str()); EXPECT_EQ(0, Command::getCommandStatus(cmd)); util.query("select * from hawqregister;", 1); @@ -166,7 +192,7 @@ TEST_F(TestHawqRegister, TestDataTypes) { util.execute("create table hawqregister(a bool, b int2, c int2, d int4, e int8, f date, g float4, h float8, i varchar, j bytea, k char, l varchar) with (appendonly=true, orientation=parquet);"); util.query("select * from hawqregister;", 0); - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_data_types.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_data_types.paq testhawqregister_testdatatypes.hawqregister", HAWQ_DB, getHdfsLocation().c_str()); EXPECT_EQ(0, Command::getCommandStatus(cmd)); util.query("select * from hawqregister;", 1); @@ -188,7 +214,7 @@ TEST_F(TestHawqRegister, TestAllNULL) { util.execute("create table hawqregister(a bool, b int2, c int2, d int4, e int8, f date, g float4, h float8, i varchar, j bytea, k char, l varchar) with (appendonly=true, orientation=parquet);"); util.query("select * from hawqregister;", 0); - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_data_types.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_data_types.paq testhawqregister_testallnull.hawqregister", HAWQ_DB, getHdfsLocation().c_str()); EXPECT_EQ(0, Command::getCommandStatus(cmd)); util.query("select * from hawqregister;", 1); @@ -222,7 +248,7 @@ TEST_F(TestHawqRegister, TestFiles) { util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); util.query("select * from hawqregister;", 0); - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_test hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_test testhawqregister_testfiles.hawqregister", HAWQ_DB, getHdfsLocation().c_str()); EXPECT_EQ(0, Command::getCommandStatus(cmd)); util.query("select * from hawqregister;", 12); @@ -245,7 +271,7 @@ TEST_F(TestHawqRegister, TestUsage1NotParquetFile) { util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); util.query("select * from hawqregister;", 0); - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_test_not_paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_test_not_paq testhawqregister_testusage1notparquetfile.hawqregister", HAWQ_DB, getHdfsLocation().c_str()); EXPECT_EQ(1, Command::getCommandStatus(cmd)); util.query("select * from hawqregister;", 0); @@ -267,21 +293,21 @@ TEST_F(TestHawqRegister, TestUsage1NotParquetTable) { util.execute("create table hawqregister(i int);"); util.query("select * from hawqregister;", 0); - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq testhawqregister_testusage1notparquettable.hawqregister", HAWQ_DB, getHdfsLocation().c_str()); EXPECT_EQ(1, Command::getCommandStatus(cmd)); util.query("select * from hawqregister;", 0); util.execute("drop table hawqregister;"); /* register a parquet file to a row hash table, should fail */ util.execute("create table hawqregister(i int) distributed by (i);"); - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq testhawqregister_testusage1notparquettable.hawqregister", HAWQ_DB, getHdfsLocation().c_str()); EXPECT_EQ(1, Command::getCommandStatus(cmd)); util.query("select * from hawqregister;", 0); util.execute("drop table hawqregister;"); /* register a parquet file to a parquet hash table, should fail */ util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet) distributed by (i);"); - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq testhawqregister_testusage1notparquettable.hawqregister", HAWQ_DB, getHdfsLocation().c_str()); EXPECT_EQ(1, Command::getCommandStatus(cmd)); util.query("select * from hawqregister;", 0); util.execute("drop table hawqregister;"); @@ -296,7 +322,7 @@ TEST_F(TestHawqRegister, TestUsage1FileNotExist) { util.execute("create table hawqregister(i int);"); util.query("select * from hawqregister;", 0); - auto cmd = hawq::test::stringFormat("hawq register -d %s -f %shawq_register_file_not_exist hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + auto cmd = hawq::test::stringFormat("hawq register -d %s -f %shawq_register_file_not_exist testhawqregister_testusage1notparquettable.hawqregister", HAWQ_DB, getHdfsLocation().c_str()); EXPECT_EQ(1, Command::getCommandStatus(cmd)); util.query("select * from hawqregister;", 0); @@ -313,7 +339,7 @@ TEST_F(TestHawqRegister, TestUsage1NoTable) { EXPECT_EQ(0, Command::getCommandStatus(cmd)); /* register a parquet file to a table not exist, should fail */ - cmd = hawq::test::stringFormat("hawq register -d %s -f %s hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s testhawqregister_testusage1notable.hawqregister", HAWQ_DB, getHdfsLocation().c_str()); EXPECT_EQ(1, Command::getCommandStatus(cmd)); } @@ -327,7 +353,7 @@ TEST_F(TestHawqRegister, TestUsage1NotHDFSPath) { util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); util.query("select * from hawqregister;", 0); - auto cmd = hawq::test::stringFormat("hawq register -d %s -f %s hawqregister", HAWQ_DB, filePath.c_str()); + auto cmd = hawq::test::stringFormat("hawq register -d %s -f %s testhawqregister_testusage1nothdfspath.hawqregister", HAWQ_DB, filePath.c_str()); EXPECT_EQ(1, Command::getCommandStatus(cmd)); util.query("select * from hawqregister;", 0); @@ -347,7 +373,7 @@ TEST_F(TestHawqRegister, TestUsage1EofSuccess) { util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); util.query("select * from hawqregister;", 0); - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister -e 0", HAWQ_DB, getHdfsLocation().c_str()); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq testhawqregister_testusage1eofsuccess.hawqregister -e 0", HAWQ_DB, getHdfsLocation().c_str()); EXPECT_EQ(0, Command::getCommandStatus(cmd)); util.query("select * from hawqregister;", 0); @@ -356,7 +382,7 @@ TEST_F(TestHawqRegister, TestUsage1EofSuccess) { cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str()); EXPECT_EQ(0, Command::getCommandStatus(cmd)); - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister -e %d", HAWQ_DB, getHdfsLocation().c_str(), size); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq testhawqregister_testusage1eofsuccess.hawqregister -e %d", HAWQ_DB, getHdfsLocation().c_str(), size); EXPECT_EQ(0, Command::getCommandStatus(cmd)); util.query("select * from hawqregister;", 3); util.execute("drop table hawqregister;"); @@ -366,7 +392,7 @@ TEST_F(TestHawqRegister, TestUsage1EofSuccess) { EXPECT_EQ(0, Command::getCommandStatus(cmd)); util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister -e %d", HAWQ_DB, getHdfsLocation().c_str(), size/2); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq testhawqregister_testusage1eofsuccess.hawqregister -e %d", HAWQ_DB, getHdfsLocation().c_str(), size/2); EXPECT_EQ(0, Command::getCommandStatus(cmd)); util.execute("drop table hawqregister;"); } @@ -384,18 +410,18 @@ TEST_F(TestHawqRegister, TestUsage1EofFailure) { util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); util.query("select * from hawqregister;", 0); - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister -e -1", HAWQ_DB, getHdfsLocation().c_str()); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq testhawqregister_testusage1eoffailure.hawqregister -e -1", HAWQ_DB, getHdfsLocation().c_str()); EXPECT_EQ(1, Command::getCommandStatus(cmd)); util.query("select * from hawqregister;", 0); /* register a parquet file with eof > filesize, should success */ int size = getFileSize(filePath.c_str()); - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister -e %d", HAWQ_DB, getHdfsLocation().c_str(), size+1); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq testhawqregister_testusage1eoffailure.hawqregister -e %d", HAWQ_DB, getHdfsLocation().c_str(), size+1); EXPECT_EQ(1, Command::getCommandStatus(cmd)); util.query("select * from hawqregister;", 0); /* register a parquet file with eof=float, should success */ - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister -e 11.1", HAWQ_DB, getHdfsLocation().c_str()); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq testhawqregister_testusage1eoffailure.hawqregister -e 11.1", HAWQ_DB, getHdfsLocation().c_str()); EXPECT_EQ(1, Command::getCommandStatus(cmd)); util.execute("drop table hawqregister;"); } @@ -416,15 +442,15 @@ TEST_F(TestHawqRegister, TestUsage1FolderFailure) { util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); /* register a empty folder, should success */ - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister", HAWQ_DB, getHdfsLocation().c_str(), folderName.c_str()); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s testhawqregister_testUsage1folderfailure.hawqregister", HAWQ_DB, getHdfsLocation().c_str(), folderName.c_str()); EXPECT_EQ(0, Command::getCommandStatus(cmd)); /* register a not exist folder, should fail */ - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister", HAWQ_DB, getHdfsLocation().c_str(), folderNameNotExist.c_str()); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s testhawqregister_testUsage1folderfailure.hawqregister", HAWQ_DB, getHdfsLocation().c_str(), folderNameNotExist.c_str()); EXPECT_EQ(1, Command::getCommandStatus(cmd)); /* register a folder with eof, should fail */ - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister -e 100", HAWQ_DB, getHdfsLocation().c_str(), folderName.c_str()); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s testhawqregister_testUsage1folderfailure.hawqregister -e 100", HAWQ_DB, getHdfsLocation().c_str(), folderName.c_str()); EXPECT_EQ(1, Command::getCommandStatus(cmd)); /* register a folder containing parquet and non-parquet files, should fail */ @@ -434,7 +460,7 @@ TEST_F(TestHawqRegister, TestUsage1FolderFailure) { cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/%s/", filePath2.c_str(), getHdfsLocation().c_str(), folderName.c_str()); EXPECT_EQ(0, Command::getCommandStatus(cmd)); - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister", HAWQ_DB, getHdfsLocation().c_str(), folderName.c_str()); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s testhawqregister_testUsage1folderfailure.hawqregister", HAWQ_DB, getHdfsLocation().c_str(), folderName.c_str()); EXPECT_EQ(1, Command::getCommandStatus(cmd)); util.execute("drop table hawqregister;"); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7363ea75/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp index 90aa8f0..c599f7c 100644 --- a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp +++ b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp @@ -33,20 +33,20 @@ TEST_F(TestHawqRegister, TestUsage2Case1IncorrectYaml) { SQLUtility util; string filePath = util.getTestRootPath() + "/ManagementTool/"; - EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_pagesize.yml xx")); - EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_rowgroupsize.yml xx")); - EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_filesize.yml xx")); - EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "wrong_schema.yml xx")); - EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_checksum.yml xx")); - EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "wrong_dfs_url.yml xx")); - EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_bucketnum.yml xx")); - EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_constraint_partition.yml xx")); + EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_pagesize.yml testhawqregister_testusage2case1incorrectyaml.xx")); + EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_rowgroupsize.yml testhawqregister_testusage2case1incorrectyaml.xx")); + EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_filesize.yml testhawqregister_testusage2case1incorrectyaml.xx")); + EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "wrong_schema.yml testhawqregister_testusage2case1incorrectyaml.xx")); + EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_checksum.yml testhawqregister_testusage2case1incorrectyaml.xx")); + EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "wrong_dfs_url.yml testhawqregister_testusage2case1incorrectyaml.xx")); + EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_bucketnum.yml testhawqregister_testusage2case1incorrectyaml.xx")); + EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_constraint_partition.yml testhawqregister_testusage2case1incorrectyaml.xx")); } TEST_F(TestHawqRegister, TestUsage2Case1MismatchFileNumber) { SQLUtility util; string filePath = util.getTestRootPath() + "/ManagementTool/"; - EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "files_incomplete.yml xx")); + EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "files_incomplete.yml testhawqregister_testusage2case1mismatchfilenumber.xx")); } @@ -317,7 +317,7 @@ TEST_F(TestHawqRegister, DISABLED_TestUsage2Case1ErrorRowgroupSize) { TEST_F(TestHawqRegister, TestUsage2Case1IncludeDirectory) { SQLUtility util; string test_root(util.getTestRootPath()); - EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hdfs dfs -put -f %s/ManagementTool/usage2case1 %s/", test_root.c_str(), getHdfsLocation().c_str()))); + EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hadoop fs -put -f %s/ManagementTool/usage2case1 %s/", test_root.c_str(), getHdfsLocation().c_str()))); string t_yml(hawq::test::stringFormat("%s/ManagementTool/usage2case1/includedirectory.yml", test_root.c_str())); EXPECT_EQ(1, Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c %s testhawqregister_testusage2case1includedirectory.nt", HAWQ_DB, t_yml.c_str()))); EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hdfs dfs -rm -r %s/usage2case1", getHdfsLocation().c_str()))); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7363ea75/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp b/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp index ad28a0a..14ea48e 100644 --- a/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp +++ b/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp @@ -111,7 +111,7 @@ void TestHawqRegister::runYamlCaseForceMode(std::string casename, std::string ym for(int i=8; i<=13; i++){ string file(hawq::test::stringFormat("%s/ManagementTool/usage2case2/", test_root.c_str())); file.append(std::to_string(i)); - EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hdfs dfs -put %s /hawq_default/16385/%s/%s/", file.c_str(), strs_src_dst["@DATABASE_OID@"].c_str(),strs_src_dst["@TABLE_OID_OLD@"].c_str()))); + EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hadoop fs -put %s /hawq_default/16385/%s/%s/", file.c_str(), strs_src_dst["@DATABASE_OID@"].c_str(),strs_src_dst["@TABLE_OID_OLD@"].c_str()))); } } //printf("%s\n", hawq::test::stringFormat("hawq register --force -d %s -c %s testhawqregister_%s.nt", HAWQ_DB, t_yml.c_str(), casename.c_str()).c_str()); @@ -366,7 +366,7 @@ TEST_F(TestHawqRegister, DISABLED_TestUsage2Case2ErrorRowgroupSize) { TEST_F(TestHawqRegister, TestUsage2Case2IncludeDirectory) { SQLUtility util; string test_root(util.getTestRootPath()); - EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hdfs dfs -put -f %s/ManagementTool/usage2case2 %s/", test_root.c_str(), getHdfsLocation().c_str()))); + EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hadoop fs -put -f %s/ManagementTool/usage2case2 %s/", test_root.c_str(), getHdfsLocation().c_str()))); string t_yml(hawq::test::stringFormat("%s/ManagementTool/usage2case2/includedirectory.yml", test_root.c_str())); EXPECT_EQ(1, Command::getCommandStatus(hawq::test::stringFormat("hawq register --force -d %s -c %s testhawqregister_testusage2case2includedirectory.nt", HAWQ_DB, t_yml.c_str()))); EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hdfs dfs -rm -r %s/usage2case2", getHdfsLocation().c_str()))); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7363ea75/tools/bin/hawqregister ---------------------------------------------------------------------- diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister index 356aef1..3477e81 100755 --- a/tools/bin/hawqregister +++ b/tools/bin/hawqregister @@ -142,6 +142,12 @@ def ispartition(yml_file): return True return False +def tablename_handler(tablename): + tmp_lst = tablename.split('.') + if len(tmp_lst) > 1: + return tmp_lst[-2], tmp_lst[-1] + return 'public', tablename + class FailureHandler(object): def __init__(self, conn): self.operations = [] @@ -197,12 +203,18 @@ class GpRegisterAccessor(object): '''execute query and return dict result''' return self.conn.query(sql).dictresult() + def get_schema_id(self, schemaname): + qry = """select oid from pg_namespace where nspname='%s';""" % schemaname + return self.exec_query(qry)[0]['oid'] + def get_table_existed(self, tablename): - qry = """select count(*) from pg_class where relname = '%s';""" % tablename.split('.')[-1].lower() + schemaname, tablename = tablename_handler(tablename) + qry = """select count(*) from pg_class where relnamespace = '%s' and relname = '%s';""" % (self.get_schema_id(schemaname), tablename) return self.exec_query(qry)[0]['count'] == 1 def get_table_column_num(self, tablename): - qry = """select count(*) from pg_attribute ,pg_class where pg_class.relname = '%s' and pg_class.oid = pg_attribute.attrelid and attnum > 0;""" % tablename.split('.')[-1].lower() + schemaname, tablename = tablename_handler(tablename) + qry = """select count(*) from pg_attribute ,pg_class where pg_class.relnamespace = '%s' and pg_class.relname = '%s' and pg_class.oid = pg_attribute.attrelid and attnum > 0;""" % (self.get_schema_id(schemaname), tablename) return self.exec_query(qry)[0]['count'] def do_create_table(self, src_table_name, tablename, schema_info, fmt, distrbution_policy, file_locations, bucket_number, partitionby, partitions_constraint, partitions_name): @@ -241,7 +253,8 @@ class GpRegisterAccessor(object): return True, query def check_hash_type(self, tablename): - qry = """select attrnums from gp_distribution_policy, pg_class where pg_class.relname = '%s' and pg_class.oid = gp_distribution_policy.localoid;""" % tablename + schemaname, tablename = tablename_handler(tablename) + qry = """select attrnums from gp_distribution_policy, pg_class where pg_class.relnamespace = '%s' and pg_class.relname = '%s' and pg_class.oid = gp_distribution_policy.localoid;""" % (self.get_schema_id(schemaname), tablename) rows = self.exec_query(qry) if len(rows) == 0: logger.error('Table %s is not an append-only table. There is no record in gp_distribution_policy table.' % tablename) @@ -252,9 +265,9 @@ class GpRegisterAccessor(object): # pg_paqseg_# def get_seg_name(self, tablename, database, fmt): - tablename = tablename.split('.')[-1] + schemaname, tablename = tablename_handler(tablename) query = ("select pg_class2.relname from pg_class as pg_class1, pg_appendonly, pg_class as pg_class2 " - "where pg_class1.relname ='%s' and pg_class1.oid = pg_appendonly.relid and pg_appendonly.segrelid = pg_class2.oid;") % tablename + "where pg_class1.relname ='%s' and pg_class1.oid = pg_appendonly.relid and pg_appendonly.segrelid = pg_class2.oid and pg_class1.relnamespace = '%s';") % (tablename, self.get_schema_id(schemaname)) rows = self.exec_query(query) if len(rows) == 0: logger.error('table "%s" not found in db "%s"' % (tablename, database)) @@ -267,7 +280,8 @@ class GpRegisterAccessor(object): return (relname, True) def get_distribution_policy_info(self, tablename): - query = "select oid from pg_class where relname = '%s';" % tablename.split('.')[-1].lower() + schemaname, tablename = tablename_handler(tablename) + query = "select oid from pg_class where relnamespace = '%s' and relname = '%s';" % (self.get_schema_id(schemaname), tablename) rows = self.exec_query(query) oid = rows[0]['oid'] query = "select * from gp_distribution_policy where localoid = '%s';" % oid @@ -276,23 +290,27 @@ class GpRegisterAccessor(object): def get_partition_info(self, tablename): ''' Get partition information from pg_partitions, return a constraint-tablename dictionary ''' - query = "SELECT partitiontablename, partitionboundary FROM pg_partitions WHERE tablename = '%s'" % tablename + schemaname, tablename = tablename_handler(tablename) + query = "SELECT partitiontablename, partitionboundary FROM pg_partitions WHERE partitionschemaname = '%s' and tablename = '%s'" % (schemaname, tablename) return self.exec_query(query) def get_partitionby(self, tablename): - query = "SELECT partitionschemaname, partitiontablename, partitionname, partitiontype, parentpartitiontablename, partitionboundary FROM pg_partitions WHERE tablename='%s';" % tablename + schemaname, tablename = tablename_handler(tablename) + query = "SELECT partitionschemaname, partitiontablename, partitionname, partitiontype, parentpartitiontablename, partitionboundary FROM pg_partitions WHERE schemaname = '%s' and tablename='%s';" % (schemaname, tablename) parition_type = self.exec_query(query)[0]['partitiontype'] - query = "SELECT columnname, partitionlevel FROM pg_partition_columns WHERE tablename='%s' ORDER BY position_in_partition_key;" % tablename + query = "SELECT columnname, partitionlevel FROM pg_partition_columns WHERE schemaname = '%s' and tablename='%s' ORDER BY position_in_partition_key;" % (schemaname, tablename) partition_columnname = self.exec_query(query)[0]['columnname'] partitionby = 'PARTITION BY %s (%s)' % (parition_type, partition_columnname) return partitionby def get_partition_num(self, tablename): - query = "SELECT partitionschemaname from pg_partitions WHERE tablename='%s';" % tablename + schemaname, tablename = tablename_handler(tablename) + query = "SELECT partitionschemaname from pg_partitions WHERE schemaname = '%s' and tablename='%s';" % (schemaname, tablename) return len(self.exec_query(query)) def get_bucket_number(self, tablename): - query = "select oid from pg_class where relname = '%s';" % tablename.split('.')[-1].lower() + schemaname, tablename = tablename_handler(tablename) + query = "select oid from pg_class where relnamespace = '%s' and relname = '%s';" % (self.get_schema_id(schemaname), tablename) rows = self.exec_query(query) oid = rows[0]['oid'] query = "select * from gp_distribution_policy where localoid = '%s';" % oid @@ -300,13 +318,14 @@ class GpRegisterAccessor(object): return rows[0]['bucketnum'] def get_metadata_from_database(self, tablename, seg_name): + schemaname, tablename = tablename_handler(tablename) query = 'select segno from pg_aoseg.%s;' % seg_name firstsegno = len(self.exec_query(query)) + 1 # get the full path of correspoding file for target table query = ("select location, gp_persistent_tablespace_node.tablespace_oid, database_oid, relfilenode from pg_class, gp_persistent_relation_node, " - "gp_persistent_tablespace_node, gp_persistent_filespace_node where relname = '%s' and pg_class.relfilenode = " + "gp_persistent_tablespace_node, gp_persistent_filespace_node where relnamespace = '%s' and relname = '%s' and pg_class.relfilenode = " "gp_persistent_relation_node.relfilenode_oid and gp_persistent_relation_node.tablespace_oid = gp_persistent_tablespace_node.tablespace_oid " - "and gp_persistent_filespace_node.filespace_oid = gp_persistent_filespace_node.filespace_oid;") % tablename.split('.')[-1] + "and gp_persistent_filespace_node.filespace_oid = gp_persistent_filespace_node.filespace_oid;") % (self.get_schema_id(schemaname), tablename) D = self.exec_query(query)[0] tabledir = '/'.join([D['location'].strip(), str(D['tablespace_oid']), str(D['database_oid']), str(D['relfilenode']), '']) return firstsegno, tabledir @@ -837,16 +856,6 @@ class HawqRegister(object): sys.exit(1) self.failure_handler.commit(('HDFSCMD', hdfscmd)) - def _delete_files_in_hdfs(self): - for fn in self.files_delete: - hdfscmd = 'hadoop fs -rm %s' % fn - sys.stdout.write('hdfscmd: "%s"\n' % hdfscmd) - result = local_ssh(hdfscmd, logger) - if result != 0: - logger.error('Fail to delete %s ' % fn) - self.failure_handler.rollback() - sys.exit(1) - def _set_modify_metadata(self, mode): segno = self.firstsegno append_eofs = self.sizes_append @@ -924,15 +933,6 @@ class HawqRegister(object): self.failure_handler.rollback() sys.exit(1) - def _delete_metadata(self): - query = "set allow_system_table_mods='dml';" - query += "begin transaction;" - segno_lst = [fn.strip().split('/')[-1] for fn in self.files_delete] - for seg in segno_lst: - query += "delete from pg_aoseg.%s where segno = '%s';" % (self.seg_name, seg) - query += "end transaction;" - return self.utility_accessor.update_catalog(query) - def _prepare_register(self): if not self.do_not_move: self._set_move_files_in_hdfs() @@ -954,13 +954,13 @@ class HawqRegisterPartition(HawqRegister): def _get_partition_info(self): dic = {} - for ele in self.accessor.get_partition_info(self.dst_table_name.split('.')[-1]): + for ele in self.accessor.get_partition_info(self.dst_table_name): dic[ele['partitionboundary']] = ele['partitiontablename'] return dic def _check_partitionby(self): def get_partitionby(): - return self.accessor.get_partitionby(self.dst_table_name.split('.')[-1]) + return self.accessor.get_partitionby(self.dst_table_name) if self.partitionby != get_partitionby(): logger.error('PartitionBy of %s is not consistent with previous partitionby.' % self.tablename) @@ -969,7 +969,7 @@ class HawqRegisterPartition(HawqRegister): def _check_partition_num(self): def get_partition_num(): - return self.accessor.get_partition_num(self.dst_table_name.split('.')[-1]) + return self.accessor.get_partition_num(self.dst_table_name) if get_partition_num() < len(self.partitions_name): logger.error('Partition Number of %s is not consistent with previous partition number.' % self.tablename) @@ -1010,13 +1010,14 @@ class HawqRegisterPartition(HawqRegister): self.queries = "set allow_system_table_mods='dml';" self.queries += "begin transaction;" self._check_duplicate_constraint() + schemaname, _ = tablename_handler(self.dst_table_name) for k, pn in enumerate(self.partitions_name): self.constraint = self.partitions_constraint[k] if not partitions.has_key(self.constraint): logger.error('Partition Constraint "%s" is not in table %s' % (self.constraint, self.tablename)) self.failure_handler.rollback() sys.exit(1) - self.tablename = partitions[self.constraint] + self.tablename = schemaname + '.' + partitions[self.constraint] self.files = self.partitions_filepaths[k] self.sizes = self.partitions_filesizes[k] self.tupcounts = self.partitions_tupcounts[k]
