Repository: incubator-hawq Updated Branches: refs/heads/master 6e921df29 -> 4af3768cd
HAWQ-1112. Error message is not accurate when hawq register with single file and the size is larger than real size. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/4af3768c Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/4af3768c Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/4af3768c Branch: refs/heads/master Commit: 4af3768cd0b44807ac1cb2859ac5a1e16a03d618 Parents: 6e921df Author: Wen Lin <[email protected]> Authored: Wed Oct 19 11:11:13 2016 +0800 Committer: Wen Lin <[email protected]> Committed: Wed Oct 19 11:11:13 2016 +0800 ---------------------------------------------------------------------- .../ManagementTool/test_hawq_register.cpp | 243 -------------- .../feature/ManagementTool/test_hawq_register.h | 12 + .../test_hawq_register_usage1.cpp | 332 +++++++++++++++++++ .../test_hawq_register_usage2_case1.cpp | 32 ++ tools/bin/hawqregister | 5 +- 5 files changed, 380 insertions(+), 244 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4af3768c/src/test/feature/ManagementTool/test_hawq_register.cpp ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/test_hawq_register.cpp b/src/test/feature/ManagementTool/test_hawq_register.cpp deleted file mode 100644 index 123f73c..0000000 --- a/src/test/feature/ManagementTool/test_hawq_register.cpp +++ /dev/null @@ -1,243 +0,0 @@ -#include <string> - -#include "gtest/gtest.h" -#include "lib/command.h" -#include "lib/sql_util.h" -#include "lib/string_util.h" -#include "lib/hdfs_config.h" -#include "test_hawq_register.h" - -using std::string; -using hawq::test::SQLUtility; -using hawq::test::Command; -using hawq::test::HdfsConfig; - -TEST_F(TestHawqRegister, TestUsage1SingleHawqFile) { - SQLUtility util; - string rootPath(util.getTestRootPath()); - string relativePath("/ManagementTool/test_hawq_register_hawq.paq"); - string filePath = rootPath + relativePath; - - auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - - util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); - util.query("select * from hawqregister;", 0); - - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - - util.query("select * from hawqregister;", 3); - util.execute("insert into hawqregister values(1);"); - util.query("select * from hawqregister;", 4); - util.execute("drop table hawqregister;"); -} - -TEST_F(TestHawqRegister, TestUsage1SingleHiveFile) { - SQLUtility util; - string rootPath(util.getTestRootPath()); - string relativePath("/ManagementTool/test_hawq_register_hive.paq"); - string filePath = rootPath + relativePath; - - auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hive.paq", filePath.c_str(), getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - - util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); - util.query("select * from hawqregister;", 0); - - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hive.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - - util.query("select * from hawqregister;", 1); - util.execute("insert into hawqregister values(1);"); - util.query("select * from hawqregister;", 2); - util.execute("drop table hawqregister;"); -} - -TEST_F(TestHawqRegister, TestDataTypes) { - SQLUtility util; - string rootPath(util.getTestRootPath()); - /* This parquet file is generate by HIVE, using the table created by */ - /* 'create table parquet(a boolean, b tinyint, c smallint, d int, e bigint, f date, g float, h double, i string, j binary, k char(10), l varchar(10)) stored as parquet;' */ - string relativePath("/ManagementTool/test_hawq_register_data_types.paq"); - string filePath = rootPath + relativePath; - - auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_data_types.paq", filePath.c_str(), getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - - util.execute("create table hawqregister(a bool, b int2, c int2, d int4, e int8, f date, g float4, h float8, i varchar, j bytea, k char, l varchar) with (appendonly=true, orientation=parquet);"); - util.query("select * from hawqregister;", 0); - - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_data_types.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - - util.query("select * from hawqregister;", 1); - util.execute("drop table hawqregister;"); -} - -TEST_F(TestHawqRegister, TestAllNULL) { - SQLUtility util; - string rootPath(util.getTestRootPath()); - /* This parquet file is generate by HIVE, using the table created by */ - /* 'create table parquet(a boolean, b tinyint, c smallint, d int, e bigint, f date, g float, h double, i string, j binary, k char(10), l varchar(10)) stored as parquet;' */ - /* with all the values set to NULL */ - string relativePath("/ManagementTool/test_hawq_register_null.paq"); - string filePath = rootPath + relativePath; - - auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_data_types.paq", filePath.c_str(), getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - - util.execute("create table hawqregister(a bool, b int2, c int2, d int4, e int8, f date, g float4, h float8, i varchar, j bytea, k char, l varchar) with (appendonly=true, orientation=parquet);"); - util.query("select * from hawqregister;", 0); - - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_data_types.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - - util.query("select * from hawqregister;", 1); - util.execute("drop table hawqregister;"); -} - -TEST_F(TestHawqRegister, TestFiles) { - SQLUtility util; - string rootPath(util.getTestRootPath()); - string relativePath("/ManagementTool/test_hawq_register_hawq.paq"); - string filePath1 = rootPath + relativePath; - relativePath = "/ManagementTool/test_hawq_register_hive.paq"; - string filePath2 = rootPath + relativePath; - - auto cmd = hawq::test::stringFormat("hadoop fs -mkdir -p %s %s/hawq_register_test/t/t", filePath1.c_str(), getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hawq1.paq", filePath1.c_str(), getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hawq2.paq", filePath1.c_str(), getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/t/hawq.paq", filePath1.c_str(), getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - - cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hive1.paq", filePath2.c_str(), getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hive2.paq", filePath2.c_str(), getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/t/hive.paq", filePath2.c_str(), getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - - util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); - util.query("select * from hawqregister;", 0); - - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_test hawqregister", HAWQ_DB, getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - - util.query("select * from hawqregister;", 12); - util.execute("insert into hawqregister values(1);"); - util.query("select * from hawqregister;", 13); - cmd = hawq::test::stringFormat("hadoop fs -rm -r %s/hawq_register_test", getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - util.execute("drop table hawqregister;"); -} - -TEST_F(TestHawqRegister, TestUsage1NotParquetFile) { - SQLUtility util; - string rootPath(util.getTestRootPath()); - string relativePath("/ManagementTool/test_hawq_register_not_paq"); - string filePath = rootPath + relativePath; - - auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test_not_paq", filePath.c_str(), getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - - util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); - util.query("select * from hawqregister;", 0); - - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_test_not_paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); - EXPECT_EQ(1, Command::getCommandStatus(cmd)); - util.query("select * from hawqregister;", 0); - - cmd = hawq::test::stringFormat("hadoop fs -rm %s/hawq_register_test_not_paq", getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - util.execute("drop table hawqregister;"); -} - -TEST_F(TestHawqRegister, TestUsage1NotParquetTable) { - SQLUtility util; - string rootPath(util.getTestRootPath()); - string relativePath("/ManagementTool/test_hawq_register_hawq.paq"); - string filePath = rootPath + relativePath; - - auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - - util.execute("create table hawqregister(i int);"); - util.query("select * from hawqregister;", 0); - - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); - EXPECT_EQ(1, Command::getCommandStatus(cmd)); - util.query("select * from hawqregister;", 0); - - cmd = hawq::test::stringFormat("hadoop fs -rm %s/hawq_register_hawq.paq", getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - util.execute("drop table hawqregister;"); -} - -TEST_F(TestHawqRegister, TestUsage1FileNotExist) { - SQLUtility util; - - util.execute("create table hawqregister(i int);"); - util.query("select * from hawqregister;", 0); - - auto cmd = hawq::test::stringFormat("hawq register -d %s -f %shawq_register_file_not_exist hawqregister", HAWQ_DB, getHdfsLocation().c_str()); - EXPECT_EQ(1, Command::getCommandStatus(cmd)); - util.query("select * from hawqregister;", 0); - - util.execute("drop table hawqregister;"); -} - -TEST_F(TestHawqRegister, TestUsage1NotHDFSPath) { - SQLUtility util; - string rootPath(util.getTestRootPath()); - string relativePath("/ManagementTool/test_hawq_register_hawq.paq"); - string filePath = rootPath + relativePath; - - auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - - util.execute("create table hawqregister(i int);"); - util.query("select * from hawqregister;", 0); - - cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); - EXPECT_EQ(1, Command::getCommandStatus(cmd)); - util.query("select * from hawqregister;", 0); - - cmd = hawq::test::stringFormat("hadoop fs -rm %s/hawq_register_hawq.paq", getHdfsLocation().c_str()); - EXPECT_EQ(0, Command::getCommandStatus(cmd)); - util.execute("drop table hawqregister;"); -} - -TEST_F(TestHawqRegister, TestEmptyTable) { - SQLUtility util; - util.execute("drop table if exists t9;"); - util.execute("create table t9(i int) with (appendonly=true, orientation=row) distributed randomly;"); - EXPECT_EQ(0, Command::getCommandStatus("hawq extract -d " + (string) HAWQ_DB + " -o t9.yml testhawqregister_testemptytable.t9")); - EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c t9.yml testhawqregister_testemptytable.nt9")); - util.query("select * from nt9;", 0); - EXPECT_EQ(0, Command::getCommandStatus("rm -rf t9.yml")); - util.execute("drop table t9;"); - util.execute("drop table nt9;"); -} - -TEST_F(TestHawqRegister, TestIncorrectYaml) { - SQLUtility util; - string filePath = util.getTestRootPath() + "/ManagementTool/"; - - EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_pagesize.yml xx")); - EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_rowgroupsize.yml xx")); - EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_filesize.yml xx")); - EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "wrong_schema.yml xx")); - EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_checksum.yml xx")); - EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "wrong_dfs_url.yml xx")); - EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_bucketnum.yml xx")); -} - -TEST_F(TestHawqRegister, TestDismatchFileNumber) { - SQLUtility util; - string filePath = util.getTestRootPath() + "/ManagementTool/"; - EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "files_incomplete.yml xx")); -} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4af3768c/src/test/feature/ManagementTool/test_hawq_register.h ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/test_hawq_register.h b/src/test/feature/ManagementTool/test_hawq_register.h index f033f6d..dfb041d 100644 --- a/src/test/feature/ManagementTool/test_hawq_register.h +++ b/src/test/feature/ManagementTool/test_hawq_register.h @@ -3,6 +3,7 @@ #include <string> #include <pwd.h> +#include <fstream> #include "lib/hdfs_config.h" #include "gtest/gtest.h" @@ -49,6 +50,17 @@ class TestHawqRegister : public ::testing::Test { return ""; } + int getFileSize(const char *file){ + if (file == NULL) + return -1; + std::ifstream stream; + stream.open(file, std::ios_base::binary); + stream.seekg(0, std::ios_base::end); + int size = stream.tellg(); + stream.close(); + return size; + } + void checkPgAOSegValue(std::string relname, std::string value, std::string fmt) { std::string reloid = getTableOid(relname); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4af3768c/src/test/feature/ManagementTool/test_hawq_register_usage1.cpp ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/test_hawq_register_usage1.cpp b/src/test/feature/ManagementTool/test_hawq_register_usage1.cpp index 1843876..a2c4a69 100644 --- a/src/test/feature/ManagementTool/test_hawq_register_usage1.cpp +++ b/src/test/feature/ManagementTool/test_hawq_register_usage1.cpp @@ -109,3 +109,335 @@ TEST_F(TestHawqRegister, TestUsage1ExpectSuccess) { register_lambda(); gc_lambda(); } + +TEST_F(TestHawqRegister, TestUsage1SingleHawqFile) { + SQLUtility util; + string rootPath(util.getTestRootPath()); + string relativePath("/ManagementTool/test_hawq_register_hawq.paq"); + string filePath = rootPath + relativePath; + + auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); + util.query("select * from hawqregister;", 0); + + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + util.query("select * from hawqregister;", 3); + util.execute("insert into hawqregister values(1);"); + util.query("select * from hawqregister;", 4); + util.execute("drop table hawqregister;"); +} + +TEST_F(TestHawqRegister, TestUsage1SingleHiveFile) { + SQLUtility util; + string rootPath(util.getTestRootPath()); + string relativePath("/ManagementTool/test_hawq_register_hive.paq"); + string filePath = rootPath + relativePath; + + auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hive.paq", filePath.c_str(), getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); + util.query("select * from hawqregister;", 0); + + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hive.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + util.query("select * from hawqregister;", 1); + util.execute("insert into hawqregister values(1);"); + util.query("select * from hawqregister;", 2); + util.execute("drop table hawqregister;"); +} + +TEST_F(TestHawqRegister, TestDataTypes) { + SQLUtility util; + string rootPath(util.getTestRootPath()); + /* This parquet file is generate by HIVE, using the table created by */ + /* 'create table parquet(a boolean, b tinyint, c smallint, d int, e bigint, f date, g float, h double, i string, j binary, k char(10), l varchar(10)) stored as parquet;' */ + string relativePath("/ManagementTool/test_hawq_register_data_types.paq"); + string filePath = rootPath + relativePath; + + auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_data_types.paq", filePath.c_str(), getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + util.execute("create table hawqregister(a bool, b int2, c int2, d int4, e int8, f date, g float4, h float8, i varchar, j bytea, k char, l varchar) with (appendonly=true, orientation=parquet);"); + util.query("select * from hawqregister;", 0); + + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_data_types.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + util.query("select * from hawqregister;", 1); + util.execute("drop table hawqregister;"); +} + +TEST_F(TestHawqRegister, TestAllNULL) { + SQLUtility util; + string rootPath(util.getTestRootPath()); + /* This parquet file is generate by HIVE, using the table created by */ + /* 'create table parquet(a boolean, b tinyint, c smallint, d int, e bigint, f date, g float, h double, i string, j binary, k char(10), l varchar(10)) stored as parquet;' */ + /* with all the values set to NULL */ + string relativePath("/ManagementTool/test_hawq_register_null.paq"); + string filePath = rootPath + relativePath; + + auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_data_types.paq", filePath.c_str(), getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + util.execute("create table hawqregister(a bool, b int2, c int2, d int4, e int8, f date, g float4, h float8, i varchar, j bytea, k char, l varchar) with (appendonly=true, orientation=parquet);"); + util.query("select * from hawqregister;", 0); + + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_data_types.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + util.query("select * from hawqregister;", 1); + util.execute("drop table hawqregister;"); +} + +TEST_F(TestHawqRegister, TestFiles) { + SQLUtility util; + string rootPath(util.getTestRootPath()); + string relativePath("/ManagementTool/test_hawq_register_hawq.paq"); + string filePath1 = rootPath + relativePath; + relativePath = "/ManagementTool/test_hawq_register_hive.paq"; + string filePath2 = rootPath + relativePath; + + auto cmd = hawq::test::stringFormat("hadoop fs -mkdir -p %s/hawq_register_test/t", getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hawq1.paq", filePath1.c_str(), getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hawq2.paq", filePath1.c_str(), getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/t/hawq.paq", filePath1.c_str(), getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hive1.paq", filePath2.c_str(), getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hive2.paq", filePath2.c_str(), getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/t/hive.paq", filePath2.c_str(), getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); + util.query("select * from hawqregister;", 0); + + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_test hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + util.query("select * from hawqregister;", 12); + util.execute("insert into hawqregister values(1);"); + util.query("select * from hawqregister;", 13); + cmd = hawq::test::stringFormat("hadoop fs -rm -r %s/hawq_register_test", getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + util.execute("drop table hawqregister;"); +} + +TEST_F(TestHawqRegister, TestUsage1NotParquetFile) { + SQLUtility util; + string rootPath(util.getTestRootPath()); + string relativePath("/ManagementTool/test_hawq_register_not_paq"); + string filePath = rootPath + relativePath; + + auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test_not_paq", filePath.c_str(), getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); + util.query("select * from hawqregister;", 0); + + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_test_not_paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + EXPECT_EQ(1, Command::getCommandStatus(cmd)); + util.query("select * from hawqregister;", 0); + + cmd = hawq::test::stringFormat("hadoop fs -rm %s/hawq_register_test_not_paq", getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + util.execute("drop table hawqregister;"); +} + +TEST_F(TestHawqRegister, TestUsage1NotParquetTable) { + SQLUtility util; + string rootPath(util.getTestRootPath()); + string relativePath("/ManagementTool/test_hawq_register_hawq.paq"); + string filePath = rootPath + relativePath; + + /* register a parquet file to a row random table, should fail */ + auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + util.execute("create table hawqregister(i int);"); + util.query("select * from hawqregister;", 0); + + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + EXPECT_EQ(1, Command::getCommandStatus(cmd)); + util.query("select * from hawqregister;", 0); + util.execute("drop table hawqregister;"); + + /* register a parquet file to a row hash table, should fail */ + util.execute("create table hawqregister(i int) distributed by (i);"); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + EXPECT_EQ(1, Command::getCommandStatus(cmd)); + util.query("select * from hawqregister;", 0); + util.execute("drop table hawqregister;"); + + /* register a parquet file to a parquet hash table, should fail */ + util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet) distributed by (i);"); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + EXPECT_EQ(1, Command::getCommandStatus(cmd)); + util.query("select * from hawqregister;", 0); + util.execute("drop table hawqregister;"); + + cmd = hawq::test::stringFormat("hadoop fs -rm %s/hawq_register_hawq.paq", getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); +} + +TEST_F(TestHawqRegister, TestUsage1FileNotExist) { + SQLUtility util; + + util.execute("create table hawqregister(i int);"); + util.query("select * from hawqregister;", 0); + + auto cmd = hawq::test::stringFormat("hawq register -d %s -f %shawq_register_file_not_exist hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + EXPECT_EQ(1, Command::getCommandStatus(cmd)); + util.query("select * from hawqregister;", 0); + + util.execute("drop table hawqregister;"); +} + +TEST_F(TestHawqRegister, TestUsage1NoTable) { + SQLUtility util; + string rootPath(util.getTestRootPath()); + string relativePath("/ManagementTool/test_hawq_register_hawq.paq"); + string filePath = rootPath + relativePath; + + auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + /* register a parquet file to a table not exist, should fail */ + cmd = hawq::test::stringFormat("hawq register -d %s -f %s hawqregister", HAWQ_DB, getHdfsLocation().c_str()); + EXPECT_EQ(1, Command::getCommandStatus(cmd)); +} + +TEST_F(TestHawqRegister, TestUsage1NotHDFSPath) { + SQLUtility util; + string rootPath(util.getTestRootPath()); + string relativePath("/ManagementTool/test_hawq_register_hawq.paq"); + string filePath = rootPath + relativePath; + + /* register a non-hdfs parquet file, should failed */ + util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); + util.query("select * from hawqregister;", 0); + + auto cmd = hawq::test::stringFormat("hawq register -d %s -f %s hawqregister", HAWQ_DB, filePath.c_str()); + EXPECT_EQ(1, Command::getCommandStatus(cmd)); + util.query("select * from hawqregister;", 0); + + util.execute("drop table hawqregister;"); +} + +TEST_F(TestHawqRegister, TestUsage1EofSuccess) { + SQLUtility util; + string rootPath(util.getTestRootPath()); + string relativePath("/ManagementTool/test_hawq_register_hawq.paq"); + string filePath = rootPath + relativePath; + + /* register a parquet file with eof=0, should success */ + auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); + util.query("select * from hawqregister;", 0); + + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister -e 0", HAWQ_DB, getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + util.query("select * from hawqregister;", 0); + + /* register a parquet file with eof=filesize, should success */ + int size = getFileSize(filePath.c_str()); + cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister -e %d", HAWQ_DB, getHdfsLocation().c_str(), size); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + util.query("select * from hawqregister;", 3); + util.execute("drop table hawqregister;"); + + /* register a parquet file with eof<filesize, should success */ + cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister -e %d", HAWQ_DB, getHdfsLocation().c_str(), size/2); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + util.execute("drop table hawqregister;"); +} + +TEST_F(TestHawqRegister, TestUsage1EofFailure) { + SQLUtility util; + string rootPath(util.getTestRootPath()); + string relativePath("/ManagementTool/test_hawq_register_hawq.paq"); + string filePath = rootPath + relativePath; + + auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + /* register a parquet file with eof=-1, should success */ + util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); + util.query("select * from hawqregister;", 0); + + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister -e -1", HAWQ_DB, getHdfsLocation().c_str()); + EXPECT_EQ(1, Command::getCommandStatus(cmd)); + util.query("select * from hawqregister;", 0); + + /* register a parquet file with eof > filesize, should success */ + int size = getFileSize(filePath.c_str()); + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister -e %d", HAWQ_DB, getHdfsLocation().c_str(), size+1); + EXPECT_EQ(1, Command::getCommandStatus(cmd)); + util.query("select * from hawqregister;", 0); + + /* register a parquet file with eof=float, should success */ + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister -e 11.1", HAWQ_DB, getHdfsLocation().c_str()); + EXPECT_EQ(1, Command::getCommandStatus(cmd)); + util.execute("drop table hawqregister;"); +} + +TEST_F(TestHawqRegister, TestUsage1FolderFailure) { + SQLUtility util; + string folderName = "usage1tmp"; + string folderNameNotExist = "usage1tmpNotExist"; + string rootPath(util.getTestRootPath()); + string relativePath("/ManagementTool/test_hawq_register_hawq.paq"); + string filePath = rootPath + relativePath; + string relativePath2("/ManagementTool/files_incomplete.yml"); + string filePath2 = rootPath + relativePath2; + + auto cmd = hawq::test::stringFormat("hdfs dfs -mkdir %s/usage1tmp", getHdfsLocation().c_str(), ""); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);"); + + /* register a empty folder, should success */ + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister", HAWQ_DB, getHdfsLocation().c_str(), folderName.c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + /* register a not exist folder, should fail */ + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister", HAWQ_DB, getHdfsLocation().c_str(), folderNameNotExist.c_str()); + EXPECT_EQ(1, Command::getCommandStatus(cmd)); + + /* register a folder with eof, should fail */ + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister -e 100", HAWQ_DB, getHdfsLocation().c_str(), folderName.c_str()); + EXPECT_EQ(1, Command::getCommandStatus(cmd)); + + /* register a folder containing parquet and non-parquet files, should fail */ + cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/%s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str(), folderName.c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/%s/", filePath2.c_str(), getHdfsLocation().c_str(), folderName.c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); + + cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister", HAWQ_DB, getHdfsLocation().c_str(), folderName.c_str()); + EXPECT_EQ(1, Command::getCommandStatus(cmd)); + + util.execute("drop table hawqregister;"); + cmd = hawq::test::stringFormat("hdfs dfs -rm -r %s/%s", getHdfsLocation().c_str(), folderName.c_str()); + EXPECT_EQ(0, Command::getCommandStatus(cmd)); +} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4af3768c/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp index 6640775..6d44ca0 100644 --- a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp +++ b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp @@ -14,6 +14,38 @@ using hawq::test::SQLUtility; using hawq::test::Command; using hawq::test::HdfsConfig; +TEST_F(TestHawqRegister, TestUsage2Case1EmptyTable) { + SQLUtility util; + util.execute("drop table if exists t9;"); + util.execute("create table t9(i int) with (appendonly=true, orientation=row) distributed randomly;"); + EXPECT_EQ(0, Command::getCommandStatus("hawq extract -d " + (string) HAWQ_DB + " -o t9.yml testhawqregister_testusage2case1emptytable.t9")); + EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c t9.yml testhawqregister_testusage2case1emptytable.nt9")); + util.query("select * from nt9;", 0); + EXPECT_EQ(0, Command::getCommandStatus("rm -rf t9.yml")); + util.execute("drop table t9;"); + util.execute("drop table nt9;"); +} + +TEST_F(TestHawqRegister, TestUsage2Case1IncorrectYaml) { + SQLUtility util; + string filePath = util.getTestRootPath() + "/ManagementTool/"; + + EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_pagesize.yml xx")); + EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_rowgroupsize.yml xx")); + EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_filesize.yml xx")); + EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "wrong_schema.yml xx")); + EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_checksum.yml xx")); + EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "wrong_dfs_url.yml xx")); + EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_bucketnum.yml xx")); +} + +TEST_F(TestHawqRegister, TestUsage2Case1MismatchFileNumber) { + SQLUtility util; + string filePath = util.getTestRootPath() + "/ManagementTool/"; + EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "files_incomplete.yml xx")); +} + + TEST_F(TestHawqRegister, TestUsage2Case1Expected) { SQLUtility util; std::vector<string> create_table_matrix = {"distributed by (i)", "distributed randomly"}; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4af3768c/tools/bin/hawqregister ---------------------------------------------------------------------- diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister index a1c8e29..245d4d8 100755 --- a/tools/bin/hawqregister +++ b/tools/bin/hawqregister @@ -498,7 +498,10 @@ class HawqRegister(object): hdfscmd = 'hadoop fs -du %s' % fn _, out, _ = local_ssh_output(hdfscmd) if self.sizes[k] > int(out.strip().split()[0]): - logger.error('File size(%s) in yaml configuration file should not exceed actual length(%s) of file %s.' % (self.sizes[k], out.strip().split()[0], fn)) + if self.mode == 'usage1': + logger.error('Specified file size(%s) should not exceed actual length(%s) of file %s.' % (self.sizes[k], out.strip().split()[0], fn)) + else: + logger.error('File size(%s) in yaml configuration file should not exceed actual length(%s) of file %s.' % (self.sizes[k], out.strip().split()[0], fn)) self.failure_handler.rollback() sys.exit(1)
