Repository: incubator-hawq Updated Branches: refs/heads/master b8c9ca9c3 -> 14cb8e091
HAWQ-753. Add parquet test using new test library. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/14cb8e09 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/14cb8e09 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/14cb8e09 Branch: refs/heads/master Commit: 14cb8e091010e6b55db4b6e1bfe3551173709320 Parents: b8c9ca9 Author: ztao1987 <[email protected]> Authored: Tue May 24 14:50:45 2016 +0800 Committer: ztao1987 <[email protected]> Committed: Wed May 25 09:45:52 2016 +0800 ---------------------------------------------------------------------- src/test/feature/lib/data-gen.cpp | 5 +- src/test/feature/lib/data-gen.h | 9 +- src/test/feature/lib/sql-util.cpp | 8 +- src/test/feature/lib/sql-util.h | 2 +- src/test/feature/parquet/test-parquet.cpp | 138 +++++++++++++++++++++++++ 5 files changed, 152 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/14cb8e09/src/test/feature/lib/data-gen.cpp ---------------------------------------------------------------------- diff --git a/src/test/feature/lib/data-gen.cpp b/src/test/feature/lib/data-gen.cpp index e2e2f71..5ec806d 100644 --- a/src/test/feature/lib/data-gen.cpp +++ b/src/test/feature/lib/data-gen.cpp @@ -135,6 +135,9 @@ std::string DataGenerator::genTableDesc(bool appendonly, std::string desc = (appendonly ? "with (appendonly = true, orientation = " : "with (appendonly = false, orientation = ") + orientation + ", compresstype = " + compresstype + - ", compresslevel = " + std::to_string(compresslevel) + ")"; + (compresslevel != -1 + ? ", compresslevel = " + std::to_string(compresslevel) + : "") + + ")"; return desc; } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/14cb8e09/src/test/feature/lib/data-gen.h ---------------------------------------------------------------------- diff --git a/src/test/feature/lib/data-gen.h b/src/test/feature/lib/data-gen.h index 25aac5a..3df0ac0 100644 --- a/src/test/feature/lib/data-gen.h +++ b/src/test/feature/lib/data-gen.h @@ -21,22 +21,23 @@ class DataGenerator { void genSimpleTable(std::string tableName, bool appendonly = true, std::string orientation = "row", - std::string compresstype = "none", int compresslevel = 0); + std::string compresstype = "none", + int compresslevel = -1); void genTableWithFullTypes(std::string tableName, bool appendonly = true, std::string orientation = "row", std::string compresstype = "none", - int compresslevel = 0); + int compresslevel = -1); void genTableWithSeries(std::string tableName, bool appendonly = true, std::string orientation = "row", std::string compresstype = "none", - int compresslevel = 0); + int compresslevel = -1); void genTableWithNull(std::string tableName, bool appendonly = true, std::string orientation = "row", std::string compresstype = "none", - int compresslevel = 0); + int compresslevel = -1); private: std::string genTableDesc(bool appendonly, std::string orientation, http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/14cb8e09/src/test/feature/lib/sql-util.cpp ---------------------------------------------------------------------- diff --git a/src/test/feature/lib/sql-util.cpp b/src/test/feature/lib/sql-util.cpp index 0714e22..6961983 100644 --- a/src/test/feature/lib/sql-util.cpp +++ b/src/test/feature/lib/sql-util.cpp @@ -33,14 +33,14 @@ SQLUtility::~SQLUtility() { } void SQLUtility::exec(const std::string &sql) { - EXPECT_EQ((conn->runSQLCommand(sql)).getLastStatus(), 0) + EXPECT_EQ(0, (conn->runSQLCommand(sql)).getLastStatus()) << conn->getLastResult(); } void SQLUtility::execute(const std::string &sql, bool check) { conn->runSQLCommand("SET SEARCH_PATH=" + schemaName + ";" + sql); EXPECT_NE(conn.get(), nullptr); - if (check) EXPECT_EQ(conn->getLastStatus(), 0) << conn->getLastResult(); + if (check) EXPECT_EQ(0, conn->getLastStatus()) << conn->getLastResult(); } void SQLUtility::query(const std::string &sql, int expectNum) { @@ -82,7 +82,7 @@ void SQLUtility::execSQLFile(const std::string &sqlFile, // outFile is located in the same folder with ansFile std::string outFileAbsPath = fp.path + "/" + fp.fileBaseName + ".out"; conn->setOutputFile(outFileAbsPath); - EXPECT_EQ(conn->runSQLFile(newSqlFile).getLastStatus(), 0); + EXPECT_EQ(0, conn->runSQLFile(newSqlFile).getLastStatus()); conn->resetOutput(); EXPECT_FALSE(conn->checkDiff(ansFileAbsPath, outFileAbsPath, true)); if (conn->checkDiff(ansFileAbsPath, outFileAbsPath, true) == false) { @@ -138,7 +138,7 @@ const PSQLQueryResult &SQLUtility::executeQuery(const std::string &sql) { return result; } -PSQL *SQLUtility::getPSQL() const { return conn.get(); } +const PSQL *SQLUtility::getPSQL() const { return conn.get(); } std::string SQLUtility::getTestRootPath() const { std::string result; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/14cb8e09/src/test/feature/lib/sql-util.h ---------------------------------------------------------------------- diff --git a/src/test/feature/lib/sql-util.h b/src/test/feature/lib/sql-util.h index b5cba5c..9fb8994 100644 --- a/src/test/feature/lib/sql-util.h +++ b/src/test/feature/lib/sql-util.h @@ -50,7 +50,7 @@ class SQLUtility { // Get PSQL connection // @return PSQL raw pointer - PSQL *getPSQL() const; + const PSQL *getPSQL() const; // Get test root dir abs path // @return path string http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/14cb8e09/src/test/feature/parquet/test-parquet.cpp ---------------------------------------------------------------------- diff --git a/src/test/feature/parquet/test-parquet.cpp b/src/test/feature/parquet/test-parquet.cpp new file mode 100644 index 0000000..2e65f0d --- /dev/null +++ b/src/test/feature/parquet/test-parquet.cpp @@ -0,0 +1,138 @@ +#include "gtest/gtest.h" + +#include "lib/data-gen.h" +#include "lib/sql-util.h" + +class TestParquet : public ::testing::Test { + public: + TestParquet() {} + ~TestParquet() {} +}; + +TEST_F(TestParquet, TestMultipleType) { + SQLUtility util; + util.execute("drop table if exists t1"); + DataGenerator dGen(&util); + dGen.genTableWithFullTypes("t1", true, "parquet"); + util.query( + "select * from t1", + "2147483647|||00:00:00||ff:89:71:45:ae:01|2000-01-01 " + "07:00:00+08||32767|t|192.168.1.255/" + "32|<(1,2),3>|[(0,0),(6,6)]|-178000000 " + "years|0|-$21,474,836.48|(200,400),(100,200)||<aa>bb</" + "aa>||123456789a|2001:db8:85a3:8d3:1319:8a2e:370:7344/" + "64|||1|0|(1,2)|4277-12-31|128|\n0|((100,123),(5,10),(7,2),(4,5))|hello " + "world||04:45:05.0012+08:40|||bbccddeeff|128||2001:db8:85a3:8d3:1319:" + "8a2e:370:7344/128|<(1,2),3>|[(0,0),(6,6)]||1|$0.00|(2,3),(0,1)|hello " + "world|<aa>bb</aa>||aaaa|2001:db8:85a3:8d3:1319:8a2e:370:7344/" + "64|0||2147483647|-Infinity|(1,2)|4277-12-31|Infinity|\n||abcd|15:01:03||" + "|2000-01-01 07:00:00+08|||t|||[(0,0),(6,6)]|-178000000 " + "years|0|-$21,474,836.48|(200,400),(100,200)||<aa>bb</" + "aa>||123456789a|2001:db8:85a3:8d3:1319:8a2e:370:7344/" + "64|||1|0|(1,2)|4277-12-31|128|\n0|((100,123),(5,10),(7,2),(4,5))|hello " + "world||04:45:05.0012+08:40|||bbccddeeff|128||2001:db8:85a3:8d3:1319:" + "8a2e:370:7344/128|<(1,2),3>||||$0.00||hello " + "world|||aaaa|2001:db8:85a3:8d3:1319:8a2e:370:7344/" + "64|0||2147483647|-Infinity|(1,2)|4277-12-31|Infinity|\n0|((100,123),(5," + "10),(7,2),(4,5))|hello " + "world||04:45:05.0012+08:40|||bbccddeeff|128||2001:db8:85a3:8d3:1319:" + "8a2e:370:7344/128|<(1,2),3>||||$0.00||hello " + "world|||||0||2147483647|-Infinity|(1,2)|4277-12-31|Infinity|\n0|((100," + "123),(5,10),(7,2),(4,5))|hello " + "world||04:45:05.0012+08:40|||bbccddeeff|128||2001:db8:85a3:8d3:1319:" + "8a2e:370:7344/128|<(1,2),3>||||||hello world|||||0||34|||||\n"); +} + +TEST_F(TestParquet, TestCompression) { + SQLUtility util; + util.execute("drop table if exists t21"); + util.execute("drop table if exists t22"); + DataGenerator dGen(&util); + dGen.genTableWithNull("t21", true, "parquet", "gzip", 9); + dGen.genTableWithNull("t22", true, "parquet", "snappy"); + util.query("select * from t21,t22", + "15||aa|15||aa|\n15||aa|||WET|\n15||aa||51||\n||WET|15||aa|\n||" + "WET|||WET|\n||WET||51||\n|51||15||aa|\n|51||||WET|\n|51|||51||" + "\n"); +} + +TEST_F(TestParquet, TestSize) { + SQLUtility util; + // value/record size equal to pagesize/rowgroupsize + util.execute("drop table if exists t31"); + util.execute( + "create table t31 (a1 char(10485760), a2 char(10485760), a3 " + "char(10485760), a4 char(10485760), a5 char(10485760), a6 " + "char(10485760), a7 char(10485760), a8 char(10485760), a9 " + "char(10485760), a10 char(10485760)) with(appendonly=true, " + "orientation=parquet, pagesize=10485760, rowgroupsize=104857600)"); + util.execute( + "insert into t31 values ( ('a'::char(10485760)), " + "('a'::char(10485760)), ('a'::char(10485760)), ('a'::char(10485760)), " + "('a'::char(10485760)), ('a'::char(10485760)), ('a'::char(10485760)), " + "('a'::char(10485760)), ('a'::char(10485760)), ('a'::char(10485760)) );", + false); + EXPECT_STREQ("ERROR: value for column \"a1\" exceeds pagesize 10485760!", + util.getPSQL()->getLastResult().substr(0, 56).c_str()); + + // single column, one data page contains several values, one rwo group + // contains several groups + util.execute("drop table if exists t32"); + util.execute("drop table if exists t33"); + util.execute( + "create table t32 ( a1 text ) with(appendonly=true, " + "orientation=parquet)"); + util.execute("insert into t32 values(repeat('parquet',100))"); + util.execute("insert into t32 values(repeat('parquet',20))"); + util.execute("insert into t32 values(repeat('parquet',30))"); + util.execute( + "create table t33 ( a1 text ) with(appendonly=true, orientation=parquet, " + "pagesize=1024, rowgroupsize=1025)"); + util.execute("insert into t33 select * from t32"); + util.query("select * from t33", 3); + + // large data insert, several column values in one page, several rows in one + // rowgroup + util.execute("drop table if exists t34"); + util.execute("drop table if exists t35"); + util.execute( + "create table t34 (a1 char(1048576), a2 char(2048576), a3 " + "char(3048576), a4 char(4048576), a5 char(5048576), a6 char(6048576), a7 " + "char(7048576), a8 char(8048576), a9 char(9048576), a10 char(9)) " + "with(appendonly=true, orientation=parquet, pagesize=10485760, " + "rowgroupsize=90874386)"); + util.execute( + "insert into t34 values ( ('a'::char(1048576)), " + "('a'::char(2048576)), ('a'::char(3048576)), ('a'::char(4048576)), " + "('a'::char(5048576)), ('a'::char(6048576)), ('a'::char(7048576)), " + "('a'::char(8048576)), ('a'::char(9048576)), ('a'::char(9)) )"); + util.execute( + "create table t35 (a1 char(1048576), a2 char(2048576), a3 " + "char(3048576), a4 char(4048576), a5 char(5048576), a6 char(6048576), a7 " + "char(7048576), a8 char(8048576), a9 char(9048576), a10 char(9)) " + "with(appendonly=true, orientation=parquet, pagesize=10485760, " + "rowgroupsize=17437200)"); + util.execute("insert into t35 select * from t34"); + util.query("select * from t35", 1); +} + +TEST_F(TestParquet, TestPartition) { + SQLUtility util; + util.execute("drop table if exists t4"); + util.execute( + "create table t4 (id SERIAL,a1 int,a2 char(5)) WITH (appendonly=true, " + "orientation=parquet,compresstype=gzip,compresslevel=1) distributed " + "randomly Partition by range(a1) (start(1) end(16) every(8)" + "WITH(appendonly = true, orientation = parquet, compresstype = " + "snappy))"); + util.execute("insert into t4(a1,a2) values(generate_series(1,5),'M')"); + util.execute( + "alter table t4 add partition new_p start(17) end (20) WITH " + "(appendonly=true, orientation=parquet, compresstype=snappy)"); + util.execute( + "alter table t4 add default partition df_p WITH " + "(appendonly=true, orientation=parquet, compresstype=gzip, " + "compresslevel=3)"); + util.execute("insert into t4(a1,a2) values(generate_series(6,25),'F')"); + util.query("select * from t4", 25); +}
