Repository: incubator-hawq Updated Branches: refs/heads/master 24a1d0114 -> 09c8b8589
HAWQ-804. Add feature test for error table with new framework Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/09c8b858 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/09c8b858 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/09c8b858 Branch: refs/heads/master Commit: 09c8b85896e00e23ab98599df3d1a54e435c76ec Parents: 24a1d01 Author: xunzhang <[email protected]> Authored: Wed Jun 15 01:06:49 2016 +0800 Committer: xunzhang <[email protected]> Committed: Wed Jun 15 13:31:13 2016 +0800 ---------------------------------------------------------------------- src/test/feature/ExternalSource/data/nation.tbl | 25 ++++ .../ExternalSource/data/nation_error50.tbl | 75 +++++++++++ .../ExternalSource/sql_source/errortbl.source | 93 ++++++++++++++ .../feature/ExternalSource/test_errortbl.cpp | 125 +++++++++++++++++++ 4 files changed, 318 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/09c8b858/src/test/feature/ExternalSource/data/nation.tbl ---------------------------------------------------------------------- diff --git a/src/test/feature/ExternalSource/data/nation.tbl b/src/test/feature/ExternalSource/data/nation.tbl new file mode 100755 index 0000000..ee71b02 --- /dev/null +++ b/src/test/feature/ExternalSource/data/nation.tbl @@ -0,0 +1,25 @@ +0|ALGERIA|0| haggle. carefully final deposits detect slyly agai +1|ARGENTINA|1|al foxes promise slyly according to the regular accounts. bold requests alon +2|BRAZIL|1|y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special +3|CANADA|1|eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold +4|EGYPT|4|y above the carefully unusual theodolites. final dugouts are quickly across the furiously regular d +5|ETHIOPIA|0|ven packages wake quickly. regu +6|FRANCE|3|refully final requests. regular, ironi +7|GERMANY|3|l platelets. regular accounts x-ray: unusual, regular acco +8|INDIA|2|ss excuses cajole slyly across the packages. deposits print aroun +9|INDONESIA|2| slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull +10|IRAN|4|efully alongside of the slyly final dependencies. +11|IRAQ|4|nic deposits boost atop the quickly final requests? quickly regula +12|JAPAN|2|ously. final, express gifts cajole a +13|JORDAN|4|ic deposits are blithely about the carefully regular pa +14|KENYA|0| pending excuses haggle furiously deposits. pending, express pinto beans wake fluffily past t +15|MOROCCO|0|rns. blithely bold courts among the closely regular packages use furiously bold platelets? +16|MOZAMBIQUE|0|s. ironic, unusual asymptotes wake blithely r +17|PERU|1|platelets. blithely pending dependencies use fluffily across the even pinto beans. carefully silent accoun +18|CHINA|2|c dependencies. furiously express notornis sleep slyly regular accounts. ideas sleep. depos +19|ROMANIA|3|ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account +20|SAUDI ARABIA|4|ts. silent requests haggle. closely express packages sleep across the blithely +21|VIETNAM|2|hely enticingly express accounts. even, final +22|RUSSIA|3| requests against the platelets use never according to the quickly regular pint +23|UNITED KINGDOM|3|eans boost carefully special requests. accounts are. carefull +24|UNITED STATES|1|y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/09c8b858/src/test/feature/ExternalSource/data/nation_error50.tbl ---------------------------------------------------------------------- diff --git a/src/test/feature/ExternalSource/data/nation_error50.tbl b/src/test/feature/ExternalSource/data/nation_error50.tbl new file mode 100755 index 0000000..6c82f16 --- /dev/null +++ b/src/test/feature/ExternalSource/data/nation_error50.tbl @@ -0,0 +1,75 @@ +0|ALGERIA|0| haggle. carefully final deposits detect slyly agai +invalid format +1|ARGENTINA|1|al foxes promise slyly according to the regular accounts. bold requests alon +invalid format +asfinvalid format +invalid format +invalid format +2|BRAZIL|1|y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special +invalid format +thwdinvalid format +invalid format +3|CANADA|1|eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold +sdhinvalid format +tjinvalid format +345invalid format +4|EGYPT|4|y above the carefully unusual theodolites. final dugouts are quickly across the furiously regular d +invalid format +thinvalid format +oinvalid format +5|ETHIOPIA|0|ven packages wake quickly. regu +6|FRANCE|3|refully final requests. regular, ironi +7|GERMANY|3|l platelets. regular accounts x-ray: unusual, regular acco +invalid format +utikinvalid format +invalid format +djinvalid format +invalid format +ykinvalid format +invalid format +yukyinvalid formatkt +yukinvalid format +kyinvalityukd format +invtykalid format +tykinvalid format +yukinvalid format +8|INDIA|2|ss excuses cajole slyly across the packages. deposits print aroun +9|INDONESIA|2| slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull +10|IRAN|4|efully alongside of the slyly final dependencies. +invalid format +invalid format +invalid format +11|IRAQ|4|nic deposits boost atop the quickly final requests? quickly regula +12|JAPAN|2|ously. final, express gifts cajole a +invalid format +invalid format +invalid format +13|JORDAN|4|ic deposits are blithely about the carefully regular pa +14|KENYA|0| pending excuses haggle furiously deposits. pending, express pinto beans wake fluffily past t +kinvalid format +kinvayklid format +invalid format +15|MOROCCO|0|rns. blithely bold courts among the closely regular packages use furiously bold platelets? +16|MOZAMBIQUE|0|s. ironic, unusual asymptotes wake blithely r +invalidk format +invalikyd format +invalid format +17|PERU|1|platelets. blithely pending dependencies use fluffily across the even pinto beans. carefully silent accoun +18|CHINA|2|c dependencies. furiously express notornis sleep slyly regular accounts. ideas sleep. depos +ktyinvalid format +invalyukid format +inykyvalid format +19|ROMANIA|3|ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account +20|SAUDI ARABIA|4|ts. silent requests haggle. closely express packages sleep across the blithely +invalid format +21|VIETNAM|2|hely enticingly express accounts. even, final +22|RUSSIA|3| requests against the platelets use never according to the quickly regular pint +iknvalid format +invalid format +ryukinvalid format +invalid formatk +invalid format +rykinvalid format +23|UNITED KINGDOM|3|eans boost carefully special requests. accounts are. carefull +24|UNITED STATES|1|y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be +invalid format \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/09c8b858/src/test/feature/ExternalSource/sql_source/errortbl.source ---------------------------------------------------------------------- diff --git a/src/test/feature/ExternalSource/sql_source/errortbl.source b/src/test/feature/ExternalSource/sql_source/errortbl.source new file mode 100644 index 0000000..aeaaff4 --- /dev/null +++ b/src/test/feature/ExternalSource/sql_source/errortbl.source @@ -0,0 +1,93 @@ +-- -------------------------------------- +-- 'gpfdist' protocol +-- -------------------------------------- + +CREATE EXTERNAL WEB TABLE gpfdist_status (x text) +execute E'( python $GPHOME/bin/lib/gppinggpfdist.py @hostname@:7070 2>&1 || echo) ' +on SEGMENT 0 +FORMAT 'text' (delimiter '|'); + +CREATE EXTERNAL WEB TABLE gpfdist_start (x text) +execute E'((@gpwhich_gpfdist@ -p 7070 -d @abs_srcdir@/data </dev/null >/dev/null 2>&1 &); sleep 2; echo "starting...") ' +on SEGMENT 0 +FORMAT 'text' (delimiter '|'); + +CREATE EXTERNAL WEB TABLE gpfdist_stop (x text) +execute E'(/bin/pkill gpfdist || killall gpfdist) > /dev/null 2>&1; echo "stopping..."' +on SEGMENT 0 +FORMAT 'text' (delimiter '|'); +-- start_ignore +select * from gpfdist_stop; +select * from gpfdist_status; +select * from gpfdist_start; +select * from gpfdist_status; +-- end_ignore + +-- readable external table with error table + +CREATE EXTERNAL TABLE EXT_NATION1 ( N_NATIONKEY INTEGER , + N_NAME CHAR(25) , + N_REGIONKEY INTEGER , + N_COMMENT VARCHAR(152)) +location ('gpfdist://@hostname@:7070/nation_error50.tbl') +FORMAT 'text' (delimiter '|') +LOG ERRORS INTO EXT_NATION_ERROR1 SEGMENT REJECT LIMIT 51; + +CREATE EXTERNAL TABLE EXT_NATION2 ( N_NATIONKEY INTEGER , + N_NAME CHAR(25) , + N_REGIONKEY INTEGER , + N_COMMENT VARCHAR(152)) +location ('gpfdist://@hostname@:7070/nation_error50.tbl') +FORMAT 'text' (delimiter '|') +LOG ERRORS INTO EXT_NATION_ERROR2 SEGMENT REJECT LIMIT 50; + +CREATE EXTERNAL TABLE EXT_NATION3 ( N_NATIONKEY INTEGER , + N_NAME CHAR(25) , + N_REGIONKEY INTEGER , + N_COMMENT VARCHAR(152)) +location ('gpfdist://@hostname@:7070/nation.tbl') +FORMAT 'text' (delimiter '|') +LOG ERRORS INTO EXT_NATION_ERROR3 SEGMENT REJECT LIMIT 50; + +-- use existing error table +CREATE EXTERNAL TABLE EXT_NATION_WITH_EXIST_ERROR_TABLE ( N_NATIONKEY INTEGER , + N_NAME CHAR(25) , + N_REGIONKEY INTEGER , + N_COMMENT VARCHAR(152)) +location ('gpfdist://@hostname@:7070/nation_error50.tbl') +FORMAT 'text' (delimiter '|') +LOG ERRORS INTO EXT_NATION_ERROR1 SEGMENT REJECT LIMIT 51; + +select * from EXT_NATION1; +select count(*) from EXT_NATION_ERROR1; -- should be 50 +select * from EXT_NATION_WITH_EXIST_ERROR_TABLE; +select count(*) from EXT_NATION_ERROR1; -- should be 100 +select * from EXT_NATION2; -- should fail +select count(*) from EXT_NATION_ERROR2; -- should be empty +select * from EXT_NATION3; +select count(*) from EXT_NATION_ERROR3; -- should be empty + +truncate EXT_NATION_ERROR1; +select * from EXT_NATION1 as x, EXT_NATION3 as y where x.n_nationkey = y.n_nationkey; +select count(*) from EXT_NATION_ERROR1; -- should be 50 + +select * from EXT_NATION1 as x, EXT_NATION1 as y where x.n_nationkey = y.n_nationkey; --should fail on self join +select * from EXT_NATION1 as x, EXT_NATION_WITH_EXIST_ERROR_TABLE as y where x.n_nationkey = y.n_nationkey; --should fail with the same error table + +-- should fail on writable external table +CREATE WRITABLE EXTERNAL TABLE EXT_NATION_WRITABLE ( N_NATIONKEY INTEGER , + N_NAME CHAR(25) , + N_REGIONKEY INTEGER , + N_COMMENT VARCHAR(152)) +LOCATION ('gpfdist://@hostname@:7070/nation_error50.tbl') +FORMAT 'text' (delimiter '|') +LOG ERRORS INTO EXT_NATION_ERROR_WRITABLE SEGMENT REJECT LIMIT 5; + +-- start_ignore +select * from gpfdist_stop; +select * from gpfdist_status; +-- end_ignore + +drop external table gpfdist_status; +drop external table gpfdist_start; +drop external table gpfdist_stop; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/09c8b858/src/test/feature/ExternalSource/test_errortbl.cpp ---------------------------------------------------------------------- diff --git a/src/test/feature/ExternalSource/test_errortbl.cpp b/src/test/feature/ExternalSource/test_errortbl.cpp new file mode 100644 index 0000000..8bd5f51 --- /dev/null +++ b/src/test/feature/ExternalSource/test_errortbl.cpp @@ -0,0 +1,125 @@ +#include "gtest/gtest.h" + +#include "lib/sql_util.h" +#include "lib/string_util.h" + +using hawq::test::SQLUtility; + +class TestErrorTable : public ::testing::Test { + public: + TestErrorTable() {} + ~TestErrorTable() {} +}; + +TEST_F(TestErrorTable, TestErrorTableAll) { + + SQLUtility util; + auto init_gpfdist = [&] () { + auto sql = "CREATE EXTERNAL WEB TABLE gpfdist_status (x text) " + "execute E'( python %s/bin/lib/gppinggpfdist.py localhost:7070 2>&1 || echo) ' " + "on SEGMENT 0 " + "FORMAT 'text' (delimiter '|');"; + auto GPHOME = getenv("GPHOME"); + util.execute(hawq::test::stringFormat(sql, GPHOME)); + + sql = "CREATE EXTERNAL WEB TABLE gpfdist_start (x text) " + "execute E'((%s/bin/gpfdist -p 7070 -d %s </dev/null >/dev/null 2>&1 &); sleep 2; echo \"starting\"...) ' " + "on SEGMENT 0 " + "FORMAT 'text' (delimiter '|');"; + std::string path = util.getTestRootPath() + "/ExternalSource/data"; + util.execute(hawq::test::stringFormat(sql, GPHOME, path.c_str())); + + util.execute( + "CREATE EXTERNAL WEB TABLE gpfdist_stop (x text) " + "execute E'(/bin/pkill gpfdist || killall gpfdist) > /dev/null 2>&1; echo stopping...' " + "on SEGMENT 0 " + "FORMAT 'text' (delimiter '|');"); + util.execute("select * from gpfdist_stop;"); + util.execute("select * from gpfdist_status;"); + util.execute("select * from gpfdist_start;"); + util.execute("select * from gpfdist_status;"); + }; + + auto finalize_gpfdist = [&] () { + util.execute("drop external table EXT_NATION_WITH_EXIST_ERROR_TABLE;"); + util.execute("drop external table EXT_NATION1;"); + util.execute("drop table EXT_NATION_ERROR1 CASCADE;"); + util.execute("drop external table EXT_NATION2;"); + util.execute("drop table EXT_NATION_ERROR2 CASCADE;"); + util.execute("drop external table EXT_NATION3;"); + util.execute("drop table EXT_NATION_ERROR3 CASCADE;"); + util.execute("select * from gpfdist_stop;"); + util.execute("select * from gpfdist_status;"); + util.execute("drop external table gpfdist_status;"); + util.execute("drop external table gpfdist_start;"); + util.execute("drop external table gpfdist_stop;"); + }; + + init_gpfdist(); + + // readable external table with error table + util.execute( + "CREATE EXTERNAL TABLE EXT_NATION1 ( N_NATIONKEY INTEGER ," + "N_NAME CHAR(25) ," + "N_REGIONKEY INTEGER ," + "N_COMMENT VARCHAR(152))" + "location ('gpfdist://localhost:7070/nation_error50.tbl')" + "FORMAT 'text' (delimiter '|')" + "LOG ERRORS INTO EXT_NATION_ERROR1 SEGMENT REJECT LIMIT 51;"); + + util.execute( + "CREATE EXTERNAL TABLE EXT_NATION2 ( N_NATIONKEY INTEGER ," + "N_NAME CHAR(25) ," + "N_REGIONKEY INTEGER ," + "N_COMMENT VARCHAR(152))" + "location ('gpfdist://localhost:7070/nation_error50.tbl')" + "FORMAT 'text' (delimiter '|')" + "LOG ERRORS INTO EXT_NATION_ERROR2 SEGMENT REJECT LIMIT 50;"); + + util.execute( + "CREATE EXTERNAL TABLE EXT_NATION3 ( N_NATIONKEY INTEGER ," + "N_NAME CHAR(25) ," + "N_REGIONKEY INTEGER ," + "N_COMMENT VARCHAR(152))" + "location ('gpfdist://localhost:7070/nation.tbl')" + "FORMAT 'text' (delimiter '|')" + "LOG ERRORS INTO EXT_NATION_ERROR3 SEGMENT REJECT LIMIT 50;"); + + // use existing error table + util.execute( + "CREATE EXTERNAL TABLE EXT_NATION_WITH_EXIST_ERROR_TABLE ( N_NATIONKEY INTEGER ," + "N_NAME CHAR(25) ," + "N_REGIONKEY INTEGER ," + "N_COMMENT VARCHAR(152))" + "location ('gpfdist://localhost:7070/nation_error50.tbl')" + "FORMAT 'text' (delimiter '|')" + "LOG ERRORS INTO EXT_NATION_ERROR1 SEGMENT REJECT LIMIT 51;"); + + util.query("select * from EXT_NATION1;", 25); + util.query("select * from EXT_NATION_ERROR1;", 50); + util.query("select * from EXT_NATION_WITH_EXIST_ERROR_TABLE;", 25); + util.query("select * from EXT_NATION_ERROR1;", 100); + util.execute("select * from EXT_NATION2;", false); + util.query("select * from EXT_NATION_ERROR2;", 0); + util.query("select * from EXT_NATION3;", 25); + util.query("select * from EXT_NATION_ERROR3;", 0); + + util.execute("truncate EXT_NATION_ERROR1;"); + util.query("select * from EXT_NATION1 as x, EXT_NATION3 as y where x.n_nationkey = y.n_nationkey;", 25); + util.query("select * from EXT_NATION_ERROR1;", 50); + + util.execute("select * from EXT_NATION1 as x, EXT_NATION1 as y where x.n_nationkey = y.n_nationkey;", false); + util.execute("select * from EXT_NATION1 as x, EXT_NATION_WITH_EXIST_ERROR_TABLE as y where x.n_nationkey = y.n_nationkey;", false); + + util.execute( + "CREATE WRITABLE EXTERNAL TABLE EXT_NATION_WRITABLE ( N_NATIONKEY INTEGER , " + "N_NAME CHAR(25) , " + "N_REGIONKEY INTEGER , " + "N_COMMENT VARCHAR(152)) " + "LOCATION ('gpfdist://localhost:7070/nation_error50.tbl') " + "FORMAT 'text' (delimiter '|') " + "LOG ERRORS INTO EXT_NATION_ERROR_WRITABLE SEGMENT REJECT LIMIT 5;", + false); + + finalize_gpfdist(); +}
