Repository: incubator-hawq Updated Branches: refs/heads/master f27bbd0d1 -> 6c295d016
HAWQ-1195. Support Two or more external tables use the same error table In this case, we should assign a new sets of seg files for INSERT INTO ERROR TABLE. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/6c295d01 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/6c295d01 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/6c295d01 Branch: refs/heads/master Commit: 6c295d016e0ce493896c353ed0212a10d7e3a696 Parents: f27bbd0 Author: Ming LI <[email protected]> Authored: Tue Dec 13 00:16:58 2016 +0800 Committer: Ming LI <[email protected]> Committed: Tue Dec 13 00:16:58 2016 +0800 ---------------------------------------------------------------------- .../access/appendonly/appendonlywriter.c | 6 +- src/backend/commands/copy.c | 4 +- src/backend/commands/tablecmds.c | 4 +- src/backend/executor/execMain.c | 12 +-- src/include/access/appendonlywriter.h | 2 +- src/test/regress/input/errortbl.source | 11 ++- src/test/regress/output/errortbl.source | 80 ++++++++++++++++++-- 7 files changed, 97 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c295d01/src/backend/access/appendonly/appendonlywriter.c ---------------------------------------------------------------------- diff --git a/src/backend/access/appendonly/appendonlywriter.c b/src/backend/access/appendonly/appendonlywriter.c index 3c7f9a2..6152166 100644 --- a/src/backend/access/appendonly/appendonlywriter.c +++ b/src/backend/access/appendonly/appendonlywriter.c @@ -977,7 +977,7 @@ addCandidateSegno(AOSegfileStatus **maxSegno4Seg, int segment_num, AOSegfileStat * */ List *SetSegnoForWrite(List *existing_segnos, Oid relid, int segment_num, - bool forNewRel, + bool forNewRel, bool reuse_segfilenum_in_same_xid, bool keepHash) { /* these vars are used in GP_ROLE_DISPATCH only */ @@ -1082,7 +1082,7 @@ List *SetSegnoForWrite(List *existing_segnos, Oid relid, int segment_num, { if (!segfilestatus->isfull) { - if (segfilestatus->xid == CurrentXid) + if (segfilestatus->xid == CurrentXid && reuse_segfilenum_in_same_xid) { has_same_txn_status = true; if(CheckSegFileForWriteIfNeeded(aoentry, segfilestatus)){ @@ -1209,7 +1209,7 @@ List *assignPerRelSegno(List *all_relids, int segment_num) n = makeNode(SegfileMapNode); n->relid = cur_relid; - n->segnos = SetSegnoForWrite(NIL, cur_relid, segment_num, false, true); + n->segnos = SetSegnoForWrite(NIL, cur_relid, segment_num, false, true, true); Assert(n->relid != InvalidOid); Assert(n->segnos != NIL); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c295d01/src/backend/commands/copy.c ---------------------------------------------------------------------- diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 9910ec3..ccdd169 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -1602,7 +1602,7 @@ DoCopy(const CopyStmt *stmt, const char *queryString) { Oid relid = RelationGetRelid(cstate->cdbsreh->errtbl); Assert(!rel_is_partitioned(relid)); - err_segnos = SetSegnoForWrite(NIL, relid, list_length(cstate->resource->segments), false, true); + err_segnos = SetSegnoForWrite(NIL, relid, list_length(cstate->resource->segments), false, true, true); if (Gp_role == GP_ROLE_DISPATCH) CreateAppendOnlyParquetSegFileForRelationOnMaster( cstate->cdbsreh->errtbl, @@ -1642,7 +1642,7 @@ DoCopy(const CopyStmt *stmt, const char *queryString) { SegfileMapNode *n = makeNode(SegfileMapNode); n->relid = RelationGetRelid(cstate->rel); - n->segnos = SetSegnoForWrite(NIL, n->relid, 1, false, true); + n->segnos = SetSegnoForWrite(NIL, n->relid, 1, false, true, true); cstate->ao_segnos = lappend(cstate->ao_segnos, n); } } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c295d01/src/backend/commands/tablecmds.c ---------------------------------------------------------------------- diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index bda1616..975e121 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -6411,7 +6411,7 @@ ATRewriteTable(AlteredTableInfo *tab, Oid OIDNewHeap) savedResource = GetActiveQueryResource(); SetActiveQueryResource(resource); - segment_segnos = SetSegnoForWrite(NIL, 0, target_segment_num, true, false); + segment_segnos = SetSegnoForWrite(NIL, 0, target_segment_num, true, true, false); /* * We create seg files for the new relation here. @@ -16682,7 +16682,7 @@ ATPExecPartSplit(Relation rel, resource = AllocateResource(QRL_ONCE, 1, 1, target_segment_num, target_segment_num, NULL, 0); savedResource = GetActiveQueryResource(); SetActiveQueryResource(resource); - segment_segnos = SetSegnoForWrite(NIL, 0, target_segment_num, true, false); + segment_segnos = SetSegnoForWrite(NIL, 0, target_segment_num, true, true, false); scantable_splits = NIL; /* create the segfiles for the new relations here */ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c295d01/src/backend/executor/execMain.c ---------------------------------------------------------------------- diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index 578b9e8..30f6d09 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -245,6 +245,7 @@ SetupSegnoForErrorTable(Node *node, QueryCxtWalkerCxt *cxt) ExternalScan *scan = (ExternalScan *)node; QueryContextInfo *info = cxt->info; List *errSegnos; + bool reuse_segfilenum_in_same_xid = true; if (NULL == node) return false; @@ -270,9 +271,8 @@ SetupSegnoForErrorTable(Node *node, QueryCxtWalkerCxt *cxt) errtbloid = lfirst_oid(c); if (errtbloid == scan->fmterrtbl) { - Relation rel = heap_open(scan->fmterrtbl, AccessShareLock); - elog(ERROR, "Two or more external tables use the same error table \"%s\" in a statement", - RelationGetRelationName(rel)); + reuse_segfilenum_in_same_xid = false; + break; } } } @@ -281,7 +281,7 @@ SetupSegnoForErrorTable(Node *node, QueryCxtWalkerCxt *cxt) * Prepare error table for insert. */ Assert(!rel_is_partitioned(scan->fmterrtbl)); - errSegnos = SetSegnoForWrite(NIL, scan->fmterrtbl, GetQEGangNum(), false, true); + errSegnos = SetSegnoForWrite(NIL, scan->fmterrtbl, GetQEGangNum(), false, reuse_segfilenum_in_same_xid, true); scan->errAosegnos = errSegnos; info->errTblOid = lcons_oid(scan->fmterrtbl, info->errTblOid); @@ -877,7 +877,7 @@ ExecutorStart(QueryDesc *queryDesc, int eflags) if (plannedstmt->intoClause != NULL) { - List *segment_segnos = SetSegnoForWrite(NIL, 0, GetQEGangNum(), true, false); + List *segment_segnos = SetSegnoForWrite(NIL, 0, GetQEGangNum(), true, true, false); prepareDispatchedCatalogSingleRelation(plannedstmt->contextdisp, plannedstmt->intoClause->oidInfo.relOid, TRUE, segment_segnos); } @@ -4456,7 +4456,7 @@ CreateIntoRel(QueryDesc *queryDesc) /* * create a list of segment file numbers for insert. */ - segnos = SetSegnoForWrite(NIL, intoRelationId, GetQEGangNum(), true, false); + segnos = SetSegnoForWrite(NIL, intoRelationId, GetQEGangNum(), true, true, false); CreateAppendOnlyParquetSegFileForRelationOnMaster(intoRelationDesc, segnos); queryDesc->plannedstmt->into_aosegnos = segnos; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c295d01/src/include/access/appendonlywriter.h ---------------------------------------------------------------------- diff --git a/src/include/access/appendonlywriter.h b/src/include/access/appendonlywriter.h index bab8739..a943bcb 100755 --- a/src/include/access/appendonlywriter.h +++ b/src/include/access/appendonlywriter.h @@ -128,7 +128,7 @@ extern Size AppendOnlyWriterShmemSize(void); extern void InitAppendOnlyWriter(void); extern Size AppendOnlyWriterShmemSize(void); extern bool TestCurrentTspSupportTruncate(Oid tsp); -extern List *SetSegnoForWrite(List *existing_segnos, Oid relid, int segment_num, bool forNewRel, bool keepHash); +extern List *SetSegnoForWrite(List *existing_segnos, Oid relid, int segment_num, bool forNewRel, bool reuse_segfilenum_in_same_xid, bool keepHash); extern List *assignPerRelSegno(List *all_rels, int segment_num); extern void UpdateMasterAosegTotals(Relation parentrel, int segno, http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c295d01/src/test/regress/input/errortbl.source ---------------------------------------------------------------------- diff --git a/src/test/regress/input/errortbl.source b/src/test/regress/input/errortbl.source index aeaaff4..bf83da7 100644 --- a/src/test/regress/input/errortbl.source +++ b/src/test/regress/input/errortbl.source @@ -71,8 +71,13 @@ truncate EXT_NATION_ERROR1; select * from EXT_NATION1 as x, EXT_NATION3 as y where x.n_nationkey = y.n_nationkey; select count(*) from EXT_NATION_ERROR1; -- should be 50 -select * from EXT_NATION1 as x, EXT_NATION1 as y where x.n_nationkey = y.n_nationkey; --should fail on self join -select * from EXT_NATION1 as x, EXT_NATION_WITH_EXIST_ERROR_TABLE as y where x.n_nationkey = y.n_nationkey; --should fail with the same error table +truncate EXT_NATION_ERROR1; +select * from EXT_NATION1 as x, EXT_NATION1 as y where x.n_nationkey = y.n_nationkey; --should not fail on self join +select count(*) from EXT_NATION_ERROR1; -- should be 100 + +truncate EXT_NATION_ERROR1; +select * from EXT_NATION1 as x, EXT_NATION_WITH_EXIST_ERROR_TABLE as y where x.n_nationkey = y.n_nationkey; --should not fail with the same error table +select count(*) from EXT_NATION_ERROR1; -- should be 100 -- should fail on writable external table CREATE WRITABLE EXTERNAL TABLE EXT_NATION_WRITABLE ( N_NATIONKEY INTEGER , @@ -90,4 +95,4 @@ select * from gpfdist_status; drop external table gpfdist_status; drop external table gpfdist_start; -drop external table gpfdist_stop; \ No newline at end of file +drop external table gpfdist_stop; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c295d01/src/test/regress/output/errortbl.source ---------------------------------------------------------------------- diff --git a/src/test/regress/output/errortbl.source b/src/test/regress/output/errortbl.source index 43137a6..8e66b11 100644 --- a/src/test/regress/output/errortbl.source +++ b/src/test/regress/output/errortbl.source @@ -230,10 +230,80 @@ select count(*) from EXT_NATION_ERROR1; -- should be 50 50 (1 row) -select * from EXT_NATION1 as x, EXT_NATION1 as y where x.n_nationkey = y.n_nationkey; --should fail on self join -ERROR: Two or more external tables use the same error table "ext_nation_error1" in a statement (execMain.c:258) -select * from EXT_NATION1 as x, EXT_NATION_WITH_EXIST_ERROR_TABLE as y where x.n_nationkey = y.n_nationkey; --should fail with the same error table -ERROR: Two or more external tables use the same error table "ext_nation_error1" in a statement (execMain.c:258) +truncate EXT_NATION_ERROR1; +select * from EXT_NATION1 as x, EXT_NATION1 as y where x.n_nationkey = y.n_nationkey; --should not fail on self join + n_nationkey | n_name | n_regionkey | n_comment | n_nationkey | n_name | n_regionkey | n_comment +-------------+---------------------------+-------------+--------------------------------------------------------------------------------------------------------------------+-------------+---------------------------+-------------+-------------------------------------------------------------------------------------------------------------------- + 1 | ARGENTINA | 1 | al foxes promise slyly according to the regular accounts. bold requests alon | 1 | ARGENTINA | 1 | al foxes promise slyly according to the regular accounts. bold requests alon + 13 | JORDAN | 4 | ic deposits are blithely about the carefully regular pa | 13 | JORDAN | 4 | ic deposits are blithely about the carefully regular pa + 15 | MOROCCO | 0 | rns. blithely bold courts among the closely regular packages use furiously bold platelets? | 15 | MOROCCO | 0 | rns. blithely bold courts among the closely regular packages use furiously bold platelets? + 17 | PERU | 1 | platelets. blithely pending dependencies use fluffily across the even pinto beans. carefully silent accoun | 17 | PERU | 1 | platelets. blithely pending dependencies use fluffily across the even pinto beans. carefully silent accoun + 9 | INDONESIA | 2 | slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull | 9 | INDONESIA | 2 | slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull + 11 | IRAQ | 4 | nic deposits boost atop the quickly final requests? quickly regula | 11 | IRAQ | 4 | nic deposits boost atop the quickly final requests? quickly regula + 23 | UNITED KINGDOM | 3 | eans boost carefully special requests. accounts are. carefull | 23 | UNITED KINGDOM | 3 | eans boost carefully special requests. accounts are. carefull + 0 | ALGERIA | 0 | haggle. carefully final deposits detect slyly agai | 0 | ALGERIA | 0 | haggle. carefully final deposits detect slyly agai + 2 | BRAZIL | 1 | y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special | 2 | BRAZIL | 1 | y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special + 14 | KENYA | 0 | pending excuses haggle furiously deposits. pending, express pinto beans wake fluffily past t | 14 | KENYA | 0 | pending excuses haggle furiously deposits. pending, express pinto beans wake fluffily past t + 16 | MOZAMBIQUE | 0 | s. ironic, unusual asymptotes wake blithely r | 16 | MOZAMBIQUE | 0 | s. ironic, unusual asymptotes wake blithely r + 8 | INDIA | 2 | ss excuses cajole slyly across the packages. deposits print aroun | 8 | INDIA | 2 | ss excuses cajole slyly across the packages. deposits print aroun + 10 | IRAN | 4 | efully alongside of the slyly final dependencies. | 10 | IRAN | 4 | efully alongside of the slyly final dependencies. + 12 | JAPAN | 2 | ously. final, express gifts cajole a | 12 | JAPAN | 2 | ously. final, express gifts cajole a + 24 | UNITED STATES | 1 | y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be | 24 | UNITED STATES | 1 | y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be + 3 | CANADA | 1 | eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold | 3 | CANADA | 1 | eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold + 5 | ETHIOPIA | 0 | ven packages wake quickly. regu | 5 | ETHIOPIA | 0 | ven packages wake quickly. regu + 7 | GERMANY | 3 | l platelets. regular accounts x-ray: unusual, regular acco | 7 | GERMANY | 3 | l platelets. regular accounts x-ray: unusual, regular acco + 19 | ROMANIA | 3 | ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account | 19 | ROMANIA | 3 | ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account + 21 | VIETNAM | 2 | hely enticingly express accounts. even, final | 21 | VIETNAM | 2 | hely enticingly express accounts. even, final + 4 | EGYPT | 4 | y above the carefully unusual theodolites. final dugouts are quickly across the furiously regular d | 4 | EGYPT | 4 | y above the carefully unusual theodolites. final dugouts are quickly across the furiously regular d + 6 | FRANCE | 3 | refully final requests. regular, ironi | 6 | FRANCE | 3 | refully final requests. regular, ironi + 18 | CHINA | 2 | c dependencies. furiously express notornis sleep slyly regular accounts. ideas sleep. depos | 18 | CHINA | 2 | c dependencies. furiously express notornis sleep slyly regular accounts. ideas sleep. depos + 20 | SAUDI ARABIA | 4 | ts. silent requests haggle. closely express packages sleep across the blithely | 20 | SAUDI ARABIA | 4 | ts. silent requests haggle. closely express packages sleep across the blithely + 22 | RUSSIA | 3 | requests against the platelets use never according to the quickly regular pint | 22 | RUSSIA | 3 | requests against the platelets use never according to the quickly regular pint +(25 rows) + +select count(*) from EXT_NATION_ERROR1; -- should be 100 + count +------- + 100 +(1 row) + +truncate EXT_NATION_ERROR1; +select * from EXT_NATION1 as x, EXT_NATION_WITH_EXIST_ERROR_TABLE as y where x.n_nationkey = y.n_nationkey; --should not fail with the same error table + n_nationkey | n_name | n_regionkey | n_comment | n_nationkey | n_name | n_regionkey | n_comment +-------------+---------------------------+-------------+--------------------------------------------------------------------------------------------------------------------+-------------+---------------------------+-------------+-------------------------------------------------------------------------------------------------------------------- + 8 | INDIA | 2 | ss excuses cajole slyly across the packages. deposits print aroun | 8 | INDIA | 2 | ss excuses cajole slyly across the packages. deposits print aroun + 10 | IRAN | 4 | efully alongside of the slyly final dependencies. | 10 | IRAN | 4 | efully alongside of the slyly final dependencies. + 12 | JAPAN | 2 | ously. final, express gifts cajole a | 12 | JAPAN | 2 | ously. final, express gifts cajole a + 24 | UNITED STATES | 1 | y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be | 24 | UNITED STATES | 1 | y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be + 1 | ARGENTINA | 1 | al foxes promise slyly according to the regular accounts. bold requests alon | 1 | ARGENTINA | 1 | al foxes promise slyly according to the regular accounts. bold requests alon + 13 | JORDAN | 4 | ic deposits are blithely about the carefully regular pa | 13 | JORDAN | 4 | ic deposits are blithely about the carefully regular pa + 15 | MOROCCO | 0 | rns. blithely bold courts among the closely regular packages use furiously bold platelets? | 15 | MOROCCO | 0 | rns. blithely bold courts among the closely regular packages use furiously bold platelets? + 17 | PERU | 1 | platelets. blithely pending dependencies use fluffily across the even pinto beans. carefully silent accoun | 17 | PERU | 1 | platelets. blithely pending dependencies use fluffily across the even pinto beans. carefully silent accoun + 4 | EGYPT | 4 | y above the carefully unusual theodolites. final dugouts are quickly across the furiously regular d | 4 | EGYPT | 4 | y above the carefully unusual theodolites. final dugouts are quickly across the furiously regular d + 6 | FRANCE | 3 | refully final requests. regular, ironi | 6 | FRANCE | 3 | refully final requests. regular, ironi + 18 | CHINA | 2 | c dependencies. furiously express notornis sleep slyly regular accounts. ideas sleep. depos | 18 | CHINA | 2 | c dependencies. furiously express notornis sleep slyly regular accounts. ideas sleep. depos + 20 | SAUDI ARABIA | 4 | ts. silent requests haggle. closely express packages sleep across the blithely | 20 | SAUDI ARABIA | 4 | ts. silent requests haggle. closely express packages sleep across the blithely + 22 | RUSSIA | 3 | requests against the platelets use never according to the quickly regular pint | 22 | RUSSIA | 3 | requests against the platelets use never according to the quickly regular pint + 9 | INDONESIA | 2 | slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull | 9 | INDONESIA | 2 | slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull + 11 | IRAQ | 4 | nic deposits boost atop the quickly final requests? quickly regula | 11 | IRAQ | 4 | nic deposits boost atop the quickly final requests? quickly regula + 23 | UNITED KINGDOM | 3 | eans boost carefully special requests. accounts are. carefull | 23 | UNITED KINGDOM | 3 | eans boost carefully special requests. accounts are. carefull + 0 | ALGERIA | 0 | haggle. carefully final deposits detect slyly agai | 0 | ALGERIA | 0 | haggle. carefully final deposits detect slyly agai + 2 | BRAZIL | 1 | y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special | 2 | BRAZIL | 1 | y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special + 14 | KENYA | 0 | pending excuses haggle furiously deposits. pending, express pinto beans wake fluffily past t | 14 | KENYA | 0 | pending excuses haggle furiously deposits. pending, express pinto beans wake fluffily past t + 16 | MOZAMBIQUE | 0 | s. ironic, unusual asymptotes wake blithely r | 16 | MOZAMBIQUE | 0 | s. ironic, unusual asymptotes wake blithely r + 3 | CANADA | 1 | eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold | 3 | CANADA | 1 | eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold + 5 | ETHIOPIA | 0 | ven packages wake quickly. regu | 5 | ETHIOPIA | 0 | ven packages wake quickly. regu + 7 | GERMANY | 3 | l platelets. regular accounts x-ray: unusual, regular acco | 7 | GERMANY | 3 | l platelets. regular accounts x-ray: unusual, regular acco + 19 | ROMANIA | 3 | ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account | 19 | ROMANIA | 3 | ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account + 21 | VIETNAM | 2 | hely enticingly express accounts. even, final | 21 | VIETNAM | 2 | hely enticingly express accounts. even, final +(25 rows) + +select count(*) from EXT_NATION_ERROR1; -- should be 100 + count +------- + 100 +(1 row) + -- should fail on writable external table CREATE WRITABLE EXTERNAL TABLE EXT_NATION_WRITABLE ( N_NATIONKEY INTEGER , N_NAME CHAR(25) , @@ -261,4 +331,4 @@ select * from gpfdist_status; -- end_ignore drop external table gpfdist_status; drop external table gpfdist_start; -drop external table gpfdist_stop; \ No newline at end of file +drop external table gpfdist_stop;
