Repository: incubator-hawq Updated Branches: refs/heads/master 833c2f97c -> 38b447d7e
HAWQ-1149. Fixed bug gp_persistent_build_all loses data in gp_relfile_node&gp_persistent_relfile_node Internally there are 3 bugs: 1) hawq2.0 changed the file path of relation file on hdfs from filespace/db/relfile.filenum to filespace/db/table/relfile/filenum. When we scan relfile on hdfs, we should change scan logic correspondingly. 2) Fetched dummy persistentTid & persistentSerialNum to PersistentRelation_MarkCreatePending() . 3) Need to reset Relation->rd_relationnodeinfo.isPresent, so that next time persistentid and serial# can be refetched during PersistentBuild_BuildDb(). Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/38b447d7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/38b447d7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/38b447d7 Branch: refs/heads/master Commit: 38b447d7e2a6a176bb41eefa7643ce14605a3f0b Parents: 833c2f9 Author: Ming LI <[email protected]> Authored: Mon Nov 14 16:21:46 2016 +0800 Committer: Ming LI <[email protected]> Committed: Mon Nov 21 16:30:58 2016 +0800 ---------------------------------------------------------------------- src/backend/cdb/cdbdatabaseinfo.c | 73 ++++++++++++++++++++++------ src/backend/cdb/cdbpersistentbuild.c | 6 +++ src/backend/cdb/cdbpersistentrelation.c | 17 ++++--- src/include/cdb/cdbpersistentrelation.h | 1 + 4 files changed, 75 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/38b447d7/src/backend/cdb/cdbdatabaseinfo.c ---------------------------------------------------------------------- diff --git a/src/backend/cdb/cdbdatabaseinfo.c b/src/backend/cdb/cdbdatabaseinfo.c index 062c4d3..4e0be41 100644 --- a/src/backend/cdb/cdbdatabaseinfo.c +++ b/src/backend/cdb/cdbdatabaseinfo.c @@ -49,7 +49,6 @@ #include "cdb/cdbpersistentfilesysobj.h" #include "commands/dbcommands.h" - /*------------------------------------------------------------------------- * Local static type declarations *------------------------------------------------------------------------- */ @@ -753,7 +752,9 @@ static void DatabaseInfo_AddFile( HTAB *dbInfoRelHashTable, Oid tablespace, char *dbDirPath, - char *name) + char *name, + bool isHdfs, + Oid table_oid) { int64 eof; int itemCount; @@ -787,7 +788,7 @@ static void DatabaseInfo_AddFile( } FileClose(file); - itemCount = sscanf(name, "%u.%u", &relfilenode, &segmentFileNum); + itemCount = sscanf(name, isHdfs?"%u/%u":"%u.%u", &relfilenode, &segmentFileNum); // UNDONE: sscanf is a rather poor scanner. // UNDONE: For right now, just assume properly named files.... @@ -796,12 +797,16 @@ static void DatabaseInfo_AddFile( DatabaseInfo_AddMiscEntry(info, tablespace, false, name); return; } - else if (itemCount == 1) + else if (itemCount == 1 && !isHdfs) segmentFileNum = 0; else Assert(itemCount == 2); - DatabaseInfo_AddRelSegFile(info, dbInfoRelHashTable, tablespace, relfilenode, segmentFileNum, eof); + if(isHdfs){ + DatabaseInfo_AddRelSegFile(info, dbInfoRelHashTable, tablespace, table_oid, relfilenode, eof); + }else{ + DatabaseInfo_AddRelSegFile(info, dbInfoRelHashTable, tablespace, relfilenode, segmentFileNum, eof); + } } /* @@ -818,9 +823,9 @@ DatabaseInfo_Scan( Oid database) { char *dbDirPath; - DIR *xldir; - struct dirent *xlde; - char fromfile[MAXPGPATH]; + DIR *xldir, *xldir1; + struct dirent *xlde,*xlde1; + char fromfile[MAXPGPATH],fromfile1[MAXPGPATH]; /* Lookup the database path and allocate a directory scan structure */ dbDirPath = GetDatabasePath( @@ -869,7 +874,9 @@ DatabaseInfo_Scan( dbInfoRelHashTable, tablespace, dbDirPath, - xlde->d_name); + xlde->d_name, + false, + InvalidOid); } }else{//on hdfs int ret = HdfsIsDirOrFile(fromfile); @@ -878,18 +885,52 @@ DatabaseInfo_Scan( (errcode_for_file_access(), errmsg("error to fetch info for path \"%s\": %m", fromfile))); }else if(ret == 0){//direcotry - DatabaseInfo_AddMiscEntry( + if(strcmp(xlde->d_name, dbDirPath) == 0){ //database level + DatabaseInfo_AddMiscEntry( info, tablespace, /* isDir */ true, xlde->d_name); + }else{ //table level, need to recursive list file in this directory + xldir1 = AllocateDir(fromfile); + if (xldir1 == NULL) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("Could not open database directory \"%s\": %m", fromfile))); + while ((xlde1 = ReadDir(xldir1, fromfile)) != NULL) + { + if (strcmp(xlde1->d_name, ".") == 0 || + strcmp(xlde1->d_name, "..") == 0) + continue; + + /* Odd... On snow leopard, we get back "/" as a subdir, which is wrong. Ingore it */ + if (xlde1->d_name[0] == '/' && xlde1->d_name[1] == '\0') + continue; + + snprintf(fromfile1, MAXPGPATH, "%s/%s", fromfile, xlde1->d_name); + + if(!IsLocalPath(fromfile1)){//on hdfs + int ret = HdfsIsDirOrFile(fromfile1); + if(ret != 1){//fetch info error + ereport(ERROR, + (errcode_for_file_access(), + errmsg("error to fetch file path \"%s\": %m", fromfile1))); + }else{ + DatabaseInfo_AddFile( + info, + dbInfoRelHashTable, + tablespace, + fromfile, + xlde1->d_name, + true, + pg_atoi(xlde->d_name, 4, 0)); + } + } + } + FreeDir(xldir1); + } }else if(ret == 1){//file - DatabaseInfo_AddFile( - info, - dbInfoRelHashTable, - tablespace, - dbDirPath, - xlde->d_name); + //just skip for PG_VERSION file } } } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/38b447d7/src/backend/cdb/cdbpersistentbuild.c ---------------------------------------------------------------------- diff --git a/src/backend/cdb/cdbpersistentbuild.c b/src/backend/cdb/cdbpersistentbuild.c index 0033769..a12bc3b 100644 --- a/src/backend/cdb/cdbpersistentbuild.c +++ b/src/backend/cdb/cdbpersistentbuild.c @@ -222,6 +222,7 @@ static void PersistentBuild_PopulateGpRelationNode( ItemPointerData persistentTid; int64 persistentSerialNum; + Relation rd; if (dbInfoRel->reltablespace == GLOBALTABLESPACE_OID && info->database != TemplateDbOid) @@ -417,6 +418,11 @@ static void PersistentBuild_PopulateGpRelationNode( persistentSerialNum); } } + // reset Relation->rd_relationnodeinfo.isPresent, so that next time persistentid and serial# can be refetched + rd = RelationIdGetRelation(relFileNode.relNode); + rd->rd_relationnodeinfo.isPresent = false; + RelationClose(rd); + (*count)++; } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/38b447d7/src/backend/cdb/cdbpersistentrelation.c ---------------------------------------------------------------------- diff --git a/src/backend/cdb/cdbpersistentrelation.c b/src/backend/cdb/cdbpersistentrelation.c index e7fd7ed..14a2cbc 100644 --- a/src/backend/cdb/cdbpersistentrelation.c +++ b/src/backend/cdb/cdbpersistentrelation.c @@ -629,7 +629,7 @@ void PersistentRelation_MarkCreatePending( relFileNode->spcNode, relFileNode->dbNode, relFileNode->relNode, - persistentTid, persistentSerialNum); + persistentTid, *persistentSerialNum); #endif /* @@ -648,14 +648,16 @@ void PersistentRelation_MarkCreatePending( void PersistentRelation_AddCreated( RelFileNode *relFileNode, ItemPointer persistentTid, + int64 *persistentSerialNum, bool flushToXLog) { WRITE_PERSISTENT_STATE_ORDERED_LOCK_DECLARE; PersistentFileSysObjName fsObjName; - ItemPointerData previousFreeTid; - int64 persistentSerialNum; + RelationDirEntry relationDirEntry; + + ItemPointerData previousFreeTid; Datum values[Natts_gp_persistent_relation_node]; if (RelFileNode_IsEmpty(relFileNode)) @@ -711,8 +713,11 @@ void PersistentRelation_AddCreated( PersistentFsObjType_RelationDir, values, flushToXLog, - persistentTid, - &persistentSerialNum); + &relationDirEntry->persistentTid, + &relationDirEntry->persistentSerialNum); + + *persistentTid = relationDirEntry->persistentTid; + *persistentSerialNum = relationDirEntry->persistentSerialNum; WRITE_PERSISTENT_STATE_ORDERED_UNLOCK; @@ -721,7 +726,7 @@ void PersistentRelation_AddCreated( elog(Persistent_DebugPrintLevel(), "Persistent relation: Add '%s', in state 'Created', serial number " INT64_FORMAT " at TID %s", PersistentFileSysObjName_ObjectName(&fsObjName), - persistentSerialNum, + *persistentSerialNum, ItemPointerToString(persistentTid)); } } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/38b447d7/src/include/cdb/cdbpersistentrelation.h ---------------------------------------------------------------------- diff --git a/src/include/cdb/cdbpersistentrelation.h b/src/include/cdb/cdbpersistentrelation.h index 292aaba..9f4e184 100644 --- a/src/include/cdb/cdbpersistentrelation.h +++ b/src/include/cdb/cdbpersistentrelation.h @@ -68,6 +68,7 @@ extern void PersistentRelation_AddCreated( RelFileNode *relFileNode, /* The tablespace, database, and relation OIDs for the create. */ ItemPointer persistentTid, + int64 *persistentSerialNum, bool flushToXLog); // -----------------------------------------------------------------------------
