Repository: incubator-hawq Updated Branches: refs/heads/master ced05d56f -> 6cefb5529
HAWQ-1113. Fix bug and add test for hawq register force mode. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/6cefb552 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/6cefb552 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/6cefb552 Branch: refs/heads/master Commit: 6cefb5529c536df99fb864105e31e083c37741ea Parents: ced05d5 Author: hzhang2 <[email protected]> Authored: Fri Oct 28 08:22:48 2016 +0800 Committer: hzhang2 <[email protected]> Committed: Fri Oct 28 08:22:48 2016 +0800 ---------------------------------------------------------------------- .../feature/ManagementTool/test_hawq_register.h | 2 +- .../test_hawq_register_usage2_case2.cpp | 14 ++++- src/test/feature/ManagementTool/usage2case2/10 | Bin 0 -> 535 bytes src/test/feature/ManagementTool/usage2case2/11 | Bin 0 -> 535 bytes src/test/feature/ManagementTool/usage2case2/12 | Bin 0 -> 535 bytes src/test/feature/ManagementTool/usage2case2/13 | Bin 0 -> 535 bytes src/test/feature/ManagementTool/usage2case2/8 | Bin 0 -> 535 bytes src/test/feature/ManagementTool/usage2case2/9 | Bin 0 -> 543 bytes .../usage2case2/same_path_yaml_config_tpl.yml | 57 +++++++++++++++++++ tools/bin/hawqregister | 35 +++++++++--- 10 files changed, 97 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/test_hawq_register.h ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/test_hawq_register.h b/src/test/feature/ManagementTool/test_hawq_register.h index 4edf2aa..dce1c8c 100644 --- a/src/test/feature/ManagementTool/test_hawq_register.h +++ b/src/test/feature/ManagementTool/test_hawq_register.h @@ -94,7 +94,7 @@ class TestHawqRegister : public ::testing::Test { void runYamlCaseTableNotExists(std::string casename, std::string ymlname, int expectederror, int checknum); void runYamlCaseTableExists(std::string casename, std::string ymlname, int isexpectederror, int checknum); - void runYamlCaseForceMode(std::string casename, std::string ymlname, int isexpectederror, int rows, int checknum); + void runYamlCaseForceMode(std::string casename, std::string ymlname, int isexpectederror, int rows, int checknum, bool samepath); private: std::unique_ptr<hawq::test::PSQL> conn; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp b/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp index 492dbd3..ad28a0a 100644 --- a/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp +++ b/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp @@ -80,7 +80,7 @@ TEST_F(TestHawqRegister, TestUsage2Case2Expected) { } } -void TestHawqRegister::runYamlCaseForceMode(std::string casename, std::string ymlname, int isexpectederror = 1, int rows = 50, int checknum = 200) { +void TestHawqRegister::runYamlCaseForceMode(std::string casename, std::string ymlname, int isexpectederror = 1, int rows = 50, int checknum = 200, bool samepathfile = false) { SQLUtility util; string test_root(util.getTestRootPath()); string t_yml_tpl(hawq::test::stringFormat("%s/ManagementTool/%s_tpl.yml", test_root.c_str(), ymlname.c_str())); @@ -107,11 +107,17 @@ void TestHawqRegister::runYamlCaseForceMode(std::string casename, std::string ym hc.getNamenodeHost(hdfs_prefix); strs_src_dst["@PORT@"]= hdfs_prefix; frep.replace(t_yml_tpl, t_yml, strs_src_dst); + if(samepathfile) { + for(int i=8; i<=13; i++){ + string file(hawq::test::stringFormat("%s/ManagementTool/usage2case2/", test_root.c_str())); + file.append(std::to_string(i)); + EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hdfs dfs -put %s /hawq_default/16385/%s/%s/", file.c_str(), strs_src_dst["@DATABASE_OID@"].c_str(),strs_src_dst["@TABLE_OID_OLD@"].c_str()))); + } + } //printf("%s\n", hawq::test::stringFormat("hawq register --force -d %s -c %s testhawqregister_%s.nt", HAWQ_DB, t_yml.c_str(), casename.c_str()).c_str()); //sleep(60); EXPECT_EQ(isexpectederror, Command::getCommandStatus(hawq::test::stringFormat("hawq register --force -d %s -c %s testhawqregister_%s.nt", HAWQ_DB, t_yml.c_str(), casename.c_str()))); util.query("select * from nt;", checknum); - EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf %s", t_yml.c_str()))); util.execute("drop table t;"); util.execute("drop table nt;"); @@ -250,6 +256,10 @@ TEST_F(TestHawqRegister, TestUsage2Case2NormalYamlConfig) { runYamlCaseForceMode("testusage2case2normalyamlconfig", "usage2case2/normal_yaml_config", 0, 50, 150); } +TEST_F(TestHawqRegister, TestUsage2Case2SamePathYamlConfig) { + runYamlCaseForceMode("testusage2case2samepathyamlconfig", "usage2case2/same_path_yaml_config", 0, 50, 332, true); +} + TEST_F(TestHawqRegister, TestUsage2Case2NormalYamlNoUpdateConfig) { runYamlCaseForceMode("testusage2case2normalyamlnoupdateconfig", "usage2case2/normal_yaml_no_update_config", 0, 50, 100); } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/usage2case2/10 ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/usage2case2/10 b/src/test/feature/ManagementTool/usage2case2/10 new file mode 100644 index 0000000..1aceb3f Binary files /dev/null and b/src/test/feature/ManagementTool/usage2case2/10 differ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/usage2case2/11 ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/usage2case2/11 b/src/test/feature/ManagementTool/usage2case2/11 new file mode 100644 index 0000000..1aceb3f Binary files /dev/null and b/src/test/feature/ManagementTool/usage2case2/11 differ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/usage2case2/12 ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/usage2case2/12 b/src/test/feature/ManagementTool/usage2case2/12 new file mode 100644 index 0000000..1aceb3f Binary files /dev/null and b/src/test/feature/ManagementTool/usage2case2/12 differ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/usage2case2/13 ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/usage2case2/13 b/src/test/feature/ManagementTool/usage2case2/13 new file mode 100644 index 0000000..1aceb3f Binary files /dev/null and b/src/test/feature/ManagementTool/usage2case2/13 differ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/usage2case2/8 ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/usage2case2/8 b/src/test/feature/ManagementTool/usage2case2/8 new file mode 100644 index 0000000..1aceb3f Binary files /dev/null and b/src/test/feature/ManagementTool/usage2case2/8 differ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/usage2case2/9 ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/usage2case2/9 b/src/test/feature/ManagementTool/usage2case2/9 new file mode 100644 index 0000000..1155c33 Binary files /dev/null and b/src/test/feature/ManagementTool/usage2case2/9 differ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/usage2case2/same_path_yaml_config_tpl.yml ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/usage2case2/same_path_yaml_config_tpl.yml b/src/test/feature/ManagementTool/usage2case2/same_path_yaml_config_tpl.yml new file mode 100644 index 0000000..9bba0aa --- /dev/null +++ b/src/test/feature/ManagementTool/usage2case2/same_path_yaml_config_tpl.yml @@ -0,0 +1,57 @@ +Bucketnum: 6 +DBVersion: PostgreSQL 8.2.15 (Greenplum Database 4.2.0 build 1) (HAWQ 2.0.1.0 build + dev) on x86_64-apple-darwin14.5.0, compiled by GCC Apple LLVM version 6.1.0 (clang-602.0.53) + (based on LLVM 3.6.0svn) compiled on Jul 6 2016 10:22:33 +DFS_URL: hdfs://@PORT@ +Distribution_Policy: DISTRIBUTED BY (i) +Encoding: UTF8 +FileFormat: Parquet +Parquet_FileLocations: + Checksum: false + CompressionLevel: 0 + CompressionType: null + EnableDictionary: false + Files: + - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/2 + size: 250 + - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/4 + size: 250 + - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/5 + size: 258 + - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/6 + size: 270 + - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/3 + size: 258 + - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/1 + size: 228 + - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/2 + size: 215 + - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/3 + size: 215 + - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/4 + size: 220 + - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/1 + size: 254 + - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/6 + size: 215 + - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/5 + size: 210 + - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/9 + size: 543 + - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/8 + size: 535 + - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/10 + size: 535 + - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/11 + size: 535 + - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/12 + size: 535 + - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/13 + size: 535 + PageSize: 1048576 + RowGroupSize: 8388608 +Parquet_Schema: +- name: i + type: int4 +TableName: public.nt +Version: 1.0.0 http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/tools/bin/hawqregister ---------------------------------------------------------------------- diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister index 46a9a8f..492b151 100755 --- a/tools/bin/hawqregister +++ b/tools/bin/hawqregister @@ -574,8 +574,10 @@ class HawqRegister(object): sys.exit(1) self.firstsegno, self.tabledir = self._get_metadata() - seg_list, existed_sizes = self._get_metadata_from_table() - existed_files = [self.tabledir + seg for seg in seg_list] + if self.yml and self.mode == 'force': + existed_files, existed_sizes = self._get_files_in_hdfs(self.tabledir) + else: + existed_files, existed_sizes = self._get_files_in_hdfs(self.filepath) # check if file numbers in hdfs is consistent with the record count of pg_aoseg. hdfs_file_no_lst = [f.split('/')[-1] for f in existed_files] for k in range(1, self.firstsegno - 1): @@ -740,13 +742,28 @@ class HawqRegister(object): catalog_lst = [str(i) for i in range(1, segno)] new_catalog_lst = [str(i) for i in range(segno, len(self.files_update) + 1)] exist_catalog_lst = [] + tmp_files_update = [f for f in self.files_update] + tmp_sizes_update = [f for f in self.sizes_update] + + tmp_tupcounts_update = [f for f in self.tupcounts_update] + tmp_eofuncompresseds_update = [f for f in self.eofuncompresseds_update] + tmp_varblockcounts_update = [f for f in self.varblockcounts_update] + for k, seg in enumerate(update_segno_lst): if seg not in catalog_lst: - self.files_same_path.append(self.files_update[k]) - self.sizes_same_path.append(self.sizes_update[k]) - self.tupcounts_same_path.append(self.tupcounts_update[k]) - self.eofuncompresseds_same_path.append(self.eofuncompresseds_update[k]) - self.varblockcounts_same_path.append(self.varblockcounts_update[k]) + + self.files_same_path.append(tmp_files_update[k]) + self.sizes_same_path.append(tmp_sizes_update[k]) + self.tupcounts_same_path.append(tmp_tupcounts_update[k]) + self.eofuncompresseds_same_path.append(tmp_eofuncompresseds_update[k]) + self.varblockcounts_same_path.append(tmp_varblockcounts_update[k]) + + self.files_update.remove(tmp_files_update[k]) + self.sizes_update.remove(tmp_sizes_update[k]) + self.tupcounts_update.remove(tmp_tupcounts_update[k]) + self.eofuncompresseds_update.remove(tmp_eofuncompresseds_update[k]) + self.varblockcounts_update.remove(tmp_varblockcounts_update[k]) + if seg in new_catalog_lst: exist_catalog_lst.append(seg) for seg in update_segno_lst: @@ -757,11 +774,13 @@ class HawqRegister(object): while (str(segno) in exist_catalog_lst): segno += 1 self.segnos_same_path.append(segno) + segno += 1 for k, f in enumerate(self.files_same_path): self.srcfiles.append(f) self.dstfiles.append(self.tabledir + str(self.segnos_same_path[k])) + segno = self.firstsegno + len(self.files_same_path) for f in self.files_append: self.srcfiles.append(f) self.dstfiles.append(self.tabledir + str(segno)) @@ -823,7 +842,7 @@ class HawqRegister(object): if len(same_path_eofs) > 0: query += 'insert into pg_aoseg.%s values(%d, %d, %d, %d)' % (self.seg_name, same_path_segno_lst[0], same_path_eofs[0], same_path_tupcounts[0], same_path_eofuncompresseds[0]) k = 0 - for same_path_eof, same_path_tupcount, same_path_eofuncompressed in zip(same_path_eofs[1:], same_path_tupcounts[1:], same_path_eofuncompresseds[1]): + for same_path_eof, same_path_tupcount, same_path_eofuncompressed in zip(same_path_eofs[1:], same_path_tupcounts[1:], same_path_eofuncompresseds[1:]): query += ',(%d, %d, %d, %d)' % (same_path_segno_lst[k + 1], same_path_eof, same_path_tupcount, same_path_eofuncompressed) k += 1 query += ';'
