Repository: incubator-hawq Updated Branches: refs/heads/master 45adcbdb3 -> 778ab999a
HAWQ-1034. Delete code for hawq register repair mode Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/778ab999 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/778ab999 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/778ab999 Branch: refs/heads/master Commit: 778ab999a95c36999430fea6ea29e7839106d9d1 Parents: 45adcbd Author: Chunling Wang <[email protected]> Authored: Mon Oct 31 13:50:14 2016 +0800 Committer: Chunling Wang <[email protected]> Committed: Mon Oct 31 13:50:14 2016 +0800 ---------------------------------------------------------------------- .../test_hawq_register_usage2_case3.cpp | 81 -------------------- .../ManagementTool/usage2case3/t_tpl_old_1.yml | 31 -------- .../ManagementTool/usage2case3/t_tpl_old_2.yml | 33 -------- .../ManagementTool/usage2case3/t_tpl_old_3.yml | 21 ----- .../ManagementTool/usage2case3/t_tpl_old_4.yml | 23 ------ tools/bin/hawqregister | 63 +++------------ 6 files changed, 12 insertions(+), 240 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/778ab999/src/test/feature/ManagementTool/test_hawq_register_usage2_case3.cpp ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/test_hawq_register_usage2_case3.cpp b/src/test/feature/ManagementTool/test_hawq_register_usage2_case3.cpp deleted file mode 100644 index 2b4d07b..0000000 --- a/src/test/feature/ManagementTool/test_hawq_register_usage2_case3.cpp +++ /dev/null @@ -1,81 +0,0 @@ -#include <string> - -#include "lib/command.h" -#include "lib/sql_util.h" -#include "lib/string_util.h" -#include "lib/hdfs_config.h" -#include "lib/file_replace.h" -#include "test_hawq_register.h" - -#include "gtest/gtest.h" - -using std::string; -using hawq::test::SQLUtility; -using hawq::test::Command; -using hawq::test::HdfsConfig; - -TEST_F(TestHawqRegister, DISABLED_TestUsage2Case3Expected) { - SQLUtility util; - string test_root(util.getTestRootPath()); - std::vector<string> create_table_matrix = {"distributed by (i)", "distributed randomly"}; - std::vector<string> fmt_matrix = {"row", "parquet"}; - std::vector<string> option_matrix = {"--repair", "-R"}; - int suffix=0; - - for (auto & opt : option_matrix) { - suffix = 0; - for (auto & ddl : create_table_matrix) { - for (auto & fmt : fmt_matrix) { - suffix++; - string t_yml_tpl_old(hawq::test::stringFormat("%s/ManagementTool/usage2case3/t_tpl_old_%s.yml", test_root.c_str(), std::to_string(suffix).c_str())); - string t_yml_old(hawq::test::stringFormat("%s/ManagementTool/usage2case3/t_old_%s.yml", test_root.c_str(), std::to_string(suffix).c_str())); - auto t = hawq::test::stringFormat("t_usage2_case3_%s", std::to_string(suffix).c_str()); - auto nt = hawq::test::stringFormat("nt_usage2_case3_%s", std::to_string(suffix).c_str()); - util.execute(hawq::test::stringFormat("drop table if exists %s;", t.c_str())); - util.execute(hawq::test::stringFormat("drop table if exists %s;", nt.c_str())); - - // hawq register -d hawq_feature_test -c t_usage2_case3_#.yml nt_usage2_case3_#, where nt_usage2_case3_# does not exist - util.execute(hawq::test::stringFormat("create table %s(i int) with (appendonly=true, orientation=%s) %s;", t.c_str(), fmt.c_str(), ddl.c_str())); - util.execute(hawq::test::stringFormat("insert into %s select generate_series(1, 100);", t.c_str())); - util.execute(hawq::test::stringFormat("insert into %s select generate_series(101, 200);", t.c_str())); - util.query(hawq::test::stringFormat("select * from %s;", t.c_str()), 200); - EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq extract -d %s -o t_%s.yml testhawqregister_testusage2case3expected.%s", HAWQ_DB, std::to_string(suffix).c_str(), t.c_str()))); - EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c t_%s.yml testhawqregister_testusage2case3expected.%s", HAWQ_DB, std::to_string(suffix).c_str(), nt.c_str()))); - util.query(hawq::test::stringFormat("select * from %s;", nt.c_str()), 200); - - // hawq register -d hawq_feature_test -c t_usage2_case3_#.yml nt_usage2_case3_#, where nt_usage2_case3_# exists - util.execute(hawq::test::stringFormat("drop table if exists %s;", t.c_str())); - util.execute(hawq::test::stringFormat("create table %s(i int) with (appendonly=true, orientation=%s) %s;", t.c_str(), fmt.c_str(), ddl.c_str())); - util.execute(hawq::test::stringFormat("insert into %s select generate_series(101, 150);", t.c_str())); - util.query(hawq::test::stringFormat("select * from %s;", t.c_str()), 50); - EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq extract -d %s -o t_%s.yml testhawqregister_testusage2case3expected.%s", HAWQ_DB, std::to_string(suffix).c_str(), t.c_str()))); - EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c t_%s.yml testhawqregister_testusage2case3expected.%s", HAWQ_DB, std::to_string(suffix).c_str(), nt.c_str()))); - util.query(hawq::test::stringFormat("select * from %s;", nt.c_str()), 250); - - // hawq register --repair/-R -d hawq_feature_test -c t_old_#.yml nt_usage2_case3_# - hawq::test::FileReplace frep; - std::unordered_map<std::string, std::string> strs_src_dst; - strs_src_dst["@DATABASE_OID@"]= getDatabaseOid(); - strs_src_dst["@TABLE_OID@"]= getTableOid(nt); - string hdfs_prefix; - hawq::test::HdfsConfig hc; - hc.getNamenodeHost(hdfs_prefix); - strs_src_dst["@PORT@"]= hdfs_prefix; - frep.replace(t_yml_tpl_old, t_yml_old, strs_src_dst); - EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq register %s -d %s -c %s testhawqregister_testusage2case3expected.%s", opt.c_str(), HAWQ_DB, t_yml_old.c_str(), nt.c_str()))); - util.query(hawq::test::stringFormat("select * from %s;", nt.c_str()), 100); - - if (fmt == "row") - checkPgAOSegValue(nt, "-1", "aoseg"); - else - checkPgAOSegValue(nt, "-1", "paqseg"); - - EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf t_%s.yml", std::to_string(suffix).c_str()))); - EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf %s", t_yml_old.c_str()))); - util.execute(hawq::test::stringFormat("drop table %s;", t.c_str())); - util.execute(hawq::test::stringFormat("drop table %s;", nt.c_str())); - } - } - } -} - http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/778ab999/src/test/feature/ManagementTool/usage2case3/t_tpl_old_1.yml ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_1.yml b/src/test/feature/ManagementTool/usage2case3/t_tpl_old_1.yml deleted file mode 100644 index 3ec2f4e..0000000 --- a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_1.yml +++ /dev/null @@ -1,31 +0,0 @@ -AO_FileLocations: - Blocksize: 32768 - Checksum: false - CompressionLevel: 0 - CompressionType: null - Files: - - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/1 - size: 176 - - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/2 - size: 168 - - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/3 - size: 192 - - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/4 - size: 168 - - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/5 - size: 192 - - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/6 - size: 216 -AO_Schema: -- name: i - type: int4 -Bucketnum: 6 -DBVersion: PostgreSQL 8.2.15 (Greenplum Database 4.2.0 build 1) (HAWQ 2.0.1.0 build - dev) on x86_64-apple-darwin14.5.0, compiled by GCC Apple LLVM version 6.1.0 (clang-602.0.53) - (based on LLVM 3.6.0svn) compiled on Jul 6 2016 10:22:33 -DFS_URL: hdfs://@PORT@ -Distribution_Policy: DISTRIBUTED BY (i) -Encoding: UTF8 -FileFormat: AO -TableName: public.t_1 -Version: 1.0.0 http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/778ab999/src/test/feature/ManagementTool/usage2case3/t_tpl_old_2.yml ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_2.yml b/src/test/feature/ManagementTool/usage2case3/t_tpl_old_2.yml deleted file mode 100644 index 1a2a459..0000000 --- a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_2.yml +++ /dev/null @@ -1,33 +0,0 @@ -Bucketnum: 6 -DBVersion: PostgreSQL 8.2.15 (Greenplum Database 4.2.0 build 1) (HAWQ 2.0.1.0 build - dev) on x86_64-apple-darwin14.5.0, compiled by GCC Apple LLVM version 6.1.0 (clang-602.0.53) - (based on LLVM 3.6.0svn) compiled on Jul 6 2016 10:22:33 -DFS_URL: hdfs://@PORT@ -Distribution_Policy: DISTRIBUTED BY (i) -Encoding: UTF8 -FileFormat: Parquet -Parquet_FileLocations: - Checksum: false - CompressionLevel: 0 - CompressionType: null - EnableDictionary: false - Files: - - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/1 - size: 254 - - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/2 - size: 250 - - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/3 - size: 258 - - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/4 - size: 250 - - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/5 - size: 258 - - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/6 - size: 270 - PageSize: 1048576 - RowGroupSize: 8388608 -Parquet_Schema: -- name: i - type: int4 -TableName: public.t_2 -Version: 1.0.0 http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/778ab999/src/test/feature/ManagementTool/usage2case3/t_tpl_old_3.yml ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_3.yml b/src/test/feature/ManagementTool/usage2case3/t_tpl_old_3.yml deleted file mode 100644 index df6730e..0000000 --- a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_3.yml +++ /dev/null @@ -1,21 +0,0 @@ -AO_FileLocations: - Blocksize: 32768 - Checksum: false - CompressionLevel: 0 - CompressionType: null - Files: - - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/1 - size: 1016 -AO_Schema: -- name: i - type: int4 -Bucketnum: 6 -DBVersion: PostgreSQL 8.2.15 (Greenplum Database 4.2.0 build 1) (HAWQ 2.0.1.0 build - dev) on x86_64-apple-darwin14.5.0, compiled by GCC Apple LLVM version 6.1.0 (clang-602.0.53) - (based on LLVM 3.6.0svn) compiled on Jul 6 2016 10:22:33 -DFS_URL: hdfs://@PORT@ -Distribution_Policy: DISTRIBUTED RANDOMLY -Encoding: UTF8 -FileFormat: AO -TableName: public.t_3 -Version: 1.0.0 http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/778ab999/src/test/feature/ManagementTool/usage2case3/t_tpl_old_4.yml ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_4.yml b/src/test/feature/ManagementTool/usage2case3/t_tpl_old_4.yml deleted file mode 100644 index 8df2afc..0000000 --- a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_4.yml +++ /dev/null @@ -1,23 +0,0 @@ -Bucketnum: 6 -DBVersion: PostgreSQL 8.2.15 (Greenplum Database 4.2.0 build 1) (HAWQ 2.0.1.0 build - dev) on x86_64-apple-darwin14.5.0, compiled by GCC Apple LLVM version 6.1.0 (clang-602.0.53) - (based on LLVM 3.6.0svn) compiled on Jul 6 2016 10:22:33 -DFS_URL: hdfs://@PORT@ -Distribution_Policy: DISTRIBUTED RANDOMLY -Encoding: UTF8 -FileFormat: Parquet -Parquet_FileLocations: - Checksum: false - CompressionLevel: 0 - CompressionType: null - EnableDictionary: false - Files: - - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/1 - size: 596 - PageSize: 1048576 - RowGroupSize: 8388608 -Parquet_Schema: -- name: i - type: int4 -TableName: public.t_4 -Version: 1.0.0 http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/778ab999/tools/bin/hawqregister ---------------------------------------------------------------------- diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister index 67a188d..b9b759e 100755 --- a/tools/bin/hawqregister +++ b/tools/bin/hawqregister @@ -18,7 +18,7 @@ # under the License. # Usage1: hawq register [-h hostname] [-p port] [-U username] [-d database] [-f filepath] [-e eof] <tablename> -# Usage2: hawq register [-h hostname] [-p port] [-U username] [-d database] [-c config] [--force] [--repair] <tablename> +# Usage2: hawq register [-h hostname] [-p port] [-U username] [-d database] [-c config] [--force] <tablename> import os import sys @@ -54,7 +54,6 @@ def option_parser(): parser.add_option('-e', '--eof', dest='filesize', type='int', default=None, help='eof of the file to be registered') parser.add_option('-c', '--config', dest='yml_config', default='', help='configuration file in YAML format') parser.add_option('-F', '--force', dest='force', action='store_true', default=False) - parser.add_option('-R', '--repair', dest='repair', action='store_true', default=False) return parser @@ -340,7 +339,7 @@ class HawqRegister(object): self.accessor = GpRegisterAccessor(conn) self.utility_accessor = GpRegisterAccessor(utility_conn) self.failure_handler = failure_handler - self.mode = self._init_mode(options.force, options.repair) + self.mode = self._init_mode(options.force) self.srcfiles = [] self.dstfiles = [] self.files_same_path = [] @@ -351,20 +350,13 @@ class HawqRegister(object): self.eofuncompresseds_same_path = [] self.segnos_same_path = [] - def _init_mode(self, force, repair): + def _init_mode(self, force): def table_existed(): return self.accessor.get_table_existed(self.dst_table_name) if self.yml: if force: return 'force' - elif repair: - logger.info('--repair mode is not supported in current release of hawq register.') - sys.exit(0) - if not table_existed(): - logger.error('--repair mode asserts the table has been already created.') - sys.exit(1) - return 'repair' else: return 'usage2_table_not_exist' else: @@ -583,9 +575,8 @@ class HawqRegister(object): self.filepath = self.files[0][:self.files[0].rfind('/')] if self.files else '' self._check_file_not_folder() self._check_database_encoding() - if self.mode != 'repair': - if not self._create_table() and self.mode != 'force': - self.mode = 'usage2_table_exist' + if not self._create_table() and self.mode != 'force': + self.mode = 'usage2_table_exist' else: if self._is_folder(self.filepath) and self.filesize: logger.error('-e option is only supported with single file case.') @@ -628,23 +619,6 @@ class HawqRegister(object): logger.error("Hawq aoseg metadata doesn't consistent with file numbers in hdfs.") self.failure_handler.rollback() sys.exit(1) - if self.mode == 'repair': - if self.tabledir.strip('/') != self.filepath.strip('/'): - logger.error("In repair mode, file path from yaml file should be the same with table's path.") - self.failure_handler.rollback() - sys.exit(1) - existed_info = {} - for k, fn in enumerate(existed_files): - existed_info[fn] = existed_sizes[k] - for k, fn in enumerate(self.files): - if fn not in existed_files: - logger.error('Can not register in repair mode since giving non-existing file: %s.' % fn) - self.failure_handler.rollback() - sys.exit(1) - if self.sizes[k] > existed_info[fn]: - logger.error('Can not register in repair mode since giving larger file size: %s' % self.sizes[k]) - self.failure_handler.rollback() - sys.exit(1) if self.mode == 'usage2_table_exist': if self.tabledir.strip('/') == self.filepath.strip('/'): @@ -689,14 +663,6 @@ class HawqRegister(object): logger.error('In force mode, you should include existing table files in yaml configuration file. Otherwise you should drop the previous table before register --force.') self.failure_handler.rollback() sys.exit(1) - elif self.mode == 'repair': - self.do_not_move = True - self.files_update, self.sizes_update = [fn for fn in self.files], [sz for sz in self.sizes] - self.files_delete = [] - for fn in existed_files: - if fn not in self.files: - self.files_delete.append(fn) - self.files_append, self.sizes_append = [], [] self._check_files_and_table_in_same_hdfs_cluster(self.filepath, self.tabledir) @@ -963,7 +929,7 @@ class HawqRegister(object): def _prepare_register(self): if not self.do_not_move: self._set_move_files_in_hdfs() - if self.mode == 'force' or self.mode == 'repair': + if self.mode == 'force': self._set_modify_metadata('force') else: self._set_modify_metadata('insert') @@ -972,10 +938,6 @@ class HawqRegister(object): if not self.do_not_move: self._move_files_in_hdfs() self._modify_metadata() - if self.mode == 'repair': - if self.files_delete: - self._delete_files_in_hdfs() - self._delete_metadata() logger.info('Hawq Register Succeed.') @@ -1033,11 +995,10 @@ class HawqRegisterPartition(HawqRegister): self._check_file_not_folder() if self.yml: self._check_database_encoding() - if self.mode != 'repair': - if not self._create_table() and self.mode != 'force': - self.mode = 'usage2_table_exist' - self._check_partitionby() - self._check_partition_num() + if not self._create_table() and self.mode != 'force': + self.mode = 'usage2_table_exist' + self._check_partitionby() + self._check_partition_num() partitions = self._get_partition_info() self.queries = "set allow_system_table_mods='dml';" self.queries += "begin transaction;" @@ -1124,10 +1085,10 @@ def test(options, args): if __name__ == '__main__': parser = option_parser() options, args = parser.parse_args() - if len(args) != 1 or (options.force and options.repair): + if len(args) != 1: parser.print_help(sys.stderr) sys.exit(1) - if (options.yml_config or options.force or options.repair) and options.filepath: + if (options.yml_config or options.force) and options.filepath: parser.print_help(sys.stderr) sys.exit(1) if local_ssh('hdfs'):
