Repository: incubator-hawq Updated Branches: refs/heads/master 778ab999a -> 7e11e66a6
HAWQ-1127. HAWQ should print error message instead of python function stack when yaml file is invalid. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/7e11e66a Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/7e11e66a Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/7e11e66a Branch: refs/heads/master Commit: 7e11e66a698dd4b87846c5680dab44df7b740ac6 Parents: 778ab99 Author: Wen Lin <[email protected]> Authored: Mon Oct 31 15:33:26 2016 +0800 Committer: Wen Lin <[email protected]> Committed: Mon Oct 31 15:33:26 2016 +0800 ---------------------------------------------------------------------- .../missing_constraint_partition.yml | 295 +++++++++++++++++++ .../test_hawq_register_usage2_case1.cpp | 1 + tools/bin/hawqregister | 69 +++-- 3 files changed, 334 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e11e66a/src/test/feature/ManagementTool/missing_constraint_partition.yml ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/missing_constraint_partition.yml b/src/test/feature/ManagementTool/missing_constraint_partition.yml new file mode 100755 index 0000000..89c537d --- /dev/null +++ b/src/test/feature/ManagementTool/missing_constraint_partition.yml @@ -0,0 +1,295 @@ +AO_FileLocations: + Blocksize: 32768 + Checksum: false + CompressionLevel: 0 + CompressionType: null + Files: + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/1 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/2 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/3 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/4 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/5 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/6 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/1 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/2 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/3 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/4 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/5 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/6 + size: 0 + tupcount: 0 + varblockcount: 0 + PartitionBy: PARTITION BY list (gender) + Partitions: + - Blocksize: 32768 + Checksum: false + CompressionLevel: 0 + CompressionType: null + Constraint: PARTITION girls VALUES('F') WITH (appendonly=true) + Files: + - eofuncompressed: 432 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD1@/1 + size: 432 + tupcount: 16 + varblockcount: 1 + - eofuncompressed: 408 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD1@/2 + size: 408 + tupcount: 15 + varblockcount: 1 + - eofuncompressed: 464 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD1@/3 + size: 464 + tupcount: 17 + varblockcount: 1 + - eofuncompressed: 408 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD1@/4 + size: 408 + tupcount: 15 + varblockcount: 1 + - eofuncompressed: 464 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD1@/5 + size: 464 + tupcount: 17 + varblockcount: 1 + - eofuncompressed: 536 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD1@/6 + size: 536 + tupcount: 20 + varblockcount: 1 + - eofuncompressed: 280 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW1@/1 + size: 280 + tupcount: 10 + varblockcount: 1 + - eofuncompressed: 224 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW1@/2 + size: 224 + tupcount: 8 + varblockcount: 1 + - eofuncompressed: 224 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW1@/3 + size: 224 + tupcount: 8 + varblockcount: 1 + - eofuncompressed: 256 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW1@/4 + size: 256 + tupcount: 9 + varblockcount: 1 + - eofuncompressed: 200 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW1@/5 + size: 200 + tupcount: 7 + varblockcount: 1 + - eofuncompressed: 224 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW1@/6 + size: 224 + tupcount: 8 + varblockcount: 1 + Name: nt_1_prt_girls + - Blocksize: 32768 + Checksum: false + CompressionLevel: 0 + CompressionType: null + Constraint: PARTITION boys VALUES('M') WITH (appendonly=true) + Files: + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD2@/1 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD2@/2 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD2@/3 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD2@/4 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD2@/5 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD2@/6 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW2@/1 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW2@/2 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW2@/3 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW2@/4 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW2@/5 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW2@/6 + size: 0 + tupcount: 0 + varblockcount: 0 + Name: t_1_prt_boys + - Blocksize: 32768 + Checksum: false + CompressionLevel: 0 + CompressionType: null + Name: nt_1_prt_boys + - Blocksize: 32768 + Checksum: false + CompressionLevel: 0 + CompressionType: null + Constraint: DEFAULT PARTITION other WITH (appendonly=true) + Files: + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD3@/1 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD3@/2 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD3@/3 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD3@/4 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD3@/5 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD3@/6 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW3@/1 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW3@/2 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW3@/3 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW3@/4 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW3@/5 + size: 0 + tupcount: 0 + varblockcount: 0 + - eofuncompressed: 0 + path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW3@/6 + size: 0 + tupcount: 0 + varblockcount: 0 + Name: nt_1_prt_other +AO_Schema: +- name: id + type: int4 +- name: rank + type: int4 +- name: year + type: int4 +- name: gender + type: bpchar +- name: count + type: int4 +Bucketnum: 6 +DBVersion: PostgreSQL 8.2.15 (Greenplum Database 4.2.0 build 1) (HAWQ 2.0.1.0 build + dev) on x86_64-apple-darwin14.5.0, compiled by GCC Apple LLVM version 6.1.0 (clang-602.0.53) + (based on LLVM 3.6.0svn) compiled on Jul 6 2016 10:22:33 +DFS_URL: hdfs://localhost:8020 +Distribution_Policy: DISTRIBUTED BY (id) +Encoding: UTF8 +FileFormat: AO +TableName: public.nt +Version: 1.0.0 http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e11e66a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp index 8411f0c..90aa8f0 100644 --- a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp +++ b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp @@ -40,6 +40,7 @@ TEST_F(TestHawqRegister, TestUsage2Case1IncorrectYaml) { EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_checksum.yml xx")); EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "wrong_dfs_url.yml xx")); EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_bucketnum.yml xx")); + EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c " + filePath + "missing_constraint_partition.yml xx")); } TEST_F(TestHawqRegister, TestUsage2Case1MismatchFileNumber) { http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e11e66a/tools/bin/hawqregister ---------------------------------------------------------------------- diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister index b9b759e..356aef1 100755 --- a/tools/bin/hawqregister +++ b/tools/bin/hawqregister @@ -443,6 +443,7 @@ class HawqRegister(object): print e self.failure_handler.rollback() sys.exit(1) + table_column_num = self.accessor.get_table_column_num(self.tablename) register_yaml_dict_check(params, table_column_num, self.tablename) partitions_filepaths = [] @@ -457,37 +458,43 @@ class HawqRegister(object): partitions_varblockcounts = [] files, sizes, tupcounts, eofuncompresseds, varblockcounts = [], [], [], [], [] - if params['FileFormat'].lower() == 'parquet': - Format = 'Parquet' - else: #AO format - Format = 'AO' - Format_FileLocations = '%s_FileLocations' % Format - partitionby = params.get(Format_FileLocations).get('PartitionBy') - if params.get(Format_FileLocations).get('Partitions') and len(params[Format_FileLocations]['Partitions']): - partitions_checksum = [d['Checksum'] for d in params[Format_FileLocations]['Partitions']] - partitions_compression_level = [d['CompressionLevel'] for d in params[Format_FileLocations]['Partitions']] - partitions_compression_type = [d['CompressionType'] for d in params[Format_FileLocations]['Partitions']] - partitions_constraint = [d['Constraint'] for d in params[Format_FileLocations]['Partitions']] - partitions_files = [d['Files'] for d in params[Format_FileLocations]['Partitions']] - if len(partitions_files): - for pfile in partitions_files: - partitions_filepaths.append([params['DFS_URL'] + item['path'] for item in pfile]) - partitions_filesizes.append([item['size'] for item in pfile]) - partitions_tupcounts.append([item['tupcount'] if item.has_key('tupcount') else -1 for item in pfile]) - partitions_eofuncompresseds.append([item['eofuncompressed'] if item.has_key('eofuncompressed') else -1 for item in pfile]) - partitions_varblockcounts.append([item['varblockcount'] if item.has_key('varblockcount') else -1 for item in pfile]) - partitions_name = [d['Name'] for d in params[Format_FileLocations]['Partitions']] - if len(params[Format_FileLocations]['Files']): - for ele in params[Format_FileLocations]['Files']: - files.append(params['DFS_URL'] + ele['path']) - sizes.append(ele['size']) - tupcounts.append(ele['tupcount'] if ele.has_key('tupcount') else -1) - eofuncompresseds.append(ele['eofuncompressed'] if ele.has_key('eofuncompressed') else -1) - varblockcounts.append(ele['varblockcount'] if ele.has_key('varblockcount') else -1) - - encoding = params['Encoding'] - bucketNum = params['Bucketnum'] if params['Distribution_Policy'].startswith('DISTRIBUTED BY') else 6 - self._set_yml_data(Format, files, sizes, tupcounts, eofuncompresseds, varblockcounts, params['TableName'], params['%s_Schema' % Format], params['Distribution_Policy'], params[Format_FileLocations], bucketNum, partitionby, partitions_constraint, partitions_name, partitions_compression_level, partitions_compression_type, partitions_checksum, partitions_filepaths, partitions_filesizes, partitions_tupcounts, partitions_eofuncompresseds, partitions_varblockcounts, encoding) + try: + if params['FileFormat'].lower() == 'parquet': + Format = 'Parquet' + else: #AO format + Format = 'AO' + Format_FileLocations = '%s_FileLocations' % Format + partitionby = params.get(Format_FileLocations).get('PartitionBy') + if params.get(Format_FileLocations).get('Partitions') and len(params[Format_FileLocations]['Partitions']): + partitions_checksum = [d['Checksum'] for d in params[Format_FileLocations]['Partitions']] + partitions_compression_level = [d['CompressionLevel'] for d in params[Format_FileLocations]['Partitions']] + partitions_compression_type = [d['CompressionType'] for d in params[Format_FileLocations]['Partitions']] + partitions_constraint = [d['Constraint'] for d in params[Format_FileLocations]['Partitions']] + partitions_files = [d['Files'] for d in params[Format_FileLocations]['Partitions']] + if len(partitions_files): + for pfile in partitions_files: + partitions_filepaths.append([params['DFS_URL'] + item['path'] for item in pfile]) + partitions_filesizes.append([item['size'] for item in pfile]) + partitions_tupcounts.append([item['tupcount'] if item.has_key('tupcount') else -1 for item in pfile]) + partitions_eofuncompresseds.append([item['eofuncompressed'] if item.has_key('eofuncompressed') else -1 for item in pfile]) + partitions_varblockcounts.append([item['varblockcount'] if item.has_key('varblockcount') else -1 for item in pfile]) + partitions_name = [d['Name'] for d in params[Format_FileLocations]['Partitions']] + if len(params[Format_FileLocations]['Files']): + for ele in params[Format_FileLocations]['Files']: + files.append(params['DFS_URL'] + ele['path']) + sizes.append(ele['size']) + tupcounts.append(ele['tupcount'] if ele.has_key('tupcount') else -1) + eofuncompresseds.append(ele['eofuncompressed'] if ele.has_key('eofuncompressed') else -1) + varblockcounts.append(ele['varblockcount'] if ele.has_key('varblockcount') else -1) + + encoding = params['Encoding'] + bucketNum = params['Bucketnum'] if params['Distribution_Policy'].startswith('DISTRIBUTED BY') else 6 + self._set_yml_data(Format, files, sizes, tupcounts, eofuncompresseds, varblockcounts, params['TableName'], params['%s_Schema' % Format], params['Distribution_Policy'], params[Format_FileLocations], bucketNum, partitionby, partitions_constraint, partitions_name, partitions_compression_level, partitions_compression_type, partitions_checksum, partitions_filepaths, partitions_filesizes, partitions_tupcounts, partitions_eofuncompresseds, partitions_varblockcounts, encoding) + + except KeyError as e: + print 'Invalid yaml file, %s is missing.' % e + self.failure_handler.rollback() + sys.exit(1) # check conflicting distributed policy
