Repository: incubator-hawq Updated Branches: refs/heads/master 676eb828c -> c8ef3a0ac
HAWQ-1099. Output yaml file should not contain Bucketnum attribute with random distributed table. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/c8ef3a0a Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/c8ef3a0a Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/c8ef3a0a Branch: refs/heads/master Commit: c8ef3a0ac847743dbed0d0715b6534e34f95d92a Parents: 676eb82 Author: xunzhang <xunzhang...@gmail.com> Authored: Thu Oct 13 18:25:40 2016 +0800 Committer: Wen Lin <w...@pivotal.io> Committed: Tue Oct 18 13:37:51 2016 +0800 ---------------------------------------------------------------------- tools/bin/hawqextract | 6 ++++-- tools/bin/hawqregister | 15 ++++++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c8ef3a0a/tools/bin/hawqextract ---------------------------------------------------------------------- diff --git a/tools/bin/hawqextract b/tools/bin/hawqextract index 28f55d6..3a6ef66 100644 --- a/tools/bin/hawqextract +++ b/tools/bin/hawqextract @@ -421,7 +421,8 @@ def extract_metadata(conn, tbname): logger.info('-- extract Distribution_Policy') metadata['Distribution_Policy'] = accessor.get_distribution_policy_info(rel_pgclass['oid'], relid) logger.info('-- extract bucket number') - metadata['Bucketnum'] = accessor.get_bucket_number(rel_pgclass['oid']) + if accessor.get_distribution_policy_info(rel_pgclass['oid'], relid).startswith('DISTRIBUTED BY'): + metadata['Bucketnum'] = accessor.get_bucket_number(rel_pgclass['oid']) def extract_Parquet_metadata(): relid = rel_pgclass['oid'] @@ -471,7 +472,8 @@ def extract_metadata(conn, tbname): logger.info('-- extract Distribution_Policy') metadata['Distribution_Policy'] = accessor.get_distribution_policy_info(rel_pgclass['oid'], relid) logger.info('-- extract bucket number') - metadata['Bucketnum'] = accessor.get_bucket_number(rel_pgclass['oid']) + if accessor.get_distribution_policy_info(rel_pgclass['oid'], relid).startswith('DISTRIBUTED BY'): + metadata['Bucketnum'] = accessor.get_bucket_number(rel_pgclass['oid']) # extract AO/Parquet specific metadata http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c8ef3a0a/tools/bin/hawqregister ---------------------------------------------------------------------- diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister index 29b3a30..a1c8e29 100755 --- a/tools/bin/hawqregister +++ b/tools/bin/hawqregister @@ -60,14 +60,18 @@ def option_parser(): def register_yaml_dict_check(D, table_column_num, src_tablename): '''check exists''' - check_list = ['DFS_URL', 'Distribution_Policy', 'FileFormat', 'TableName', 'Bucketnum'] + check_list = ['DFS_URL', 'Distribution_Policy', 'FileFormat', 'TableName'] for attr in check_list: if D.get(attr) is None: logger.error('Wrong configuration yaml file format: "%s" attribute does not exist.\n See example in "hawq register --help".' % attr) sys.exit(1) - if D['Bucketnum'] <= 0: - logger.error('Bucketnum should not be zero, please check your yaml configuration file.') - sys.exit(1) + if D['Distribution_Policy'].startswith('DISTRIBUTED BY'): + if D.get('Bucketnum') is None: + logger.error('Wrong configuration yaml file format: "%s" attribute does not exist.\n See example in "hawq register --help".' % attr) + sys.exit(1) + if D['Bucketnum'] <= 0: + logger.error('Bucketnum should not be zero, please check your yaml configuration file.') + sys.exit(1) if D['FileFormat'] in ['Parquet', 'AO']: prefix = D['FileFormat'] local_check_list = ['%s_FileLocations' % prefix, '%s_Schema' % prefix] @@ -432,7 +436,8 @@ class HawqRegister(object): if len(params[Format_FileLocations]['Files']): files, sizes = [params['DFS_URL'] + d['path'] for d in params[Format_FileLocations]['Files']], [d['size'] for d in params[Format_FileLocations]['Files']] encoding = params['Encoding'] - self._set_yml_dataa(Format, files, sizes, params['TableName'], params['%s_Schema' % Format], params['Distribution_Policy'], params[Format_FileLocations], params['Bucketnum'], partitionby,\ + bucketNum = params['Bucketnum'] if params['Distribution_Policy'].startswith('DISTRIBUTED BY') else 6 + self._set_yml_dataa(Format, files, sizes, params['TableName'], params['%s_Schema' % Format], params['Distribution_Policy'], params[Format_FileLocations], bucketNum, partitionby,\ partitions_constraint, partitions_name, partitions_compression_level, partitions_compression_type, partitions_checksum, partitions_filepaths, partitions_filesizes, encoding)