khtruong commented on a change in pull request #7413: feature: see Presto row
and array data types
URL:
https://github.com/apache/incubator-superset/pull/7413#discussion_r279843023
##########
File path: superset/db_engine_specs.py
##########
@@ -814,6 +822,178 @@ def get_view_names(cls, inspector, schema):
"""
return []
+ @classmethod
+ def _create_column_info(cls, column: RowProxy, name: str, data_type: str)
-> dict:
+ """
+ Create column info object
+ :param column: column object
+ :param name: column name
+ :param data_type: column data type
+ :return: column info object
+ """
+ return {
+ 'name': name,
+ 'type': data_type,
+ # newer Presto no longer includes this column
+ 'nullable': getattr(column, 'Null', True),
+ 'default': None,
+ }
+
+ @classmethod
+ def _get_full_name(cls, names: list) -> str:
+ """
+ Get the full column name
+ :param names: list of all individual column names
+ :return: full column name
+ """
+ return '.'.join(row_type[0] for row_type in names if row_type[0] is
not None)
+
+ @classmethod
+ def _has_nested_data_types(cls, component_type: str) -> bool:
+ """
+ Check if string contains a data type. We determine if there is a data
type by
+ whitespace or multiple data types by commas
+ :param component_type: data type
+ :return: boolean
+ """
+ comma_regex = r',(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)'
+ white_space_regex = r'\s(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)'
+ return re.search(comma_regex, component_type) is not None \
+ or re.search(white_space_regex, component_type) is not None
+
+ @classmethod
+ def _split_data_type(cls, data_type: str, delimiter: str) -> list:
+ """
+ Split data type based on given delimiter. Do not split the string if
the
+ delimiter is enclosed in quotes
+ :param data_type: data type
+ :param delimiter: string separator (i.e. open parenthesis, closed
parenthesis,
+ comma, whitespace)
+ :return:list of strings after breaking it by the delimiter
+ """
+ return re.split(
+ r'{}(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)'.format(delimiter), data_type)
+
+ @classmethod
+ def _parse_structural_column(cls, column: RowProxy, result: list) -> None:
+ """
+ Parse a row or array column
+ :param column: column
+ :param result: list tracking the results
+ """
+ full_data_type = '{} {}'.format(column.Column, column.Type)
+ # split on open parenthesis ( to get the structural
+ # data type and its component types
+ data_types = cls._split_data_type(full_data_type, r'\(')
+ stack: list = []
+ for data_type in data_types:
+ # split on closed parenthesis ) to track which component
+ # types belong to what structural data type
+ inner_types = cls._split_data_type(data_type, r'\)')
+ for inner_type in inner_types:
+ # We have finished parsing multiple structural data types
+ if not inner_type and len(stack) > 0:
+ stack.pop()
+ elif cls._has_nested_data_types(inner_type):
+ # split on comma , to get individual data types
+ single_fields = cls._split_data_type(inner_type, ', ')
+ for single_field in single_fields:
+ # If component type starts with a comma, the first
single field
+ # will be an empty string. Disregard this empty string.
+ if not single_field:
+ continue
+ # split on whitespace to get field name and data type
+ field_info = cls._split_data_type(single_field, r'\s')
+ # check if there is a structural data type within
+ # overall structural data type
+ if field_info[1] == 'array' or field_info[1] == 'row':
+ stack.append((field_info[0], field_info[1]))
+ full_parent_path = cls._get_full_name(stack)
+ result.append(cls._create_column_info(
+ column, full_parent_path,
+ presto_type_map[field_info[1]]()))
+ else: # otherwise this field is a basic data type
+ full_parent_path = cls._get_full_name(stack)
+ column_name = '{}.{}'.format(full_parent_path,
field_info[0])
+ result.append(cls._create_column_info(
+ column, column_name,
presto_type_map[field_info[1]]()))
+ # If the component type ends with a structural data type,
do not pop
+ # the stack. We have run across a structural data type
within the
+ # overall structural data type. Otherwise, we have
completely parsed
+ # through the entire structural data type and can move on.
+ if not (inner_type.endswith('array') or
inner_type.endswith('row')):
+ stack.pop()
+ # We have an array of row objects (i.e. array(row(...)))
+ elif 'array' == inner_type or 'row' == inner_type:
+ # Push a dummy object to represent the structural data type
+ stack.append((None, inner_type))
+ # We have an array of a basic data types(i.e. array(varchar)).
+ elif len(stack) > 0:
+ # Because it is an array of a basic data type. We have
finished
+ # parsing the structural data type and can move on.
+ stack.pop()
+
+ @classmethod
+ def _show_columns(cls, inspector: Inspector, table_name: str, schema: str)
-> list:
+ """
+ Show presto column names
+ :param inspector: object that performs database schema inspection
+ :param table_name: table name
+ :param schema: schema name
+ :return: list of column objects
+ """
+ quote = inspector.engine.dialect.identifier_preparer.quote_identifier
+ full_table = quote(table_name)
+ if schema:
+ full_table = '{}.{}'.format(quote(schema), full_table)
+ columns = inspector.bind.execute('SHOW COLUMNS FROM
{}'.format(full_table))
Review comment:
We use pyhive to get the columns, which does not handle complex types. It
will detect row and array objects as unknown types and assign it as a null data
type.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
