Fokko commented on code in PR #5447:
URL: https://github.com/apache/iceberg/pull/5447#discussion_r940156778
##########
python/pyiceberg/catalog/hive.py:
##########
@@ -159,16 +156,56 @@ def _construct_parameters(metadata_location: str,
previous_metadata_location: Op
def _annotate_namespace(database: HiveDatabase, properties: Properties) ->
HiveDatabase:
params = {}
for key, value in properties.items():
- if key == "comment":
+ if key == COMMENT:
database.description = value
- elif key == "location":
+ elif key == LOCATION:
database.locationUri = value
else:
params[key] = value
database.parameters = params
return database
+hive_primitive_types = {
+ BooleanType: "boolean",
+ IntegerType: "int",
+ LongType: "bigint",
+ FloatType: "float",
+ DoubleType: "double",
+ DateType: "date",
+ TimeType: "string",
+ TimestampType: "timestamp",
+ StringType: "string",
+ UUIDType: "string",
+ BinaryType: "binary",
+ FixedType: "binary",
+}
+
+
+class HiveSchemaConstructor(SchemaVisitor[str]):
+ def schema(self, schema: Schema, struct_result: str) -> str:
+ return struct_result
+
+ def struct(self, struct: StructType, field_results: List[str]) -> str:
+ return f"struct<{', '.join(field_results)}>"
+
+ def field(self, field: NestedField, field_result: str) -> str:
+ return f"{field.name}: {field_result}"
Review Comment:
Ahh, great catch! It was writing fine, but when reading the table it would
throw an error:
```
Error: Error while processing statement: FAILED: Execution Error, return
code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Error: type expected at the
position 83 of
'string:int:boolean:array<string>:map<string,map<string,int>>:array<struct<latitude:
float,longitude: float>>:struct<name: string,age: int>' but ' float' is found.
(state=08S01,code=1)
```
Blame it on validate on read.
After removing all the spaces, it works fine:
```
+-------------------------------+----------------------------------------------------+----------------------------------------------------+
| col_name | data_type
| comment |
+-------------------------------+----------------------------------------------------+----------------------------------------------------+
| # col_name | data_type
| comment |
| foo | string
| |
| bar | int
| |
| baz | boolean
| |
| qux | array<string>
| |
| quux | map<string,map<string,int>>
| |
| location |
array<struct<latitude:float,longitude:float>> |
|
| person | struct<name:string,age:int>
| |
| | NULL
| NULL |
| # Detailed Table Information | NULL
| NULL |
| Database: | nyc
| NULL |
| OwnerType: | USER
| NULL |
| Owner: | fokkodriesprong
| NULL |
| CreateTime: | Mon Aug 08 11:58:50 UTC 2022
| NULL |
| LastAccessTime: | Mon Aug 08 11:58:50 UTC 2022
| NULL |
| Retention: | 0
| NULL |
| Location: |
file:/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/nyc.db/complex |
NULL |
| Table Type: | EXTERNAL_TABLE
| NULL |
| Table Parameters: | NULL
| NULL |
| | EXTERNAL
| TRUE |
| | metadata_location
|
file:/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/nyc.db/complex/metadata/3400b5e4-d454-4788-b269-18ea6a4beac0.metadata.json
|
| | numFiles
| 0 |
| | table_type
| ICEBERG |
| | totalSize
| 0 |
| | transient_lastDdlTime
| 1659959930 |
| | NULL
| NULL |
| # Storage Information | NULL
| NULL |
| SerDe Library: |
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe | NULL
|
| InputFormat: | org.apache.hadoop.mapred.FileInputFormat
| NULL |
| OutputFormat: | org.apache.hadoop.mapred.FileOutputFormat
| NULL |
| Compressed: | No
| NULL |
| Num Buckets: | 0
| NULL |
| Bucket Columns: | []
| NULL |
| Sort Columns: | []
| NULL |
+-------------------------------+----------------------------------------------------+----------------------------------------------------+
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
