silvermissile opened a new issue #2673: URL: https://github.com/apache/iceberg/issues/2673
I test create external table using hive on an Exiting iceberg table ,but happen to `MetaException(message:org.apache.hadoop.hive.serde2.SerDeException Please provide an existing table or a valid schema)` same as here https://github.com/apache/iceberg/issues/1884 I'm not sure whether I understand this doc correctly http://iceberg.apache.org/hive/ . My attempt is as follows: ### 1. create iceberg table ``` java SparkSession spark = SparkSession.builder() .master("local[*]") .appName("iceberg demo") .enableHiveSupport().getOrCreate(); Configuration hadoopConfiguration = spark.sparkContext().hadoopConfiguration(); HadoopCatalog catalog = new HadoopCatalog(hadoopConfiguration, "hdfs:///data/hive236/warehouse"); Map<String, String> tableProperties = new HashMap<>(); List<Types.NestedField> fields = new LinkedList<Types.NestedField>(); fields.add(Types.NestedField.optional(1, "city_name", Types.StringType.get())); fields.add(Types.NestedField.optional(2, "item_id", Types.LongType.get())); fields.add(Types.NestedField.optional(3, "price", Types.DoubleType.get())); fields.add(Types.NestedField.optional(4, "time_stamp", Types.LongType.get())); fields.add(Types.NestedField.optional(5, "user_id", Types.LongType.get())); Schema schema = new Schema(fields); TableIdentifier tableIdentifier = TableIdentifier.of(Namespace.of("tmp"), "test_hive_ex"); catalog.createTable(tableIdentifier, schema, null, tableProperties); ``` ### 2. verify table on hdfs ``` sh hdfs dfs -du -h /data/hive236/warehouse/tmp/test_hive_ex/ 1.1 K 3.3 K /data/hive236/warehouse/tmp/test_hive_ex/metadata hdfs dfs -cat /data/hive236/warehouse/tmp/test_hive_ex/metadata/version-hint.text 1 ``` `hdfs dfs -cat /data/hive236/warehouse/tmp/test_hive_ex/metadata/v1.metadata.json` ``` json { "format-version" : 1, "table-uuid" : "38f8c458-5d56-43bd-8bb9-ef52ed8e4cf3", "location" : "hdfs:///data/hive236/warehouse/tmp/test_hive_ex", "last-updated-ms" : 1622777931088, "last-column-id" : 5, "schema" : { "type" : "struct", "fields" : [ { "id" : 1, "name" : "city_name", "required" : false, "type" : "string" }, { "id" : 2, "name" : "item_id", "required" : false, "type" : "long" }, { "id" : 3, "name" : "price", "required" : false, "type" : "double" }, { "id" : 4, "name" : "time_stamp", "required" : false, "type" : "long" }, { "id" : 5, "name" : "user_id", "required" : false, "type" : "long" } ] }, "partition-spec" : [ ], "default-spec-id" : 0, "partition-specs" : [ { "spec-id" : 0, "fields" : [ ] } ], "default-sort-order-id" : 0, "sort-orders" : [ { "order-id" : 0, "fields" : [ ] } ], "properties" : { }, "current-snapshot-id" : -1, "snapshots" : [ ], "snapshot-log" : [ ], "metadata-log" : [ ] } ``` ### 3. create external table ``` sh CREATE EXTERNAL table tmp.test_hive_ex STORED BY 'org.apache.iceberg.mr.hive.HiveIcebergStorageHandler' LOCATION '/data/hive236/warehouse/tmp/test_hive_ex'; or CREATE EXTERNAL TABLE tmp.test_hive_ex STORED BY 'org.apache.iceberg.mr.hive.HiveIcebergStorageHandler' LOCATION '/data/hive236/warehouse/tmp/test_hive_ex' TBLPROPERTIES ( 'iceberg.mr.catalog'='hadoop', 'iceberg.mr.catalog.hadoop.warehouse.location'='/data/hive236/warehouse') ; Error: org.apache.hive.service.cli.HiveSQLException: Error while processing statement: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. java.lang.RuntimeException: MetaException(message:org.apache.hadoop.hive.serde2.SerDeException Please provide an existing table or a valid schema) at org.apache.hive.service.cli.operation.Operation.toSQLException(Operation.java:380) at org.apache.hive.service.cli.operation.SQLOperation.runQuery(SQLOperation.java:257) at org.apache.hive.service.cli.operation.SQLOperation.access$800(SQLOperation.java:91) at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork$1.run(SQLOperation.java:348) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1917) at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork.run(SQLOperation.java:362) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.RuntimeException: MetaException(message:org.apache.hadoop.hive.serde2.SerDeException Please provide an existing table or a valid schema) at org.apache.hadoop.hive.ql.metadata.Hive.createTable(Hive.java:862) at org.apache.hadoop.hive.ql.metadata.Hive.createTable(Hive.java:867) at org.apache.hadoop.hive.ql.exec.DDLTask.createTable(DDLTask.java:4356) at org.apache.hadoop.hive.ql.exec.DDLTask.execute(DDLTask.java:354) at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:199) at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:100) at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:2183) at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1839) at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1526) at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1237) at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1232) at org.apache.hive.service.cli.operation.SQLOperation.runQuery(SQLOperation.java:255) ... 11 more Caused by: java.lang.RuntimeException: MetaException(message:org.apache.hadoop.hive.serde2.SerDeException Please provide an existing table or a valid schema) at org.apache.hadoop.hive.ql.metadata.Table.getDeserializerFromMetaStore(Table.java:283) at org.apache.hadoop.hive.ql.metadata.Table.getDeserializer(Table.java:263) at org.apache.hadoop.hive.ql.metadata.Hive.createTable(Hive.java:833) ... 22 more Caused by: MetaException(message:org.apache.hadoop.hive.serde2.SerDeException Please provide an existing table or a valid schema) at org.apache.hadoop.hive.metastore.MetaStoreUtils.getDeserializer(MetaStoreUtils.java:458) at org.apache.hadoop.hive.metastore.MetaStoreUtils.getDeserializer(MetaStoreUtils.java:437) at org.apache.hadoop.hive.ql.metadata.Table.getDeserializerFromMetaStore(Table.java:281) ... 24 more (state=08S01,code=1) ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
