[
https://issues.apache.org/jira/browse/HIVE-27669?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Work on HIVE-27669 started by Harshal Patel.
--------------------------------------------
> Hive Acid CTAS fails incremental if no of rows inserted is > INT_MAX
> --------------------------------------------------------------------
>
> Key: HIVE-27669
> URL: https://issues.apache.org/jira/browse/HIVE-27669
> Project: Hive
> Issue Type: Bug
> Reporter: Harshal Patel
> Assignee: Harshal Patel
> Priority: Major
>
> * If a Table is created using CTAS with rows > INT_MAX then beeline eats up
> the thrown error
> * As replication also uses the same infra it should also do the same
> instead of failing with NumberFormatException
> *Note:* This is happening in the customer's environment consistently but we
> are not able to reproduce it. So, we have gone through the whole code flow
> and handled the error accordingly.
>
> Error message while incremental replication:
> {code:java}
> 4:12:03.230 PM INFO Driver [Scheduled Query
> Executor(schedule:repl_sample_acid_1, execution_id:49625)]: Starting task
> [Stage-10066:REPL_STATE_LOG] in serial mode4:12:03.231 PM INFO
> ReplState [Scheduled Query Executor(schedule:repl_sample_acid_1,
> execution_id:49625)]: REPL::EVENT_LOAD:
> {"dbName":"sample","eventId":"50442182","eventType":"EVENT_ALLOC_WRITE_ID","eventsLoadProgress":"2443/20424","loadTime":1687187523,"eventDuration":"159
> ms"}4:12:03.231 PM INFO Driver [Scheduled Query
> Executor(schedule:repl_sample_acid_1, execution_id:49625)]: Starting task
> [Stage-10067:COLUMNSTATS] in serial mode4:12:03.488 PM INFO Driver
> [Scheduled Query Executor(schedule:repl_sample_acid_1, execution_id:49625)]:
> Starting task [Stage-10068:DEPENDENCY_COLLECTION] in serial mode4:12:03.488
> PM INFO Driver [Scheduled Query Executor(schedule:repl_sample_acid_1,
> execution_id:49625)]: Starting task [Stage-10069:DDL] in serial
> mode4:12:03.504 PM INFO Driver [Scheduled Query
> Executor(schedule:repl_sample_acid_1, execution_id:49625)]: Starting task
> [Stage-10070:REPL_STATE_LOG] in serial mode4:12:03.504 PM INFO
> ReplState [Scheduled Query Executor(schedule:repl_sample_acid_1,
> execution_id:49625)]: REPL::EVENT_LOAD:
> {"dbName":"sample","eventId":"50442183","eventType":"EVENT_UPDATE_TABLE_COL_STAT","eventsLoadProgress":"2444/20424","loadTime":1687187523,"eventDuration":"273
> ms"}4:12:03.504 PM INFO Driver [Scheduled Query
> Executor(schedule:repl_sample_acid_1, execution_id:49625)]: Starting task
> [Stage-10071:DDL] in serial mode4:12:03.596 PM ERROR Task
> [Scheduled Query Executor(schedule:repl_sample_acid_1, execution_id:49625)]:
> Failedorg.apache.hadoop.hive.ql.metadata.HiveException: Unable to alter
> table. java.lang.NumberFormatException: For input string: "5744479373" at
> org.apache.hadoop.hive.ql.metadata.Hive.alterTable(Hive.java:854)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.ddl.table.create.CreateTableOperation.createTableReplaceMode(CreateTableOperation.java:127)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.ddl.table.create.CreateTableOperation.execute(CreateTableOperation.java:90)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.ddl.DDLTask.execute(DDLTask.java:82)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:213)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:105)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.Executor.launchTask(Executor.java:357)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.Executor.launchTasks(Executor.java:330)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.Executor.runTasks(Executor.java:246)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.Executor.execute(Executor.java:109)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:772)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.Driver.run(Driver.java:511)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.Driver.run(Driver.java:505)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:166)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:232)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.scheduled.ScheduledQueryExecutionService$ScheduledQueryExecutor.processQuery(ScheduledQueryExecutionService.java:240)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.scheduled.ScheduledQueryExecutionService$ScheduledQueryExecutor.run(ScheduledQueryExecutionService.java:193)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> ~[?:1.8.0_342] at
> java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_342]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> ~[?:1.8.0_342] at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> ~[?:1.8.0_342] at java.lang.Thread.run(Thread.java:750)
> ~[?:1.8.0_342]Caused by: org.apache.hadoop.hive.metastore.api.MetaException:
> java.lang.NumberFormatException: For input string: "5744479373" at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_req_result$alter_table_req_resultStandardScheme.read(ThriftHiveMetastore.java)
> ~[hive-standalone-metastore-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_req_result$alter_table_req_resultStandardScheme.read(ThriftHiveMetastore.java)
> ~[hive-standalone-metastore-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_req_result.read(ThriftHiveMetastore.java)
> ~[hive-standalone-metastore-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5]
> at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:88)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_alter_table_req(ThriftHiveMetastore.java:2628)
> ~[hive-standalone-metastore-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.alter_table_req(ThriftHiveMetastore.java:2615)
> ~[hive-standalone-metastore-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5]
> at
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient.alter_table(HiveMetaStoreClient.java:494)
> ~[hive-standalone-metastore-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5]
> at
> org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.alter_table(SessionHiveMetaStoreClient.java:447)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> sun.reflect.GeneratedMethodAccessor456.invoke(Unknown Source) ~[?:?] at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> ~[?:1.8.0_342] at java.lang.reflect.Method.invoke(Method.java:498)
> ~[?:1.8.0_342] at
> org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:213)
> ~[hive-standalone-metastore-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5]
> at com.sun.proxy.$Proxy52.alter_table(Unknown Source) ~[?:?] at
> sun.reflect.GeneratedMethodAccessor456.invoke(Unknown Source) ~[?:?] at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> ~[?:1.8.0_342] at java.lang.reflect.Method.invoke(Method.java:498)
> ~[?:1.8.0_342] at
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient$SynchronizedHandler.invoke(HiveMetaStoreClient.java:3546)
> ~[hive-standalone-metastore-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5]
> at com.sun.proxy.$Proxy52.alter_table(Unknown Source) ~[?:?] at
> org.apache.hadoop.hive.ql.metadata.Hive.alterTable(Hive.java:850)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] ... 21
> more4:12:03.600 PM ERROR Task [Scheduled Query
> Executor(schedule:repl_sample_acid_1, execution_id:49625)]: DDLTask failed,
> DDL Operation: class
> org.apache.hadoop.hive.ql.ddl.table.create.CreateTableOperationorg.apache.hadoop.hive.ql.metadata.HiveException:
> Unable to alter table. java.lang.NumberFormatException: For input string:
> "5744479373" at
> org.apache.hadoop.hive.ql.metadata.Hive.alterTable(Hive.java:854)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.ddl.table.create.CreateTableOperation.createTableReplaceMode(CreateTableOperation.java:127)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.ddl.table.create.CreateTableOperation.execute(CreateTableOperation.java:90)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.ddl.DDLTask.execute(DDLTask.java:82)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:213)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:105)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.Executor.launchTask(Executor.java:357)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.Executor.launchTasks(Executor.java:330)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.Executor.runTasks(Executor.java:246)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.Executor.execute(Executor.java:109)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:772)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.Driver.run(Driver.java:511)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.Driver.run(Driver.java:505)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:166)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:232)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.scheduled.ScheduledQueryExecutionService$ScheduledQueryExecutor.processQuery(ScheduledQueryExecutionService.java:240)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.ql.scheduled.ScheduledQueryExecutionService$ScheduledQueryExecutor.run(ScheduledQueryExecutionService.java:193)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> ~[?:1.8.0_342] at
> java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_342]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> ~[?:1.8.0_342] at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> ~[?:1.8.0_342] at java.lang.Thread.run(Thread.java:750)
> ~[?:1.8.0_342]Caused by: org.apache.hadoop.hive.metastore.api.MetaException:
> java.lang.NumberFormatException: For input string: "5744479373" at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_req_result$alter_table_req_resultStandardScheme.read(ThriftHiveMetastore.java)
> ~[hive-standalone-metastore-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_req_result$alter_table_req_resultStandardScheme.read(ThriftHiveMetastore.java)
> ~[hive-standalone-metastore-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_req_result.read(ThriftHiveMetastore.java)
> ~[hive-standalone-metastore-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5]
> at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:88)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_alter_table_req(ThriftHiveMetastore.java:2628)
> ~[hive-standalone-metastore-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.alter_table_req(ThriftHiveMetastore.java:2615)
> ~[hive-standalone-metastore-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5]
> at
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient.alter_table(HiveMetaStoreClient.java:494)
> ~[hive-standalone-metastore-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5]
> at
> org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.alter_table(SessionHiveMetaStoreClient.java:447)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] at
> sun.reflect.GeneratedMethodAccessor456.invoke(Unknown Source) ~[?:?] at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> ~[?:1.8.0_342] at java.lang.reflect.Method.invoke(Method.java:498)
> ~[?:1.8.0_342] at
> org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:213)
> ~[hive-standalone-metastore-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5]
> at com.sun.proxy.$Proxy52.alter_table(Unknown Source) ~[?:?] at
> sun.reflect.GeneratedMethodAccessor456.invoke(Unknown Source) ~[?:?] at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> ~[?:1.8.0_342] at java.lang.reflect.Method.invoke(Method.java:498)
> ~[?:1.8.0_342] at
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient$SynchronizedHandler.invoke(HiveMetaStoreClient.java:3546)
> ~[hive-standalone-metastore-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5]
> at com.sun.proxy.$Proxy52.alter_table(Unknown Source) ~[?:?] at
> org.apache.hadoop.hive.ql.metadata.Hive.alterTable(Hive.java:850)
> ~[hive-exec-3.1.3000.7.1.8.15-5.jar:3.1.3000.7.1.8.15-5] ... 21
> more4:12:03.681 PM INFO ReplicationMetricCollector [Scheduled
> Query Executor(schedule:repl_sample_acid_1, execution_id:49625)]: End
> FAILED_ADMIN4:12:03.681 PM INFO ReOptimizePlugin [Scheduled
> Query Executor(schedule:repl_sample_acid_1, execution_id:49625)]:
> ReOptimization: retryPossible: false4:12:03.685 PM ERROR Driver [Scheduled
> Query Executor(schedule:repl_sample_acid_1, execution_id:49625)]: FAILED:
> Execution Error, return code 40013 from
> org.apache.hadoop.hive.ql.ddl.DDLTask. Unable to alter table.
> java.lang.NumberFormatException: For input string: "5744479373"4:12:03.686 PM
> INFO Driver [Scheduled Query Executor(schedule:repl_sample_acid_1,
> execution_id:49625)]: Completed executing
> command(queryId=eup1hive_20230619160006_d5a7e666-480f-4f55-a083-5aabcee41711);
> Time taken: 711.336 seconds4:12:03.686 PM INFO Driver [Scheduled
> Query Executor(schedule:repl_sample_acid_1, execution_id:49625)]: OK {code}
>
>
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
