[
https://issues.apache.org/jira/browse/IMPALA-12621?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17821006#comment-17821006
]
Quanlong Huang commented on IMPALA-12621:
-----------------------------------------
Saw this again in https://jenkins.impala.io/job/ubuntu-20.04-from-scratch/1823/
But on an INSERT statement in
TestIcebergTable.test_using_upper_lower_bound_metrics
{noformat}
query_test/test_iceberg.py:1014: in test_using_upper_lower_bound_metrics
use_db=unique_database)
common/impala_test_suite.py:725: in run_test_case
result = exec_fn(query, user=test_section.get('USER', '').strip() or None)
common/impala_test_suite.py:660: in __exec_in_impala
result = self.__execute_query(target_impalad_client, query, user=user)
common/impala_test_suite.py:1003: in __execute_query
return impalad_client.execute(query, user=user)
common/impala_connection.py:214: in execute
return self.__beeswax_client.execute(sql_stmt, user=user)
beeswax/impala_beeswax.py:191: in execute
handle = self.__execute_query(query_string.strip(), user=user)
beeswax/impala_beeswax.py:369: in __execute_query
self.wait_for_finished(handle)
beeswax/impala_beeswax.py:390: in wait_for_finished
raise ImpalaBeeswaxException("Query aborted:" + error_log, None)
E ImpalaBeeswaxException: ImpalaBeeswaxException:
E Query aborted:RuntimeIOException: Failed to write json to file:
hdfs://localhost:20500/test-warehouse/test_using_upper_lower_bound_metrics_95c209f8.db/ice_types3/metadata/00001-3c31e838-641a-4a4b-a6da-c8b2ccfc5948.metadata.json
E CAUSED BY: IOException: The stream is closed{noformat}
The query is
{noformat}
I0227 03:30:42.723153 802189 Frontend.java:2072]
0e47e5b251a8d900:9830d1d300000000] Analyzing query: insert into ice_types3
values
(0, 123.456, 1234567890.123, 1234567890123456789.012),
(0, 123.457, 1234567890.124, 1234567890123456789.013),
(0, 123.458, 1234567890.125, 1234567890123456789.014),
(1, 123.459, 1234567890.126, 1234567890123456789.015),
(1, 333.333, 3333333333.333, 3333333333333333333.333),
(1, 341.234, 3412345678.901, 3412345678901234567.89),
(2, 341.234, 3412345678.901, 3412345678901234567.89),
(2, 123456.789, 123456789012345.678,
12345678901234567890123456789012345.678),
(2, 123456.790, 123456789012345.679,
12345678901234567890123456789012345.68) db:
test_using_upper_lower_bound_metrics_95c209f8{noformat}
The catalogd logs:
{noformat}
I0227 03:30:48.779507 572803 JniUtil.java:166]
0e47e5b251a8d900:9830d1d300000000] updateCatalog request: Update catalog for
test_using_upper_lower_bound_metrics_95c209f8.ice_types3
I0227 03:30:48.855067 572803 SnapshotProducer.java:400]
0e47e5b251a8d900:9830d1d300000000] Committed snapshot 5973762531632606267
(MergeAppend)
I0227 03:30:48.857475 572803 LoggingMetricsReporter.java:38]
0e47e5b251a8d900:9830d1d300000000] Received metrics report:
CommitReport{tableName=ImpalaHiveCatalog.test_using_upper_lower_bound_metrics_95c209f8.ice_types3,
snapshotId=5973762531632606267, sequenceNumber=0, operation=append,
commitMetrics=CommitMetricsResult{totalDuration=TimerResult{timeUnit=NANOSECONDS,
totalDuration=PT0.070239109S, count=1}, attempts=CounterResult{unit=COUNT,
value=1}, addedDataFiles=CounterResult{unit=COUNT, value=3},
removedDataFiles=null, totalDataFiles=CounterResult{unit=COUNT, value=3},
addedDeleteFiles=null, addedEqualityDeleteFiles=null,
addedPositionalDeleteFiles=null, removedDeleteFiles=null,
removedEqualityDeleteFiles=null, removedPositionalDeleteFiles=null,
totalDeleteFiles=CounterResult{unit=COUNT, value=0},
addedRecords=CounterResult{unit=COUNT, value=9}, removedRecords=null,
totalRecords=CounterResult{unit=COUNT, value=9},
addedFilesSizeInBytes=CounterResult{unit=BYTES, value=3765},
removedFilesSizeInBytes=null, totalFilesSizeInBytes=CounterResult{unit=BYTES,
value=3765}, addedPositionalDeletes=null, removedPositionalDeletes=null,
totalPositionalDeletes=CounterResult{unit=COUNT, value=0},
addedEqualityDeletes=null, removedEqualityDeletes=null,
totalEqualityDeletes=CounterResult{unit=COUNT, value=0}},
metadata={iceberg-version=Apache Iceberg 1.3.1.7.2.18.0-369 (commit
5dfbdbef8d7ff80b06f755e49d607ac24c0fa46f)}}
I0227 03:30:48.857812 572803 CatalogServiceCatalog.java:1185]
0e47e5b251a8d900:9830d1d300000000] Added catalog version 8578 in table's
test_using_upper_lower_bound_metrics_95c209f8.ice_types3 in-flight events
W0227 03:30:48.895732 855438 DataStreamer.java:829] DataStreamer Exception
Java exception follows:
java.nio.channels.ClosedByInterruptException
at
java.nio.channels.spi.AbstractInterruptibleChannel.end(AbstractInterruptibleChannel.java:202)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:477)
at
org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:63)
at
org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141)
at
org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:159)
at
org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:117)
at
java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
at java.io.DataOutputStream.flush(DataOutputStream.java:123)
at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:778)
E0227 03:30:48.903847 572803 JniUtil.java:183]
0e47e5b251a8d900:9830d1d300000000] Error in Update catalog for
test_using_upper_lower_bound_metrics_95c209f8.ice_types3. Time spent: 124ms
I0227 03:30:48.904982 572803 jni-util.cc:302]
0e47e5b251a8d900:9830d1d300000000]
org.apache.iceberg.exceptions.RuntimeIOException: Failed to write json to file:
hdfs://localhost:20500/test-warehouse/test_using_upper_lower_bound_metrics_95c209f8.db/ice_types3/metadata/00001-3c31e838-641a-4a4b-a6da-c8b2ccfc5948.metadata.json
at
org.apache.iceberg.TableMetadataParser.internalWrite(TableMetadataParser.java:132)
at
org.apache.iceberg.TableMetadataParser.overwrite(TableMetadataParser.java:114)
at
org.apache.iceberg.BaseMetastoreTableOperations.writeNewMetadata(BaseMetastoreTableOperations.java:170)
at
org.apache.iceberg.BaseMetastoreTableOperations.writeNewMetadataIfRequired(BaseMetastoreTableOperations.java:160)
at
org.apache.iceberg.hive.HiveTableOperations.doCommit(HiveTableOperations.java:185)
at
org.apache.iceberg.BaseMetastoreTableOperations.commit(BaseMetastoreTableOperations.java:135)
at
org.apache.iceberg.BaseTransaction.lambda$commitSimpleTransaction$5(BaseTransaction.java:422)
at
org.apache.iceberg.util.Tasks$Builder.runTaskWithRetry(Tasks.java:413)
at
org.apache.iceberg.util.Tasks$Builder.runSingleThreaded(Tasks.java:219)
at org.apache.iceberg.util.Tasks$Builder.run(Tasks.java:203)
at org.apache.iceberg.util.Tasks$Builder.run(Tasks.java:196)
at
org.apache.iceberg.BaseTransaction.commitSimpleTransaction(BaseTransaction.java:418)
at
org.apache.iceberg.BaseTransaction.commitTransaction(BaseTransaction.java:302)
at
org.apache.impala.service.CatalogOpExecutor.updateCatalogImpl(CatalogOpExecutor.java:7114)
at
org.apache.impala.service.CatalogOpExecutor.updateCatalog(CatalogOpExecutor.java:6838)
at
org.apache.impala.service.JniCatalog.lambda$updateCatalog$15(JniCatalog.java:500)
at
org.apache.impala.service.JniCatalogOp.lambda$execAndSerialize$1(JniCatalogOp.java:90)
at org.apache.impala.service.JniCatalogOp.execOp(JniCatalogOp.java:58)
at
org.apache.impala.service.JniCatalogOp.execAndSerialize(JniCatalogOp.java:89)
at
org.apache.impala.service.JniCatalogOp.execAndSerialize(JniCatalogOp.java:100)
at
org.apache.impala.service.JniCatalog.execAndSerialize(JniCatalog.java:243)
at
org.apache.impala.service.JniCatalog.execAndSerialize(JniCatalog.java:257)
at
org.apache.impala.service.JniCatalog.updateCatalog(JniCatalog.java:499)
Caused by: java.io.IOException: The stream is closed
at
org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:118)
at
java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
at java.io.DataOutputStream.flush(DataOutputStream.java:123)
at java.io.FilterOutputStream.close(FilterOutputStream.java:158)
at
org.apache.hadoop.hdfs.DataStreamer.closeStream(DataStreamer.java:1015)
at
org.apache.hadoop.hdfs.DataStreamer.closeInternal(DataStreamer.java:851)
at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:846)
Suppressed: java.io.IOException: The stream is closed
at
org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:118)
at
java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
at
java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
at java.io.FilterOutputStream.close(FilterOutputStream.java:158)
at java.io.FilterOutputStream.close(FilterOutputStream.java:159)
... 3 more{noformat}
In the NameNode logs, the file is closed by DFSClient_NONMAPREDUCE_299053853_1
{noformat}
2024-02-27 03:30:48,869 INFO org.apache.hadoop.hdfs.StateChange: BLOCK*
allocate blk_1073780244_39421, replicas=127.0.0.1:31002, 127.0.0.1:31000,
127.0.0.1:31001 for
/test-warehouse/test_using_upper_lower_bound_metrics_95c209f8.db/ice_types3/metadata/00001-3c31e838-641a-4a4b-a6da-c8b2ccfc5948.metadata.json
2024-02-27 03:30:48,891 INFO org.apache.hadoop.hdfs.StateChange: DIR*
completeFile:
/test-warehouse/test_using_upper_lower_bound_metrics_95c209f8.db/ice_types3/metadata/00001-3c31e838-641a-4a4b-a6da-c8b2ccfc5948.metadata.json
is closed by DFSClient_NONMAPREDUCE_299053853_1{noformat}
CC [~boroknagyz], [~gaborkaszab]
> Flaky tests due to "Failed to write json to file" in creating Iceberg tables
> ----------------------------------------------------------------------------
>
> Key: IMPALA-12621
> URL: https://issues.apache.org/jira/browse/IMPALA-12621
> Project: IMPALA
> Issue Type: Bug
> Reporter: Quanlong Huang
> Priority: Critical
>
> There are some flaky tests that could fail in creating Iceberg tables. E.g.
> saw TestIcebergTable.test_create_table_like_table failed in
> [https://jenkins.impala.io/job/ubuntu-20.04-dockerised-tests/930/]
> {code:java}
> query_test/test_iceberg.py:937: in test_create_table_like_table
> use_db=unique_database)
> common/impala_test_suite.py:720: in run_test_case
> result = exec_fn(query, user=test_section.get('USER', '').strip() or None)
> common/impala_test_suite.py:658: in __exec_in_impala
> result = self.__execute_query(target_impalad_client, query, user=user)
> common/impala_test_suite.py:994: in __execute_query
> return impalad_client.execute(query, user=user)
> common/impala_connection.py:214: in execute
> return self.__beeswax_client.execute(sql_stmt, user=user)
> beeswax/impala_beeswax.py:191: in execute
> handle = self.__execute_query(query_string.strip(), user=user)
> beeswax/impala_beeswax.py:369: in __execute_query
> self.wait_for_finished(handle)
> beeswax/impala_beeswax.py:390: in wait_for_finished
> raise ImpalaBeeswaxException("Query aborted:" + error_log, None)
> E ImpalaBeeswaxException: ImpalaBeeswaxException:
> E Query aborted:ImpalaRuntimeException: Error making 'createTable' RPC to
> Hive Metastore:
> E CAUSED BY: RuntimeIOException: Failed to write json to file:
> hdfs://172.18.0.1:20500/test-warehouse/test_create_table_like_table_260cb6ec.db/ice_hadoop_tbl_no_part/metadata/3db9a772-baf3-4cd8-9e6b-273ed2839ebb.metadata.json
> E CAUSED BY: IOException: The stream is closed{code}
> The CreateTable command is
> {noformat}
> create table ice_hadoop_tbl_no_part(
> id INT comment "this is id",
> col_bool BOOLEAN,
> col_int INT,
> col_bigint BIGINT,
> col_float FLOAT,
> col_double DOUBLE,
> col_dc1 DECIMAL(9, 3),
> col_dc2 DECIMAL(18, 3),
> col_dc3 DECIMAL(38, 3),
> col_date DATE,
> col_timestamp TIMESTAMP,
> col_string STRING,
> col_struct STRUCT < f1: BIGINT,
> f2: BIGINT >,
> col_array ARRAY < STRING >,
> col_map MAP < STRING,
> array < STRING > >
> ) stored as iceberg tblproperties('iceberg.catalog' = 'hadoop.tables');
> {noformat}
> There are several JIRAs tracking the same failure, e.g. IMPALA-10540,
> IMPALA-12261, IMPALA-10924, IMPALA-10633.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
