[
https://issues.apache.org/jira/browse/HADOOP-17207?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17177705#comment-17177705
]
Steve Loughran commented on HADOOP-17207:
-----------------------------------------
{code}
ERROR] testProxyConnection(org.apache.hadoop.fs.s3a.ITestS3AConfiguration)
Time elapsed: 5.954 s <<< FAILURE!
java.lang.AssertionError: Expected a
org.apache.hadoop.fs.s3a.AWSClientIOException to be thrown, but got the result:
: "expected failure creating FS when using proxy 127.0.0.1:1 got
S3AFileSystem{uri=s3a://stevel-london,
workingDir=s3a://stevel-london/user/stevel, inputPolicy=normal,
partSize=8388608, enableMultiObjectsDelete=true, maxKeys=5000,
readAhead=524288, blockSize=33554432, multiPartThreshold=134217728,
serverSideEncryptionAlgorithm='NONE',
blockFactory=ByteBufferBlockFactory{buffersOutstanding=0},
metastore=NullMetadataStore, authoritativeStore=false,
authoritativePath=[s3a://stevel-london/tables/], useListV1=false,
magicCommitter=true,
boundedExecutor=BlockingThreadPoolExecutorService{SemaphoredDelegatingExecutor{permitCount=92,
available=92, waiting=0}, activeCount=0},
unboundedExecutor=java.util.concurrent.ThreadPoolExecutor@42b93282[Running,
pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 0],
credentials=AWSCredentialProviderList[refcount= 1:
[TemporaryAWSCredentialsProvider, SimpleAWSCredentialsProvider,
EnvironmentVariableCredentialsProvider,
org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider@3e77e2aa],
delegation tokens=disabled, DirectoryMarkerRetention{policy='authoritative'},
statistics {0 bytes read, 0 bytes written, 0 read ops, 0 large read ops, 0
write ops}, metrics {{Context=s3aFileSystem}
{s3aFileSystemId=07604028-9219-47b5-a453-07900b8d53e7} {bucket=stevel-london}
{stream_opened=0} {stream_close_operations=0} {stream_closed=0}
{stream_aborted=0} {stream_seek_operations=0} {stream_read_exceptions=0}
{stream_forward_seek_operations=0} {stream_backward_seek_operations=0}
{stream_bytes_skipped_on_seek=0} {stream_bytes_backwards_on_seek=0}
{stream_bytes_read=0} {stream_read_operations=0}
{stream_read_fully_operations=0} {stream_read_operations_incomplete=0}
{stream_bytes_read_in_close=0} {stream_bytes_discarded_in_abort=0}
{files_created=0} {files_copied=0} {files_copied_bytes=0} {files_deleted=0}
{fake_directories_deleted=0} {directories_created=0} {directories_deleted=0}
{ignored_errors=0} {op_copy_from_local_file=0} {op_create=0}
{op_create_non_recursive=0} {op_delete=0} {op_exists=0}
{op_get_delegation_token=0} {op_get_file_checksum=0} {op_get_file_status=0}
{op_glob_status=0} {op_is_directory=0} {op_is_file=0} {op_list_files=0}
{op_list_located_status=0} {op_list_status=0} {op_mkdirs=0} {op_open=0}
{op_rename=0} {object_copy_requests=0} {object_delete_requests=0}
{object_list_requests=0} {object_continue_list_requests=0}
{object_metadata_requests=0} {object_multipart_aborted=0} {object_put_bytes=0}
{object_put_requests=0} {object_put_requests_completed=0}
{object_select_requests=0} {stream_read_version_mismatches=0}
{stream_write_failures=0} {stream_write_block_uploads=0}
{stream_write_block_uploads_committed=0} {stream_write_block_uploads_aborted=0}
{stream_write_total_time=0} {stream_write_total_data=0}
{committer_commits_created=0} {committer_commits_completed=0}
{committer_jobs_completed=0} {committer_jobs_failed=0}
{committer_tasks_completed=0} {committer_tasks_failed=0}
{committer_bytes_committed=0} {committer_bytes_uploaded=0}
{committer_commits_failed=0} {committer_commits_aborted=0}
{committer_commits_reverted=0} {committer_magic_files_created=0}
{s3guard_metadatastore_put_path_request=0}
{s3guard_metadatastore_initialization=0}
{s3guard_metadatastore_record_deletes=0} {s3guard_metadatastore_record_reads=0}
{s3guard_metadatastore_record_writes=0} {s3guard_metadatastore_retry=0}
{s3guard_metadatastore_throttled=0}
{s3guard_metadatastore_authoritative_directories_updated=0}
{store_io_throttled=0} {delegation_tokens_issued=0} {files_delete_rejected=0}
{multipart_instantiated=0} {multipart_part_put=0} {multipart_part_put_bytes=0}
{multipart_upload_aborted=0} {multipart_upload_abort_under_path_invoked=0}
{multipart_upload_completed=0} {multipart_upload_started=0}
{object_put_requests_active=0} {object_put_bytes_pending=0}
{stream_write_block_uploads_active=0} {stream_write_block_uploads_pending=0}
{stream_write_block_uploads_data_pending=0}
{S3guard_metadatastore_put_path_latencyNumOps=0}
{S3guard_metadatastore_put_path_latency50thPercentileLatency=0}
{S3guard_metadatastore_put_path_latency75thPercentileLatency=0}
{S3guard_metadatastore_put_path_latency90thPercentileLatency=0}
{S3guard_metadatastore_put_path_latency95thPercentileLatency=0}
{S3guard_metadatastore_put_path_latency99thPercentileLatency=0}
{S3guard_metadatastore_throttle_rateNumEvents=0}
{S3guard_metadatastore_throttle_rate50thPercentileFrequency (Hz)=0}
{S3guard_metadatastore_throttle_rate75thPercentileFrequency (Hz)=0}
{S3guard_metadatastore_throttle_rate90thPercentileFrequency (Hz)=0}
{S3guard_metadatastore_throttle_rate95thPercentileFrequency (Hz)=0}
{S3guard_metadatastore_throttle_rate99thPercentileFrequency (Hz)=0}
{Store_io_throttle_rateNumEvents=0}
{Store_io_throttle_rate50thPercentileFrequency (Hz)=0}
{Store_io_throttle_rate75thPercentileFrequency (Hz)=0}
{Store_io_throttle_rate90thPercentileFrequency (Hz)=0}
{Store_io_throttle_rate95thPercentileFrequency (Hz)=0}
{Store_io_throttle_rate99thPercentileFrequency (Hz)=0} }}"
at
org.apache.hadoop.test.LambdaTestUtils.intercept(LambdaTestUtils.java:499)
at
org.apache.hadoop.test.LambdaTestUtils.intercept(LambdaTestUtils.java:384)
at
org.apache.hadoop.fs.s3a.ITestS3AConfiguration.expectFSCreateFailure(ITestS3AConfiguration.java:159)
at
org.apache.hadoop.fs.s3a.ITestS3AConfiguration.testProxyConnection(ITestS3AConfiguration.java:134)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
at
org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298)
at
org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
[ERROR]
testAutomaticProxyPortSelection(org.apache.hadoop.fs.s3a.ITestS3AConfiguration)
Time elapsed: 0.063 s <<< FAILURE!
java.lang.AssertionError: Expected a
org.apache.hadoop.fs.s3a.AWSClientIOException to be thrown, but got the result:
: "expected failure creating FS Expected a connection error for proxy server
got S3AFileSystem{uri=s3a://stevel-london,
workingDir=s3a://stevel-london/user/stevel, inputPolicy=normal,
partSize=8388608, enableMultiObjectsDelete=true, maxKeys=5000,
readAhead=524288, blockSize=33554432, multiPartThreshold=134217728,
serverSideEncryptionAlgorithm='NONE',
blockFactory=ByteBufferBlockFactory{buffersOutstanding=0},
metastore=NullMetadataStore, authoritativeStore=false,
authoritativePath=[s3a://stevel-london/tables/], useListV1=false,
magicCommitter=true,
boundedExecutor=BlockingThreadPoolExecutorService{SemaphoredDelegatingExecutor{permitCount=92,
available=92, waiting=0}, activeCount=0},
unboundedExecutor=java.util.concurrent.ThreadPoolExecutor@68bf4e89[Running,
pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 0],
credentials=AWSCredentialProviderList[refcount= 1:
[TemporaryAWSCredentialsProvider, SimpleAWSCredentialsProvider,
EnvironmentVariableCredentialsProvider,
org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider@8f88339],
delegation tokens=disabled, DirectoryMarkerRetention{policy='authoritative'},
statistics {0 bytes read, 0 bytes written, 0 read ops, 0 large read ops, 0
write ops}, metrics {{Context=s3aFileSystem}
{s3aFileSystemId=719e24a3-3c95-4b97-bc9b-7e2ca6ac84de} {bucket=stevel-london}
{stream_opened=0} {stream_close_operations=0} {stream_closed=0}
{stream_aborted=0} {stream_seek_operations=0} {stream_read_exceptions=0}
{stream_forward_seek_operations=0} {stream_backward_seek_operations=0}
{stream_bytes_skipped_on_seek=0} {stream_bytes_backwards_on_seek=0}
{stream_bytes_read=0} {stream_read_operations=0}
{stream_read_fully_operations=0} {stream_read_operations_incomplete=0}
{stream_bytes_read_in_close=0} {stream_bytes_discarded_in_abort=0}
{files_created=0} {files_copied=0} {files_copied_bytes=0} {files_deleted=0}
{fake_directories_deleted=0} {directories_created=0} {directories_deleted=0}
{ignored_errors=0} {op_copy_from_local_file=0} {op_create=0}
{op_create_non_recursive=0} {op_delete=0} {op_exists=0}
{op_get_delegation_token=0} {op_get_file_checksum=0} {op_get_file_status=0}
{op_glob_status=0} {op_is_directory=0} {op_is_file=0} {op_list_files=0}
{op_list_located_status=0} {op_list_status=0} {op_mkdirs=0} {op_open=0}
{op_rename=0} {object_copy_requests=0} {object_delete_requests=0}
{object_list_requests=0} {object_continue_list_requests=0}
{object_metadata_requests=0} {object_multipart_aborted=0} {object_put_bytes=0}
{object_put_requests=0} {object_put_requests_completed=0}
{object_select_requests=0} {stream_read_version_mismatches=0}
{stream_write_failures=0} {stream_write_block_uploads=0}
{stream_write_block_uploads_committed=0} {stream_write_block_uploads_aborted=0}
{stream_write_total_time=0} {stream_write_total_data=0}
{committer_commits_created=0} {committer_commits_completed=0}
{committer_jobs_completed=0} {committer_jobs_failed=0}
{committer_tasks_completed=0} {committer_tasks_failed=0}
{committer_bytes_committed=0} {committer_bytes_uploaded=0}
{committer_commits_failed=0} {committer_commits_aborted=0}
{committer_commits_reverted=0} {committer_magic_files_created=0}
{s3guard_metadatastore_put_path_request=0}
{s3guard_metadatastore_initialization=0}
{s3guard_metadatastore_record_deletes=0} {s3guard_metadatastore_record_reads=0}
{s3guard_metadatastore_record_writes=0} {s3guard_metadatastore_retry=0}
{s3guard_metadatastore_throttled=0}
{s3guard_metadatastore_authoritative_directories_updated=0}
{store_io_throttled=0} {delegation_tokens_issued=0} {files_delete_rejected=0}
{multipart_instantiated=0} {multipart_part_put=0} {multipart_part_put_bytes=0}
{multipart_upload_aborted=0} {multipart_upload_abort_under_path_invoked=0}
{multipart_upload_completed=0} {multipart_upload_started=0}
{object_put_requests_active=0} {object_put_bytes_pending=0}
{stream_write_block_uploads_active=0} {stream_write_block_uploads_pending=0}
{stream_write_block_uploads_data_pending=0}
{S3guard_metadatastore_put_path_latencyNumOps=0}
{S3guard_metadatastore_put_path_latency50thPercentileLatency=0}
{S3guard_metadatastore_put_path_latency75thPercentileLatency=0}
{S3guard_metadatastore_put_path_latency90thPercentileLatency=0}
{S3guard_metadatastore_put_path_latency95thPercentileLatency=0}
{S3guard_metadatastore_put_path_latency99thPercentileLatency=0}
{S3guard_metadatastore_throttle_rateNumEvents=0}
{S3guard_metadatastore_throttle_rate50thPercentileFrequency (Hz)=0}
{S3guard_metadatastore_throttle_rate75thPercentileFrequency (Hz)=0}
{S3guard_metadatastore_throttle_rate90thPercentileFrequency (Hz)=0}
{S3guard_metadatastore_throttle_rate95thPercentileFrequency (Hz)=0}
{S3guard_metadatastore_throttle_rate99thPercentileFrequency (Hz)=0}
{Store_io_throttle_rateNumEvents=0}
{Store_io_throttle_rate50thPercentileFrequency (Hz)=0}
{Store_io_throttle_rate75thPercentileFrequency (Hz)=0}
{Store_io_throttle_rate90thPercentileFrequency (Hz)=0}
{Store_io_throttle_rate95thPercentileFrequency (Hz)=0}
{Store_io_throttle_rate99thPercentileFrequency (Hz)=0} }}"
at
org.apache.hadoop.test.LambdaTestUtils.intercept(LambdaTestUtils.java:499)
at
org.apache.hadoop.test.LambdaTestUtils.intercept(LambdaTestUtils.java:384)
at
org.apache.hadoop.fs.s3a.ITestS3AConfiguration.expectFSCreateFailure(ITestS3AConfiguration.java:159)
at
org.apache.hadoop.fs.s3a.ITestS3AConfiguration.testAutomaticProxyPortSelection(ITestS3AConfiguration.java:187)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
at
org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298)
at
org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
{code}
> ITestS3AConfiguration.testProxyConnection failing when s3a bucket probe
> disabled
> ---------------------------------------------------------------------------------
>
> Key: HADOOP-17207
> URL: https://issues.apache.org/jira/browse/HADOOP-17207
> Project: Hadoop Common
> Issue Type: Sub-task
> Components: fs/s3
> Affects Versions: 3.4.0
> Reporter: Steve Loughran
> Priority: Minor
>
> Failure of
> {{testProxyConnection(org.apache.hadoop.fs.s3a.ITestS3AConfiguration)}} when
> bucket probes set to
> {code}
> <property>
> <name>fs.s3a.bucket.probe</name>
> <value>0</value>
> </property>
> {code}
> and s3guard disabled
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
