steveloughran commented on PR #7334:
URL: https://github.com/apache/hadoop/pull/7334#issuecomment-2681786677
I got failures on the vector read stuff. As I noted in the tests, a separate
contract tests for the analytics stream is needed.
Prefetch was set everywhere
```
<property>
<name>fs.s3a.input.stream.type</name>
<value>prefetch</value>
</property>
```
note you can subclass `assumeEnabled()` for contract tests, invoke the
superclass and then add more assumptions.
I have suggested using getConfiguration() in some tests, but I see that
actually we don't export that from the contract tests. I think we should, but
there'll be problems with subclasses. Better to have the superclass setup run
then ask for its conf from the contract
```
setup() {
super.setup()
conf = getContract().getConf();
}
```
*or* the fileystem `getFilesystem.getConf()` which is this gives you the
fully resolved config of the FS
```
[ERROR] ITestS3AContractVectoredRead.testVectoredReadWholeFilePlusOne
[ERROR] Run 1: Expected a java.io.EOFException to be thrown, but got the
result: : "triggered read of 1 ranges against
org.apache.hadoop.fs.FSDataInputStream@114f7a7e:
ObjectInputStream{streamType=Prefetch,
uri='s3a://stevel-london/job-00-fork-0009/test/vectored_file.txt',
contentLength=65536, inputPolicy=random,
vectoredIOContext=VectoredIOContext{minSeekForVectorReads=0,
maxReadSizeForVectorReads=2097152, vectoredActiveRangeReads=4}}
org.apache.hadoop.fs.s3a.prefetch.S3APrefetchingInputStream@3a0c0511{counters=((stream_read_version_mismatches=0)
(stream_read_bytes_discarded_in_close=0) (stream_read_operations=1)
(stream_read_fully_operations=0) (stream_read_block_read.failures=0)
(stream_read_seek_forward_operations=0)
(stream_read_vectored_combined_ranges=0) (stream_read_seek_policy_changed=2)
(stream_read_total_bytes=65536) (action_http_get_request.failures=0)
(stream_read_exceptions=0) (stream_read_vectored_read_bytes_discarded=0)
(stream_read_remote_stream_aborted.failures
=0) (stream_read_bytes_discarded_in_abort=0)
(stream_read_seek_bytes_discarded=0) (stream_file_cache_eviction=0)
(stream_read_bytes=65536) (stream_evict_blocks_from_cache=0)
(stream_read_opened=1) (action_file_opened=1) (stream_leaks=0)
(stream_read_operations_incomplete=0) (stream_read_prefetch_operations=0)
(action_executor_acquired.failures=0)
(stream_read_remote_stream_drain.failures=0)
(stream_read_vectored_incoming_ranges=0) (action_http_get_request=1)
(stream_read_seek_operations=0) (stream_read_analytics_opened=0)
(stream_read_block_acquire_read=0) (stream_read_prefetch_operations.failures=0)
(stream_read_closed=1) (action_file_opened.failures=0)
(stream_read_close_operations=0) (stream_file_cache_eviction.failures=0)
(stream_read_bytes_backwards_on_seek=0) (action_executor_acquired=0)
(stream_read_seek_backward_operations=0) (stream_read_remote_stream_drain=1)
(stream_read_unbuffered=0) (stream_aborted=0)
(stream_read_seek_bytes_skipped=0) (stream_read_remote_stream_aborted
=0) (stream_read_block_read=1) (stream_read_block_acquire_read.failures=0)
(stream_read_vectored_operations=0));
gauges=((stream_read_active_prefetch_operations=0)
(stream_read_active_memory_in_use=0) (stream_read_blocks_in_cache=0)
(stream_read_gauge_input_policy=1));
minimums=((action_file_opened.min=0)
(stream_read_remote_stream_aborted.failures.min=-1)
(stream_read_remote_stream_drain.failures.min=-1)
(stream_read_remote_stream_aborted.min=-1)
(stream_read_block_acquire_read.failures.min=-1)
(action_http_get_request.min=45) (stream_read_block_acquire_read.min=-1)
(stream_read_block_read.failures.min=-1)
(stream_read_prefetch_operations.min=-1)
(stream_read_prefetch_operations.failures.min=-1)
(action_executor_acquired.min=-1) (stream_file_cache_eviction.min=-1)
(action_http_get_request.failures.min=-1)
(stream_read_remote_stream_drain.min=0) (stream_read_block_read.min=170)
(stream_file_cache_eviction.failures.min=-1)
(action_executor_acquired.failures.min=-1)
(action_file_opened.failures.min=-1));
maximums=((stream_read_prefetch_operations.max=-1)
(action_executor_acquired.max=-1) (stream_read_block_read.failures.max=-1)
(action_http_get_request.max=45) (stream_file_cache_eviction.failures.max=-1)
(stream_read_prefetch_operations.failures.max=-1)
(stream_read_block_acquire_read.max=-1)
(action_http_get_request.failures.max=-1) (action_file_opened.failures.max=-1)
(stream_read_block_acquire_read.failures.max=-1)
(action_executor_acquired.failures.max=-1)
(stream_read_remote_stream_drain.failures.max=-1) (action_file_opened.max=0)
(stream_read_remote_stream_drain.max=0)
(stream_read_remote_stream_aborted.max=-1) (stream_file_cache_eviction.max=-1)
(stream_read_block_read.max=170)
(stream_read_remote_stream_aborted.failures.max=-1));
means=((stream_read_remote_stream_drain.mean=(samples=1, sum=0,
mean=0.0000)) (stream_read_prefetch_operations.mean=(samples=0, sum=0,
mean=0.0000)) (stream_read_block_acquire_read.failures.mean=(samples=0, sum=0,
mean=0.0000)) (stream_read_block_read.mean=(samples=1, sum=170, mean=170.0000))
(action_executor_acquired.failures.mean=(samples=0, sum=0, mean=0.0000))
(action_file_opened.mean=(samples=1, sum=0, mean=0.0000))
(stream_read_remote_stream_aborted.failures.mean=(samples=0, sum=0,
mean=0.0000)) (action_file_opened.failures.mean=(samples=0, sum=0,
mean=0.0000)) (action_http_get_request.failures.mean=(samples=0, sum=0,
mean=0.0000)) (stream_read_prefetch_operations.failures.mean=(samples=0, sum=0,
mean=0.0000)) (stream_read_block_acquire_read.mean=(samples=0, sum=0,
mean=0.0000)) (stream_file_cache_eviction.mean=(samples=0, sum=0, mean=0.0000))
(stream_file_cache_eviction.failures.mean=(samples=0, sum=0, mean=0.0000))
(action_http_get_request.mean=(samples=1, sum=45, mean=45.
0000)) (action_executor_acquired.mean=(samples=0, sum=0, mean=0.0000))
(stream_read_remote_stream_drain.failures.mean=(samples=0, sum=0, mean=0.0000))
(stream_read_block_read.failures.mean=(samples=0, sum=0, mean=0.0000))
(stream_read_remote_stream_aborted.mean=(samples=0, sum=0, mean=0.0000)));
}"
[ERROR] Run 2: Expected a java.io.EOFException to be thrown, but got the
result: : "triggered read of 1 ranges against
org.apache.hadoop.fs.FSDataInputStream@5b4f1c30:
ObjectInputStream{streamType=Prefetch,
uri='s3a://stevel-london/job-00-fork-0009/test/vectored_file.txt',
contentLength=65536, inputPolicy=random,
vectoredIOContext=VectoredIOContext{minSeekForVectorReads=0,
maxReadSizeForVectorReads=2097152, vectoredActiveRangeReads=4}}
org.apache.hadoop.fs.s3a.prefetch.S3APrefetchingInputStream@57150126{counters=((stream_read_vectored_read_bytes_discarded=0)
(stream_read_vectored_combined_ranges=0) (stream_read_seek_operations=0)
(stream_read_seek_bytes_discarded=0) (stream_read_opened=1)
(stream_read_remote_stream_drain=1) (action_executor_acquired.failures=0)
(stream_evict_blocks_from_cache=0) (stream_read_vectored_incoming_ranges=0)
(stream_read_fully_operations=0) (stream_read_block_read.failures=0)
(stream_read_bytes_discarded_in_abort=0) (stream_read_bytes_discarded_in_clo
se=0) (action_executor_acquired=0) (stream_read_operations=1)
(action_http_get_request=1) (stream_read_bytes=65536)
(stream_read_operations_incomplete=0) (stream_aborted=0)
(action_file_opened.failures=0) (stream_file_cache_eviction.failures=0)
(stream_read_prefetch_operations.failures=0)
(stream_read_bytes_backwards_on_seek=0) (stream_read_exceptions=0)
(stream_read_remote_stream_drain.failures=0) (action_file_opened=1)
(stream_read_block_acquire_read.failures=0) (stream_read_close_operations=0)
(stream_read_unbuffered=0) (stream_read_seek_bytes_skipped=0)
(stream_read_vectored_operations=0) (stream_read_version_mismatches=0)
(stream_read_block_read=1) (stream_read_total_bytes=65536)
(stream_read_seek_policy_changed=2) (stream_read_seek_backward_operations=0)
(stream_read_analytics_opened=0) (stream_leaks=0)
(stream_read_prefetch_operations=0) (stream_read_seek_forward_operations=0)
(stream_read_closed=1) (stream_file_cache_eviction=0)
(action_http_get_request.failures=0) (stream_r
ead_remote_stream_aborted=0) (stream_read_remote_stream_aborted.failures=0)
(stream_read_block_acquire_read=0));
gauges=((stream_read_active_prefetch_operations=0)
(stream_read_active_memory_in_use=0) (stream_read_blocks_in_cache=0)
(stream_read_gauge_input_policy=1));
minimums=((action_executor_acquired.failures.min=-1)
(action_file_opened.failures.min=-1)
(stream_read_block_acquire_read.failures.min=-1)
(action_http_get_request.min=55) (action_file_opened.min=0)
(stream_read_prefetch_operations.failures.min=-1)
(stream_read_block_acquire_read.min=-1)
(action_http_get_request.failures.min=-1)
(stream_read_prefetch_operations.min=-1) (stream_read_block_read.min=192)
(stream_read_remote_stream_aborted.failures.min=-1)
(stream_read_remote_stream_aborted.min=-1)
(stream_read_block_read.failures.min=-1)
(stream_read_remote_stream_drain.failures.min=-1)
(stream_file_cache_eviction.failures.min=-1)
(stream_read_remote_stream_drain.min=0) (stream_file_cache_eviction.min=-1)
(action_executor_acquired.min=-1));
maximums=((action_file_opened.max=0)
(stream_read_remote_stream_aborted.max=-1)
(stream_read_block_acquire_read.failures.max=-1)
(action_http_get_request.failures.max=-1)
(action_executor_acquired.failures.max=-1)
(stream_read_remote_stream_drain.failures.max=-1)
(stream_file_cache_eviction.failures.max=-1)
(stream_file_cache_eviction.max=-1)
(stream_read_prefetch_operations.failures.max=-1)
(action_http_get_request.max=55) (stream_read_block_read.max=192)
(stream_read_remote_stream_drain.max=0) (action_file_opened.failures.max=-1)
(action_executor_acquired.max=-1) (stream_read_prefetch_operations.max=-1)
(stream_read_block_read.failures.max=-1)
(stream_read_block_acquire_read.max=-1)
(stream_read_remote_stream_aborted.failures.max=-1));
means=((action_executor_acquired.failures.mean=(samples=0, sum=0,
mean=0.0000)) (stream_read_remote_stream_aborted.failures.mean=(samples=0,
sum=0, mean=0.0000)) (stream_read_prefetch_operations.failures.mean=(samples=0,
sum=0, mean=0.0000)) (stream_read_remote_stream_aborted.mean=(samples=0, sum=0,
mean=0.0000)) (stream_read_prefetch_operations.mean=(samples=0, sum=0,
mean=0.0000)) (stream_file_cache_eviction.failures.mean=(samples=0, sum=0,
mean=0.0000)) (action_http_get_request.failures.mean=(samples=0, sum=0,
mean=0.0000)) (action_http_get_request.mean=(samples=1, sum=55, mean=55.0000))
(stream_read_block_read.mean=(samples=1, sum=192, mean=192.0000))
(stream_file_cache_eviction.mean=(samples=0, sum=0, mean=0.0000))
(action_file_opened.mean=(samples=1, sum=0, mean=0.0000))
(stream_read_remote_stream_drain.failures.mean=(samples=0, sum=0, mean=0.0000))
(stream_read_block_acquire_read.failures.mean=(samples=0, sum=0, mean=0.0000))
(action_executor_acquired.mean=(samples=0, sum=0
, mean=0.0000)) (stream_read_remote_stream_drain.mean=(samples=1, sum=0,
mean=0.0000)) (stream_read_block_read.failures.mean=(samples=0, sum=0,
mean=0.0000)) (action_file_opened.failures.mean=(samples=0, sum=0,
mean=0.0000)) (stream_read_block_acquire_read.mean=(samples=0, sum=0,
mean=0.0000)));
}"
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
