alamb commented on code in PR #5746:
URL: https://github.com/apache/arrow-rs/pull/5746#discussion_r1597421777
##########
object_store/src/upload.rs:
##########
@@ -246,4 +250,69 @@ mod tests {
assert!(write.wait_for_capacity(10).now_or_never().is_none());
write.wait_for_capacity(10).await.unwrap()
}
+
+ #[derive(Debug, Default)]
+ struct InstrumentedUpload {
+ chunks: Arc<Mutex<Vec<PutPayload>>>,
+ }
+
+ #[async_trait]
+ impl MultipartUpload for InstrumentedUpload {
+ fn put_part(&mut self, data: PutPayload) -> UploadPart {
+ self.chunks.lock().push(data);
+ futures::future::ready(Ok(())).boxed()
+ }
+
+ async fn complete(&mut self) -> Result<PutResult> {
+ Ok(PutResult {
+ e_tag: None,
+ version: None,
+ })
+ }
+
+ async fn abort(&mut self) -> Result<()> {
+ unimplemented!()
+ }
+ }
+
+ #[tokio::test]
+ async fn test_write_multipart() {
+ let mut rng = StdRng::seed_from_u64(42);
+
+ for method in [0.0, 0.5, 1.0] {
+ for _ in 0..10 {
+ for chunk_size in [1, 17, 23] {
+ let upload = Box::<InstrumentedUpload>::default();
+ let chunks = Arc::clone(&upload.chunks);
+ let mut write =
WriteMultipart::new_with_chunk_size(upload, chunk_size);
+
+ let mut expected = Vec::with_capacity(1024);
+
+ for _ in 0..50 {
+ let chunk_size = rng.gen_range(0..30);
+ let data: Vec<_> = (0..chunk_size).map(|_|
rng.gen()).collect();
+ expected.extend_from_slice(&data);
+
+ match rng.gen_bool(method) {
+ true => write.put(data.into()),
+ false => write.write(&data),
+ }
+ }
+ write.finish().await.unwrap();
+
+ let chunks = chunks.lock();
+
+ let actual: Vec<_> =
chunks.iter().flatten().flatten().copied().collect();
Review Comment:
Yes you are right -- sorry I missed that
I was thinking that `for chunk in chunks.iter().take(chunks.len() - 1) {`
would work even if there were only one chunk. However, I see now that
`assert_eq!(chunk.content_length(), chunk_size)` verifies that each chunk is
exactly `chunk_size` bytes so if for example `new_with_chunks_size` hadn't
worked, this assert would fail
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
