alamb commented on code in PR #4792:
URL: https://github.com/apache/arrow-rs/pull/4792#discussion_r1319105415
##########
object_store/src/local.rs:
##########
@@ -800,14 +788,14 @@ impl AsyncWrite for LocalUpload {
fn poll_flush(
self: Pin<&mut Self>,
_cx: &mut std::task::Context<'_>,
- ) -> std::task::Poll<Result<(), io::Error>> {
Review Comment:
most of these changes are unrelated, driveby cleanups, right?
##########
object_store/src/local.rs:
##########
@@ -1560,6 +1561,25 @@ mod not_wasm_tests {
let expected_data = Bytes::from("arbitrarydatagnzarbitrarydatagnz");
assert_eq!(&*read_data, expected_data);
}
+
+ #[tokio::test]
+ async fn test_cleanup_intermediate_files() {
+ let root = TempDir::new().unwrap();
+ let integration =
LocalFileSystem::new_with_prefix(root.path()).unwrap();
+
+ let location = Path::from("some_file");
+ let (_, mut writer) =
integration.put_multipart(&location).await.unwrap();
+ writer.write_all(b"hello").await.unwrap();
+
+ let file_count = std::fs::read_dir(root.path()).unwrap().count();
+ assert_eq!(file_count, 1);
+ drop(writer);
+
+ tokio::time::sleep(Duration::from_millis(1)).await;
Review Comment:
if we find that this fails intermittently, the other classic pattern is to
add a poll loop with maximum wait time
##########
object_store/src/local.rs:
##########
@@ -274,13 +274,15 @@ impl ObjectStore for LocalFileSystem {
maybe_spawn_blocking(move || {
let (mut file, suffix) = new_staged_upload(&path)?;
let staging_path = staged_upload_path(&path, &suffix);
-
file.write_all(&bytes)
- .context(UnableToCopyDataToFileSnafu)?;
-
- std::fs::rename(staging_path,
path).context(UnableToRenameFileSnafu)?;
-
- Ok(())
+ .context(UnableToCopyDataToFileSnafu)
+ .and_then(|_| {
+ std::fs::rename(&staging_path,
&path).context(UnableToRenameFileSnafu)
Review Comment:
I don't have a great idea either
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
