bryanck commented on code in PR #7976:
URL: https://github.com/apache/iceberg/pull/7976#discussion_r1289412845
##########
core/src/main/java/org/apache/iceberg/io/ResolvingFileIO.java:
##########
@@ -23,32 +23,44 @@
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
+import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.CatalogUtil;
import org.apache.iceberg.exceptions.ValidationException;
import org.apache.iceberg.hadoop.HadoopConfigurable;
import org.apache.iceberg.hadoop.SerializableConfiguration;
+import
org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
import org.apache.iceberg.relocated.com.google.common.base.Joiner;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Iterators;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.util.SerializableMap;
import org.apache.iceberg.util.SerializableSupplier;
+import org.apache.iceberg.util.Tasks;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** FileIO implementation that uses location scheme to choose the correct
FileIO implementation. */
-public class ResolvingFileIO implements FileIO, HadoopConfigurable {
+public class ResolvingFileIO implements FileIO, HadoopConfigurable,
SupportsBulkOperations {
private static final Logger LOG =
LoggerFactory.getLogger(ResolvingFileIO.class);
+ private static final Integer BATCH_SIZE = 100_000;
private static final String FALLBACK_IMPL =
"org.apache.iceberg.hadoop.HadoopFileIO";
private static final String S3_FILE_IO_IMPL =
"org.apache.iceberg.aws.s3.S3FileIO";
private static final String GCS_FILE_IO_IMPL =
"org.apache.iceberg.gcp.gcs.GCSFileIO";
+ private static final String IN_MEMORY_IO_IMPL =
"org.apache.iceberg.inmemory.InMemoryFileIO";
private static final Map<String, String> SCHEME_TO_FILE_IO =
ImmutableMap.of(
- "s3", S3_FILE_IO_IMPL,
- "s3a", S3_FILE_IO_IMPL,
- "s3n", S3_FILE_IO_IMPL,
- "gs", GCS_FILE_IO_IMPL);
+ "s3",
+ S3_FILE_IO_IMPL,
+ "s3a",
+ S3_FILE_IO_IMPL,
+ "s3n",
+ S3_FILE_IO_IMPL,
+ "gs",
+ GCS_FILE_IO_IMPL,
+ "inmemory",
Review Comment:
Is this needed? You could use a mock/spy for testing.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
