dramaticlly commented on code in PR #7976:
URL: https://github.com/apache/iceberg/pull/7976#discussion_r1289568391
##########
core/src/main/java/org/apache/iceberg/io/ResolvingFileIO.java:
##########
@@ -23,32 +23,44 @@
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
+import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.CatalogUtil;
import org.apache.iceberg.exceptions.ValidationException;
import org.apache.iceberg.hadoop.HadoopConfigurable;
import org.apache.iceberg.hadoop.SerializableConfiguration;
+import
org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
import org.apache.iceberg.relocated.com.google.common.base.Joiner;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Iterators;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.util.SerializableMap;
import org.apache.iceberg.util.SerializableSupplier;
+import org.apache.iceberg.util.Tasks;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** FileIO implementation that uses location scheme to choose the correct
FileIO implementation. */
-public class ResolvingFileIO implements FileIO, HadoopConfigurable {
+public class ResolvingFileIO implements FileIO, HadoopConfigurable,
SupportsBulkOperations {
private static final Logger LOG =
LoggerFactory.getLogger(ResolvingFileIO.class);
+ private static final Integer BATCH_SIZE = 100_000;
private static final String FALLBACK_IMPL =
"org.apache.iceberg.hadoop.HadoopFileIO";
private static final String S3_FILE_IO_IMPL =
"org.apache.iceberg.aws.s3.S3FileIO";
private static final String GCS_FILE_IO_IMPL =
"org.apache.iceberg.gcp.gcs.GCSFileIO";
+ private static final String IN_MEMORY_IO_IMPL =
"org.apache.iceberg.inmemory.InMemoryFileIO";
private static final Map<String, String> SCHEME_TO_FILE_IO =
ImmutableMap.of(
- "s3", S3_FILE_IO_IMPL,
- "s3a", S3_FILE_IO_IMPL,
- "s3n", S3_FILE_IO_IMPL,
- "gs", GCS_FILE_IO_IMPL);
+ "s3",
+ S3_FILE_IO_IMPL,
+ "s3a",
+ S3_FILE_IO_IMPL,
+ "s3n",
+ S3_FILE_IO_IMPL,
+ "gs",
+ GCS_FILE_IO_IMPL,
+ "inmemory",
Review Comment:
removed this mapping as I can mock the `io` method to return desired FileIO
instance.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
