bvaradar commented on a change in pull request #600: Timeline Service with
Incremental View Syncing support
URL: https://github.com/apache/incubator-hudi/pull/600#discussion_r276443809
##########
File path:
hoodie-common/src/main/java/com/uber/hoodie/common/table/view/HoodieTableFileSystemView.java
##########
@@ -16,99 +16,88 @@
package com.uber.hoodie.common.table.view;
-import com.google.common.collect.ImmutableMap;
+import com.google.common.base.Preconditions;
import com.uber.hoodie.common.model.CompactionOperation;
-import com.uber.hoodie.common.model.FileSlice;
-import com.uber.hoodie.common.model.HoodieDataFile;
import com.uber.hoodie.common.model.HoodieFileGroup;
import com.uber.hoodie.common.model.HoodieFileGroupId;
-import com.uber.hoodie.common.model.HoodieLogFile;
import com.uber.hoodie.common.table.HoodieTableMetaClient;
import com.uber.hoodie.common.table.HoodieTimeline;
import com.uber.hoodie.common.table.TableFileSystemView;
-import com.uber.hoodie.common.table.timeline.HoodieInstant;
-import com.uber.hoodie.common.util.CompactionUtils;
-import com.uber.hoodie.common.util.FSUtils;
+import com.uber.hoodie.common.util.Option;
import com.uber.hoodie.common.util.collection.Pair;
-import com.uber.hoodie.exception.HoodieIOException;
import java.io.IOException;
-import java.io.Serializable;
import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.function.Predicate;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
/**
- * Common abstract implementation for multiple TableFileSystemView
Implementations. 2 possible
- * implementations are ReadOptimizedView and RealtimeView <p> Concrete
implementations extending
- * this abstract class, should only implement getDataFilesInPartition which
includes files to be
- * included in the view
- *
+ * TableFileSystemView Implementations based on in-memory storage.
* @see TableFileSystemView
* @since 0.3.0
*/
-public class HoodieTableFileSystemView implements TableFileSystemView,
- TableFileSystemView.ReadOptimizedView,
- TableFileSystemView.RealtimeView, Serializable {
+public class HoodieTableFileSystemView extends
IncrementalTimelineSyncFileSystemView {
private static Logger log =
LogManager.getLogger(HoodieTableFileSystemView.class);
- protected HoodieTableMetaClient metaClient;
- // This is the commits that will be visible for all views extending this view
- protected HoodieTimeline visibleActiveTimeline;
-
// mapping from partition paths to file groups contained within them
- protected HashMap<String, List<HoodieFileGroup>> partitionToFileGroupsMap;
- // mapping from file id to the file group.
- protected HashMap<HoodieFileGroupId, HoodieFileGroup> fileGroupMap;
+ protected Map<String, List<HoodieFileGroup>> partitionToFileGroupsMap;
/**
* PartitionPath + File-Id to pending compaction instant time
*/
- private final Map<HoodieFileGroupId, Pair<String, CompactionOperation>>
fgIdToPendingCompaction;
+ private Map<HoodieFileGroupId, Pair<String, CompactionOperation>>
fgIdToPendingCompaction;
+
+ /**
+ * Flag to determine if closed
+ */
+ private boolean closed = false;
+
+ public HoodieTableFileSystemView() {
Review comment:
Made it default scope. Used by subclass : SpillableMapBasedFileSystemView
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
