RussellSpitzer commented on code in PR #4759:
URL: https://github.com/apache/iceberg/pull/4759#discussion_r902797301
##########
core/src/main/java/org/apache/iceberg/actions/SortStrategy.java:
##########
@@ -19,27 +19,57 @@
package org.apache.iceberg.actions;
+import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.SortOrder;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.FluentIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
+import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
+import org.apache.iceberg.util.BinPacking;
+import org.apache.iceberg.util.BinPacking.ListPacker;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.SortStrategyUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* A rewrite strategy for data files which aims to reorder data with data
files to optimally lay them out
* in relation to a column. For example, if the Sort strategy is used on a set
of files which is ordered
- * by column x and original has files File A (x: 0 - 50), File B ( x: 10 - 40)
and File C ( x: 30 - 60),
- * this Strategy will attempt to rewrite those files into File A' (x: 0-20),
File B' (x: 21 - 40),
- * File C' (x: 41 - 60).
+ * by column x and original has files File A (x: 0 - 50), File B ( x: 10 -
40), File C ( x: 30 - 60),
+ * File D ( x: 61 - 80), and File E ( x: 81 - 100), this Strategy will attempt
to rewrite files A, B and C
+ * into File A' (x: 0-20), File B' (x: 21 - 40), File C' (x: 41 - 60) and keep
File D and E unchanged.
* <p>
- * Currently the there is no file overlap detection and we will rewrite all
files if {@link SortStrategy#REWRITE_ALL}
- * is true (default: false). If this property is disabled any files that would
be chosen by
- * {@link BinPackStrategy} will be rewrite candidates.
+ * When the {@link BinPackStrategy#REWRITE_ALL} flag is, all files are
selected for rewrite. Otherwise,
+ * only unsorted files are selected. Rewrite will be applied to those selected
files if
+ * 1. There are a certain number of mis-sized data files or
+ * 2. Those data files do not have sortedness score good enough.
* <p>
- * In the future other algorithms for determining files to rewrite will be
provided.
*/
public abstract class SortStrategy extends BinPackStrategy {
+ private static final Logger LOG =
LoggerFactory.getLogger(SortStrategy.class);
+ /**
+ * Rewrites if the ratio of mis-sized files to total files is over this
threshold.
+ * The value should be between 0.0 and 1.0
+ */
+ public static final String MIS_SIZED_RATIO_THRESHOLD =
"mis-sized-ratio-threshold";
+ public static final double MIS_SIZED_RATIO_THRESHOLD_DEFAULT = 0.05;
+
+ /**
+ * Rewrites if the sortedness score of given files is below this threshold.
Review Comment:
Score is not defined yet, I think as we discussed before it may be worth
while to just have this be something that's literally countable like "number of
overlaps" or rather than something which is abstract.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
