This is an automated email from the ASF dual-hosted git repository.
jenniferdai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new a1fff55 Ensuring partition config is valid during Hadoop preprocess
(#4463)
a1fff55 is described below
commit a1fff551d143e99719e66cdf73273602fc993a2f
Author: Jennifer Dai <[email protected]>
AuthorDate: Tue Jul 23 14:57:37 2019 -0700
Ensuring partition config is valid during Hadoop preprocess (#4463)
* Check to make sure the partition algorithm is valid.
---
.../java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java | 7 +++++++
1 file changed, 7 insertions(+)
diff --git
a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java
b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java
index a3f3901..65fbd73 100644
---
a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java
+++
b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java
@@ -41,6 +41,7 @@ import org.apache.pinot.common.config.IndexingConfig;
import org.apache.pinot.common.config.SegmentPartitionConfig;
import org.apache.pinot.common.config.TableConfig;
import org.apache.pinot.common.config.TableCustomConfig;
+import org.apache.pinot.core.data.partition.PartitionFunctionFactory;
import org.apache.pinot.hadoop.utils.PushLocation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -286,6 +287,12 @@ public class SegmentPreprocessingJob extends
BaseSegmentJob {
.format("Partition column: %s is not found from the schema of input
files.", _partitionColumn));
Preconditions.checkArgument(_numberOfPartitions > 0,
String.format("Number of partitions should be positive. Current value: %s",
_numberOfPartitions));
Preconditions.checkArgument(_partitionFunction != null, "Partition
function should not be null!");
+ try {
+
PartitionFunctionFactory.PartitionFunctionType.fromString(_partitionFunction);
+ } catch (IllegalArgumentException e) {
+ _logger.error("Partition function needs to be one of Modulo, Murmur,
ByteArray, HashCode, it is currently {}", _partitionColumn);
+ throw new IllegalArgumentException(e);
+ }
}
if (_enableSorting) {
Preconditions.checkArgument(_sortedColumn != null, "Sorted column should
not be null!");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
