jackjlli commented on a change in pull request #4462: Time-aware resizing
URL: https://github.com/apache/incubator-pinot/pull/4462#discussion_r307068393
##########
File path:
pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentPreprocessingMapper.java
##########
@@ -29,24 +30,49 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.pinot.core.segment.name.NormalizedDateSegmentNameGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.pinot.hadoop.job.InternalConfigConstants.*;
import static org.apache.pinot.hadoop.job.JobConfigConstants.*;
public class SegmentPreprocessingMapper extends Mapper<AvroKey<GenericRecord>,
NullWritable, AvroKey<GenericRecord>, AvroValue<GenericRecord>> {
private static final Logger LOGGER =
LoggerFactory.getLogger(SegmentPreprocessingMapper.class);
private String _sortedColumn = null;
+ private String _timeColumn = null;
private Schema _outputKeySchema;
private Schema _outputSchema;
private boolean _enablePartition = false;
+ private String _sampleNormalizedTimeColumnValue = null;
+ private NormalizedDateSegmentNameGenerator
_normalizedDateSegmentNameGenerator = null;
+ private boolean _isAppend = false;
@Override
public void setup(final Context context) {
Configuration configuration = context.getConfiguration();
- String sortedColumn = configuration.get("sorted.column");
+ _isAppend = configuration.get(IS_APPEND).equalsIgnoreCase("true");
+
+ if (_isAppend) {
+ // Get time column name
+ _timeColumn = configuration.get(TIME_COLUMN_CONFIG);
+
+ // Get sample time column value
+ String timeColumnValue = configuration.get(TIME_COLUMN_VALUE);
+
+ String pushFrequency = configuration.get(SEGMENT_PUSH_FREQUENCY);
+ String timeType = configuration.get(SEGMENT_TIME_TYPE);
+ String timeFormat = configuration.get(SEGMENT_TIME_FORMAT);
+ TimeUnit timeUnit = TimeUnit.valueOf(timeType
+ );
Review comment:
No need to put it to a different line
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
