Jackie-Jiang commented on a change in pull request #4577: Adding integration
test for Hadoop pre-process job
URL: https://github.com/apache/incubator-pinot/pull/4577#discussion_r320995822
##########
File path:
pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentPreprocessingMapper.java
##########
@@ -43,30 +44,32 @@
private Schema _outputKeySchema;
private Schema _outputSchema;
private boolean _enablePartitioning;
- private String _sampleNormalizedTimeColumnValue = null;
- private NormalizedDateSegmentNameGenerator
_normalizedDateSegmentNameGenerator = null;
private boolean _isAppend = false;
+ private String _timeColumnValue;
+ private String _pushFrequency;
+ private String _timeFormat;
+ private TimeUnit _timeUnit;
+ private String _tableName;
@Override
public void setup(final Context context) {
Configuration configuration = context.getConfiguration();
+ _tableName = configuration.get(JobConfigConstants.SEGMENT_TABLE_NAME);
+
_isAppend =
configuration.get(InternalConfigConstants.IS_APPEND).equalsIgnoreCase("true");
if (_isAppend) {
// Get time column name
_timeColumn =
configuration.get(InternalConfigConstants.TIME_COLUMN_CONFIG);
// Get sample time column value
- String timeColumnValue =
configuration.get(InternalConfigConstants.TIME_COLUMN_VALUE);
+ _timeColumnValue =
configuration.get(InternalConfigConstants.TIME_COLUMN_VALUE);
- String pushFrequency =
configuration.get(InternalConfigConstants.SEGMENT_PUSH_FREQUENCY);
+ _pushFrequency =
configuration.get(InternalConfigConstants.SEGMENT_PUSH_FREQUENCY);
String timeType =
configuration.get(InternalConfigConstants.SEGMENT_TIME_TYPE);
- String timeFormat =
configuration.get(InternalConfigConstants.SEGMENT_TIME_FORMAT);
- TimeUnit timeUnit = TimeUnit.valueOf(timeType);
- // Normalize time column value
- _normalizedDateSegmentNameGenerator = new
NormalizedDateSegmentNameGenerator(pushFrequency, timeUnit, timeFormat);
Review comment:
Why changing this? We can still create name generator and sample value here
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
