Jackie-Jiang commented on a change in pull request #4646: Enabling alternative
controller rest API classes in preprocess
URL: https://github.com/apache/incubator-pinot/pull/4646#discussion_r328402163
##########
File path:
pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/BaseSegmentJob.java
##########
@@ -37,13 +41,57 @@
protected final Logger _logger = LoggerFactory.getLogger(getClass());
protected final Properties _properties;
protected final Configuration _conf;
+ protected final List<PushLocation> _pushLocations;
+ protected final String _rawTableName;
protected BaseSegmentJob(Properties properties) {
_properties = properties;
_conf = new Configuration();
setConf(_conf);
Utils.logVersions();
logProperties();
+
+ // Optional push location and table parameters. If set, will use the table
config and schema from the push hosts.
+ String pushHostsString =
_properties.getProperty(JobConfigConstants.PUSH_TO_HOSTS);
+ String pushPortString =
_properties.getProperty(JobConfigConstants.PUSH_TO_PORT);
+ if (pushHostsString != null && pushPortString != null) {
+ _pushLocations =
+ PushLocation.getPushLocations(StringUtils.split(pushHostsString,
','), Integer.parseInt(pushPortString));
+ } else {
+ _pushLocations = null;
+ }
+
+ _rawTableName =
Preconditions.checkNotNull(_properties.getProperty(JobConfigConstants.SEGMENT_TABLE_NAME));
+
+ }
+
+ @Nullable
+ protected TableConfig getTableConfig()
+ throws IOException {
+ try (ControllerRestApi controllerRestApi = getControllerRestApi()) {
+ return controllerRestApi != null ? controllerRestApi.getTableConfig() :
null;
+ }
+ }
+
+ /**
+ * This method is currently implemented in SegmentCreationJob and
SegmentPreprocessingJob due to a dependency on
+ * the hadoop filesystem, which we can only get once the job begins to run.
+ * We return null here to make it clear that for now, all implementations of
this method have to support
+ * reading from a schema file. In the future, we hope to deprecate reading
the schema from the schema file in favor
+ * of mandating that a schema is pushed to the controller.
+ * @return
Review comment:
Remove this line
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
