imay commented on a change in pull request #1235: Support SSL authentication
with Kafka in routine load job
URL: https://github.com/apache/incubator-doris/pull/1235#discussion_r290355134
##########
File path:
fe/src/main/java/org/apache/doris/load/routineload/KafkaRoutineLoadJob.java
##########
@@ -305,17 +322,41 @@ public static KafkaRoutineLoadJob
fromCreateStmt(CreateRoutineLoadStmt stmt) thr
db.readUnlock();
}
- // TODO(ml): check partition
-
// init kafka routine load job
long id = Catalog.getInstance().getNextId();
KafkaRoutineLoadJob kafkaRoutineLoadJob = new KafkaRoutineLoadJob(id,
stmt.getName(),
db.getClusterName(), db.getId(), tableId,
stmt.getKafkaBrokerList(), stmt.getKafkaTopic());
kafkaRoutineLoadJob.setOptional(stmt);
+ kafkaRoutineLoadJob.checkCustomProperties();
+ kafkaRoutineLoadJob.checkCustomPartition();
return kafkaRoutineLoadJob;
}
+ private void checkCustomPartition() throws UserException {
+ if (customKafkaPartitions.isEmpty()) {
+ return;
+ }
+ List<Integer> allKafkaPartitions = getAllKafkaPartitions();
+ for (Integer customPartition : customKafkaPartitions) {
+ if (!allKafkaPartitions.contains(customPartition)) {
+ throw new LoadException("there is a custom kafka partition " +
customPartition
+ + " which is invalid for topic " + topic);
+ }
+ }
+ }
+
+ private void checkCustomProperties() throws DdlException {
+ SmallFileMgr smallFileMgr =
Catalog.getCurrentCatalog().getSmallFileMgr();
+ for (Map.Entry<String, String> entry :
customKafkaProperties.entrySet()) {
+ if (entry.getValue().startsWith("FILE:")) {
+ String file =
entry.getValue().substring(entry.getValue().indexOf(":") + 1);
+ // check and save file to disk
+ smallFileMgr.saveToFile(dbId, KAFKA_FILE_CATALOG, file);
Review comment:
I don't know why we should save it to file?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]