vinothchandar commented on a change in pull request #2136:
URL: https://github.com/apache/hudi/pull/2136#discussion_r528873727
##########
File path:
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java
##########
@@ -85,4 +92,56 @@ public static HoodieRecord getTaggedRecord(HoodieRecord
inputRecord, Option<Hood
}
return record;
}
+
+ /**
+ * Check compatible between new writeIndexType and indexType already in
hoodie.properties.
+ * @param writeIndexType new indexType
+ * @param persistIndexType indexType already in hoodie.properties
+ */
+ public static void checkIndexTypeCompatible(IndexType writeIndexType,
IndexType persistIndexType) {
+ boolean isTypeCompatible = false;
+ switch (persistIndexType) {
+ case GLOBAL_BLOOM:
+ isTypeCompatible = writeIndexType.equals(IndexType.GLOBAL_BLOOM)
+ || writeIndexType.equals(IndexType.BLOOM)
+ || writeIndexType.equals(IndexType.SIMPLE)
+ || writeIndexType.equals(IndexType.GLOBAL_SIMPLE);
+ break;
+ case GLOBAL_SIMPLE:
+ isTypeCompatible = writeIndexType.equals(IndexType.GLOBAL_SIMPLE)
+ || writeIndexType.equals(IndexType.GLOBAL_BLOOM)
+ || writeIndexType.equals(IndexType.BLOOM)
+ || writeIndexType.equals(IndexType.SIMPLE);
+ break;
+ case SIMPLE:
+ isTypeCompatible = writeIndexType.equals(IndexType.SIMPLE)
+ || writeIndexType.equals(IndexType.BLOOM);
+ break;
+ case BLOOM:
+ isTypeCompatible = writeIndexType.equals(IndexType.BLOOM)
+ || writeIndexType.equals(IndexType.SIMPLE);
+ break;
+ case INMEMORY:
+ isTypeCompatible = writeIndexType.equals(IndexType.INMEMORY);
+ LOG.error("PersistIndexType INMEMORY can not be used in production");
Review comment:
LOG.warn
##########
File path:
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/index/state/FlinkInMemoryStateIndex.java
##########
@@ -132,4 +132,9 @@ public boolean canIndexLogFiles() {
public boolean isImplicitWithStorage() {
return false;
}
+
+ @Override
+ public IndexType indexType() {
+ return IndexType.INMEMORY;
Review comment:
does this mean, this cannot be used in production as well? the Flink In
Memory state is backed up/restored etc. no?
##########
File path:
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkRDDWriteClient.java
##########
@@ -88,7 +91,18 @@ public static SparkConf registerClasses(SparkConf conf) {
@Override
protected HoodieIndex<T, JavaRDD<HoodieRecord<T>>, JavaRDD<HoodieKey>,
JavaRDD<WriteStatus>> createIndex(HoodieWriteConfig writeConfig) {
- return SparkHoodieIndex.createIndex(config);
+ String persistIndexType = null;
+ try {
+ persistIndexType =
this.createMetaClient(false).getTableConfig().getProperties().getProperty(HoodieIndexConfig.INDEX_TYPE_PROP);
Review comment:
this is the metaClient creation we wanted to avoid?
##########
File path:
hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java
##########
@@ -237,7 +241,8 @@ public void refreshTimeline() throws IOException {
} else {
this.commitTimelineOpt = Option.empty();
HoodieTableMetaClient.initTableType(new
Configuration(jssc.hadoopConfiguration()), cfg.targetBasePath,
- HoodieTableType.valueOf(cfg.tableType), cfg.targetTableName,
"archived", cfg.payloadClassName, cfg.baseFileFormat);
+ HoodieTableType.valueOf(cfg.tableType), cfg.targetTableName,
"archived", cfg.payloadClassName, cfg.baseFileFormat,
+ HoodieIndexUtils.getIndexType(this.getHoodieClientConfig((Schema)
null)).name());
Review comment:
why not reuse `this.hoodieClientConfig`
##########
File path:
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkTable.java
##########
@@ -67,6 +69,12 @@ protected HoodieSparkTable(HoodieWriteConfig config,
HoodieEngineContext context
@Override
protected HoodieIndex<T, JavaRDD<HoodieRecord<T>>, JavaRDD<HoodieKey>,
JavaRDD<WriteStatus>> getIndex(HoodieWriteConfig config, HoodieEngineContext
context) {
- return SparkHoodieIndex.createIndex(config);
+ String persistIndexType =
this.metaClient.getTableConfig().getProperties().getProperty(HoodieIndexConfig.INDEX_TYPE_PROP);
Review comment:
this part does not need the `TableNotFoundException` checking?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
