This is an automated email from the ASF dual-hosted git repository.
codope pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new e05e758e55e [HUDI-8541] Change to relative paths for the index defs
json (#12309)
e05e758e55e is described below
commit e05e758e55edeeb046fa5c63136280f5a2602aaa
Author: Lokesh Jain <[email protected]>
AuthorDate: Fri Nov 22 14:20:12 2024 +0530
[HUDI-8541] Change to relative paths for the index defs json (#12309)
---
.../action/index/functional/BaseHoodieIndexClient.java | 6 ++++--
.../org/apache/hudi/common/table/HoodieTableConfig.java | 8 ++++----
.../apache/hudi/common/table/HoodieTableMetaClient.java | 15 ++++++++-------
.../hudi/common/table/TestHoodieTableMetaClient.java | 2 +-
.../main/java/org/apache/hudi/HoodieSparkIndexClient.java | 2 +-
.../spark/sql/hudi/command/index/TestSecondaryIndex.scala | 5 ++++-
6 files changed, 22 insertions(+), 16 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/functional/BaseHoodieIndexClient.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/functional/BaseHoodieIndexClient.java
index 0a3113ce6b8..786b36a823f 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/functional/BaseHoodieIndexClient.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/functional/BaseHoodieIndexClient.java
@@ -19,8 +19,10 @@
package org.apache.hudi.table.action.index.functional;
+import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.storage.StoragePath;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,8 +49,8 @@ public abstract class BaseHoodieIndexClient {
metaClient.buildIndexDefinition(indexName, indexType, columns, options);
// update table config if necessary
String indexMetaPath = metaClient.getIndexDefinitionPath();
- if
(!metaClient.getTableConfig().getProps().containsKey(HoodieTableConfig.INDEX_DEFINITION_PATH)
|| !metaClient.getTableConfig().getIndexDefinitionPath().isPresent()) {
-
metaClient.getTableConfig().setValue(HoodieTableConfig.INDEX_DEFINITION_PATH,
indexMetaPath);
+ if
(!metaClient.getTableConfig().getProps().containsKey(HoodieTableConfig.RELATIVE_INDEX_DEFINITION_PATH)
|| !metaClient.getTableConfig().getRelativeIndexDefinitionPath().isPresent()) {
+
metaClient.getTableConfig().setValue(HoodieTableConfig.RELATIVE_INDEX_DEFINITION_PATH,
FSUtils.getRelativePartitionPath(metaClient.getBasePath(), new
StoragePath(indexMetaPath)));
HoodieTableConfig.update(metaClient.getStorage(),
metaClient.getMetaPath(), metaClient.getTableConfig().getProps());
}
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
index 9a0773eedd7..0f9749e9407 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
@@ -331,11 +331,11 @@ public class HoodieTableConfig extends HoodieConfig {
.sinceVersion("0.13.0")
.withDocumentation("The metadata of secondary indexes");
- public static final ConfigProperty<String> INDEX_DEFINITION_PATH =
ConfigProperty
+ public static final ConfigProperty<String> RELATIVE_INDEX_DEFINITION_PATH =
ConfigProperty
.key("hoodie.table.index.defs.path")
.noDefaultValue()
.sinceVersion("1.0.0")
- .withDocumentation("Absolute path where the index definitions are
stored");
+ .withDocumentation("Relative path to table base path where the index
definitions are stored");
private static final String TABLE_CHECKSUM_FORMAT = "%s.%s"; //
<database_name>.<table_name>
@@ -970,8 +970,8 @@ public class HoodieTableConfig extends HoodieConfig {
/**
* @returns the index definition path.
*/
- public Option<String> getIndexDefinitionPath() {
- return Option.ofNullable(getString(INDEX_DEFINITION_PATH));
+ public Option<String> getRelativeIndexDefinitionPath() {
+ return Option.ofNullable(getString(RELATIVE_INDEX_DEFINITION_PATH));
}
/**
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
index 288005ea2b3..442d0d81be6 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
@@ -196,7 +196,8 @@ public class HoodieTableMetaClient implements Serializable {
}
public String getIndexDefinitionPath() {
- return tableConfig.getIndexDefinitionPath()
+ return tableConfig.getRelativeIndexDefinitionPath()
+ .map(definitionPath -> new StoragePath(basePath,
definitionPath).toString())
.orElseGet(() -> metaPath + StoragePath.SEPARATOR +
HoodieTableMetaClient.INDEX_DEFINITION_FOLDER_NAME
+ StoragePath.SEPARATOR +
HoodieTableMetaClient.INDEX_DEFINITION_FILE_NAME);
}
@@ -254,14 +255,14 @@ public class HoodieTableMetaClient implements
Serializable {
if (indexMetadataOpt.isPresent() &&
!indexMetadataOpt.get().getIndexDefinitions().isEmpty()) {
return indexMetadataOpt;
}
- if (tableConfig.getIndexDefinitionPath().isPresent() &&
StringUtils.nonEmpty(tableConfig.getIndexDefinitionPath().get())) {
+ if (tableConfig.getRelativeIndexDefinitionPath().isPresent() &&
StringUtils.nonEmpty(tableConfig.getRelativeIndexDefinitionPath().get())) {
StoragePath indexDefinitionPath =
- new StoragePath(tableConfig.getIndexDefinitionPath().get());
+ new StoragePath(basePath,
tableConfig.getRelativeIndexDefinitionPath().get());
try {
return Option.of(HoodieIndexMetadata.fromJson(
new String(FileIOUtils.readDataFromPath(storage,
indexDefinitionPath).get())));
} catch (IOException e) {
- throw new HoodieIOException("Could not load functional index metadata
at path: " + tableConfig.getIndexDefinitionPath().get(), e);
+ throw new HoodieIOException("Could not load functional index metadata
at path: " + tableConfig.getRelativeIndexDefinitionPath().get(), e);
}
}
return Option.empty();
@@ -1302,8 +1303,8 @@ public class HoodieTableMetaClient implements
Serializable {
if
(hoodieConfig.contains(HoodieTableConfig.MULTIPLE_BASE_FILE_FORMATS_ENABLE)) {
setMultipleBaseFileFormatsEnabled(hoodieConfig.getBoolean(HoodieTableConfig.MULTIPLE_BASE_FILE_FORMATS_ENABLE));
}
- if (hoodieConfig.contains(HoodieTableConfig.INDEX_DEFINITION_PATH)) {
-
setIndexDefinitionPath(hoodieConfig.getString(HoodieTableConfig.INDEX_DEFINITION_PATH));
+ if
(hoodieConfig.contains(HoodieTableConfig.RELATIVE_INDEX_DEFINITION_PATH)) {
+
setIndexDefinitionPath(hoodieConfig.getString(HoodieTableConfig.RELATIVE_INDEX_DEFINITION_PATH));
}
return this;
}
@@ -1420,7 +1421,7 @@ public class HoodieTableMetaClient implements
Serializable {
tableConfig.setValue(HoodieTableConfig.MULTIPLE_BASE_FILE_FORMATS_ENABLE,
Boolean.toString(multipleBaseFileFormatsEnabled));
}
if (null != indexDefinitionPath) {
- tableConfig.setValue(HoodieTableConfig.INDEX_DEFINITION_PATH,
indexDefinitionPath);
+ tableConfig.setValue(HoodieTableConfig.RELATIVE_INDEX_DEFINITION_PATH,
indexDefinitionPath);
}
return tableConfig.getProps();
}
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
index db28066ebaa..171230a3dda 100644
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
@@ -240,7 +240,7 @@ public class TestHoodieTableMetaClient extends
HoodieCommonTestHarness {
assertEquals(metaClient.getMetaPath() + "/.index_defs/index.json",
metaClient.getIndexDefinitionPath());
String randomDefinitionPath = "/a/b/c";
-
metaClient.getTableConfig().setValue(HoodieTableConfig.INDEX_DEFINITION_PATH.key(),
"/a/b/c");
+
metaClient.getTableConfig().setValue(HoodieTableConfig.RELATIVE_INDEX_DEFINITION_PATH.key(),
"/a/b/c");
assertEquals(randomDefinitionPath, metaClient.getIndexDefinitionPath());
}
}
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/HoodieSparkIndexClient.java
b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/HoodieSparkIndexClient.java
index 2637ce3d45d..c2eb464463d 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/HoodieSparkIndexClient.java
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/HoodieSparkIndexClient.java
@@ -94,7 +94,7 @@ public class HoodieSparkIndexClient extends
BaseHoodieIndexClient {
throw new HoodieMetadataIndexException("Not eligible for indexing: " +
indexType + ", indexName: " + userIndexName);
}
- if (!metaClient.getTableConfig().getIndexDefinitionPath().isPresent()
+ if
(!metaClient.getTableConfig().getRelativeIndexDefinitionPath().isPresent()
|| !metaClient.getIndexMetadata().isPresent()
||
!metaClient.getIndexMetadata().get().getIndexDefinitions().containsKey(fullIndexName))
{
LOG.info("Index definition is not present. Registering the index first");
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestSecondaryIndex.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestSecondaryIndex.scala
index d0ce0dd0e72..99c1a53a009 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestSecondaryIndex.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestSecondaryIndex.scala
@@ -31,7 +31,7 @@ import
org.apache.hudi.metadata.HoodieMetadataPayload.SECONDARY_INDEX_RECORD_KEY
import org.apache.hudi.metadata.SecondaryIndexKeyUtils
import org.apache.spark.sql.SaveMode
import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase
-import org.junit.jupiter.api.Assertions.assertTrue
+import org.junit.jupiter.api.Assertions.{assertFalse, assertTrue}
import java.util.concurrent.atomic.AtomicInteger
import scala.collection.JavaConverters._
@@ -130,7 +130,10 @@ class TestSecondaryIndex extends HoodieSparkSqlTestBase {
.setBasePath(basePath)
.setConf(HoodieTestUtils.getDefaultStorageConf)
.build()
+
assertFalse(metaClient.getTableConfig.getRelativeIndexDefinitionPath.get().contains(metaClient.getBasePath))
+
assertTrue(metaClient.getIndexDefinitionPath.contains(metaClient.getBasePath.toString))
val indexDefinition =
metaClient.getIndexMetadata.get().getIndexDefinitions.values().stream().findFirst().get()
+
metaClient.getTableConfig.setMetadataPartitionState(metaClient,
indexDefinition.getIndexName, false)
checkAnswer(s"drop index idx_price on $tableName")()
checkAnswer(s"show indexes from $tableName")(
