This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 3ddf6285d52e fix(timeline-service): support response charset for
Timeline Server interaction (#19753)
3ddf6285d52e is described below
commit 3ddf6285d52e69ce842ac9197509dee854ca1acf
Author: Lokesh Jain <[email protected]>
AuthorDate: Thu Aug 27 05:55:12 2026 +0530
fix(timeline-service): support response charset for Timeline Server
interaction (#19753)
Co-authored-by: Y Ethan Guo <[email protected]>
---
.../client/HoodieTableServiceManagerClient.java | 7 ++-
.../client/embedded/EmbeddedTimelineService.java | 1 +
.../table/timeline/TimelineServiceClient.java | 7 ++-
.../table/view/FileSystemViewStorageConfig.java | 19 ++++++++
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 51 ++++++++++++++++++++++
5 files changed, 83 insertions(+), 2 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTableServiceManagerClient.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTableServiceManagerClient.java
index ec7dd1b30b0e..96ed1f789dde 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTableServiceManagerClient.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTableServiceManagerClient.java
@@ -21,6 +21,7 @@ package org.apache.hudi.client;
import org.apache.hudi.common.config.HoodieTableServiceManagerConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.util.ClusteringUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.RetryHelper;
@@ -34,6 +35,7 @@ import org.apache.http.client.utils.URIBuilder;
import java.io.IOException;
import java.net.URI;
+import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;
@@ -78,6 +80,7 @@ public class HoodieTableServiceManagerClient {
private final String basePath;
private final String dbName;
private final String tableName;
+ private final Charset responseCharset;
public HoodieTableServiceManagerClient(HoodieTableMetaClient metaClient,
HoodieTableServiceManagerConfig config) {
this.basePath = metaClient.getBasePath().toString();
@@ -86,6 +89,8 @@ public class HoodieTableServiceManagerClient {
this.uri = config.getTableServiceManagerURIs();
this.config = config;
this.metaClient = metaClient;
+ this.responseCharset = Charset.forName(
+
config.getStringOrDefault(FileSystemViewStorageConfig.REMOTE_RESPONSE_CHARSET));
}
private String executeRequest(String requestPath, Map<String, String>
queryParameters) throws IOException {
@@ -99,7 +104,7 @@ public class HoodieTableServiceManagerClient {
int connectionRetryDelay = config.getConnectionRetryDelay();
RetryHelper<String, IOException> retryHelper = new
RetryHelper<>(connectionRetryDelay, requestRetryLimit, connectionRetryDelay,
RETRY_EXCEPTIONS);
- return retryHelper.tryWith(() ->
Request.Get(url).connectTimeout(timeoutMs).socketTimeout(timeoutMs).execute().returnContent().asString()).start();
+ return retryHelper.tryWith(() ->
Request.Get(url).connectTimeout(timeoutMs).socketTimeout(timeoutMs).execute().returnContent().asString(responseCharset)).start();
}
private Map<String, String> getParamsWithAdditionalParams(String[]
paramNames, String[] paramVals) {
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java
index 495346a328d3..13a075d09f19 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java
@@ -214,6 +214,7 @@ public class EmbeddedTimelineService {
.withRemoteTimelineInitialRetryIntervalMs(clientWriteConfig.getClientSpecifiedViewStorageConfig().getRemoteTimelineInitialRetryIntervalMs())
.withRemoteTimelineClientMaxRetryIntervalMs(clientWriteConfig.getClientSpecifiedViewStorageConfig().getRemoteTimelineClientMaxRetryIntervalMs())
.withRemoteTimelineClientRetryExceptions(clientWriteConfig.getClientSpecifiedViewStorageConfig().getRemoteTimelineClientRetryExceptions())
+
.withRemoteResponseCharset(clientWriteConfig.getClientSpecifiedViewStorageConfig().getRemoteResponseCharset())
.build();
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineServiceClient.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineServiceClient.java
index f26b1fa97316..14c3a17b721e 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineServiceClient.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineServiceClient.java
@@ -23,7 +23,9 @@ import
org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.utils.URIBuilder;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
+import java.nio.charset.Charset;
import java.util.concurrent.TimeUnit;
/**
@@ -39,12 +41,14 @@ public class TimelineServiceClient extends
TimelineServiceClientBase {
protected final String timelineServerHost;
protected final int timelineServerPort;
protected final int timeoutMs;
+ protected final String responseCharsetName;
public TimelineServiceClient(FileSystemViewStorageConfig config) {
super(config);
this.timelineServerHost = config.getRemoteViewServerHost();
this.timelineServerPort = config.getRemoteViewServerPort();
this.timeoutMs = (int)
TimeUnit.SECONDS.toMillis(config.getRemoteTimelineClientTimeoutSecs());
+ this.responseCharsetName =
config.getStringOrDefault(FileSystemViewStorageConfig.REMOTE_RESPONSE_CHARSET);
}
@Override
@@ -59,7 +63,8 @@ public class TimelineServiceClient extends
TimelineServiceClientBase {
String url = builder.toString();
log.debug("Sending request : ({})", url);
org.apache.http.client.fluent.Response response = get(request.getMethod(),
url, timeoutMs);
- return new Response(response.returnContent().asStream());
+ return new Response(new ByteArrayInputStream(
+
response.returnContent().asString(Charset.forName(responseCharsetName)).getBytes(Charset.forName(responseCharsetName))));
}
private org.apache.http.client.fluent.Response get(RequestMethod method,
String url, int timeoutMs) throws IOException {
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java
index 19741cd8ad3e..ae08886ed537 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java
@@ -27,6 +27,7 @@ import org.apache.hudi.common.util.ValidationUtils;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Properties;
import java.util.stream.Collectors;
@@ -166,6 +167,15 @@ public class FileSystemViewStorageConfig extends
HoodieConfig {
.withDocumentation("The class name of the Exception that needs to be
retried, separated by commas. "
+ "Default is empty which means retry all the IOException and
RuntimeException from Remote Request.");
+ public static final ConfigProperty<String> REMOTE_RESPONSE_CHARSET =
ConfigProperty
+ .key("hoodie.filesystem.view.remote.response.charset")
+ .defaultValue(StandardCharsets.ISO_8859_1.name())
+ .markAdvanced()
+ .sinceVersion("1.3.0")
+ .withDocumentation("Charset used for decoding HTTP responses from the
timeline server. "
+ + "Set to 'UTF-8' if partition paths or file paths contain non-ASCII
characters (e.g. Unicode). "
+ + "Default is ISO-8859-1 for backwards compatibility.");
+
public static final ConfigProperty<String> REMOTE_BACKUP_VIEW_ENABLE =
ConfigProperty
.key("hoodie.filesystem.remote.backup.view.enable")
.defaultValue("true") // Need to be disabled only for tests.
@@ -267,6 +277,10 @@ public class FileSystemViewStorageConfig extends
HoodieConfig {
return FileSystemViewStorageType.valueOf(getString(SECONDARY_VIEW_TYPE));
}
+ public String getRemoteResponseCharset() {
+ return getStringOrDefault(REMOTE_RESPONSE_CHARSET);
+ }
+
public boolean shouldEnableBackupForRemoteFileSystemView() {
return getBoolean(REMOTE_BACKUP_VIEW_ENABLE);
}
@@ -374,6 +388,11 @@ public class FileSystemViewStorageConfig extends
HoodieConfig {
return this;
}
+ public Builder withRemoteResponseCharset(String charset) {
+ fileSystemViewStorageConfig.setValue(REMOTE_RESPONSE_CHARSET, charset);
+ return this;
+ }
+
public Builder withEnableBackupForRemoteFileSystemView(boolean enable) {
fileSystemViewStorageConfig.setValue(REMOTE_BACKUP_VIEW_ENABLE,
Boolean.toString(enable));
return this;
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
index 393030dd83b3..e4ae6bbdd086 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
@@ -1420,6 +1420,57 @@ def testBulkInsertForDropPartitionColumn(): Unit = {
val tableMetaClient = createMetaClient(spark, tempBasePath)
new TableSchemaResolver(tableMetaClient).getTableSchema(false)
}
+
+ /**
+ * Test that upsert works correctly when partition path contains Unicode
characters.
+ * Reproduces a bug where UTF-8 bytes for characters like "ü" (U+00FC) get
misinterpreted
+ * as Latin-1 during the String-to-Path round-trip, causing file-not-found
errors on the
+ * second write.
+ */
+ @Test
+ def testUpsertWithUnicodePartitionPath(): Unit = {
+ val options = Map(
+ DataSourceWriteOptions.TABLE_TYPE.key ->
HoodieTableType.COPY_ON_WRITE.name(),
+ DataSourceWriteOptions.PRECOMBINE_FIELD.key -> "ts",
+ DataSourceWriteOptions.RECORDKEY_FIELD.key -> "uuid",
+ DataSourceWriteOptions.PARTITIONPATH_FIELD.key -> "company",
+ DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME.key ->
"org.apache.hudi.keygen.SimpleKeyGenerator",
+ HoodieWriteConfig.TBL_NAME.key -> "hoodie_test",
+ "hoodie.insert.shuffle.parallelism" -> "1",
+ "hoodie.upsert.shuffle.parallelism" -> "1",
+ "hoodie.filesystem.view.remote.response.charset" -> "UTF-8"
+ )
+
+ // Unicode partition value containing German umlaut ü (U+00FC)
+ val unicodePartition = "M\u00fcnchen"
+
+ // First write - insert with Overwrite
+ val df = spark.createDataFrame(Seq(
+ ("id1", 100L, unicodePartition),
+ ("id2", 200L, unicodePartition)
+ )).toDF("uuid", "ts", "company")
+
+ df.write.format("hudi")
+ .options(options)
+ .mode(SaveMode.Overwrite)
+ .save(tempBasePath)
+
+ // Second write - upsert with Append (triggers reading existing Parquet
data)
+ val dfUpdate = spark.createDataFrame(Seq(
+ ("id1", 300L, unicodePartition),
+ ("id2", 400L, unicodePartition)
+ )).toDF("uuid", "ts", "company")
+
+ dfUpdate.write.format("hudi")
+ .options(options)
+ .mode(SaveMode.Append)
+ .save(tempBasePath)
+
+ // Verify upserted data can be read back
+ val dfResult = spark.read.format("hudi").load(tempBasePath)
+ assert(dfResult.count() == 2)
+ assert(dfResult.where("ts >= 300").count() == 2)
+ }
}
object TestHoodieSparkSqlWriter {
