This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 52ec8673 fix(spec): normalize file-format options to lowercase (#663)
52ec8673 is described below
commit 52ec8673d4cc01f065911ea8c540c70405a34074
Author: jackylee <[email protected]>
AuthorDate: Tue Aug 4 13:33:07 2026 +0800
fix(spec): normalize file-format options to lowercase (#663)
---
crates/paimon/src/spec/core_options.rs | 70 +++++++++++++++++++++---
crates/paimon/src/table/data_evolution_writer.rs | 4 +-
crates/paimon/src/table/format_table_scan.rs | 2 +-
crates/paimon/src/table/table_write.rs | 2 +-
4 files changed, 64 insertions(+), 14 deletions(-)
diff --git a/crates/paimon/src/spec/core_options.rs
b/crates/paimon/src/spec/core_options.rs
index 20216368..dfb64911 100644
--- a/crates/paimon/src/spec/core_options.rs
+++ b/crates/paimon/src/spec/core_options.rs
@@ -974,11 +974,15 @@ impl<'a> CoreOptions<'a> {
///
/// Java leaves this unset by default. When present, vector columns are
/// written to files named `*.vector.<format>`.
- pub fn vector_file_format(&self) -> Option<&str> {
+ /// Dedicated vector file format, normalized to lowercase.
+ ///
+ /// Mirrors Java `CoreOptions.normalizeFileFormat`, which lowercases every
+ /// file-format option so the value can be used directly in a file name.
+ pub fn vector_file_format(&self) -> Option<String> {
self.options
.get(VECTOR_FILE_FORMAT_OPTION)
- .map(String::as_str)
- .filter(|format| !format.trim().is_empty())
+ .map(|format| format.trim().to_ascii_lowercase())
+ .filter(|format| !format.is_empty())
}
pub fn vector_target_file_size(&self) -> i64 {
@@ -990,11 +994,12 @@ impl<'a> CoreOptions<'a> {
/// File format for data files (e.g. "parquet", "orc", "avro", "vortex").
/// Default is "parquet".
- pub fn file_format(&self) -> &str {
+ pub fn file_format(&self) -> String {
self.options
.get(FILE_FORMAT_OPTION)
- .map(String::as_str)
- .unwrap_or("parquet")
+ .map(|format| format.trim().to_ascii_lowercase())
+ .filter(|format| !format.is_empty())
+ .unwrap_or_else(|| "parquet".to_string())
}
/// File compression codec (e.g. "lz4", "zstd", "snappy", "none").
@@ -1027,10 +1032,11 @@ impl<'a> CoreOptions<'a> {
///
/// When `changelog-file.format` is not configured, Java Paimon falls back
/// to the table `file.format`.
- pub fn changelog_file_format(&self) -> &str {
+ pub fn changelog_file_format(&self) -> String {
self.options
.get(CHANGELOG_FILE_FORMAT_OPTION)
- .map(String::as_str)
+ .map(|format| format.trim().to_ascii_lowercase())
+ .filter(|format| !format.is_empty())
.unwrap_or_else(|| self.file_format())
}
@@ -1638,6 +1644,52 @@ mod tests {
}
}
+ #[test]
+ fn test_file_formats_are_normalized_to_lowercase() {
+ // Java routes every file-format option through
+ // `CoreOptions.normalizeFileFormat`, which lowercases it. The value
ends
+ // up in the data file name, so an uppercase option would otherwise
write
+ // `data-<uuid>-0.PARQUET` where Java writes `.parquet`.
+ for raw in ["PARQUET", "Parquet", " parquet ", "parquet"] {
+ let options = HashMap::from([(FILE_FORMAT_OPTION.to_string(),
raw.to_string())]);
+ assert_eq!(
+ CoreOptions::new(&options).file_format(),
+ "parquet",
+ "file.format = {raw:?} should normalize to \"parquet\""
+ );
+ }
+
+ let options = HashMap::from([(
+ CHANGELOG_FILE_FORMAT_OPTION.to_string(),
+ " AVRO ".to_string(),
+ )]);
+ assert_eq!(CoreOptions::new(&options).changelog_file_format(), "avro");
+
+ let options =
+ HashMap::from([(VECTOR_FILE_FORMAT_OPTION.to_string(),
"VORTEX".to_string())]);
+ assert_eq!(
+ CoreOptions::new(&options).vector_file_format().as_deref(),
+ Some("vortex")
+ );
+ }
+
+ #[test]
+ fn test_file_format_defaults_and_blank_handling() {
+ let empty = HashMap::new();
+ assert_eq!(CoreOptions::new(&empty).file_format(), "parquet");
+ // `changelog-file.format` falls back to `file.format`, still
normalized.
+ let options = HashMap::from([(FILE_FORMAT_OPTION.to_string(),
"ORC".to_string())]);
+ assert_eq!(CoreOptions::new(&options).changelog_file_format(), "orc");
+
+ // A blank value is not a format: fall back rather than return "".
+ for raw in ["", " "] {
+ let options = HashMap::from([(FILE_FORMAT_OPTION.to_string(),
raw.to_string())]);
+ assert_eq!(CoreOptions::new(&options).file_format(), "parquet");
+ let options =
HashMap::from([(VECTOR_FILE_FORMAT_OPTION.to_string(), raw.to_string())]);
+ assert_eq!(CoreOptions::new(&options).vector_file_format(), None);
+ }
+ }
+
#[test]
fn test_parse_memory_size() {
assert_eq!(parse_memory_size("1024"), Some(1024));
@@ -1950,7 +2002,7 @@ mod tests {
),
]);
let custom_core = CoreOptions::new(&custom_options);
- assert_eq!(custom_core.vector_file_format(), Some("vortex"));
+ assert_eq!(custom_core.vector_file_format().as_deref(),
Some("vortex"));
assert_eq!(custom_core.vector_target_file_size(), 64 * 1024 * 1024);
}
diff --git a/crates/paimon/src/table/data_evolution_writer.rs
b/crates/paimon/src/table/data_evolution_writer.rs
index 47752e3f..5e16524f 100644
--- a/crates/paimon/src/table/data_evolution_writer.rs
+++ b/crates/paimon/src/table/data_evolution_writer.rs
@@ -980,9 +980,7 @@ impl DataEvolutionPartialWriter {
fields: &[DataField],
core_options: &CoreOptions<'_>,
) -> Result<Vec<PartialWriteSet>> {
- let vector_file_format = core_options
- .vector_file_format()
- .map(|format| format.trim().to_ascii_lowercase());
+ let vector_file_format = core_options.vector_file_format();
let mut normal_fields = Vec::new();
let mut normal_columns = Vec::new();
let mut normal_indices = Vec::new();
diff --git a/crates/paimon/src/table/format_table_scan.rs
b/crates/paimon/src/table/format_table_scan.rs
index 4190526a..d3dbff36 100644
--- a/crates/paimon/src/table/format_table_scan.rs
+++ b/crates/paimon/src/table/format_table_scan.rs
@@ -65,7 +65,7 @@ impl<'a> FormatTableScan<'a> {
async fn plan_inner(&self, trace: Option<&mut ScanTrace>) ->
crate::Result<Plan> {
let core_options = CoreOptions::new(self.table.schema().options());
- let format_extension =
supported_format_table_extension(core_options.file_format())?;
+ let format_extension =
supported_format_table_extension(&core_options.file_format())?;
let schema_id = self.table.schema().id();
let table_path = core_options
.path()
diff --git a/crates/paimon/src/table/table_write.rs
b/crates/paimon/src/table/table_write.rs
index 490de635..bb152a67 100644
--- a/crates/paimon/src/table/table_write.rs
+++ b/crates/paimon/src/table/table_write.rs
@@ -208,7 +208,7 @@ impl TableWrite {
let file_compression = core_options.file_compression().to_string();
let file_compression_zstd_level =
core_options.file_compression_zstd_level();
let file_format = core_options.file_format().to_string();
- let vector_file_format =
core_options.vector_file_format().map(str::to_string);
+ let vector_file_format = core_options.vector_file_format();
let changelog_file_prefix =
core_options.changelog_file_prefix().to_string();
let changelog_file_format =
core_options.changelog_file_format().to_string();
let changelog_file_compression =
core_options.changelog_file_compression().to_string();
