This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-12837-73da49f80ff2aa0a0e96605de45d31d72d22f9a3 in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit 9f741b9c64220db31bd94c62dd0749d0b71e9358 Author: Goutam Adwant <[email protected]> AuthorDate: Thu Nov 5 14:04:23 2026 +0000 [Fix][Format] Apply configured CSV defaults to null and missing fields (#12837) Signed-off-by: Goutam Adwant <[email protected]> --- docs/en/connectors/source/FtpFile.md | 5 +- docs/en/connectors/source/HdfsFile.md | 2 +- docs/en/connectors/source/LocalFile.md | 5 +- docs/en/connectors/source/ObsFile.md | 2 +- docs/en/connectors/source/OssFile.md | 2 +- docs/en/connectors/source/OssJindoFile.md | 5 +- docs/en/connectors/source/S3File.md | 2 +- docs/en/connectors/source/SftpFile.md | 2 +- docs/en/connectors/source/SmbFile.md | 2 +- .../introduction/concepts/incompatible-changes.md | 11 + docs/zh/connectors/source/FtpFile.md | 5 +- docs/zh/connectors/source/HdfsFile.md | 2 +- docs/zh/connectors/source/LocalFile.md | 5 +- docs/zh/connectors/source/ObsFile.md | 2 +- docs/zh/connectors/source/OssFile.md | 2 +- docs/zh/connectors/source/OssJindoFile.md | 2 +- docs/zh/connectors/source/S3File.md | 2 +- docs/zh/connectors/source/SftpFile.md | 2 +- docs/zh/connectors/source/SmbFile.md | 2 +- .../introduction/concepts/incompatible-changes.md | 8 + .../file/source/reader/CsvReadStrategy.java | 1 + .../file/source/reader/CsvReadStrategyTest.java | 56 +++ .../src/test/resources/csv/default-values.csv | 3 + .../e2e/connector/file/local/LocalFileIT.java | 5 + .../src/test/resources/csv/default-values.csv | 3 + .../resources/csv/default_values_to_assert.conf | 57 +++ .../format/csv/CsvDeserializationSchema.java | 251 ++++++++++++- .../format/csv/CsvTextFormatSchemaTest.java | 402 +++++++++++++++++++++ 28 files changed, 815 insertions(+), 33 deletions(-) diff --git a/docs/en/connectors/source/FtpFile.md b/docs/en/connectors/source/FtpFile.md index d7302d7f0e..29e239f86e 100644 --- a/docs/en/connectors/source/FtpFile.md +++ b/docs/en/connectors/source/FtpFile.md @@ -516,8 +516,9 @@ The encoding of the file to read. This param will be parsed by `Charset.forName( ### null_format [string] -Only used when file_format_type is text. -null_format to define which strings can be represented as null. +Used when `file_format_type` is `text` or `csv`. Defines which field value represents null. +For CSV with a configured `schema.columns[].defaultValue`, a matching `null_format` value or a column absent from the file header uses that default. An empty CSV field still produces null unless it matches `null_format`; columns without a default keep their existing behavior. +CSV defaults are converted to the column type when the reader is created. A default that cannot be converted fails the job even if no field needs it. DATE and TIMESTAMP defaults may use a different supported format from the file data. e.g: `\N` diff --git a/docs/en/connectors/source/HdfsFile.md b/docs/en/connectors/source/HdfsFile.md index f7d0bfa94b..6be5933f9d 100644 --- a/docs/en/connectors/source/HdfsFile.md +++ b/docs/en/connectors/source/HdfsFile.md @@ -82,7 +82,7 @@ Read data from hdfs file system. | compress_codec | string | no | none | The compress codec of files | | archive_compress_codec | string | no | none | Which archive compress codec the files used. Supported: `none` `zip` `tar` `tar.gz` `gz`. | | encoding | string | no | UTF-8 | | -| null_format | string | no | - | Only used when file_format_type is `text` or `csv`. null_format to define which strings can be represented as null. e.g: `\N` | +| null_format | string | no | - | Used when `file_format_type` is `text` or `csv`. Defines which field value represents null. For CSV with a configured `schema.columns[].defaultValue`, a matching `null_format` value or a column absent from the file header uses that default. An empty CSV field still produces null unless it matches `null_format`; columns without a default keep their existing behavior. CSV defaults are converted to the column [...] | excel_use_header_line | boolean | no | false | Use the first Excel row as column headers for name-based mapping. Only used when file_format_type is `excel`. | | markdown_rag_metadata_enabled | boolean | no | false | Whether to append RAG-oriented metadata columns when reading markdown files. Only valid when file_format_type is `markdown`. | | pdf_rag_metadata_enabled | boolean | no | false | Whether to append RAG-oriented metadata columns when reading PDF files. Only valid when file_format_type is `pdf`. | diff --git a/docs/en/connectors/source/LocalFile.md b/docs/en/connectors/source/LocalFile.md index 395b7c8fbf..59f6a17981 100644 --- a/docs/en/connectors/source/LocalFile.md +++ b/docs/en/connectors/source/LocalFile.md @@ -483,8 +483,9 @@ The encoding of the file to read. This param will be parsed by `Charset.forName( ### null_format [string] -Only used when file_format_type is `text` or `csv`. -null_format to define which strings can be represented as null. +Used when `file_format_type` is `text` or `csv`. Defines which field value represents null. +For CSV with a configured `schema.columns[].defaultValue`, a matching `null_format` value or a column absent from the file header uses that default. An empty CSV field still produces null unless it matches `null_format`; columns without a default keep their existing behavior. +CSV defaults are validated when the reader is created. DATE and TIMESTAMP defaults may use a different supported format from the file data. e.g: `\N` diff --git a/docs/en/connectors/source/ObsFile.md b/docs/en/connectors/source/ObsFile.md index b0db487279..0df2b5ef5d 100644 --- a/docs/en/connectors/source/ObsFile.md +++ b/docs/en/connectors/source/ObsFile.md @@ -88,7 +88,7 @@ It only supports hadoop version **2.9.X+**. | compress_codec | string | no | none | Which compress codec the files used. | | archive_compress_codec | string | no | none | Which archive compress codec the files used. Supported: `none` `zip` `tar` `tar.gz` `gz`. | | encoding | string | no | UTF-8 | File encoding, only used when `file_format_type` is `json`, `text`, `csv`, or `xml`. | -| null_format | string | no | - | Only used when file_format_type is text. null_format to define which strings can be represented as null. e.g: `\N` | +| null_format | string | no | - | Used when `file_format_type` is `text` or `csv`. Defines which field value represents null. For CSV with a configured `schema.columns[].defaultValue`, a matching `null_format` value or a column absent from the file header uses that default. An empty CSV field still produces null unless it matches `null_format`; columns without a default keep their existing behavior. CSV defaults are converted to the column type when the reader is created. A default that [...] | binary_chunk_size | int | no | 1024 | Only used when file_format_type is binary. The chunk size (in bytes) for reading binary files. Default is 1024 bytes. Larger values may improve performance for large files but use more memory. | | binary_complete_file_mode | boolean | no | false | Only used when file_format_type is binary. Whether to read the complete file as a single chunk instead of splitting into chunks. When enabled, the entire file content will be read into memory at once. Default is false. | | file_filter_pattern | string | no | - | Filter pattern, which used for filtering files. | diff --git a/docs/en/connectors/source/OssFile.md b/docs/en/connectors/source/OssFile.md index ad60d4b6dd..d7ed8c158f 100644 --- a/docs/en/connectors/source/OssFile.md +++ b/docs/en/connectors/source/OssFile.md @@ -212,7 +212,7 @@ If you assign file type to `parquet` `orc`, schema option not required, connecto | compress_codec | string | no | none | Which compress codec the files used. | | archive_compress_codec | string | no | none | Which archive compress codec the files used. Supported: `none` `zip` `tar` `tar.gz` `gz`. | | encoding | string | no | UTF-8 | File encoding, only used when `file_format_type` is `json`, `text`, `csv`, or `xml`. | -| null_format | string | no | - | Only used when file_format_type is text. null_format to define which strings can be represented as null. e.g: `\N` | +| null_format | string | no | - | Used when `file_format_type` is `text` or `csv`. Defines which field value represents null. For CSV with a configured `schema.columns[].defaultValue`, a matching `null_format` value or a column absent from the file header uses that default. An empty CSV field still produces null unless it matches `null_format`; columns without a default keep their existing behavior. CSV defaults are converted to the column type whe [...] | binary_chunk_size | int | no | 1024 | Only used when file_format_type is binary. The chunk size (in bytes) for reading binary files. Default is 1024 bytes. Larger values may improve performance for large files but use more memory. | | binary_complete_file_mode | boolean | no | false | Only used when file_format_type is binary. Whether to read the complete file as a single chunk instead of splitting into chunks. When enabled, the entire file content will be read into memory at once. Default is false. | | discovery_mode | string | no | once | File discovery mode. Supported values: `once` (default), `continuous`. Continuous mode periodically scans the path and currently requires `sync_mode=update` and `file_format_type=binary`. | diff --git a/docs/en/connectors/source/OssJindoFile.md b/docs/en/connectors/source/OssJindoFile.md index 6b648bae14..46028cdaf8 100644 --- a/docs/en/connectors/source/OssJindoFile.md +++ b/docs/en/connectors/source/OssJindoFile.md @@ -454,8 +454,9 @@ The encoding of the file to read. This param will be parsed by `Charset.forName( ### null_format [string] -Only used when file_format_type is text. -null_format to define which strings can be represented as null. +Used when `file_format_type` is `text` or `csv`. Defines which field value represents null. +For CSV with a configured `schema.columns[].defaultValue`, a matching `null_format` value or a column absent from the file header uses that default. An empty CSV field still produces null unless it matches `null_format`; columns without a default keep their existing behavior. +CSV defaults are converted to the column type when the reader is created. A default that cannot be converted fails the job even if no field needs it. DATE and TIMESTAMP defaults may use a different supported format from the file data. e.g: `\N` diff --git a/docs/en/connectors/source/S3File.md b/docs/en/connectors/source/S3File.md index 6a9ea3f487..78844607ec 100644 --- a/docs/en/connectors/source/S3File.md +++ b/docs/en/connectors/source/S3File.md @@ -261,7 +261,7 @@ If you assign file type to `parquet` `orc`, schema option not required, connecto | enable_file_split | boolean | no | false | Turn on logical file split to improve parallelism for huge files. Only supported for `text`/`csv`/`json`/`parquet` and non-compressed format. | | file_split_size | long | no | 134217728 | Split size in bytes when `enable_file_split=true`. For `text`/`csv`/`json`, the split end will be aligned to the next `row_delimiter`. For `parquet`, the split unit is RowGroup and will never break a RowGroup. | | encoding | string | no | UTF-8 | [...] -| null_format | string | no | - | Only used when file_format_type is text. null_format to define which strings can be represented as null. e.g: `\N` [...] +| null_format | string | no | - | Used when `file_format_type` is `text` or `csv`. Defines which field value represents null. For CSV with a configured `schema.columns[].defaultValue`, a matching `null_format` value or a column absent from the file header uses that default. An empty CSV field still produces null unless it matches `null_format`; columns without a default keep their existing behavior. CSV defaul [...] | binary_chunk_size | int | no | 1024 | Only used when file_format_type is binary. The chunk size (in bytes) for reading binary files. Default is 1024 bytes. Larger values may improve performance for large files but use more memory. [...] | binary_complete_file_mode | boolean | no | false | Only used when file_format_type is binary. Whether to read the complete file as a single chunk instead of splitting into chunks. When enabled, the entire file content will be read into memory at once. Default is false. [...] | discovery_mode | string | no | once | File discovery mode. Supported values: `once` (default), `continuous`. When `continuous`, the source periodically scans the path and processes new or changed files as an unbounded source. Continuous mode currently requires `sync_mode=update` and `file_format_type=binary`. | diff --git a/docs/en/connectors/source/SftpFile.md b/docs/en/connectors/source/SftpFile.md index eb04505c68..6eccb62533 100644 --- a/docs/en/connectors/source/SftpFile.md +++ b/docs/en/connectors/source/SftpFile.md @@ -113,7 +113,7 @@ The File does not have a specific type list, and we can indicate which SeaTunnel | compress_codec | String | No | None | The compress codec of files and the details that supported as the following shown: <br/> - txt: `lzo` `None` <br/> - json: `lzo` `None` <br/> - csv: `lzo` `None` <br/> - orc: `lzo` `snappy` `lz4` `zlib` `None` <br/> - parquet: `lzo` `snappy` `lz4` `gzip` `brotli` `zstd` `None` <br/> Tips: excel type does Not support any compression format | | archive_compress_codec | string | no | none | Which archive compress codec the files used. Supported: `none` `zip` `tar` `tar.gz` `gz`. | | encoding | string | no | UTF-8 | File encoding, only used when `file_format_type` is `json`, `text`, `csv`, or `xml`. | -| null_format | string | no | - | Only used when file_format_type is text. null_format to define which strings can be represented as null. e.g: `\N` | +| null_format | string | no | - | Used when `file_format_type` is `text` or `csv`. Defines which field value represents null. For CSV with a configured `schema.columns[].defaultValue`, a matching `null_format` value or a column absent from the file header uses that default. An empty CSV field still produces null unless it matches `null_format`; columns without a default keep their existing behavior. CSV defaults are converted to the colum [...] | binary_chunk_size | int | no | 1024 | Only used when file_format_type is binary. The chunk size (in bytes) for reading binary files. Default is 1024 bytes. Larger values may improve performance for large files but use more memory. | | binary_complete_file_mode | boolean | no | false | Only used when file_format_type is binary. Whether to read the complete file as a single chunk instead of splitting into chunks. When enabled, the entire file content will be read into memory at once. Default is false. | | discovery_mode | string | no | once | File discovery mode. Supported values: `once` (default), `continuous`. When `continuous`, the source keeps scanning the path and processes new/changed files at runtime (unbounded). In the current implementation, `continuous` requires `sync_mode=update` (binary only). | diff --git a/docs/en/connectors/source/SmbFile.md b/docs/en/connectors/source/SmbFile.md index f4c7a6f3e3..52e0a403f4 100644 --- a/docs/en/connectors/source/SmbFile.md +++ b/docs/en/connectors/source/SmbFile.md @@ -104,7 +104,7 @@ The File does not have a specific type list, and we can indicate which SeaTunnel | compress_codec | String | No | None | The compress codec of files | | archive_compress_codec | String | No | none | Which archive compress codec the files used. Supported: `none` `zip` `tar` `tar.gz` `gz`. | | encoding | String | No | UTF-8 | The encoding of the file to read | -| null_format | String | No | - | Only used when file_format_type is text. Define which strings can be represented as null, e.g. `\N`. | +| null_format | String | No | - | Used when `file_format_type` is `text` or `csv`. Defines which field value represents null. For CSV with a configured `schema.columns[].defaultValue`, a matching `null_format` value or a column absent from the file header uses that default. An empty CSV field still produces null unless it matches `null_format`; columns without a default keep their existing behavior. CSV defaults are converted to the colum [...] | binary_chunk_size | int | no | 1024 | Only used when file_format_type is binary. The chunk size (in bytes) for reading binary files. Larger values may improve performance for large files but use more memory. | | binary_complete_file_mode | boolean | no | false | Only used when file_format_type is binary. Whether to read the complete file as a single chunk instead of splitting into chunks. | | filename_extension | String | No | - | Filter filename extension, which used for filtering files with specific extension. Example: `csv` `.txt` `json` `.xml`. | diff --git a/docs/en/introduction/concepts/incompatible-changes.md b/docs/en/introduction/concepts/incompatible-changes.md index e98a52c541..5ca2420243 100644 --- a/docs/en/introduction/concepts/incompatible-changes.md +++ b/docs/en/introduction/concepts/incompatible-changes.md @@ -464,6 +464,17 @@ or state format migration is required. ### Format Changes +- **CSV source defaults for missing and configured null fields**: When a CSV source declares + `schema.columns[].defaultValue`, a column absent from the file header or a field matching + `null_format` now receives that default instead of null. Jobs without configured defaults are + unchanged, and an empty field remains null unless it matches `null_format`. If downstream logic + intentionally relied on null despite a configured default, remove that default from the source + schema before upgrading. + Configured defaults are now converted to the column type when the CSV reader is created. + A default that cannot be converted, such as `CURRENT_TIMESTAMP` for a TIMESTAMP column, now + fails the job at reader creation instead of being ignored, even if no field needs the default. + Defaults must be literal values, not SQL expressions. Fix or remove invalid defaults before upgrading. + - **Breaking Change: JSON serialization of numeric fields now follows the runtime value type** - **Affected component**: `seatunnel-formats/seatunnel-format-json` (`RowToJsonConverters`) - affects every connector that serializes rows with the JSON format (for example Kafka, RabbitMQ, Pulsar, and file JSON sinks) - **Description**: Previously, a field declared as a numeric type in the catalog (`TINYINT`, `SMALLINT`, `INT`, `BIGINT`, `FLOAT`, `DOUBLE`, `DECIMAL`) was serialized by blindly casting the runtime value to the Java type implied by the declared type (for example `(long) value` for `BIGINT`). In multi-table jobs (for example CDC jobs writing JSON to RabbitMQ/Kafka) where several tables share one catalog schema but carry different physical column types, a `String` or `BigDecimal` runtime [...] diff --git a/docs/zh/connectors/source/FtpFile.md b/docs/zh/connectors/source/FtpFile.md index be15ba625f..9b34e9e625 100644 --- a/docs/zh/connectors/source/FtpFile.md +++ b/docs/zh/connectors/source/FtpFile.md @@ -489,8 +489,9 @@ POI 引擎允许读取的最大 Excel 文件大小,单位为字节。默认值 ### null_format [string] -仅在文件格式类型为 text 时使用。 -用于定义哪些字符串可以表示为 null。 +当 `file_format_type` 为 `text` 或 `csv` 时使用。指定表示 null 的字段值。 +对于配置了 `schema.columns[].defaultValue` 的 CSV,如果字段值匹配 `null_format`,或文件表头缺少该列,则使用配置的默认值。空 CSV 字段仍为 null,除非它与 `null_format` 匹配;未配置默认值的列保持原有行为。 +CSV 默认值在创建读取器时转换为列类型。无法转换的默认值会导致作业失败,即使没有字段需要使用该默认值。DATE 和 TIMESTAMP 默认值可以使用与文件数据不同的受支持格式。 例如:`\N` diff --git a/docs/zh/connectors/source/HdfsFile.md b/docs/zh/connectors/source/HdfsFile.md index ff113557c2..36df709042 100644 --- a/docs/zh/connectors/source/HdfsFile.md +++ b/docs/zh/connectors/source/HdfsFile.md @@ -82,7 +82,7 @@ import ChangeLog from '../changelog/connector-file-hadoop.md'; | compress_codec | string | 否 | none | 文件的压缩编解码器 | | archive_compress_codec | string | 否 | none | 文件使用的归档压缩编解码器。支持:`none` `zip` `tar` `tar.gz` `gz`。 | | encoding | string | 否 | UTF-8 | | -| null_format | string | 否 | - | 仅在 file_format_type 为 `text` 或 `csv` 时使用。null_format 定义哪些字符串可以表示为 null。例如:`\N` | +| null_format | string | 否 | - | 当 `file_format_type` 为 `text` 或 `csv` 时使用。指定表示 null 的字段值。 对于配置了 `schema.columns[].defaultValue` 的 CSV,如果字段值匹配 `null_format`,或文件表头缺少该列,则使用配置的默认值。空 CSV 字段仍为 null,除非它与 `null_format` 匹配;未配置默认值的列保持原有行为。 CSV 默认值在创建读取器时转换为列类型。无法转换的默认值会导致作业失败,即使没有字段需要使用该默认值。DATE 和 TIMESTAMP 默认值可以使用与文件数据不同的受支持格式。 例如:`\N` | | excel_use_header_line | boolean | 否 | false | 使用 Excel 第一行作为表头按名称映射字段。仅在 file_format_type 为 `excel` 时使用。 | | markdown_rag_metadata_enabled | boolean | 否 | false | 读取 markdown 文件时是否追加面向 RAG 的元数据列。仅在 file_format_type 为 `markdown` 时有效。 | | pdf_rag_metadata_enabled | boolean | 否 | false | 读取 PDF 文件时是否追加面向 RAG 的元数据列。仅在 file_format_type 为 `pdf` 时有效。 | diff --git a/docs/zh/connectors/source/LocalFile.md b/docs/zh/connectors/source/LocalFile.md index 18e14f9b9b..84c2c02d22 100644 --- a/docs/zh/connectors/source/LocalFile.md +++ b/docs/zh/connectors/source/LocalFile.md @@ -482,8 +482,9 @@ abc.* ### null_format [string] -仅在 file_format_type 为 `text` 或 `csv` 时使用。 -null_format 定义哪些字符串可以表示为 null。 +当 `file_format_type` 为 `text` 或 `csv` 时使用。指定表示 null 的字段值。 +对于配置了 `schema.columns[].defaultValue` 的 CSV,如果字段值匹配 `null_format`,或文件表头缺少该列,则使用配置的默认值。空 CSV 字段仍为 null,除非它与 `null_format` 匹配;未配置默认值的列保持原有行为。 +CSV 默认值在创建读取器时进行校验。DATE 和 TIMESTAMP 默认值可以使用与文件数据不同的受支持格式。 例如:`\N` diff --git a/docs/zh/connectors/source/ObsFile.md b/docs/zh/connectors/source/ObsFile.md index 4fc5a7dfd7..cc56deffae 100644 --- a/docs/zh/connectors/source/ObsFile.md +++ b/docs/zh/connectors/source/ObsFile.md @@ -91,7 +91,7 @@ import ChangeLog from '../changelog/connector-file-obs.md'; | compress_codec | string | 否 | none | 文件使用的压缩编解码器。 | | archive_compress_codec | string | 否 | none | 文件使用的归档压缩编解码器。支持:`none` `zip` `tar` `tar.gz` `gz`。 | | encoding | string | 否 | UTF-8 | 文件编码,仅在 `file_format_type` 为 `json`、`text`、`csv` 或 `xml` 时使用。 | -| null_format | string | 否 | - | 仅在file_format_type为text时使用。null_format用于定义哪些字符串可以表示为null。例如:`\N` | +| null_format | string | 否 | - | 当 `file_format_type` 为 `text` 或 `csv` 时使用。指定表示 null 的字段值。 对于配置了 `schema.columns[].defaultValue` 的 CSV,如果字段值匹配 `null_format`,或文件表头缺少该列,则使用配置的默认值。空 CSV 字段仍为 null,除非它与 `null_format` 匹配;未配置默认值的列保持原有行为。 CSV 默认值在创建读取器时转换为列类型。无法转换的默认值会导致作业失败,即使没有字段需要使用该默认值。DATE 和 TIMESTAMP 默认值可以使用与文件数据不同的受支持格式。 例如:`\N` | | binary_chunk_size | int | 否 | 1024 | 仅在file_format_type为binary时使用。读取二进制文件的块大小(以字节为单位)。默认为1024字节。较大的值可能会提高大文件的性能,但会使用更多内存。 | | binary_complete_file_mode | boolean | 否 | false | 仅在file_format_type为binary时使用。是否将完整文件作为单个块读取,而不是分割成块。启用时,整个文件内容将一次性读入内存。默认为false。 | | file_filter_pattern | string | 否 | - | 用于过滤文件的模式。 | diff --git a/docs/zh/connectors/source/OssFile.md b/docs/zh/connectors/source/OssFile.md index 54ba631ea7..f7c759c544 100644 --- a/docs/zh/connectors/source/OssFile.md +++ b/docs/zh/connectors/source/OssFile.md @@ -212,7 +212,7 @@ schema { | compress_codec | string | 否 | none | 文件使用的压缩编解码器。 | | archive_compress_codec | string | 否 | none | 文件使用的归档压缩编解码器。支持:`none` `zip` `tar` `tar.gz` `gz`。 | | encoding | string | 否 | UTF-8 | 文件编码,仅在 `file_format_type` 为 `json`、`text`、`csv` 或 `xml` 时使用。 | -| null_format | string | 否 | - | 仅在file_format_type为text时使用。null_format用于定义哪些字符串可以表示为null。例如:`\N` | +| null_format | string | 否 | - | 当 `file_format_type` 为 `text` 或 `csv` 时使用。指定表示 null 的字段值。 对于配置了 `schema.columns[].defaultValue` 的 CSV,如果字段值匹配 `null_format`,或文件表头缺少该列,则使用配置的默认值。空 CSV 字段仍为 null,除非它与 `null_format` 匹配;未配置默认值的列保持原有行为。 CSV 默认值在创建读取器时转换为列类型。无法转换的默认值会导致作业失败,即使没有字段需要使用该默认值。DATE 和 TIMESTAMP 默认值可以使用与文件数据不同的受支持格式。 例如:`\N` | | binary_chunk_size | int | 否 | 1024 | 仅在file_format_type为binary时使用。读取二进制文件的块大小(以字节为单位)。默认为1024字节。较大的值可能会提高大文件的性能,但会使用更多内存。 | | binary_complete_file_mode | boolean | 否 | false | 仅在file_format_type为binary时使用。是否将完整文件作为单个块读取,而不是分割成块。启用时,整个文件内容将一次性读入内存。默认为false。 | | discovery_mode | string | 否 | once | 文件发现模式,支持 `once`(默认)和 `continuous`。continuous 模式会定期扫描路径,目前需要同时配置 `sync_mode=update` 和 `file_format_type=binary`。 | diff --git a/docs/zh/connectors/source/OssJindoFile.md b/docs/zh/connectors/source/OssJindoFile.md index b8584da979..23a8ff2199 100644 --- a/docs/zh/connectors/source/OssJindoFile.md +++ b/docs/zh/connectors/source/OssJindoFile.md @@ -86,7 +86,7 @@ import ChangeLog from '../changelog/connector-file-oss-jindo.md'; | compress_codec | string | 否 | none | 读取文件时使用的压缩格式。 | | archive_compress_codec | string | 否 | none | 读取归档文件时使用的归档压缩格式。 | | encoding | string | 否 | UTF-8 | 读取 `json`、`text`、`csv` 或 `xml` 文件时使用的字符编码。 | -| null_format | string | 否 | - | 仅用于 `text` 格式,指定应解析为 `null` 的字符串。 | +| null_format | string | 否 | - | 当 `file_format_type` 为 `text` 或 `csv` 时使用。指定表示 null 的字段值。 对于配置了 `schema.columns[].defaultValue` 的 CSV,如果字段值匹配 `null_format`,或文件表头缺少该列,则使用配置的默认值。空 CSV 字段仍为 null,除非它与 `null_format` 匹配;未配置默认值的列保持原有行为。 CSV 默认值在创建读取器时转换为列类型。无法转换的默认值会导致作业失败,即使没有字段需要使用该默认值。DATE 和 TIMESTAMP 默认值可以使用与文件数据不同的受支持格式。 例如:`\N` | | binary_chunk_size | int | 否 | 1024 | 仅在 file_format_type 为 binary 时使用。读取二进制文件的块大小(以字节为单位)。默认为 1024 字节。较大的值可能会提高大文件的性能,但会使用更多内存。 | | binary_complete_file_mode | boolean | 否 | false | 仅在 file_format_type 为 binary 时使用。是否将完整文件作为单个块读取,而不是分割成块。 | | common-options | | 否 | - | Source 插件通用参数,详见 [Source Common Options](../common-options/source-common-options.md)。 | diff --git a/docs/zh/connectors/source/S3File.md b/docs/zh/connectors/source/S3File.md index 08209a1b19..3e2c5cd6de 100644 --- a/docs/zh/connectors/source/S3File.md +++ b/docs/zh/connectors/source/S3File.md @@ -253,7 +253,7 @@ schema { | enable_file_split | boolean | 否 | false | 开启大文件拆分以提升并行度。仅支持 `text`/`csv`/`json`/`parquet` 且非压缩格式(`compress_codec=none` 且 `archive_compress_codec=none`)。 | | file_split_size | long | 否 | 134217728 | `enable_file_split=true` 时生效,单位字节。`text`/`csv`/`json` 按 `file_split_size` 拆分并对齐到下一个 `row_delimiter`;`parquet` 以 RowGroup 为拆分单位,不会切开 RowGroup。 | | encoding | string | 否 | UTF-8 | | -| null_format | string | 否 | - | 仅在file_format_type为text时使用。null_format用于定义哪些字符串可以表示为null。例如:`\N` | +| null_format | string | 否 | - | 当 `file_format_type` 为 `text` 或 `csv` 时使用。指定表示 null 的字段值。 对于配置了 `schema.columns[].defaultValue` 的 CSV,如果字段值匹配 `null_format`,或文件表头缺少该列,则使用配置的默认值。空 CSV 字段仍为 null,除非它与 `null_format` 匹配;未配置默认值的列保持原有行为。 CSV 默认值在创建读取器时转换为列类型。无法转换的默认值会导致作业失败,即使没有字段需要使用该默认值。DATE 和 TIMESTAMP 默认值可以使用与文件数据不同的受支持格式。 例如:`\N` | | binary_chunk_size | int | 否 | 1024 | 仅在file_format_type为binary时使用。读取二进制文件的块大小(以字节为单位)。默认为1024字节。较大的值可能会提高大文件的性能,但会使用更多内存。 | | binary_complete_file_mode | boolean | 否 | false | 仅在file_format_type为binary时使用。是否将完整文件作为单个块读取,而不是分割成块。启用时,整个文件内容将一次性读入内存。默认为false。 | | discovery_mode | string | 否 | once | 文件发现模式,支持 `once`(默认)和 `continuous`。continuous 模式会定期扫描路径并处理新增或变更对象,目前需要同时配置 `sync_mode=update` 和 `file_format_type=binary`。 | diff --git a/docs/zh/connectors/source/SftpFile.md b/docs/zh/connectors/source/SftpFile.md index c94e04060f..7465c5ca06 100644 --- a/docs/zh/connectors/source/SftpFile.md +++ b/docs/zh/connectors/source/SftpFile.md @@ -109,7 +109,7 @@ import ChangeLog from '../changelog/connector-file-sftp.md'; | compress_codec | String | 否 | None | 文件的压缩编解码器,支持的详细信息如下所示:<br/> - txt: `lzo` `None` <br/> - json: `lzo` `None` <br/> - csv: `lzo` `None` <br/> - orc: `lzo` `snappy` `lz4` `zlib` `None` <br/> - parquet: `lzo` `snappy` `lz4` `gzip` `brotli` `zstd` `None` <br/> 提示:excel类型不支持任何压缩格式 | | archive_compress_codec | string | 否 | none | 文件使用的归档压缩编解码器。支持:`none` `zip` `tar` `tar.gz` `gz`。 | | encoding | string | 否 | UTF-8 | 文件编码,仅在 `file_format_type` 为 `json`、`text`、`csv` 或 `xml` 时使用。 | -| null_format | string | 否 | - | 仅在file_format_type为text时使用。null_format用于定义哪些字符串可以表示为null。例如:`\N` | +| null_format | string | 否 | - | 当 `file_format_type` 为 `text` 或 `csv` 时使用。指定表示 null 的字段值。 对于配置了 `schema.columns[].defaultValue` 的 CSV,如果字段值匹配 `null_format`,或文件表头缺少该列,则使用配置的默认值。空 CSV 字段仍为 null,除非它与 `null_format` 匹配;未配置默认值的列保持原有行为。 CSV 默认值在创建读取器时转换为列类型。无法转换的默认值会导致作业失败,即使没有字段需要使用该默认值。DATE 和 TIMESTAMP 默认值可以使用与文件数据不同的受支持格式。 例如:`\N` | | binary_chunk_size | int | 否 | 1024 | 仅在file_format_type为binary时使用。读取二进制文件的块大小(以字节为单位)。默认为1024字节。较大的值可能会提高大文件的性能,但会使用更多内存。 | | binary_complete_file_mode | boolean | 否 | false | 仅在file_format_type为binary时使用。是否将完整文件作为单个块读取,而不是分割成块。启用时,整个文件内容将一次性读入内存。默认为false。 | | discovery_mode | string | 否 | once | 文件发现模式,支持:`once`(默认)、`continuous`。continuous 模式下将周期性扫描并处理新/变更文件(无界)。当前实现中 continuous 需要配合 `sync_mode=update`(仅 binary)使用,以避免重复传输。 | diff --git a/docs/zh/connectors/source/SmbFile.md b/docs/zh/connectors/source/SmbFile.md index 578efa01b7..fef2c6cc83 100644 --- a/docs/zh/connectors/source/SmbFile.md +++ b/docs/zh/connectors/source/SmbFile.md @@ -104,7 +104,7 @@ import ChangeLog from '../changelog/connector-file-smb.md'; | compress_codec | String | 否 | None | 文件的压缩编解码器 | | archive_compress_codec | String | 否 | none | 文件使用的归档压缩编解码器。支持:`none` `zip` `tar` `tar.gz` `gz`。 | | encoding | String | 否 | UTF-8 | 读取文件时使用的编码 | -| null_format | String | 否 | - | 仅在 file_format_type 为 text 时使用。定义哪些字符串可以表示为 null,例如 `\N` | +| null_format | String | 否 | - | 当 `file_format_type` 为 `text` 或 `csv` 时使用。指定表示 null 的字段值。 对于配置了 `schema.columns[].defaultValue` 的 CSV,如果字段值匹配 `null_format`,或文件表头缺少该列,则使用配置的默认值。空 CSV 字段仍为 null,除非它与 `null_format` 匹配;未配置默认值的列保持原有行为。 CSV 默认值在创建读取器时转换为列类型。无法转换的默认值会导致作业失败,即使没有字段需要使用该默认值。DATE 和 TIMESTAMP 默认值可以使用与文件数据不同的受支持格式。 例如:`\N` | | binary_chunk_size | int | 否 | 1024 | 仅在 file_format_type 为 binary 时使用。读取二进制文件的块大小(以字节为单位)。较大的值可能会提高大文件的性能,但会使用更多内存。 | | binary_complete_file_mode | boolean | 否 | false | 仅在 file_format_type 为 binary 时使用。是否将完整文件作为单个块读取,而不是分割成块。 | | filename_extension | String | 否 | - | 文件扩展名过滤,用于过滤特定扩展名的文件。例如:`csv` `.txt` `json` `.xml` | diff --git a/docs/zh/introduction/concepts/incompatible-changes.md b/docs/zh/introduction/concepts/incompatible-changes.md index 2c5f9190f6..9800b82878 100644 --- a/docs/zh/introduction/concepts/incompatible-changes.md +++ b/docs/zh/introduction/concepts/incompatible-changes.md @@ -393,6 +393,14 @@ Zeta SQL ARRAY 函数现在将 TINYINT 元素声明为 `ARRAY<TINYINT>`,与实 ### 格式变更 +- **CSV Source 对缺失字段和已配置的 null 值应用默认值**:配置了 `schema.columns[].defaultValue` 时, + 如果文件表头缺少该列,或字段值与 `null_format` 匹配,读取结果现在使用默认值而不是 null。 + 未配置默认值的作业保持不变;空字段仍为 null,除非它与 `null_format` 匹配。 + 如果下游逻辑原本依赖“配置了默认值但仍输出 null”的行为,升级前请移除 Source Schema 中的该默认值。 + 配置的默认值现在会在创建 CSV 读取器时转换为列类型。无法转换的默认值(例如 TIMESTAMP 列的 + `CURRENT_TIMESTAMP`)现在会在创建读取器时导致作业失败,而不是被忽略,即使没有字段需要使用该默认值。 + 默认值必须是字面量,不能是 SQL 表达式。升级前请修正或移除无效的默认值。 + - **破坏性变更:JSON 数值字段的序列化改为按运行时实际类型处理** - **影响范围**:`seatunnel-formats/seatunnel-format-json`(`RowToJsonConverters`)--影响所有以 JSON 格式序列化行的连接器(例如 Kafka、RabbitMQ、Pulsar 及文件 JSON Sink)。 - **变更说明**:以前,目录 Schema 中声明为数值类型(`TINYINT`、`SMALLINT`、`INT`、`BIGINT`、`FLOAT`、`DOUBLE`、`DECIMAL`)的字段,序列化时会把运行时值强制转换为声明类型对应的 Java 类型(例如 `BIGINT` 直接 `(long) value`)。在多表作业(例如多表 CDC 作业写 JSON 到 RabbitMQ/Kafka)中,多张表共享同一份目录 Schema 但物理列类型不一致时,`String` 或 `BigDecimal` 运行时值会抛出原始 `ClassCastException` 并导致作业失败。现在数值字段按运行时实际类型序列化:任意数值包装类型(`Byte`、`Short`、`Integer`、`Long`、`Float`、`Double`、`BigInteger`、`BigDecimal`)输出为对应的 JSON 数字;可解析为数字的字符串会解析成 JSON 数字,无法解析的文本则输出为 JSON 字符串;声明为 `DECIMAL` 的字段遇到 `Float`/`Dou [...] diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/CsvReadStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/CsvReadStrategy.java index 6f52c45566..619c38d4e6 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/CsvReadStrategy.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/CsvReadStrategy.java @@ -292,6 +292,7 @@ public class CsvReadStrategy extends AbstractReadStrategy { initFormatter(); CsvDeserializationSchema.Builder builder = CsvDeserializationSchema.builder() + .setCatalogTable(catalogTable) .delimiter(getDelimiter()) .csvLineProcessor(processor) .nullFormat( diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/CsvReadStrategyTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/CsvReadStrategyTest.java index 965e6a714b..86ff64c1c8 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/CsvReadStrategyTest.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/CsvReadStrategyTest.java @@ -17,11 +17,16 @@ package org.apache.seatunnel.connectors.seatunnel.file.source.reader; +import org.apache.seatunnel.shade.com.typesafe.config.Config; import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.source.Collector; import org.apache.seatunnel.api.table.catalog.CatalogTable; import org.apache.seatunnel.api.table.catalog.CatalogTableUtil; +import org.apache.seatunnel.api.table.catalog.TableIdentifier; +import org.apache.seatunnel.api.table.catalog.TablePath; +import org.apache.seatunnel.api.table.catalog.schema.ReadonlyConfigParser; import org.apache.seatunnel.api.table.type.BasicType; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; @@ -43,6 +48,8 @@ import java.io.IOException; import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; +import java.time.LocalDate; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -201,6 +208,55 @@ public class CsvReadStrategyTest { Assertions.assertArrayEquals(new Object[] {2, "Bob", 20}, rows.get(1).getFields()); } + @Test + public void testConfiguredDefaultsFromLocalCsvFile() throws Exception { + URL resource = CsvReadStrategyTest.class.getResource("/csv/default-values.csv"); + String path = Paths.get(resource.toURI()).toString(); + Config config = + ConfigFactory.parseString( + "csv_use_header_line = true\n" + + "null_format = \"NULL\"\n" + + "schema { columns = [" + + "{ name = \"id\", type = \"int\", nullable = false, defaultValue = 7 }," + + "{ name = \"name\", type = \"string\", nullable = false, defaultValue = \"unknown\" }," + + "{ name = \"optional\", type = \"string\", nullable = true }," + + "{ name = \"date\", type = \"date\", defaultValue = \"2020-01-01\" }," + + "{ name = \"timestamp\", type = \"timestamp\", defaultValue = \"2020-01-01 12:34:56\" }" + + "] }"); + CatalogTable catalogTable = + CatalogTable.of( + TableIdentifier.of("test", TablePath.DEFAULT), + new ReadonlyConfigParser().parse(ReadonlyConfig.fromConfig(config)), + new HashMap<>(), + Collections.emptyList(), + "test"); + try (CsvReadStrategy csvReadStrategy = new CsvReadStrategy()) { + csvReadStrategy.init(new LocalFileSystemConf.LocalConf(FS_DEFAULT_NAME_DEFAULT)); + csvReadStrategy.getFileNamesByPath(path); + csvReadStrategy.setPluginConfig(config); + csvReadStrategy.setCatalogTable(catalogTable); + TestCollector collector = new TestCollector(); + csvReadStrategy.read(path, "", collector); + + Assertions.assertEquals(2, collector.getRows().size()); + Assertions.assertEquals(7, collector.getRows().get(0).getField(0)); + Assertions.assertEquals("unknown", collector.getRows().get(0).getField(1)); + Assertions.assertEquals("keep", collector.getRows().get(0).getField(2)); + Assertions.assertEquals(8, collector.getRows().get(1).getField(0)); + Assertions.assertEquals("unknown", collector.getRows().get(1).getField(1)); + Assertions.assertNull(collector.getRows().get(1).getField(2)); + Assertions.assertEquals( + LocalDate.of(2020, 1, 1), collector.getRows().get(0).getField(3)); + Assertions.assertEquals( + LocalDateTime.of(2024, 2, 3, 4, 5, 6), collector.getRows().get(0).getField(4)); + Assertions.assertEquals( + LocalDate.of(2024, 2, 3), collector.getRows().get(1).getField(3)); + Assertions.assertEquals( + LocalDateTime.of(2020, 1, 1, 12, 34, 56), + collector.getRows().get(1).getField(4)); + } + } + @Test public void testReadCsv() throws Exception { URL resource = CsvReadStrategyTest.class.getResource("/test.csv"); diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/csv/default-values.csv b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/csv/default-values.csv new file mode 100644 index 0000000000..fc3fcc05a1 --- /dev/null +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/csv/default-values.csv @@ -0,0 +1,3 @@ +id,optional,date,timestamp +NULL,keep,NULL,2024/02/03 04:05:06 +8,,2024/02/03,NULL diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java index 34ead22e85..97a7462fa0 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java @@ -328,6 +328,10 @@ public class LocalFileIT extends TestSuiteBase { "/csv/csv_with_header2.csv", "/seatunnel/read/csv/header/csv_with_header2.csv", container); + ContainerUtil.copyFileIntoContainers( + "/csv/default-values.csv", + "/seatunnel/read/csv/defaults/default-values.csv", + container); Path csvWithPreamble = Files.createTempFile("csv-preamble-", ".csv"); try { @@ -382,6 +386,7 @@ public class LocalFileIT extends TestSuiteBase { helper.execute("/csv/local_csv_to_assert.conf"); helper.execute("/csv/local_csv_enable_split_to_assert.conf"); helper.execute("/csv/csv_with_header_to_assert.conf"); + helper.execute("/csv/default_values_to_assert.conf"); helper.execute("/csv/breakline_csv_to_assert.conf"); } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/csv/default-values.csv b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/csv/default-values.csv new file mode 100644 index 0000000000..c09b0e288f --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/csv/default-values.csv @@ -0,0 +1,3 @@ +id,optional,date,timestamp +NULL,keep,NULL,2020/01/01 12:34:56 +NULL,keep,2020/01/01,NULL diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/csv/default_values_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/csv/default_values_to_assert.conf new file mode 100644 index 0000000000..8c26597a25 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/csv/default_values_to_assert.conf @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" +} + +source { + LocalFile { + path = "/seatunnel/read/csv/defaults" + file_format_type = csv + csv_use_header_line = true + null_format = "NULL" + schema { + columns = [ + { name = "id", type = "int", nullable = false, defaultValue = 7 } + { name = "name", type = "string", nullable = false, defaultValue = "unknown" } + { name = "optional", type = "string", nullable = true } + { name = "date", type = "date", defaultValue = "2020-01-01" } + { name = "timestamp", type = "timestamp", defaultValue = "2020-01-01 12:34:56" } + ] + } + } +} + +sink { + Assert { + rules { + row_rules = [ + { rule_type = MIN_ROW, rule_value = 2 } + { rule_type = MAX_ROW, rule_value = 2 } + ] + field_rules = [ + { field_name = id, field_type = int, field_value = [{ equals_to = 7 }] } + { field_name = name, field_type = string, field_value = [{ equals_to = "unknown" }] } + { field_name = optional, field_type = string, field_value = [{ equals_to = "keep" }] } + { field_name = date, field_type = date, field_value = [{ rule_type = NOT_NULL, equals_to = "2020-01-01" }] } + { field_name = timestamp, field_type = timestamp, field_value = [{ rule_type = NOT_NULL, equals_to = "2020-01-01T12:34:56" }] } + ] + } + } +} diff --git a/seatunnel-formats/seatunnel-format-csv/src/main/java/org/apache/seatunnel/format/csv/CsvDeserializationSchema.java b/seatunnel-formats/seatunnel-format-csv/src/main/java/org/apache/seatunnel/format/csv/CsvDeserializationSchema.java index ba3b7be105..0bd12caaeb 100644 --- a/seatunnel-formats/seatunnel-format-csv/src/main/java/org/apache/seatunnel/format/csv/CsvDeserializationSchema.java +++ b/seatunnel-formats/seatunnel-format-csv/src/main/java/org/apache/seatunnel/format/csv/CsvDeserializationSchema.java @@ -20,6 +20,7 @@ package org.apache.seatunnel.format.csv; import org.apache.seatunnel.shade.org.apache.commons.lang3.StringUtils; import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.catalog.Column; import org.apache.seatunnel.api.table.catalog.TablePath; import org.apache.seatunnel.api.table.type.ArrayType; import org.apache.seatunnel.api.table.type.MapType; @@ -40,7 +41,9 @@ import org.apache.seatunnel.format.csv.processor.DefaultCsvLineProcessor; import lombok.NonNull; import java.io.IOException; +import java.io.ObjectInputStream; import java.io.Serializable; +import java.lang.reflect.Array; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; import java.time.LocalDate; @@ -55,6 +58,7 @@ import java.time.temporal.ChronoField; import java.time.temporal.TemporalAccessor; import java.time.temporal.TemporalQueries; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; @@ -67,6 +71,7 @@ public class CsvDeserializationSchema implements Serializable { private final String nullFormat; private final CsvLineProcessor processor; private final CatalogTable catalogTable; + private final Object[] defaultValues; @SuppressWarnings("MagicNumber") public static final DateTimeFormatter TIME_FORMAT = @@ -75,7 +80,8 @@ public class CsvDeserializationSchema implements Serializable { .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) .toFormatter(); - public Map<String, DateTimeFormatter> fieldFormatterMap = new HashMap<>(); + // Formatters are derived from file data and are not serializable. + public transient Map<String, DateTimeFormatter> fieldFormatterMap = new HashMap<>(); private CsvDeserializationSchema( @NonNull SeaTunnelRowType seaTunnelRowType, @@ -90,6 +96,30 @@ public class CsvDeserializationSchema implements Serializable { this.nullFormat = nullFormat; this.processor = processor; this.catalogTable = catalogTable; + Map<String, Column> columnsByName = new HashMap<>(); + if (catalogTable != null) { + for (Column column : catalogTable.getTableSchema().getColumns()) { + columnsByName.put(column.getName(), column); + } + } + this.defaultValues = new Object[seaTunnelRowType.getTotalFields()]; + for (int i = 0; i < defaultValues.length; i++) { + String fieldName = seaTunnelRowType.getFieldName(i); + defaultValues[i] = + resolveDefaultValue( + columnsByName.get(fieldName), + seaTunnelRowType.getFieldType(i), + fieldName); + } + } + + /** + * Restores the converted defaults, but rebuilds the transient, non-serializable formatter + * cache. File-data formats are learned again from incoming values after deserialization. + */ + private void readObject(ObjectInputStream input) throws IOException, ClassNotFoundException { + input.defaultReadObject(); + fieldFormatterMap = new HashMap<>(); } public static Builder builder() { @@ -185,10 +215,11 @@ public class CsvDeserializationSchema implements Serializable { Object[] objects = new Object[seaTunnelRowType.getTotalFields()]; for (int i = 0; i < objects.length; i++) { String fieldValue = splitsMap.get(i); - if (StringUtils.isBlank(fieldValue)) { + if (fieldValue == null || StringUtils.equals(fieldValue, nullFormat)) { + objects[i] = getDefaultValue(i); continue; } - if (StringUtils.equals(fieldValue, nullFormat)) { + if (StringUtils.isBlank(fieldValue)) { continue; } objects[i] = @@ -207,6 +238,186 @@ public class CsvDeserializationSchema implements Serializable { return seaTunnelRow; } + private Object getDefaultValue(int fieldIndex) { + return copyDefaultValue(defaultValues[fieldIndex]); + } + + /** Resolves defaults once, independently of the format learned from file data. */ + private Object resolveDefaultValue( + Column column, SeaTunnelDataType<?> fieldType, String fieldName) { + if (column == null || column.getDefaultValue() == null) { + return null; + } + try { + return convertDefaultValue(column.getDefaultValue(), fieldType, fieldName); + } catch (RuntimeException ignored) { + // Parser exceptions can include the configured value in their messages. + throw new IllegalArgumentException( + String.format( + "Invalid defaultValue for CSV column [%s] of type [%s]", + fieldName, fieldType)); + } + } + + /** + * Copies cached mutable defaults for each output row, including nested bytes, arrays, map keys + * and values, and row fields. Immutable scalar and temporal values can be shared. + */ + private Object copyDefaultValue(Object value) { + if (value == null) { + return null; + } + if (value instanceof byte[]) { + return ((byte[]) value).clone(); + } + if (value.getClass().isArray()) { + int length = Array.getLength(value); + Object copy = Array.newInstance(value.getClass().getComponentType(), length); + for (int i = 0; i < length; i++) { + Array.set(copy, i, copyDefaultValue(Array.get(value, i))); + } + return copy; + } + if (value instanceof Map) { + Map<Object, Object> copy = new LinkedHashMap<>(); + ((Map<?, ?>) value) + .forEach( + (key, mapValue) -> + copy.put(copyDefaultValue(key), copyDefaultValue(mapValue))); + return copy; + } + if (value instanceof SeaTunnelRow) { + Object[] fields = ((SeaTunnelRow) value).getFields(); + return new SeaTunnelRow((Object[]) copyDefaultValue(fields)); + } + return value; + } + + /** + * Converts a configured literal once to its declared type. Collections, arrays, maps and rows + * are converted recursively; supported typed temporal and byte values avoid string conversion. + * Text literals use default parsing without reading or populating the file-data formatter + * cache. Conversion failures are reported by {@link #resolveDefaultValue} with column context. + */ + private Object convertDefaultValue( + Object value, SeaTunnelDataType<?> fieldType, String fieldName) { + if (value == null) { + return null; + } + switch (fieldType.getSqlType()) { + case STRING: + return value.toString(); + case BOOLEAN: + if (!"true".equalsIgnoreCase(value.toString()) + && !"false".equalsIgnoreCase(value.toString())) { + throw new IllegalArgumentException("Boolean default must be true or false"); + } + return Boolean.parseBoolean(value.toString()); + case DATE: + if (value instanceof LocalDate) { + return value; + } + break; + case TIME: + if (value instanceof LocalTime) { + return value; + } + break; + case TIMESTAMP: + if (value instanceof LocalDateTime) { + return value; + } + break; + case TIMESTAMP_TZ: + if (value instanceof OffsetDateTime) { + return value; + } + break; + case BYTES: + if (value instanceof byte[]) { + return ((byte[]) value).clone(); + } + if (value instanceof String) { + return ((String) value).getBytes(StandardCharsets.UTF_8); + } + break; + case ARRAY: + if (value instanceof Collection || value.getClass().isArray()) { + ArrayType<?, ?> arrayType = (ArrayType<?, ?>) fieldType; + SeaTunnelDataType<?> elementType = arrayType.getElementType(); + int length = + value instanceof Collection + ? ((Collection<?>) value).size() + : Array.getLength(value); + Object result = Array.newInstance(elementType.getTypeClass(), length); + int index = 0; + if (value instanceof Collection) { + for (Object element : (Collection<?>) value) { + Array.set( + result, + index++, + convertDefaultValue(element, elementType, fieldName)); + } + } else { + for (int i = 0; i < length; i++) { + Array.set( + result, + i, + convertDefaultValue( + Array.get(value, i), elementType, fieldName)); + } + } + return result; + } + break; + case MAP: + if (value instanceof Map) { + MapType<?, ?> mapType = (MapType<?, ?>) fieldType; + Map<Object, Object> result = new LinkedHashMap<>(); + ((Map<?, ?>) value) + .forEach( + (key, mapValue) -> + result.put( + convertDefaultValue( + key, mapType.getKeyType(), fieldName), + convertDefaultValue( + mapValue, + mapType.getValueType(), + fieldName))); + return result; + } + break; + case ROW: + if (value instanceof Map || value instanceof SeaTunnelRow) { + SeaTunnelRowType rowType = (SeaTunnelRowType) fieldType; + Object[] fields = new Object[rowType.getTotalFields()]; + for (int i = 0; i < fields.length; i++) { + Object fieldValue = + value instanceof Map + ? ((Map<?, ?>) value).get(rowType.getFieldName(i)) + : ((SeaTunnelRow) value).getField(i); + fields[i] = + convertDefaultValue( + fieldValue, + rowType.getFieldType(i), + fieldName + "." + rowType.getFieldName(i)); + } + return new SeaTunnelRow(fields); + } + break; + case NULL: + return null; + default: + break; + } + Object converted = convert(value.toString(), fieldType, 0, fieldName, true); + if (converted == null) { + throw new IllegalArgumentException( + "Default value cannot be converted to CSV field type"); + } + return converted; + } + public SeaTunnelDataType<SeaTunnelRow> getProducedType() { return seaTunnelRowType; } @@ -230,6 +441,15 @@ public class CsvDeserializationSchema implements Serializable { private Object convert( String field, SeaTunnelDataType<?> fieldType, int level, String fieldName) { + return convert(field, fieldType, level, fieldName, false); + } + + private Object convert( + String field, + SeaTunnelDataType<?> fieldType, + int level, + String fieldName, + boolean isDefaultValue) { if (StringUtils.isBlank(field)) { return null; } @@ -239,7 +459,8 @@ public class CsvDeserializationSchema implements Serializable { String[] elements = field.split(separators[level + 1]); ArrayList<Object> objectArrayList = new ArrayList<>(); for (String element : elements) { - objectArrayList.add(convert(element, elementType, level + 1, fieldName)); + objectArrayList.add( + convert(element, elementType, level + 1, fieldName, isDefaultValue)); } switch (elementType.getSqlType()) { case STRING: @@ -283,11 +504,18 @@ public class CsvDeserializationSchema implements Serializable { for (String kv : kvs) { String[] splits = kv.split(separators[level + 2]); if (splits.length < 2) { - objectMap.put(convert(splits[0], keyType, level + 1, fieldName), null); + objectMap.put( + convert(splits[0], keyType, level + 1, fieldName, isDefaultValue), + null); } else { objectMap.put( - convert(splits[0], keyType, level + 1, fieldName), - convert(splits[1], valueType, level + 1, fieldName)); + convert(splits[0], keyType, level + 1, fieldName, isDefaultValue), + convert( + splits[1], + valueType, + level + 1, + fieldName, + isDefaultValue)); } } return objectMap; @@ -314,11 +542,13 @@ public class CsvDeserializationSchema implements Serializable { case BYTES: return field.getBytes(StandardCharsets.UTF_8); case DATE: - return parseDate(field, fieldName); + return isDefaultValue ? DateUtils.parse(field) : parseDate(field, fieldName); case TIME: return parseTime(field); case TIMESTAMP: - return parseTimestamp(field, fieldName); + return isDefaultValue + ? DateTimeUtils.parse(field) + : parseTimestamp(field, fieldName); case TIMESTAMP_TZ: return parseTimestampTz(field, fieldName); case ROW: @@ -332,7 +562,8 @@ public class CsvDeserializationSchema implements Serializable { splitsMap.get(i), ((SeaTunnelRowType) fieldType).getFieldType(i), level + 1, - fieldName + "." + eleFieldNames[i]); + fieldName + "." + eleFieldNames[i], + isDefaultValue); } return new SeaTunnelRow(objects); default: diff --git a/seatunnel-formats/seatunnel-format-csv/src/test/java/org/apache/seatunnel/format/csv/CsvTextFormatSchemaTest.java b/seatunnel-formats/seatunnel-format-csv/src/test/java/org/apache/seatunnel/format/csv/CsvTextFormatSchemaTest.java index 04f35ce554..4cbc47f7d8 100644 --- a/seatunnel-formats/seatunnel-format-csv/src/test/java/org/apache/seatunnel/format/csv/CsvTextFormatSchemaTest.java +++ b/seatunnel-formats/seatunnel-format-csv/src/test/java/org/apache/seatunnel/format/csv/CsvTextFormatSchemaTest.java @@ -17,11 +17,17 @@ package org.apache.seatunnel.format.csv; +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.catalog.PhysicalColumn; +import org.apache.seatunnel.api.table.catalog.TableIdentifier; +import org.apache.seatunnel.api.table.catalog.TablePath; +import org.apache.seatunnel.api.table.catalog.TableSchema; import org.apache.seatunnel.api.table.type.ArrayType; import org.apache.seatunnel.api.table.type.BasicType; import org.apache.seatunnel.api.table.type.DecimalType; import org.apache.seatunnel.api.table.type.LocalTimeType; import org.apache.seatunnel.api.table.type.MapType; +import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; @@ -33,14 +39,23 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.PrintWriter; +import java.io.StringWriter; import java.math.BigDecimal; import java.nio.file.Path; import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.LocalTime; import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TimeZone; @@ -299,6 +314,393 @@ public class CsvTextFormatSchemaTest { } } + @Test + public void testConfiguredDefaultsForNullMarkerAndMissingColumn() throws Exception { + CatalogTable catalogTable = + CatalogTable.of( + TableIdentifier.of("test", TablePath.DEFAULT), + TableSchema.builder() + .column( + PhysicalColumn.of( + "id", + BasicType.INT_TYPE, + (Long) null, + false, + 7, + null)) + .column( + PhysicalColumn.of( + "name", + BasicType.STRING_TYPE, + (Long) null, + false, + "unknown", + null)) + .column( + PhysicalColumn.of( + "optional", + BasicType.STRING_TYPE, + (Long) null, + true, + null, + null)) + .build(), + new HashMap<>(), + Collections.emptyList(), + "test"); + CsvDeserializationSchema schema = + CsvDeserializationSchema.builder() + .seaTunnelRowType(catalogTable.getSeaTunnelRowType()) + .setCatalogTable(catalogTable) + .nullFormat("NULL") + .build(); + + SeaTunnelRow row = schema.getSeaTunnelRow(Collections.singletonMap(0, "NULL")); + Assertions.assertEquals(7, row.getField(0)); + Assertions.assertEquals("unknown", row.getField(1)); + Assertions.assertNull(row.getField(2)); + + Map<Integer, String> emptyValues = new HashMap<>(); + emptyValues.put(0, ""); + emptyValues.put(1, ""); + SeaTunnelRow emptyRow = schema.getSeaTunnelRow(emptyValues); + Assertions.assertNull(emptyRow.getField(0)); + Assertions.assertNull(emptyRow.getField(1)); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream output = new ObjectOutputStream(bytes)) { + output.writeObject(schema); + } + CsvDeserializationSchema restored; + try (ObjectInputStream input = + new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + restored = (CsvDeserializationSchema) input.readObject(); + } + Assertions.assertEquals( + 7, restored.getSeaTunnelRow(Collections.singletonMap(0, "NULL")).getField(0)); + } + + @Test + public void testDateDefaultBeforeDataWithDifferentFormat() { + assertTemporalDefaultWithDifferentDataFormat(LocalTimeType.LOCAL_DATE_TYPE, true); + } + + @Test + public void testDateDefaultAfterDataWithDifferentFormat() { + assertTemporalDefaultWithDifferentDataFormat(LocalTimeType.LOCAL_DATE_TYPE, false); + } + + @Test + public void testTimestampDefaultBeforeDataWithDifferentFormat() { + assertTemporalDefaultWithDifferentDataFormat(LocalTimeType.LOCAL_DATE_TIME_TYPE, true); + } + + @Test + public void testTimestampDefaultAfterDataWithDifferentFormat() { + assertTemporalDefaultWithDifferentDataFormat(LocalTimeType.LOCAL_DATE_TIME_TYPE, false); + } + + private void assertTemporalDefaultWithDifferentDataFormat( + SeaTunnelDataType<?> type, boolean defaultFirst) { + boolean date = type.equals(LocalTimeType.LOCAL_DATE_TYPE); + CsvDeserializationSchema schema = + schemaWithDefault(type, date ? "2020-01-01" : "2020-01-01 12:34:56"); + Object expectedDefault = + date ? LocalDate.of(2020, 1, 1) : LocalDateTime.of(2020, 1, 1, 12, 34, 56); + String data = date ? "2024/02/03" : "2024/02/03 04:05:06"; + Object expectedData = + date ? LocalDate.of(2024, 2, 3) : LocalDateTime.of(2024, 2, 3, 4, 5, 6); + if (defaultFirst) { + assertEquals( + expectedDefault, schema.getSeaTunnelRow(Collections.emptyMap()).getField(0)); + } + assertEquals( + expectedData, + schema.getSeaTunnelRow(Collections.singletonMap(0, data)).getField(0)); + assertEquals( + expectedDefault, + schema.getSeaTunnelRow(Collections.singletonMap(0, "NULL")).getField(0)); + assertEquals( + expectedData, + schema.getSeaTunnelRow(Collections.singletonMap(0, data)).getField(0)); + } + + @Test + public void testTemporalDefaultsSurviveSerializationAfterData() throws Exception { + for (boolean date : new boolean[] {true, false}) { + CsvDeserializationSchema schema = + schemaWithDefault( + date + ? LocalTimeType.LOCAL_DATE_TYPE + : LocalTimeType.LOCAL_DATE_TIME_TYPE, + date ? "2020-01-01" : "2020-01-01 12:34:56"); + Map<Integer, String> data = + Collections.singletonMap(0, date ? "2024/02/03" : "2024/02/03 04:05:06"); + Object expectedDefault = + date ? LocalDate.of(2020, 1, 1) : LocalDateTime.of(2020, 1, 1, 12, 34, 56); + assertEquals( + expectedDefault, + roundTrip(schema).getSeaTunnelRow(Collections.emptyMap()).getField(0)); + SeaTunnelRow expected = schema.getSeaTunnelRow(data); + CsvDeserializationSchema restored = roundTrip(schema); + assertEquals(expected.getField(0), restored.getSeaTunnelRow(data).getField(0)); + assertEquals( + expectedDefault, restored.getSeaTunnelRow(Collections.emptyMap()).getField(0)); + assertEquals( + expected.getField(0), roundTrip(restored).getSeaTunnelRow(data).getField(0)); + } + } + + @Test + public void testTypedTemporalDefaultsArePreserved() throws Exception { + SeaTunnelDataType<?>[] types = { + LocalTimeType.LOCAL_DATE_TYPE, + LocalTimeType.LOCAL_TIME_TYPE, + LocalTimeType.LOCAL_DATE_TIME_TYPE, + LocalTimeType.OFFSET_DATE_TIME_TYPE + }; + Object[] values = { + LocalDate.of(2020, 1, 1), + LocalTime.of(12, 34), + LocalDateTime.of(2020, 1, 1, 12, 34), + OffsetDateTime.of(2020, 1, 1, 12, 34, 0, 0, ZoneOffset.ofHours(9)) + }; + for (int i = 0; i < types.length; i++) { + CsvDeserializationSchema schema = schemaWithDefault(types[i], values[i]); + Assertions.assertSame( + values[i], schema.getSeaTunnelRow(Collections.emptyMap()).getField(0)); + assertEquals( + values[i], + roundTrip(schema).getSeaTunnelRow(Collections.emptyMap()).getField(0)); + } + } + + @Test + public void testStructuredTemporalDefaultsUseIndependentFormats() { + ArrayType<LocalDate[], LocalDate> dates = + new ArrayType<>(LocalDate[].class, LocalTimeType.LOCAL_DATE_TYPE); + LocalDate[] expected = {LocalDate.of(2020, 1, 1), LocalDate.of(2020, 2, 3)}; + for (Object value : + new Object[] { + Arrays.asList("2020-01-01", "2020/02/03"), "2020-01-01\u00022020/02/03" + }) { + CsvDeserializationSchema schema = schemaWithDefault(dates, value); + Assertions.assertArrayEquals( + expected, + (LocalDate[]) schema.getSeaTunnelRow(Collections.emptyMap()).getField(0)); + Assertions.assertArrayEquals( + new LocalDate[] {LocalDate.of(2024, 2, 3)}, + (LocalDate[]) + schema.getSeaTunnelRow(Collections.singletonMap(0, "2024.02.03")) + .getField(0)); + } + } + + @Test + public void testTimestampTzDefaultDoesNotAffectDataParser() { + CsvDeserializationSchema schema = + schemaWithDefault(LocalTimeType.OFFSET_DATE_TIME_TYPE, "2020-01-01T12:34:56+09:00"); + OffsetDateTime expectedDefault = OffsetDateTime.parse("2020-01-01T12:34:56+09:00"); + assertEquals(expectedDefault, schema.getSeaTunnelRow(Collections.emptyMap()).getField(0)); + assertEquals( + OffsetDateTime.of(2024, 2, 3, 4, 5, 6, 0, ZoneOffset.UTC), + schema.getSeaTunnelRow(Collections.singletonMap(0, "2024/02/03 04:05:06")) + .getField(0)); + assertEquals(expectedDefault, schema.getSeaTunnelRow(Collections.emptyMap()).getField(0)); + } + + @Test + public void testInvalidTemporalDefaultDoesNotExposeValue() { + for (SeaTunnelDataType<?> type : + new SeaTunnelDataType<?>[] { + LocalTimeType.LOCAL_DATE_TYPE, LocalTimeType.LOCAL_DATE_TIME_TYPE + }) { + String sensitiveDefault = "secret-invalid-temporal-default"; + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> schemaWithDefault(type, sensitiveDefault)); + StringWriter stackTrace = new StringWriter(); + exception.printStackTrace(new PrintWriter(stackTrace)); + Assertions.assertTrue(stackTrace.toString().contains("value")); + Assertions.assertTrue(stackTrace.toString().contains(type.toString())); + Assertions.assertFalse(stackTrace.toString().contains(sensitiveDefault)); + } + } + + private CsvDeserializationSchema schemaWithDefault(SeaTunnelDataType<?> type, Object value) { + CatalogTable table = + CatalogTable.of( + TableIdentifier.of("test", TablePath.DEFAULT), + TableSchema.builder() + .column( + PhysicalColumn.of( + "value", type, (Long) null, true, value, null)) + .build(), + new HashMap<>(), + Collections.emptyList(), + "test"); + return CsvDeserializationSchema.builder() + .seaTunnelRowType(table.getSeaTunnelRowType()) + .setCatalogTable(table) + .nullFormat("NULL") + .build(); + } + + private CsvDeserializationSchema roundTrip(CsvDeserializationSchema schema) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream output = new ObjectOutputStream(bytes)) { + output.writeObject(schema); + } + try (ObjectInputStream input = + new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + return (CsvDeserializationSchema) input.readObject(); + } + } + + @Test + public void testInvalidConfiguredDefaultIsReportedWithColumnContext() { + CatalogTable catalogTable = + CatalogTable.of( + TableIdentifier.of("test", TablePath.DEFAULT), + TableSchema.builder() + .column( + PhysicalColumn.of( + "enabled", + BasicType.BOOLEAN_TYPE, + (Long) null, + false, + "not-a-boolean", + null)) + .build(), + new HashMap<>(), + Collections.emptyList(), + "test"); + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + CsvDeserializationSchema.builder() + .seaTunnelRowType(catalogTable.getSeaTunnelRowType()) + .setCatalogTable(catalogTable) + .build()); + Assertions.assertTrue(exception.getMessage().contains("enabled")); + Assertions.assertFalse(exception.getMessage().contains("not-a-boolean")); + } + + @Test + public void testInvalidNumericDefaultDoesNotExposeValueInStackTrace() { + String sensitiveDefault = "secret-csv-default"; + CatalogTable catalogTable = + CatalogTable.of( + TableIdentifier.of("test", TablePath.DEFAULT), + TableSchema.builder() + .column( + PhysicalColumn.of( + "count", + BasicType.INT_TYPE, + (Long) null, + false, + sensitiveDefault, + null)) + .build(), + new HashMap<>(), + Collections.emptyList(), + "test"); + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + CsvDeserializationSchema.builder() + .seaTunnelRowType(catalogTable.getSeaTunnelRowType()) + .setCatalogTable(catalogTable) + .build()); + StringWriter stackTrace = new StringWriter(); + exception.printStackTrace(new PrintWriter(stackTrace)); + Assertions.assertTrue(stackTrace.toString().contains("count")); + Assertions.assertTrue(stackTrace.toString().contains("INT")); + Assertions.assertFalse(stackTrace.toString().contains(sensitiveDefault)); + } + + @Test + public void testStructuredDefaultsAreIndependentBetweenRows() throws Exception { + SeaTunnelRowType nestedRowType = + new SeaTunnelRowType( + new String[] {"numbers"}, + new SeaTunnelDataType<?>[] {ArrayType.INT_ARRAY_TYPE}); + Map<String, Object> mapDefault = new HashMap<>(); + mapDefault.put("count", 3); + Map<String, Object> rowDefault = new HashMap<>(); + rowDefault.put("numbers", Arrays.asList(4, 5)); + CatalogTable catalogTable = + CatalogTable.of( + TableIdentifier.of("test", TablePath.DEFAULT), + TableSchema.builder() + .column( + PhysicalColumn.of( + "arrays", + new ArrayType<>( + Integer[][].class, + ArrayType.INT_ARRAY_TYPE), + (Long) null, + true, + Arrays.asList( + Arrays.asList(1, 2), Arrays.asList(3)), + null)) + .column( + PhysicalColumn.of( + "map", + new MapType<>( + BasicType.STRING_TYPE, BasicType.INT_TYPE), + (Long) null, + true, + mapDefault, + null)) + .column( + PhysicalColumn.of( + "row", + nestedRowType, + (Long) null, + true, + rowDefault, + null)) + .column( + PhysicalColumn.of( + "bytes", + PrimitiveByteArrayType.INSTANCE, + (Long) null, + true, + new byte[] {1, 2}, + null)) + .build(), + new HashMap<>(), + Collections.emptyList(), + "test"); + CsvDeserializationSchema schema = + CsvDeserializationSchema.builder() + .seaTunnelRowType(catalogTable.getSeaTunnelRowType()) + .setCatalogTable(catalogTable) + .build(); + mapDefault.clear(); + rowDefault.clear(); + + SeaTunnelRow first = schema.getSeaTunnelRow(Collections.emptyMap()); + ((Integer[][]) first.getField(0))[0][0] = 99; + ((Map<?, ?>) first.getField(1)).clear(); + ((Integer[]) ((SeaTunnelRow) first.getField(2)).getField(0))[0] = 99; + ((byte[]) first.getField(3))[0] = 99; + + for (CsvDeserializationSchema copy : + new CsvDeserializationSchema[] {schema, roundTrip(schema)}) { + SeaTunnelRow second = copy.getSeaTunnelRow(Collections.emptyMap()); + Assertions.assertEquals(1, ((Integer[][]) second.getField(0))[0][0]); + Assertions.assertEquals(3, ((Map<?, ?>) second.getField(1)).get("count")); + Assertions.assertEquals( + 4, ((Integer[]) ((SeaTunnelRow) second.getField(2)).getField(0))[0]); + Assertions.assertArrayEquals(new byte[] {1, 2}, (byte[]) second.getField(3)); + } + } + @Test void testTimestampTzRoundTrip() throws IOException { SeaTunnelRowType rowType =
