hubgeter commented on code in PR #54591:
URL: https://github.com/apache/doris/pull/54591#discussion_r2268517247
##########
be/src/vec/exec/format/parquet/schema_desc.cpp:
##########
@@ -251,72 +251,6 @@ std::pair<DataTypePtr, bool>
FieldDescriptor::get_doris_type(
return ans;
}
-// Copy from org.apache.iceberg.avro.AvroSchemaUtil#validAvroName
-static bool is_valid_avro_name(const std::string& name) {
- size_t length = name.length();
- char first = name[0];
- if (!isalpha(first) && first != '_') {
- return false;
- }
-
- for (size_t i = 1; i < length; i++) {
- char character = name[i];
- if (!isalpha(character) && !isdigit(character) && character != '_') {
- return false;
- }
- }
- return true;
-}
-
-// Copy from org.apache.iceberg.avro.AvroSchemaUtil#sanitize
-static void sanitize_avro_name(std::ostringstream& buf, char character) {
- if (isdigit(character)) {
- buf << '_' << character;
- } else {
- std::stringstream ss;
- ss << std::hex << (int)character;
- std::string hex_str = ss.str();
- buf << "_x" << doris::to_lower(hex_str);
- }
-}
-
-// Copy from org.apache.iceberg.avro.AvroSchemaUtil#sanitize
-static std::string sanitize_avro_name(const std::string& name) {
- std::ostringstream buf;
- size_t length = name.length();
- char first = name[0];
- if (!isalpha(first) && first != '_') {
- sanitize_avro_name(buf, first);
- } else {
- buf << first;
- }
-
- for (size_t i = 1; i < length; i++) {
- char character = name[i];
- if (!isalpha(character) && !isdigit(character) && character != '_') {
- sanitize_avro_name(buf, character);
- } else {
- buf << character;
- }
- }
- return buf.str();
-}
-
-void FieldDescriptor::iceberg_sanitize(const std::vector<std::string>&
read_columns) {
Review Comment:
This code is currently unused.
reason :
Prior to pull request https://github.com/apache/doris/pull/27108, the
Iceberg Parquet reader parsed the `field ID` and `file name` from the Parquet
file's [Properties
`iceberg.schema`](https://github.com/morningman/doris/blob/52995c528ea9be3deb2df817e4d1c7bb0505eca3/be/src/vec/exec/format/table/iceberg_reader.cpp#L511).
Since `iceberg.schema` stores the table name, which is not in valid `Avro`
format, the PR #27108 needs to convert it (`sanitize_avro_name`).
In the current master code implementation, the `field ID` comes from the
`schema`.
<img width="2856" height="708" alt="Image"
src="https://github.com/user-attachments/assets/d69c4327-0e5d-4b20-a7d1-8620c5f4aad3";
/>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
