github-actions[bot] commented on code in PR #65501:
URL: https://github.com/apache/doris/pull/65501#discussion_r3566289713
##########
be/src/format_v2/delimited_text/csv_reader.cpp:
##########
@@ -174,77 +172,40 @@ void CsvReader::_split_line(const Slice& line) {
return;
}
- // The text line reader is responsible for split boundaries and multi-line
quoted fields.
- // Field slicing still happens here because FileScannerV2 asks columns by
file-local id, so we
- // must be able to materialize only the requested CSV ordinals without
building a row object.
- // Example: for `1,"a,b",10` and column separator `,`, this loop returns
three slices:
- // `1`, `a,b`, and `10`; the comma inside quotes does not create an extra
field.
- bool in_quote = false;
- bool escaped = false;
- size_t start = 0;
- size_t i = 0;
- while (i < line.size) {
- const char ch = line.data[i];
- if (_enclose != 0) {
- if (escaped) {
- escaped = false;
- ++i;
- continue;
- }
- if (_escape != 0 && ch == _escape) {
- escaped = true;
- ++i;
- continue;
- }
- if (ch == _enclose) {
- if (in_quote && i + 1 < line.size && line.data[i + 1] ==
_enclose) {
- i += 2;
- continue;
- }
- in_quote = !in_quote;
- ++i;
- continue;
- }
+ const auto append_value = [&](size_t value_start, size_t value_len) {
+ while (_trim_tailing_spaces && value_len > 0 &&
+ line.data[value_start + value_len - 1] == ' ') {
+ --value_len;
}
- if (!in_quote && starts_with_at(line, i, _value_separator)) {
- size_t value_start = start;
- size_t value_len = i - start;
- while (_trim_tailing_spaces && value_len > 0 &&
- line.data[value_start + value_len - 1] == ' ') {
- --value_len;
- }
- if (_trim_double_quotes && value_len > 1 && line.data[value_start]
== '"' &&
- line.data[value_start + value_len - 1] == '"') {
- ++value_start;
- value_len -= 2;
- } else if (_enclose != 0 && value_len > 1 &&
line.data[value_start] == _enclose &&
- line.data[value_start + value_len - 1] == _enclose) {
- ++value_start;
- value_len -= 2;
- }
- _split_values.emplace_back(line.data + value_start, value_len);
- i += _value_separator.size();
- start = i;
- continue;
+ if (_enclose != 0 && value_len > 1 && line.data[value_start] ==
_enclose &&
+ line.data[value_start + value_len - 1] == _enclose) {
+ ++value_start;
+ value_len -= 2;
}
- ++i;
- }
+ _split_values.emplace_back(line.data + value_start, value_len);
+ };
- size_t value_start = start;
- size_t value_len = line.size - start;
- while (_trim_tailing_spaces && value_len > 0 && line.data[value_start +
value_len - 1] == ' ') {
- --value_len;
- }
- if (_trim_double_quotes && value_len > 1 && line.data[value_start] == '"'
&&
- line.data[value_start + value_len - 1] == '"') {
- ++value_start;
- value_len -= 2;
- } else if (_enclose != 0 && value_len > 1 && line.data[value_start] ==
_enclose &&
- line.data[value_start + value_len - 1] == _enclose) {
- ++value_start;
- value_len -= 2;
+ size_t value_start = 0;
+ if (_enclose_reader_ctx != nullptr) {
+ for (const size_t separator_position :
_enclose_reader_ctx->column_sep_positions()) {
+ DORIS_CHECK_LE(value_start, separator_position);
+ DORIS_CHECK_LE(separator_position, line.size);
+ append_value(value_start, separator_position - value_start);
+ value_start = separator_position + _value_separator.size();
Review Comment:
The separator positions used here are collected before `_remove_bom()`
strips a first data-line BOM, so a headerless file whose first field is quoted
can still be parsed as unquoted. With double quote as `enclose`, bytes `EF BB
BF 22 61 2C 62 22 2C 32 0A` (BOM + `"a,b",2`) reach
`EncloseCsvLineReaderCtx::_on_start()` with the BOM at byte 0. That takes
NORMAL state and records the comma inside `"a,b"` as a column separator;
`_on_bom_removed(3)` only shifts those already-wrong offsets, and this loop
splits the post-BOM slice into `"a`, `b"`, and `2`. If the first quoted field
contains a newline, the logical row can also be terminated early before BOM
stripping runs. Please strip or skip the initial UTF-8 BOM before the enclosed
state machine decides the first field state, and add v2 coverage for
BOM-prefixed quoted first fields containing a comma/newline.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
