HuaHuaY commented on code in PR #51286:
URL: https://github.com/apache/arrow/pull/51286#discussion_r4441044224
##########
cpp/src/arrow/csv/chunker_test.cc:
##########
@@ -193,6 +193,49 @@ TEST_P(BaseChunkerTest, QuotingNewline) {
}
}
+TEST_P(BaseChunkerTest, MultiDelimiter) {
+ if (!options_.newlines_in_values) {
+ GTEST_SKIP() << "Quoted newlines require newlines_in_values";
+ }
+ options_.delimiter_string = "||";
+ options_.escaping = true;
+ MakeChunker();
+
+ // The escape breaks the delimiter prefix, so the quote before "me" is
literal.
+ auto partial = std::make_shared<Buffer>("na|\\x|\"me|");
+ auto block =
std::make_shared<Buffer>("|mes|sage||\"hello\nworld\"||42\nnext||row\n");
+ std::shared_ptr<Buffer> completion;
+ std::shared_ptr<Buffer> rest;
+ ASSERT_OK(chunker_->ProcessWithPartial(partial, block, &completion, &rest));
+ ASSERT_EQ(completion->ToString(), "|mes|sage||\"hello\nworld\"||42\n");
+ ASSERT_EQ(rest->ToString(), "next||row\n");
+
+ partial = std::make_shared<Buffer>("name|");
+ block = std::make_shared<Buffer>("|message");
+ ASSERT_RAISES(Invalid,
+ chunker_->ProcessWithPartial(partial, block, &completion,
&rest));
+
+ // Long ordinary fields make the bulk filter profitable.
+ for (size_t padding : {0, 256}) {
+ const std::string first_field = std::string(padding, 'a') + "a|b";
+ const std::string row = first_field + "||\"c\n d\"||e\n";
+ const std::string next_row = std::string(padding, 'n') + "next||row\n";
+ MakeChunker();
+ const auto lengths = {row.size(), next_row.size()};
+ AssertChunking(*chunker_, row + next_row, lengths);
+
+ for (size_t split :
+ {first_field.find('|') + 1, first_field.size() + 1,
first_field.size() + 2}) {
Review Comment:
I have changed to `for (size_t split = padding + 1; split < row.size();
++split)`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
