This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new ce94fbabae GH-34823: [C++][ORC] Fix ORC CHAR type mapping (#34836)
ce94fbabae is described below
commit ce94fbabae53fb53fef600f224dd44fc69b713b4
Author: Gang Wu <[email protected]>
AuthorDate: Mon Apr 3 13:54:10 2023 +0800
GH-34823: [C++][ORC] Fix ORC CHAR type mapping (#34836)
### Rationale for this change
Currently the ORC CHAR type is mapped to Arrow FIXED_SIZE_BINARY type.
However, the meaning of length differs between these types. The length of ORC
CHAR type means number of UTF8 characters while that of Arrow FIXED_SIZE_BINARY
means number of bytes.
### What changes are included in this PR?
Use Arrow String type to read from ORC CHAR type.
### Are these changes tested?
Make sure all tests pass.
### Are there any user-facing changes?
Yes, the doc has been updated as well.
* Closes: #34823
Lead-authored-by: Gang Wu <[email protected]>
Co-authored-by: Sutou Kouhei <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
cpp/src/arrow/adapters/orc/adapter_test.cc | 53 ++++++++++++++++++++++++++++++
cpp/src/arrow/adapters/orc/util.cc | 22 ++-----------
docs/source/cpp/orc.rst | 9 +++--
3 files changed, 61 insertions(+), 23 deletions(-)
diff --git a/cpp/src/arrow/adapters/orc/adapter_test.cc
b/cpp/src/arrow/adapters/orc/adapter_test.cc
index ff194e945d..890e9e8972 100644
--- a/cpp/src/arrow/adapters/orc/adapter_test.cc
+++ b/cpp/src/arrow/adapters/orc/adapter_test.cc
@@ -453,6 +453,59 @@ TEST(TestAdapterRead, ReadIntAndStringFileMultipleStripes)
{
EXPECT_EQ(num_rows / reader_batch_size, batches);
}
+TEST(TestAdapterRead, ReadCharAndVarcharType) {
+ MemoryOutputStream mem_stream(kDefaultMemStreamSize);
+ auto orc_type =
liborc::Type::buildTypeFromString("struct<c1:char(6),c2:varchar(6)>");
+ auto writer = CreateWriter(/*stripe_size=*/1024, *orc_type, &mem_stream);
+
+ constexpr int64_t row_count = 2;
+ auto batch = writer->createRowBatch(row_count);
+ auto struct_batch =
internal::checked_cast<liborc::StructVectorBatch*>(batch.get());
+
+ // Verify that longer data will be truncated by ORC char and varchar types.
+ // In addition, ORC char type will pad the data with spaces if the data is
shorter.
+ const std::vector<std::string> data = {"abcd", "ABCDEFGH"};
+ const std::vector<std::vector<std::string>> expected_data = {{"abcd ",
"ABCDEF"},
+ {"abcd",
"ABCDEF"}};
+
+ for (uint64_t col = 0; col < orc_type->getSubtypeCount(); ++col) {
+ auto str_batch =
+
internal::checked_cast<liborc::StringVectorBatch*>(struct_batch->fields[col]);
+ str_batch->hasNulls = false;
+ str_batch->numElements = row_count;
+ for (int64_t row = 0; row < row_count; ++row) {
+ str_batch->data[row] = const_cast<char*>(data[row].c_str());
+ str_batch->length[row] = static_cast<int64_t>(data[row].size());
+ }
+ }
+ batch->numElements = row_count;
+ writer->add(*batch);
+ writer->close();
+
+ std::shared_ptr<io::RandomAccessFile>
in_stream(std::make_shared<io::BufferReader>(
+ reinterpret_cast<const uint8_t*>(mem_stream.getData()),
+ static_cast<int64_t>(mem_stream.getLength())));
+ ASSERT_OK_AND_ASSIGN(
+ auto reader, adapters::orc::ORCFileReader::Open(in_stream,
default_memory_pool()));
+ ASSERT_EQ(row_count, reader->NumberOfRows());
+ ASSERT_EQ(1, reader->NumberOfStripes());
+
+ EXPECT_OK_AND_ASSIGN(auto stripe_reader,
reader->NextStripeReader(row_count));
+ std::shared_ptr<RecordBatch> record_batch;
+ ASSERT_OK(stripe_reader->ReadNext(&record_batch));
+ ASSERT_NE(nullptr, record_batch);
+ ASSERT_EQ(row_count, record_batch->num_rows());
+
+ for (int col = 0; col < record_batch->num_columns(); ++col) {
+ auto str_array =
checked_pointer_cast<StringArray>(record_batch->column(col));
+ for (int row = 0; row < row_count; ++row) {
+ EXPECT_EQ(expected_data[col][row], str_array->GetString(row));
+ }
+ }
+ ASSERT_OK(stripe_reader->ReadNext(&record_batch));
+ ASSERT_EQ(nullptr, record_batch);
+}
+
// Trivial
class TestORCWriterTrivialNoWrite : public ::testing::Test {};
diff --git a/cpp/src/arrow/adapters/orc/util.cc
b/cpp/src/arrow/adapters/orc/util.cc
index 6e9f9a288b..d09ddb0d19 100644
--- a/cpp/src/arrow/adapters/orc/util.cc
+++ b/cpp/src/arrow/adapters/orc/util.cc
@@ -238,22 +238,6 @@ Status AppendBinaryBatch(liborc::ColumnVectorBatch*
column_vector_batch, int64_t
return Status::OK();
}
-Status AppendFixedBinaryBatch(liborc::ColumnVectorBatch* column_vector_batch,
- int64_t offset, int64_t length, ArrayBuilder*
abuilder) {
- auto builder = checked_cast<FixedSizeBinaryBuilder*>(abuilder);
- auto batch = checked_cast<liborc::StringVectorBatch*>(column_vector_batch);
-
- const bool has_nulls = batch->hasNulls;
- for (int64_t i = offset; i < length + offset; i++) {
- if (!has_nulls || batch->notNull[i]) {
- RETURN_NOT_OK(builder->Append(batch->data[i]));
- } else {
- RETURN_NOT_OK(builder->AppendNull());
- }
- }
- return Status::OK();
-}
-
Status AppendDecimalBatch(const liborc::Type* type,
liborc::ColumnVectorBatch* column_vector_batch,
int64_t offset,
int64_t length, ArrayBuilder* abuilder) {
@@ -370,13 +354,12 @@ Status AppendBatch(const liborc::Type* type,
liborc::ColumnVectorBatch* batch,
double>(batch, offset, length, builder);
case liborc::BOOLEAN:
return AppendBoolBatch(batch, offset, length, builder);
+ case liborc::CHAR:
case liborc::VARCHAR:
case liborc::STRING:
return AppendBinaryBatch<StringBuilder>(batch, offset, length, builder);
case liborc::BINARY:
return AppendBinaryBatch<BinaryBuilder>(batch, offset, length, builder);
- case liborc::CHAR:
- return AppendFixedBinaryBatch(batch, offset, length, builder);
case liborc::DATE:
return AppendNumericBatchCast<Date32Builder, int32_t,
liborc::LongVectorBatch,
int64_t>(batch, offset, length, builder);
@@ -1113,13 +1096,12 @@ Result<std::shared_ptr<DataType>> GetArrowType(const
liborc::Type* type) {
return float32();
case liborc::DOUBLE:
return float64();
+ case liborc::CHAR:
case liborc::VARCHAR:
case liborc::STRING:
return utf8();
case liborc::BINARY:
return binary();
- case liborc::CHAR:
- return fixed_size_binary(static_cast<int>(type->getMaximumLength()));
case liborc::TIMESTAMP:
// Values of TIMESTAMP type are stored in the writer timezone in the Orc
file.
// Values are read back in the reader timezone. However, the writer
timezone
diff --git a/docs/source/cpp/orc.rst b/docs/source/cpp/orc.rst
index 58f9fffc62..a9e5f02865 100644
--- a/docs/source/cpp/orc.rst
+++ b/docs/source/cpp/orc.rst
@@ -85,10 +85,10 @@ Here are a list of ORC types and mapped Arrow types.
+-------------------+-----------------------------------+-----------+
| DATE | Date32 | |
+-------------------+-----------------------------------+-----------+
-| VARCHAR | String | |
+| VARCHAR | String | \(3) |
++-------------------+-----------------------------------+-----------+
+| CHAR | String | \(3) |
+-------------------+-----------------------------------+-----------+
-
-*Unsupported ORC types:* CHAR.
* \(1) On the read side the ORC type is read as the first corresponding Arrow
type in the table.
@@ -97,6 +97,9 @@ Here are a list of ORC types and mapped Arrow types.
as the Arrow Timestamp type with :cpp:enumerator:`arrow::TimeUnit::NANO` and
timezone is set to
UTC for ORC TIMESTAMP_INSTANT type only.
+* \(3) On the read side both ORC CHAR and VARCHAR types are read as the Arrow
String type. ORC CHAR
+ and VARCHAR types are not supported on the write side.
+
Compression
-----------
