divjotarora commented on code in PR #50916:
URL: https://github.com/apache/arrow/pull/50916#discussion_r3825681829
##########
cpp/src/parquet/arrow/schema_internal.cc:
##########
@@ -207,6 +207,8 @@ Result<std::shared_ptr<ArrowType>> FromFLBA(
return ::arrow::extension::uuid();
}
+ return ::arrow::fixed_size_binary(physical_length);
+ case LogicalType::Type::TIMESTAMP:
return ::arrow::fixed_size_binary(physical_length);
Review Comment:
Ack, added `convert_flba_timestamps` and `flba_timestamp_clamp_on_overflow`
properties to control conversion from FLBA(12) --> Arrow timestamps and
clamping to min/max int64 vs. erroring for values out of the int64 range,
respectively
##########
cpp/src/parquet/statistics.cc:
##########
@@ -463,6 +463,56 @@ struct RebindLogical<Float16LogicalType> {
using c_type = DType::c_type;
};
+// Tag type for FLBA(12) timestamps.
+struct Flba12TimestampType {};
+
+// Max / min representable signed 96-bit two's-complement, little-endian.
+constexpr uint8_t kFlba12SignedMax[12] = {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F};
+constexpr uint8_t kFlba12SignedMin[12] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x80};
+
+template <>
+struct CompareHelper<Flba12TimestampType, /*is_signed=*/true> {
+ using T = FLBA;
+
+ // Seed for the running minimum is the maximum value; for the maximum, the
minimum
+ // value.
+ static T DefaultMin() { return T{kFlba12SignedMax}; }
+ static T DefaultMax() { return T{kFlba12SignedMin}; }
+
+ static T Coalesce(T val, T fallback) { return val.ptr == nullptr ? fallback
: val; }
+
+ // Signed little-endian comparison.
+ // Differing signs: negative (MSB >= 0x80) is smaller.
+ // Same sign: unsigned scan and comparison.
+ static inline bool Compare(int /*type_length*/, const T& a, const T& b) {
+ const bool a_neg = (a.ptr[11] & 0x80) != 0;
+ const bool b_neg = (b.ptr[11] & 0x80) != 0;
+ if (a_neg != b_neg) return a_neg;
+ for (int i = 11; i >= 0; --i) if (a.ptr[i] != b.ptr[i]) return a.ptr[i] <
b.ptr[i];
Review Comment:
Done
##########
cpp/src/parquet/reader_test.cc:
##########
@@ -1769,6 +1771,66 @@ TEST(TestByteStreamSplit, ExtendedIntegrationFile) {
}
#endif // ARROW_WITH_ZLIB
+TEST(TestFileReader, TestFlba12Timestamp) {
+ auto file = ParquetFileReader::OpenFile(flba12_timestamp());
+
+ const int64_t kNumRows = 6;
+ // Row indices of the minimum (year 0001) and maximum (year 9999) values.
+ const int kMinRow = 5;
+ const int kMaxRow = 4;
+
+ auto metadata = file->metadata();
+ ASSERT_EQ(kNumRows, metadata->num_rows());
+ ASSERT_EQ(3, metadata->num_columns());
+ ASSERT_EQ(1, metadata->num_row_groups());
+
+ const struct {
+ const char* name;
+ LogicalType::TimeUnit::unit unit;
+ } columns[] = {
+ {"timestamp_millis", LogicalType::TimeUnit::MILLIS},
+ {"timestamp_micros", LogicalType::TimeUnit::MICROS},
+ {"timestamp_nanos", LogicalType::TimeUnit::NANOS},
+ };
+
+ auto rg_reader = file->RowGroup(0);
+ for (int c = 0; c < 3; ++c) {
+ const auto* descr = metadata->schema()->Column(c);
+ ASSERT_EQ(columns[c].name, descr->name());
+ ASSERT_EQ(Type::FIXED_LEN_BYTE_ARRAY, descr->physical_type());
+ ASSERT_EQ(12, descr->type_length());
+ ASSERT_EQ(SortOrder::SIGNED, descr->sort_order());
+ ASSERT_EQ(ColumnOrder::TYPE_DEFINED_ORDER,
descr->column_order().get_order());
+
+ const auto& logical_type = descr->logical_type();
+ ASSERT_EQ(LogicalType::Type::TIMESTAMP, logical_type->type());
+ const auto& ts =
+ ::arrow::internal::checked_cast<const
TimestampLogicalType&>(*logical_type);
+ ASSERT_TRUE(ts.is_adjusted_to_utc());
+ ASSERT_EQ(columns[c].unit, ts.time_unit());
+
+ std::string min_value, max_value;
+ {
+ auto col_reader =
+
checked_pointer_cast<TypedColumnReader<FLBAType>>(rg_reader->Column(c));
+ std::vector<FLBA> values(kNumRows);
+ int64_t values_read = 0;
+ int64_t levels_read =
+ col_reader->ReadBatch(kNumRows, nullptr, nullptr, values.data(),
&values_read);
+ ASSERT_EQ(kNumRows, levels_read);
+ ASSERT_EQ(kNumRows, values_read);
+ min_value.assign(reinterpret_cast<const char*>(values[kMinRow].ptr), 12);
+ max_value.assign(reinterpret_cast<const char*>(values[kMaxRow].ptr), 12);
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
