lucasfang commented on code in PR #534: URL: https://github.com/apache/paimon-cpp/pull/534#discussion_r4522120761
########## src/paimon/core/catalog/table_query_auth_result.cpp: ########## @@ -0,0 +1,1369 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/catalog/table_query_auth_result.h" + +#include <algorithm> +#include <cmath> +#include <cstdint> +#include <deque> +#include <limits> +#include <optional> +#include <string_view> +#include <utility> + +#include "arrow/api.h" +#include "arrow/util/decimal.h" +#include "fmt/format.h" +#include "fmt/ranges.h" +#include "paimon/common/data/binary_string.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/common/types/data_type_json_parser.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/field_type_utils.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/data/decimal.h" +#include "paimon/data/timestamp.h" +#include "paimon/defs.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/predicate/predicate_utils.h" +#include "rapidjson/document.h" + +namespace paimon { +namespace { + +using ReferencedTypes = std::map<std::string, std::shared_ptr<arrow::DataType>>; + +/// Fails when two rules read one column as different types. +Status RecordReferencedType(const std::string& name, const std::shared_ptr<arrow::DataType>& type, + ReferencedTypes* types) { + auto [iter, inserted] = types->emplace(name, type); + if (!inserted && !iter->second->Equals(*type)) { + return Status::Invalid(fmt::format("Query auth rules read column '{}' as both {} and {}.", + name, iter->second->ToString(), type->ToString())); + } + return Status::OK(); +} + +Result<std::string> GetString(const rapidjson::Value& object, const char* key, + const std::string& what) { + auto iter = object.FindMember(key); + if (iter == object.MemberEnd() || !iter->value.IsString()) { + return Status::Invalid(fmt::format("{} needs a string member '{}'.", what, key)); + } + return std::string(iter->value.GetString(), iter->value.GetStringLength()); +} + +/// A Java `FieldRef`: `{"index": 0, "name": "f0", "type": "INT"}`. +struct FieldRef { + int32_t index = 0; + std::string name; + std::shared_ptr<arrow::DataType> type; + /// `UNKNOWN` for a type a predicate cannot compare, such as TIME, which a mask still copies. + FieldType field_type = FieldType::UNKNOWN; +}; + +Result<FieldRef> ParseFieldRef(const rapidjson::Value& value, ReferencedTypes* types) { + if (!value.IsObject()) { + return Status::Invalid("A field reference of a query auth rule must be a JSON object."); + } + FieldRef ref; + PAIMON_ASSIGN_OR_RAISE(ref.name, GetString(value, "name", "A field reference")); + if (ref.name.empty()) { + return Status::Invalid("A field reference of a query auth rule cannot have an empty name."); + } + auto index_iter = value.FindMember("index"); + if (index_iter != value.MemberEnd() && index_iter->value.IsInt()) { + ref.index = index_iter->value.GetInt(); + } + auto type_iter = value.FindMember("type"); + if (type_iter == value.MemberEnd() || type_iter->value.IsNull()) { + return Status::Invalid( + fmt::format("The field reference to '{}' of a query auth rule has no type.", ref.name)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Field> field, + DataTypeJsonParser::ParseType(ref.name, type_iter->value)); Review Comment: **[P2] Return a parse error for an out-of-range decimal precision.** A catalog rule with `fieldRef.type` set to `DECIMAL(39,0)` reaches `DataTypeJsonParser::ParseType` here, before the field reference is checked against the table schema. The shared parser calls `arrow::decimal128`, and Arrow 17’s constructor uses a fatal check for precision above 38, even in release builds. One malformed rule can therefore terminate the process serving other queries. Please use `Decimal128Type::Make` in the shared parser, propagate its error, and add a test that parsing this rule returns an error. Java already reports invalid precision as an exception. [C++ decimal parser](https://github.com/apache/paimon-cpp/blob/5f40ceef425fa4024a75dd49039f47a7801c20fe/src/paimon/common/types/data_type_json_parser.cpp#L564-L579) · [Arrow constructor and checked factory](https://github.com/apache/arrow/blob/6a2e19a852b367c72d7b12da4d104456491ed8b7/cpp/src/arrow/type.cc#L1465-L1476) · [Java precision check](ht tps://github.com/apache/paimon/blob/59ad511d4cf567f33a17c1e5a2f8a450eae47004/paimon-api/src/main/java/org/apache/paimon/types/DecimalType.java#L37-L65) ########## src/paimon/core/catalog/table_query_auth_result.cpp: ########## @@ -0,0 +1,1369 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/catalog/table_query_auth_result.h" + +#include <algorithm> +#include <cmath> +#include <cstdint> +#include <deque> +#include <limits> +#include <optional> +#include <string_view> +#include <utility> + +#include "arrow/api.h" +#include "arrow/util/decimal.h" +#include "fmt/format.h" +#include "fmt/ranges.h" +#include "paimon/common/data/binary_string.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/common/types/data_type_json_parser.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/field_type_utils.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/data/decimal.h" +#include "paimon/data/timestamp.h" +#include "paimon/defs.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/predicate/predicate_utils.h" +#include "rapidjson/document.h" + +namespace paimon { +namespace { + +using ReferencedTypes = std::map<std::string, std::shared_ptr<arrow::DataType>>; + +/// Fails when two rules read one column as different types. +Status RecordReferencedType(const std::string& name, const std::shared_ptr<arrow::DataType>& type, + ReferencedTypes* types) { + auto [iter, inserted] = types->emplace(name, type); + if (!inserted && !iter->second->Equals(*type)) { + return Status::Invalid(fmt::format("Query auth rules read column '{}' as both {} and {}.", + name, iter->second->ToString(), type->ToString())); + } + return Status::OK(); +} + +Result<std::string> GetString(const rapidjson::Value& object, const char* key, + const std::string& what) { + auto iter = object.FindMember(key); + if (iter == object.MemberEnd() || !iter->value.IsString()) { + return Status::Invalid(fmt::format("{} needs a string member '{}'.", what, key)); + } + return std::string(iter->value.GetString(), iter->value.GetStringLength()); +} + +/// A Java `FieldRef`: `{"index": 0, "name": "f0", "type": "INT"}`. +struct FieldRef { + int32_t index = 0; + std::string name; + std::shared_ptr<arrow::DataType> type; + /// `UNKNOWN` for a type a predicate cannot compare, such as TIME, which a mask still copies. + FieldType field_type = FieldType::UNKNOWN; +}; + +Result<FieldRef> ParseFieldRef(const rapidjson::Value& value, ReferencedTypes* types) { + if (!value.IsObject()) { + return Status::Invalid("A field reference of a query auth rule must be a JSON object."); + } + FieldRef ref; + PAIMON_ASSIGN_OR_RAISE(ref.name, GetString(value, "name", "A field reference")); + if (ref.name.empty()) { + return Status::Invalid("A field reference of a query auth rule cannot have an empty name."); + } + auto index_iter = value.FindMember("index"); + if (index_iter != value.MemberEnd() && index_iter->value.IsInt()) { + ref.index = index_iter->value.GetInt(); + } + auto type_iter = value.FindMember("type"); + if (type_iter == value.MemberEnd() || type_iter->value.IsNull()) { + return Status::Invalid( + fmt::format("The field reference to '{}' of a query auth rule has no type.", ref.name)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Field> field, + DataTypeJsonParser::ParseType(ref.name, type_iter->value)); + ref.type = field->type(); + Result<FieldType> field_type = FieldTypeUtils::ConvertToFieldType(field); + if (field_type.ok()) { + ref.field_type = field_type.value(); + } + PAIMON_RETURN_NOT_OK(RecordReferencedType(ref.name, ref.type, types)); + return ref; +} + +bool IsDigit(char c) { + return c >= '0' && c <= '9'; +} + +/// Reads exactly `count` digits of `text` at `*pos`. +bool ReadDigits(std::string_view text, size_t* pos, size_t count, int64_t* value) { + if (text.size() - *pos < count) { + return false; + } + int64_t result = 0; + for (size_t i = 0; i < count; ++i) { + if (!IsDigit(text[*pos + i])) { + return false; + } + result = result * 10 + (text[*pos + i] - '0'); + } + *pos += count; + *value = result; + return true; +} + +bool ReadChar(std::string_view text, size_t* pos, char expected) { + if (*pos >= text.size() || text[*pos] != expected) { + return false; + } + ++*pos; + return true; +} + +/// Days from 1970-01-01 to a date of the proleptic Gregorian calendar, as Java +/// `LocalDate.toEpochDay()`. +int64_t EpochDay(int64_t year, int64_t month, int64_t day) { + const int64_t y = month <= 2 ? year - 1 : year; + const int64_t era = (y >= 0 ? y : y - 399) / 400; + const int64_t year_of_era = y - era * 400; + const int64_t day_of_year = (153 * (month > 2 ? month - 3 : month + 9) + 2) / 5 + day - 1; + const int64_t day_of_era = + year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year; + return era * 146097 + day_of_era - 719468; +} + +/// Reads a date as Java `DateTimeFormatter.ISO_LOCAL_DATE` parses it: a year of four digits, of +/// more only after a `+`, or negative after a `-`, then the `-MM-dd` of a real date. +bool ReadIsoDate(std::string_view text, size_t* pos, int64_t* epoch_day) { + const bool positive = *pos < text.size() && text[*pos] == '+'; + const bool negative = *pos < text.size() && text[*pos] == '-'; + if (positive || negative) { + ++*pos; + } + const size_t start = *pos; + int64_t year = 0; + while (*pos < text.size() && *pos - start < 10 && IsDigit(text[*pos])) { + year = year * 10 + (text[(*pos)++] - '0'); + } + const size_t digits = *pos - start; + if (digits < 4 || (positive && digits == 4) || (!positive && !negative && digits > 4) || + (negative && year == 0) || year > 999999999) { + return false; + } + if (negative) { + year = -year; + } + int64_t month = 0; + int64_t day = 0; + if (!ReadChar(text, pos, '-') || !ReadDigits(text, pos, 2, &month) || + !ReadChar(text, pos, '-') || !ReadDigits(text, pos, 2, &day)) { + return false; + } + static constexpr int64_t kDaysInMonth[] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; + const bool leap = (year % 4 == 0 && year % 100 != 0) || year % 400 == 0; + if (month < 1 || month > 12 || day < 1 || + day > kDaysInMonth[month - 1] + (month == 2 && leap ? 1 : 0)) { + return false; + } + *epoch_day = EpochDay(year, month, day); + return true; +} + +/// Reads a time as Java `DateTimeFormatter.ISO_LOCAL_TIME` parses it: `HH:mm`, optionally followed +/// by `:ss` and then by a fraction of one to nine digits. `seconds_required` makes only `:ss` +/// mandatory; the fraction stays optional. +bool ReadIsoTime(std::string_view text, size_t* pos, bool seconds_required, int64_t* second_of_day, + int64_t* nanos) { + int64_t hour = 0; + int64_t minute = 0; + int64_t second = 0; + *nanos = 0; + if (!ReadDigits(text, pos, 2, &hour) || !ReadChar(text, pos, ':') || + !ReadDigits(text, pos, 2, &minute) || hour > 23 || minute > 59) { + return false; + } + if (ReadChar(text, pos, ':')) { + if (!ReadDigits(text, pos, 2, &second) || second > 59) { + return false; + } + if (ReadChar(text, pos, '.')) { + size_t digits = 0; + // Accumulate at most ten digits, so an overlong fraction cannot overflow. + while (*pos < text.size() && IsDigit(text[*pos]) && digits < 10) { + *nanos = *nanos * 10 + (text[(*pos)++] - '0'); + ++digits; + } + if (digits == 0 || digits > 9) { + return false; + } + for (; digits < 9; ++digits) { + *nanos *= 10; + } + } + } else if (seconds_required) { + return false; + } + *second_of_day = hour * 3600 + minute * 60 + second; + return true; +} + +/// Parses the text Java `LocalDate.parse()` reads. +Result<int32_t> ParseDateLiteral(std::string_view text, const FieldRef& ref) { + size_t pos = 0; + int64_t epoch_day = 0; + if (!ReadIsoDate(text, &pos, &epoch_day) || pos != text.size() || + epoch_day < std::numeric_limits<int32_t>::min() || + epoch_day > std::numeric_limits<int32_t>::max()) { + return Status::Invalid( + fmt::format("Cannot parse '{}' as a date literal of column '{}'.", text, ref.name)); + } + return static_cast<int32_t>(epoch_day); +} + +/// Parses the ISO text Java `LocalDateTime.toString()` writes for a TIMESTAMP column, or the +/// text `Instant.toString()` writes with a `Z` for a TIMESTAMP_LTZ one. Expanded years are +/// supported; the instant must lie within the range of epoch milliseconds. +Result<Literal> ParseTimestampLiteral(std::string_view text, const FieldRef& ref) { + const bool instant = !checked_cast<const arrow::TimestampType&>(*ref.type).timezone().empty(); + size_t pos = 0; + int64_t epoch_day = 0; + int64_t second_of_day = 0; + int64_t nanos = 0; + if (!ReadIsoDate(text, &pos, &epoch_day) || !ReadChar(text, &pos, 'T') || + !ReadIsoTime(text, &pos, /*seconds_required=*/instant, &second_of_day, &nanos) || + (instant && !ReadChar(text, &pos, 'Z')) || pos != text.size()) { + return Status::Invalid(fmt::format( + "Cannot parse '{}' as a timestamp literal of column '{}'.", text, ref.name)); + } + const int64_t seconds = epoch_day * 86400 + second_of_day; + // Check the complete value: the fractional milliseconds can bring the first second into + // range, and only part of the last second fits. Widen before multiplying to avoid overflow. + const Decimal::int128_t millis = + static_cast<Decimal::int128_t>(seconds) * 1000 + nanos / 1000000; + if (millis > std::numeric_limits<int64_t>::max() || Review Comment: **[P1] Check the nanosecond range before building the comparison scalar.** The cutoff `2500-01-01T00:00:00.000000001` fits this millisecond check, but its fractional nanosecond makes the read-side comparison build an `int64` epoch-nanosecond scalar, which overflows. A regression case is `ts > cutoff OR allowed = true` on a `TIMESTAMP(9)` batch with two rows dated 2020, one allowed and one denied. The allowed row keeps the file eligible for scanning. On builds where the conversion wraps, the denied row can also pass. Please use checked arithmetic when constructing the scalar and fall back to comparing milliseconds and the sub-millisecond remainder when it is out of range. [C++ scalar conversion](https://github.com/apache/paimon-cpp/blob/5f40ceef425fa4024a75dd49039f47a7801c20fe/src/paimon/common/predicate/literal_converter.cpp#L399-L411) · [Java comparison](https://github.com/apache/paimon/blob/59ad511d4cf567f33a17c1e5a2f8a450eae47004/paimon-common/src/main/java/org/apache/paimon/d ata/Timestamp.java#L128-L135) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
