SteNicholas commented on code in PR #534:
URL: https://github.com/apache/paimon-cpp/pull/534#discussion_r4522545971


##########
src/paimon/core/catalog/table_query_auth_result.cpp:
##########
@@ -0,0 +1,1369 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/catalog/table_query_auth_result.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <deque>
+#include <limits>
+#include <optional>
+#include <string_view>
+#include <utility>
+
+#include "arrow/api.h"
+#include "arrow/util/decimal.h"
+#include "fmt/format.h"
+#include "fmt/ranges.h"
+#include "paimon/common/data/binary_string.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_type_json_parser.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/field_type_utils.h"
+#include "paimon/common/utils/rapidjson_util.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/data/decimal.h"
+#include "paimon/data/timestamp.h"
+#include "paimon/defs.h"
+#include "paimon/predicate/literal.h"
+#include "paimon/predicate/predicate.h"
+#include "paimon/predicate/predicate_builder.h"
+#include "paimon/predicate/predicate_utils.h"
+#include "rapidjson/document.h"
+
+namespace paimon {
+namespace {
+
+using ReferencedTypes = std::map<std::string, 
std::shared_ptr<arrow::DataType>>;
+
+/// Fails when two rules read one column as different types.
+Status RecordReferencedType(const std::string& name, const 
std::shared_ptr<arrow::DataType>& type,
+                            ReferencedTypes* types) {
+    auto [iter, inserted] = types->emplace(name, type);
+    if (!inserted && !iter->second->Equals(*type)) {
+        return Status::Invalid(fmt::format("Query auth rules read column '{}' 
as both {} and {}.",
+                                           name, iter->second->ToString(), 
type->ToString()));
+    }
+    return Status::OK();
+}
+
+Result<std::string> GetString(const rapidjson::Value& object, const char* key,
+                              const std::string& what) {
+    auto iter = object.FindMember(key);
+    if (iter == object.MemberEnd() || !iter->value.IsString()) {
+        return Status::Invalid(fmt::format("{} needs a string member '{}'.", 
what, key));
+    }
+    return std::string(iter->value.GetString(), iter->value.GetStringLength());
+}
+
+/// A Java `FieldRef`: `{"index": 0, "name": "f0", "type": "INT"}`.
+struct FieldRef {
+    int32_t index = 0;
+    std::string name;
+    std::shared_ptr<arrow::DataType> type;
+    /// `UNKNOWN` for a type a predicate cannot compare, such as TIME, which a 
mask still copies.
+    FieldType field_type = FieldType::UNKNOWN;
+};
+
+Result<FieldRef> ParseFieldRef(const rapidjson::Value& value, ReferencedTypes* 
types) {
+    if (!value.IsObject()) {
+        return Status::Invalid("A field reference of a query auth rule must be 
a JSON object.");
+    }
+    FieldRef ref;
+    PAIMON_ASSIGN_OR_RAISE(ref.name, GetString(value, "name", "A field 
reference"));
+    if (ref.name.empty()) {
+        return Status::Invalid("A field reference of a query auth rule cannot 
have an empty name.");
+    }
+    auto index_iter = value.FindMember("index");
+    if (index_iter != value.MemberEnd() && index_iter->value.IsInt()) {
+        ref.index = index_iter->value.GetInt();
+    }
+    auto type_iter = value.FindMember("type");
+    if (type_iter == value.MemberEnd() || type_iter->value.IsNull()) {
+        return Status::Invalid(
+            fmt::format("The field reference to '{}' of a query auth rule has 
no type.", ref.name));
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Field> field,
+                           DataTypeJsonParser::ParseType(ref.name, 
type_iter->value));
+    ref.type = field->type();
+    Result<FieldType> field_type = FieldTypeUtils::ConvertToFieldType(field);
+    if (field_type.ok()) {
+        ref.field_type = field_type.value();
+    }
+    PAIMON_RETURN_NOT_OK(RecordReferencedType(ref.name, ref.type, types));
+    return ref;
+}
+
+bool IsDigit(char c) {
+    return c >= '0' && c <= '9';
+}
+
+/// Reads exactly `count` digits of `text` at `*pos`.
+bool ReadDigits(std::string_view text, size_t* pos, size_t count, int64_t* 
value) {
+    if (text.size() - *pos < count) {
+        return false;
+    }
+    int64_t result = 0;
+    for (size_t i = 0; i < count; ++i) {
+        if (!IsDigit(text[*pos + i])) {
+            return false;
+        }
+        result = result * 10 + (text[*pos + i] - '0');
+    }
+    *pos += count;
+    *value = result;
+    return true;
+}
+
+bool ReadChar(std::string_view text, size_t* pos, char expected) {
+    if (*pos >= text.size() || text[*pos] != expected) {
+        return false;
+    }
+    ++*pos;
+    return true;
+}
+
+/// Days from 1970-01-01 to a date of the proleptic Gregorian calendar, as Java
+/// `LocalDate.toEpochDay()`.
+int64_t EpochDay(int64_t year, int64_t month, int64_t day) {
+    const int64_t y = month <= 2 ? year - 1 : year;
+    const int64_t era = (y >= 0 ? y : y - 399) / 400;
+    const int64_t year_of_era = y - era * 400;
+    const int64_t day_of_year = (153 * (month > 2 ? month - 3 : month + 9) + 
2) / 5 + day - 1;
+    const int64_t day_of_era =
+        year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year;
+    return era * 146097 + day_of_era - 719468;
+}
+
+/// Reads a date as Java `DateTimeFormatter.ISO_LOCAL_DATE` parses it: a year 
of four digits, of
+/// more only after a `+`, or negative after a `-`, then the `-MM-dd` of a 
real date.
+bool ReadIsoDate(std::string_view text, size_t* pos, int64_t* epoch_day) {
+    const bool positive = *pos < text.size() && text[*pos] == '+';
+    const bool negative = *pos < text.size() && text[*pos] == '-';
+    if (positive || negative) {
+        ++*pos;
+    }
+    const size_t start = *pos;
+    int64_t year = 0;
+    while (*pos < text.size() && *pos - start < 10 && IsDigit(text[*pos])) {
+        year = year * 10 + (text[(*pos)++] - '0');
+    }
+    const size_t digits = *pos - start;
+    if (digits < 4 || (positive && digits == 4) || (!positive && !negative && 
digits > 4) ||
+        (negative && year == 0) || year > 999999999) {
+        return false;
+    }
+    if (negative) {
+        year = -year;
+    }
+    int64_t month = 0;
+    int64_t day = 0;
+    if (!ReadChar(text, pos, '-') || !ReadDigits(text, pos, 2, &month) ||
+        !ReadChar(text, pos, '-') || !ReadDigits(text, pos, 2, &day)) {
+        return false;
+    }
+    static constexpr int64_t kDaysInMonth[] = {31, 28, 31, 30, 31, 30, 31, 31, 
30, 31, 30, 31};
+    const bool leap = (year % 4 == 0 && year % 100 != 0) || year % 400 == 0;
+    if (month < 1 || month > 12 || day < 1 ||
+        day > kDaysInMonth[month - 1] + (month == 2 && leap ? 1 : 0)) {
+        return false;
+    }
+    *epoch_day = EpochDay(year, month, day);
+    return true;
+}
+
+/// Reads a time as Java `DateTimeFormatter.ISO_LOCAL_TIME` parses it: 
`HH:mm`, optionally followed
+/// by `:ss` and then by a fraction of one to nine digits. `seconds_required` 
makes only `:ss`
+/// mandatory; the fraction stays optional.
+bool ReadIsoTime(std::string_view text, size_t* pos, bool seconds_required, 
int64_t* second_of_day,
+                 int64_t* nanos) {
+    int64_t hour = 0;
+    int64_t minute = 0;
+    int64_t second = 0;
+    *nanos = 0;
+    if (!ReadDigits(text, pos, 2, &hour) || !ReadChar(text, pos, ':') ||
+        !ReadDigits(text, pos, 2, &minute) || hour > 23 || minute > 59) {
+        return false;
+    }
+    if (ReadChar(text, pos, ':')) {
+        if (!ReadDigits(text, pos, 2, &second) || second > 59) {
+            return false;
+        }
+        if (ReadChar(text, pos, '.')) {
+            size_t digits = 0;
+            // Accumulate at most ten digits, so an overlong fraction cannot 
overflow.
+            while (*pos < text.size() && IsDigit(text[*pos]) && digits < 10) {
+                *nanos = *nanos * 10 + (text[(*pos)++] - '0');
+                ++digits;
+            }
+            if (digits == 0 || digits > 9) {
+                return false;
+            }
+            for (; digits < 9; ++digits) {
+                *nanos *= 10;
+            }
+        }
+    } else if (seconds_required) {
+        return false;
+    }
+    *second_of_day = hour * 3600 + minute * 60 + second;
+    return true;
+}
+
+/// Parses the text Java `LocalDate.parse()` reads.
+Result<int32_t> ParseDateLiteral(std::string_view text, const FieldRef& ref) {
+    size_t pos = 0;
+    int64_t epoch_day = 0;
+    if (!ReadIsoDate(text, &pos, &epoch_day) || pos != text.size() ||
+        epoch_day < std::numeric_limits<int32_t>::min() ||
+        epoch_day > std::numeric_limits<int32_t>::max()) {
+        return Status::Invalid(
+            fmt::format("Cannot parse '{}' as a date literal of column '{}'.", 
text, ref.name));
+    }
+    return static_cast<int32_t>(epoch_day);
+}
+
+/// Parses the ISO text Java `LocalDateTime.toString()` writes for a TIMESTAMP 
column, or the
+/// text `Instant.toString()` writes with a `Z` for a TIMESTAMP_LTZ one. 
Expanded years are
+/// supported; the instant must lie within the range of epoch milliseconds.
+Result<Literal> ParseTimestampLiteral(std::string_view text, const FieldRef& 
ref) {
+    const bool instant = !checked_cast<const 
arrow::TimestampType&>(*ref.type).timezone().empty();
+    size_t pos = 0;
+    int64_t epoch_day = 0;
+    int64_t second_of_day = 0;
+    int64_t nanos = 0;
+    if (!ReadIsoDate(text, &pos, &epoch_day) || !ReadChar(text, &pos, 'T') ||
+        !ReadIsoTime(text, &pos, /*seconds_required=*/instant, &second_of_day, 
&nanos) ||
+        (instant && !ReadChar(text, &pos, 'Z')) || pos != text.size()) {
+        return Status::Invalid(fmt::format(
+            "Cannot parse '{}' as a timestamp literal of column '{}'.", text, 
ref.name));
+    }
+    const int64_t seconds = epoch_day * 86400 + second_of_day;
+    // Check the complete value: the fractional milliseconds can bring the 
first second into
+    // range, and only part of the last second fits. Widen before multiplying 
to avoid overflow.
+    const Decimal::int128_t millis =
+        static_cast<Decimal::int128_t>(seconds) * 1000 + nanos / 1000000;
+    if (millis > std::numeric_limits<int64_t>::max() ||

Review Comment:
   Fixed in 4d194c4e. `LiteralConverter::ConvertLiteralsToArray()` now builds a 
timestamp with checked arithmetic and fails when the value does not fit int64 
in the literal's time unit, instead of wrapping it. A negative millisecond 
value lends one unit to the sub-millisecond remainder first, so the earliest 
representable instants still convert. When a timestamp can't be converted, 
`NullFalseLeafBinaryFunction` and `IN`/`NOT IN` fall back to comparing 
`Timestamp`s row by row, by milliseconds and then nanos of millisecond, as Java 
`Timestamp.compareTo` does.
   
   `TableQueryAuthResultTest.ParsesTemporalLiteralsAsJava` adds your case: `ts 
> '2500-01-01T00:00:00.000000001' OR allowed = true` on a `TIMESTAMP(9)` batch 
with two 2020 rows, one allowed and one denied. Only the allowed row passes, 
and the same holds for `>=`. It also checks `<`, `!=` and `IN` with the same 
cutoff. `LiteralConverterTest.TestLiteralsToArrayRejectsTimestampsOutOfRange` 
checks that the `INT64_MAX` and `INT64_MIN` epoch nanoseconds convert, that one 
nanosecond beyond either fails, and that the 2500 cutoff fails in nanoseconds 
but still converts in microseconds.



##########
src/paimon/core/catalog/table_query_auth_result.cpp:
##########
@@ -0,0 +1,1369 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/catalog/table_query_auth_result.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <deque>
+#include <limits>
+#include <optional>
+#include <string_view>
+#include <utility>
+
+#include "arrow/api.h"
+#include "arrow/util/decimal.h"
+#include "fmt/format.h"
+#include "fmt/ranges.h"
+#include "paimon/common/data/binary_string.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_type_json_parser.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/field_type_utils.h"
+#include "paimon/common/utils/rapidjson_util.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/data/decimal.h"
+#include "paimon/data/timestamp.h"
+#include "paimon/defs.h"
+#include "paimon/predicate/literal.h"
+#include "paimon/predicate/predicate.h"
+#include "paimon/predicate/predicate_builder.h"
+#include "paimon/predicate/predicate_utils.h"
+#include "rapidjson/document.h"
+
+namespace paimon {
+namespace {
+
+using ReferencedTypes = std::map<std::string, 
std::shared_ptr<arrow::DataType>>;
+
+/// Fails when two rules read one column as different types.
+Status RecordReferencedType(const std::string& name, const 
std::shared_ptr<arrow::DataType>& type,
+                            ReferencedTypes* types) {
+    auto [iter, inserted] = types->emplace(name, type);
+    if (!inserted && !iter->second->Equals(*type)) {
+        return Status::Invalid(fmt::format("Query auth rules read column '{}' 
as both {} and {}.",
+                                           name, iter->second->ToString(), 
type->ToString()));
+    }
+    return Status::OK();
+}
+
+Result<std::string> GetString(const rapidjson::Value& object, const char* key,
+                              const std::string& what) {
+    auto iter = object.FindMember(key);
+    if (iter == object.MemberEnd() || !iter->value.IsString()) {
+        return Status::Invalid(fmt::format("{} needs a string member '{}'.", 
what, key));
+    }
+    return std::string(iter->value.GetString(), iter->value.GetStringLength());
+}
+
+/// A Java `FieldRef`: `{"index": 0, "name": "f0", "type": "INT"}`.
+struct FieldRef {
+    int32_t index = 0;
+    std::string name;
+    std::shared_ptr<arrow::DataType> type;
+    /// `UNKNOWN` for a type a predicate cannot compare, such as TIME, which a 
mask still copies.
+    FieldType field_type = FieldType::UNKNOWN;
+};
+
+Result<FieldRef> ParseFieldRef(const rapidjson::Value& value, ReferencedTypes* 
types) {
+    if (!value.IsObject()) {
+        return Status::Invalid("A field reference of a query auth rule must be 
a JSON object.");
+    }
+    FieldRef ref;
+    PAIMON_ASSIGN_OR_RAISE(ref.name, GetString(value, "name", "A field 
reference"));
+    if (ref.name.empty()) {
+        return Status::Invalid("A field reference of a query auth rule cannot 
have an empty name.");
+    }
+    auto index_iter = value.FindMember("index");
+    if (index_iter != value.MemberEnd() && index_iter->value.IsInt()) {
+        ref.index = index_iter->value.GetInt();
+    }
+    auto type_iter = value.FindMember("type");
+    if (type_iter == value.MemberEnd() || type_iter->value.IsNull()) {
+        return Status::Invalid(
+            fmt::format("The field reference to '{}' of a query auth rule has 
no type.", ref.name));
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Field> field,
+                           DataTypeJsonParser::ParseType(ref.name, 
type_iter->value));

Review Comment:
   Fixed in 4d194c4e. `DataTypeJsonParser` now builds a decimal through 
`arrow::Decimal128Type::Make()` and returns its error, so `DECIMAL(39, 0)` no 
longer aborts the process. It also rejects a scale outside `[0, precision]`, as 
Java `DecimalType` does; before, Arrow silently accepted it.
   
   `TableQueryAuthResultTest.RefusesMalformedRuleJson` checks that `DECIMAL(39, 
0)` returns `Decimal precision out of range`, both in a row filter's `fieldRef` 
and in a `FIELD_REF` mask. `DataTypeJsonParserTest.ParseTypeAtomicTypeSuccess` 
covers the accepted bounds `DECIMAL(1, 0)` and `DECIMAL(38, 38)` and rejects 
`DECIMAL(0)`, `DECIMAL(39)`, `NUMERIC(100, 2)`, `DECIMAL(10, 11)` and 
`DECIMAL(38, 39)`.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to