SteNicholas commented on code in PR #534:
URL: https://github.com/apache/paimon-cpp/pull/534#discussion_r4509912374


##########
src/paimon/core/table/source/query_auth_table_read.cpp:
##########
@@ -0,0 +1,463 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/table/source/query_auth_table_read.h"
+
+#include <cstdint>
+#include <utility>
+
+#include "arrow/api.h"
+#include "arrow/c/abi.h"
+#include "arrow/c/bridge.h"
+#include "arrow/compute/cast.h"
+#include "fmt/format.h"
+#include "paimon/common/reader/concat_batch_reader.h"
+#include "paimon/common/reader/predicate_batch_reader.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/object_utils.h"
+#include "paimon/common/utils/scope_guard.h"
+#include "paimon/core/casting/casting_utils.h"
+#include "paimon/core/table/source/query_auth_split.h"
+#include "paimon/defs.h"
+#include "paimon/predicate/predicate.h"
+#include "paimon/predicate/predicate_builder.h"
+#include "paimon/predicate/predicate_utils.h"
+#include "paimon/status.h"
+
+namespace paimon {
+namespace {
+
+/// The column as a mask or a predicate reads it: dictionary-decoded, and cast 
to a string when it
+/// is a large string.
+Result<std::shared_ptr<arrow::Array>> DecodeDictionaryAndLargeString(
+    const std::shared_ptr<arrow::Array>& column, arrow::MemoryPool* pool) {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> decoded,
+                           CastingUtils::DecodeDictionary(column, pool));
+    if (decoded->type_id() == arrow::Type::LARGE_STRING) {
+        return CastingUtils::Cast(decoded, arrow::utf8(), 
arrow::compute::CastOptions::Safe(),
+                                  pool);
+    }
+    return decoded;
+}
+
+/// The type `DecodeDictionaryAndLargeString()` gives a column of `type`.
+std::shared_ptr<arrow::DataType> DecodedType(const 
std::shared_ptr<arrow::DataType>& type) {
+    std::shared_ptr<arrow::DataType> value_type = type;
+    if (value_type->id() == arrow::Type::DICTIONARY) {
+        value_type = checked_cast<const 
arrow::DictionaryType&>(*value_type).value_type();
+    }
+    if (value_type->id() == arrow::Type::LARGE_STRING) {
+        return arrow::utf8();
+    }
+    return value_type;
+}
+
+}  // namespace
+
+QueryAuthBatchReader::QueryAuthBatchReader(
+    std::unique_ptr<BatchReader> reader,
+    std::map<std::string, std::shared_ptr<MaskTransform>> masks,
+    std::set<std::string> dropped_fields, std::set<std::string> decoded_fields,
+    std::shared_ptr<arrow::MemoryPool> arrow_pool)
+    : reader_(std::move(reader)),
+      masks_(std::move(masks)),
+      dropped_fields_(std::move(dropped_fields)),
+      decoded_fields_(std::move(decoded_fields)),
+      arrow_pool_(std::move(arrow_pool)) {
+    decode_inputs_ = decoded_fields_;
+    for (const auto& [target, mask] : masks_) {
+        for (const std::string& input : mask->InputFields()) {
+            decode_inputs_.insert(input);
+        }
+    }
+}
+
+Result<std::shared_ptr<arrow::StructArray>> QueryAuthBatchReader::DecodeInputs(
+    const std::shared_ptr<arrow::StructArray>& batch) const {
+    arrow::FieldVector fields = batch->struct_type()->fields();
+    arrow::ArrayVector children = batch->fields();
+    bool decoded_any = false;
+    for (size_t i = 0; i < fields.size(); ++i) {
+        if (decode_inputs_.count(fields[i]->name()) == 0) {
+            continue;
+        }
+        Result<std::shared_ptr<arrow::Array>> result =
+            DecodeDictionaryAndLargeString(children[i], arrow_pool_.get());
+        if (!result.ok()) {
+            return result.status().WithMessage(
+                fmt::format("Query auth read cannot decode column '{}': {}", 
fields[i]->name(),
+                            result.status().message()));
+        }
+        std::shared_ptr<arrow::Array> decoded = std::move(result).value();
+        if (decoded != children[i]) {
+            fields[i] = fields[i]->WithType(decoded->type());
+            children[i] = std::move(decoded);
+            decoded_any = true;
+        }
+    }
+    if (!decoded_any) {
+        return batch;
+    }
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::StructArray> 
decoded_batch,
+                                      arrow::StructArray::Make(children, 
fields));
+    return decoded_batch;
+}
+
+Result<BatchReader::ReadBatch> QueryAuthBatchReader::NextBatch() {
+    PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader_->NextBatch());
+    if (BatchReader::IsEofBatch(batch)) {
+        return batch;
+    }
+    auto& [c_array, c_schema] = batch;
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
+                                      arrow::ImportArray(c_array.get(), 
c_schema.get()));
+    if (array->type_id() != arrow::Type::STRUCT) {
+        return Status::Invalid("a query-auth read requires struct batches");
+    }
+    auto struct_array = checked_pointer_cast<arrow::StructArray>(array);
+    if (struct_array->null_count() != 0) {
+        return Status::Invalid("a query-auth read does not support null rows 
in a batch");
+    }
+    const auto& struct_type = *struct_array->struct_type();
+    std::shared_ptr<arrow::StructArray> decoded = struct_array;
+    if (!decode_inputs_.empty()) {
+        PAIMON_ASSIGN_OR_RAISE(decoded, DecodeInputs(struct_array));
+    }
+    arrow::FieldVector fields;
+    arrow::ArrayVector children;
+    for (int32_t i = 0; i < struct_type.num_fields(); ++i) {
+        const std::shared_ptr<arrow::Field>& field = struct_type.field(i);
+        if (dropped_fields_.count(field->name()) > 0) {
+            continue;
+        }
+        auto mask = masks_.find(field->name());
+        if (mask == masks_.end()) {
+            const arrow::StructArray& source =
+                decoded_fields_.count(field->name()) > 0 ? *decoded : 
*struct_array;
+            fields.push_back(source.struct_type()->field(i));
+            children.push_back(source.field(i));
+            continue;
+        }
+        const std::shared_ptr<arrow::DataType> target_type = 
DecodedType(field->type());
+        Result<std::shared_ptr<arrow::Array>> result =
+            mask->second->Evaluate(*decoded, target_type, arrow_pool_.get());
+        if (!result.ok()) {
+            return result.status().WithMessage(
+                fmt::format("Query auth mask {} of column '{}' failed: {}", 
mask->second->Name(),
+                            field->name(), result.status().message()));
+        }
+        std::shared_ptr<arrow::Array> masked = std::move(result).value();
+        if (masked->length() != struct_array->length()) {
+            return Status::Invalid(fmt::format("the mask of column '{}' 
produced {} rows for {}",
+                                               field->name(), masked->length(),
+                                               struct_array->length()));
+        }
+        if (!masked->type()->Equals(*target_type)) {
+            return Status::Invalid(fmt::format("the mask of column '{}' 
produced {} for {}",
+                                               field->name(), 
masked->type()->ToString(),
+                                               target_type->ToString()));
+        }
+        // A mask may null a column that holds no nulls.
+        fields.push_back(field->WithType(target_type)->WithNullable(true));
+        children.push_back(std::move(masked));
+    }
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::StructArray> 
output,
+                                      arrow::StructArray::Make(children, 
fields));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*output, c_array.get(), 
c_schema.get()));
+    // The masked buffers come from `arrow_pool_`, which must outlive the 
batch, not this reader.
+    PAIMON_RETURN_NOT_OK(AddArrowArrayLifetime(c_array.get(), c_schema.get(), 
arrow_pool_));
+    return batch;
+}
+
+Result<std::unique_ptr<TableRead>> QueryAuthTableRead::Create(
+    const std::shared_ptr<ReadContext>& context, const 
std::shared_ptr<TableSchema>& table_schema,
+    const std::shared_ptr<arrow::MemoryPool>& arrow_pool, InnerReadFactory 
create_inner) {
+    if (context->HasVectorOrFullTextSearch() || 
context->EnablePreciseFileIndexEvaluate()) {
+        return Status::NotImplemented(
+            "Search is not supported on a query-auth table: the index ranks 
raw values, which a "
+            "column mask invalidates.");
+    }
+    if (context->HasReadSchema()) {
+        return Status::NotImplemented(
+            "An explicit read schema is not supported on a query-auth table: 
the rules apply to "
+            "whole columns; use SetReadFieldNames() or SetReadFieldIds() 
instead.");
+    }
+    if (context->GetRealtimeContext() != nullptr) {
+        return Status::NotImplemented("Real-time reads are not supported on a 
query-auth table.");
+    }
+    std::optional<std::vector<std::string>> output_fields;
+    if (!context->GetReadFieldIds().empty()) {
+        std::vector<std::string> names;
+        for (int32_t id : context->GetReadFieldIds()) {
+            Result<DataField> field = table_schema->GetField(id);
+            if (!field.ok()) {
+                return Status::NotImplemented(fmt::format(
+                    "Field {} is not a column of the table schema, which is 
all a query-auth read "
+                    "can authorize.",
+                    id));
+            }
+            names.push_back(field.value().Name());
+        }
+        output_fields = std::move(names);
+    } else if (!context->GetReadFieldNames().empty()) {
+        output_fields = context->GetReadFieldNames();
+    }
+    std::set<std::string> predicate_fields;
+    if (context->GetPredicate() != nullptr) {
+        PAIMON_RETURN_NOT_OK(
+            PredicateUtils::GetAllNames(context->GetPredicate(), 
&predicate_fields));
+    }
+    std::vector<std::string> requested = 
output_fields.value_or(std::vector<std::string>());
+    requested.insert(requested.end(), predicate_fields.begin(), 
predicate_fields.end());
+    for (const std::string& name : requested) {
+        if (!table_schema->GetField(name).ok()) {
+            return Status::NotImplemented(fmt::format(
+                "Column '{}' is not a column of the table schema, which is all 
a query-auth read "
+                "can authorize.",
+                name));
+        }
+    }
+    std::unique_ptr<QueryAuthTableRead> read(new QueryAuthTableRead(
+        context, table_schema, arrow_pool, std::move(create_inner), 
std::move(output_fields)));
+    // Built now, so a read the table cannot serve fails here rather than at 
the first split.
+    PAIMON_RETURN_NOT_OK(read->GetOrCreateInnerRead({}).status());
+    return std::unique_ptr<TableRead>(std::move(read));
+}
+
+QueryAuthTableRead::QueryAuthTableRead(const std::shared_ptr<ReadContext>& 
context,
+                                       const std::shared_ptr<TableSchema>& 
table_schema,
+                                       const 
std::shared_ptr<arrow::MemoryPool>& arrow_pool,
+                                       InnerReadFactory create_inner,
+                                       std::optional<std::vector<std::string>> 
output_fields)
+    : context_(context),
+      table_schema_(table_schema),
+      arrow_pool_(arrow_pool),
+      create_inner_(std::move(create_inner)),
+      output_fields_(std::move(output_fields)) {}
+
+Result<std::shared_ptr<Predicate>> QueryAuthTableRead::PostMaskFilter(
+    const TableQueryAuthResult& rules) const {
+    const std::shared_ptr<Predicate>& predicate = context_->GetPredicate();
+    if (predicate == nullptr || context_->EnablePredicateFilter()) {
+        return predicate;
+    }
+    // As Java `ReadTransform`: the scan did not prune by the conjuncts on 
masked columns, so the
+    // read evaluates them even when the caller evaluates the rest of its 
predicate itself.
+    const std::set<std::string> masked_fields = rules.MaskedFields();
+    std::vector<std::shared_ptr<Predicate>> masked_conjuncts;
+    for (const std::shared_ptr<Predicate>& conjunct : 
PredicateUtils::SplitAnd(predicate)) {
+        PAIMON_ASSIGN_OR_RAISE(bool masked,
+                               PredicateUtils::ContainAnyField(conjunct, 
masked_fields));
+        if (masked) {
+            masked_conjuncts.push_back(conjunct);
+        }
+    }
+    if (masked_conjuncts.empty()) {
+        return std::shared_ptr<Predicate>();
+    }
+    return PredicateBuilder::And(masked_conjuncts);
+}
+
+Result<TableRead*> QueryAuthTableRead::GetOrCreateInnerRead(
+    const std::vector<std::string>& extra_fields) {
+    auto iter = inner_reads_.find(extra_fields);
+    if (iter != inner_reads_.end()) {
+        return iter->second.get();
+    }
+    std::vector<std::string> read_field_names = context_->GetReadFieldNames();
+    std::vector<int32_t> read_field_ids = context_->GetReadFieldIds();
+    if (!read_field_ids.empty()) {
+        for (const std::string& name : extra_fields) {
+            PAIMON_ASSIGN_OR_RAISE(DataField field, 
table_schema_->GetField(name));
+            read_field_ids.push_back(field.Id());
+        }
+    } else if (!read_field_names.empty()) {
+        read_field_names.insert(read_field_names.end(), extra_fields.begin(), 
extra_fields.end());
+    }
+    // Without the predicate: file statistics and indexes hold raw values.
+    auto inner_context = std::make_unique<ReadContext>(
+        context_->GetPath(), context_->GetBranch(), read_field_names, 
read_field_ids,
+        /*predicate=*/nullptr, /*vector_search=*/nullptr, 
/*full_text_search=*/nullptr,
+        /*enable_precise_file_index_evaluate=*/false, 
/*enable_predicate_filter=*/false,
+        context_->EnablePrefetch(), context_->EnableLateMaterializing(),
+        context_->GetPrefetchBatchCount(), 
context_->GetPrefetchMaxParallelNum(),
+        context_->EnableMultiThreadRowToBatch(), 
context_->GetRowToBatchThreadNumber(),
+        context_->GetSpecificTableSchema(), context_->GetMemoryPool(), 
context_->GetExecutor(),
+        context_->GetSpecificFileSystem(), 
context_->GetFileSystemSchemeToIdentifierMap(),
+        /*realtime_context=*/nullptr, context_->GetOptions(), 
context_->ReadAheadCacheEnabled(),
+        context_->GetCacheConfig(), context_->GetCache(), 
context_->GetFormatTable(),
+        context_->GetWarmupLevel(), context_->GetCatalog(), 
context_->GetIdentifier());
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> read,
+                           create_inner_(std::move(inner_context)));
+    TableRead* raw_read = read.get();
+    inner_reads_.emplace(extra_fields, std::move(read));
+    return raw_read;
+}
+
+Status QueryAuthTableRead::ValidateSplit(const QueryAuthSplit& split,
+                                         const std::set<std::string>& 
filter_fields) const {
+    if (!split.GetScope()) {
+        return Status::Invalid(
+            "The split does not record which columns its query auth rules were 
authorized for, "
+            "as a split Java planned does not; refusing to read it. Plan the 
scan with "
+            "paimon-cpp.");
+    }
+    const QueryAuthScope& scope = split.GetScope().value();
+    const TableQueryAuthResult& rules = *split.GetAuthResult();
+    const std::vector<DataField>& fields = table_schema_->Fields();
+    // The field ids are checked first, so a renamed column is reported as 
such rather than as a
+    // missing one.
+    PAIMON_RETURN_NOT_OK(rules.ValidateFieldIds(scope.rule_field_ids, fields));
+    PAIMON_RETURN_NOT_OK(rules.ValidateAgainstSchema(fields));
+    // The columns the rules read themselves are the catalog's own and need no 
authorization.
+    for (const DataField& field : fields) {
+        const bool read = filter_fields.count(field.Name()) > 0 || 
!output_fields_ ||
+                          ObjectUtils::Contains(output_fields_.value(), 
field.Name());
+        if (!read) {
+            continue;
+        }
+        auto authorized = scope.authorized_field_ids.find(field.Name());
+        if (authorized == scope.authorized_field_ids.end()) {
+            return Status::Invalid(
+                fmt::format("Column '{}' is read, but the scan did not ask the 
catalog to "
+                            "authorize it: name it in 
ScanContextBuilder::SetReadFieldNames().",
+                            field.Name()));
+        }
+        if (authorized->second != field.Id()) {
+            return Status::Invalid(fmt::format(
+                "Column '{}' is read as field {}, but the scan authorized 
field {} under that "
+                "name: the schema changed since the plan. Plan the scan 
again.",
+                field.Name(), field.Id(), authorized->second));
+        }
+    }
+    return Status::OK();
+}
+
+Result<std::unique_ptr<BatchReader>> QueryAuthTableRead::CreateReader(
+    const std::shared_ptr<Split>& split) {
+    auto auth_split = std::dynamic_pointer_cast<QueryAuthSplit>(split);
+    if (auth_split == nullptr || auth_split->GetAuthResult() == nullptr) {
+        return Status::Invalid(fmt::format(
+            "A split of a table with '{}=true' must be planned by a scan of 
the table through its "
+            "catalog, which attaches the row filters and column masks it is 
read with.",
+            Options::QUERY_AUTH_ENABLED));
+    }
+    const TableQueryAuthResult& rules = *auth_split->GetAuthResult();
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Predicate> filter, 
PostMaskFilter(rules));
+    std::set<std::string> filter_fields;
+    if (filter != nullptr) {
+        PAIMON_RETURN_NOT_OK(PredicateUtils::GetAllNames(filter, 
&filter_fields));
+    }
+    Status valid = ValidateSplit(*auth_split, filter_fields);
+    if (!valid.ok()) {
+        return valid.WithMessage(fmt::format("Query auth read of {} (schema 
{}): {}",
+                                             context_->GetPath(), 
table_schema_->Id(),
+                                             valid.message()));
+    }
+
+    if (rules.RejectsAllRows()) {
+        return 
std::unique_ptr<BatchReader>(std::make_unique<ConcatBatchReader>(
+            std::vector<std::unique_ptr<BatchReader>>(), arrow_pool_));
+    }
+
+    // The columns read only for the rules or for the predicate, dropped from 
the output.
+    std::vector<std::string> extra_fields;
+    if (output_fields_) {
+        std::set<std::string> output(output_fields_->begin(), 
output_fields_->end());
+        std::vector<std::string> visible = output_fields_.value();
+        for (const std::string& field : filter_fields) {
+            if (output.count(field) == 0) {
+                visible.push_back(field);
+            }
+        }
+        std::set<std::string> required = rules.RequiredFields(visible);
+        required.insert(filter_fields.begin(), filter_fields.end());
+        for (const DataField& field : table_schema_->Fields()) {
+            if (required.count(field.Name()) > 0 && output.count(field.Name()) 
== 0) {
+                extra_fields.push_back(field.Name());
+            }
+        }
+    }
+    PAIMON_ASSIGN_OR_RAISE(TableRead * inner, 
GetOrCreateInnerRead(extra_fields));
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
+                           inner->CreateReader(auth_split->GetSplit()));
+    // Wrapping it can fail, and `PredicateBatchReader::Create()` keeps the 
reader it refuses.
+    ScopeGuard close_guard([&reader]() {
+        if (reader) {
+            reader->Close();
+        }
+    });
+
+    // The row filter sees the raw values, the predicate of the read the 
masked ones. Each reads
+    // its columns decoded, which `PredicateBatchReader` requires.
+    const std::map<std::string, std::shared_ptr<MaskTransform>> no_masks;
+    if (rules.RowFilter() != nullptr) {
+        reader = std::make_unique<QueryAuthBatchReader>(std::move(reader), 
no_masks,
+                                                        
/*dropped_fields=*/std::set<std::string>(),
+                                                        
rules.RowFilterFields(), arrow_pool_);
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr<PredicateBatchReader> filtered,
+            PredicateBatchReader::Create(std::move(reader), rules.RowFilter(), 
arrow_pool_));
+        reader = std::move(filtered);

Review Comment:
   Fixed in 46778d17. With a row filter, the `PredicateBatchReader` is now 
wrapped in `RowFilterBoundaryReader`, which serves `NextBatchWithBitmap()` from 
the compacted `NextBatch()`. Rows the filter drops are removed from the Arrow 
array itself, not only cleared in the bitmap. 
`QueryAuthTableReadTest.TestRowFilterRemovesRowsFromTheBatchWithBitmap` reads a 
batch with tenants A, B and A, calls `NextBatchWithBitmap()` through both 
`CreateReader()` overloads, and checks that the returned array holds only A's 
rows.



##########
src/paimon/core/table/system/system_table.cpp:
##########
@@ -174,6 +176,23 @@ std::optional<SystemTableFactory> 
FindSystemTableFactory(const std::string& syst
     return std::nullopt;
 }
 
+// The system tables that read the rows or the file statistics of a query-auth 
table are refused
+// as soon as they are loaded: `$files` and `$binlog` as in Java, and 
`$audit_log` and `$ro`, whose
+// reads would not apply the rules. The other system tables report only 
metadata.
+Status ValidateQueryAuthSystemTable(const std::string& system_table_name,
+                                    const std::shared_ptr<TableSchema>& 
table_schema,
+                                    const std::map<std::string, std::string>& 
dynamic_options) {
+    std::string normalized_name = StringUtils::ToLowerCase(system_table_name);
+    if (table_schema == nullptr || (normalized_name != FilesSystemTable::kName 
&&

Review Comment:
   Fixed in 46778d17. `$manifests` is now refused on a query-auth table, like 
`$files`, `$binlog`, `$audit_log` and `$ro`, because `min_partition_stats` and 
`max_partition_stats` hold raw partition values. For the same reason, the 
global `sys.partitions` table now leaves out query-auth tables, since its 
`partition_name` exposes the same values. The remaining per-table system tables 
(`$options`, `$snapshots`, `$schemas`, `$tags`, `$branches`, `$consumers`) hold 
no partition or column values. 
`TableTest.TestQueryAuthTableRefusesWhatCannotApplyItsRules` and its 
system-table sibling now include `manifests`. 
`SystemTableReadInteTest.TestReadGlobalPartitions` adds a query-auth table with 
a `secret` partition and checks that only the other table's partition is listed.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to