This is an automated email from the ASF dual-hosted git repository. HappenLee pushed a commit to branch paimon-blob-4.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit e49d3a547e5c075a384a3f9e343bdb176b19e7da Author: linzhenqi <[email protected]> AuthorDate: Thu Oct 22 02:30:27 2026 +0800 [improvement](function) Rename LIST_FILE to LIST_FILES and return directories LIST_FILES returns path, size and nullable FILE metadata. Non-recursive listing emits immediate directories with zero size and NULL FILE values. Recursive listing, now the default, emits files and skips directory markers. Carry native S3 and Azure hierarchy prefixes through the paginated listing. Use LIST_FILES in SQL. The result includes a BIGINT size column and nullable file column. The recursive property defaults to true; false includes direct subdirectories. Ordinary zero-byte files are returned in both modes. - Test: Full BE/FE build; ListFilesRowsTest with coverage; real MinIO P0 regression including 1005-entry directory and file listings; three docs SQL examples; clang-format 16 and build hygiene checks. - Behavior changed: Yes, function name, result schema and directory listing. - Does this need documentation: Yes, current and 4.x English/Chinese pages updated in the local doris-website repository. (cherry picked from commit e2196f37952c8dd168d5b728b2aaddb31000199a) --- be/src/core/value/file_value.cpp | 2 +- .../common/data_gen_functions/vlist_file_tvf.cpp | 32 +- be/src/io/fs/azure_obj_storage_client.cpp | 4 +- be/src/io/fs/obj_storage_client.h | 1 + be/src/io/fs/s3_obj_storage_client.cpp | 4 + .../data_gen_functions/vlist_file_tvf_test.cpp | 7 +- .../data_gen_functions/vlist_files_rows_test.cpp | 102 +++++ be/test/io/fs/azure_obj_storage_client_test.cpp | 17 +- .../doris/catalog/BuiltinTableValuedFunctions.java | 4 +- .../table/{ListFile.java => ListFiles.java} | 20 +- .../visitor/TableValuedFunctionVisitor.java | 6 +- ...tion.java => ListFilesTableValuedFunction.java} | 19 +- .../doris/tablefunction/TableValuedFunctionIf.java | 4 +- ....java => ListFilesTableValuedFunctionTest.java} | 35 +- .../test_list_file_resource_local.out | 165 -------- .../datatype_p0/complex_types/test_list_files.out | 88 ++++ .../test_list_file_resource_local.groovy | 447 --------------------- .../complex_types/test_list_files.groovy | 101 +++++ 18 files changed, 384 insertions(+), 674 deletions(-) diff --git a/be/src/core/value/file_value.cpp b/be/src/core/value/file_value.cpp index 91951205691..e4b59824308 100644 --- a/be/src/core/value/file_value.cpp +++ b/be/src/core/value/file_value.cpp @@ -208,7 +208,7 @@ bool is_valid_file_content_type(std::string_view content_type) { } std::string_view infer_file_content_type_from_name(std::string_view name) { - // Shared by TO_FILE and list_file. Keep inference independent of URI rewriting. + // Shared by TO_FILE and list_files. Keep inference independent of URI rewriting. static constexpr std::pair<std::string_view, std::string_view> TYPES[] = { {".csv", "text/csv"}, {".tsv", "text/tab-separated-values"}, diff --git a/be/src/exec/common/data_gen_functions/vlist_file_tvf.cpp b/be/src/exec/common/data_gen_functions/vlist_file_tvf.cpp index ebb2e9d7b9e..21a0d992344 100644 --- a/be/src/exec/common/data_gen_functions/vlist_file_tvf.cpp +++ b/be/src/exec/common/data_gen_functions/vlist_file_tvf.cpp @@ -49,7 +49,7 @@ Status VListFileTVF::set_scan_ranges(const std::vector<TScanRangeParams>& scan_r _path.prefix += '/'; } _path.delimiter = params.recursive ? "" : "/"; - static constexpr std::array<std::string_view, 2> NAMES {"path", "file"}; + static constexpr std::array<std::string_view, 3> NAMES {"path", "size", "file"}; _column_indices.clear(); for (const auto* slot : _tuple_desc->slots()) { const auto* const field = std::ranges::find(NAMES, slot->col_name()); @@ -81,6 +81,9 @@ Status VListFileTVF::_fetch_next_page(RuntimeState* state) { return {page.resp.status.code, std::move(page.resp.status.msg)}; } _objects = std::move(page.objects); + for (auto& prefix : page.common_prefixes) { + _objects.emplace_back(ObjectMeta {.key = std::move(prefix)}); + } _next_index = 0; _continuation_token = std::move(page.continuation_token); _has_more = page.has_more; @@ -117,19 +120,26 @@ Status VListFileTVF::get_next(RuntimeState* state, Block* block, bool* eos) { } RETURN_IF_CANCELLED(state); const auto& object = _objects[_next_index++]; - if (object.key.ends_with('/')) { + const bool is_directory = object.key.ends_with('/'); + // The prefix marker names the listed directory itself; recursive scans omit all markers. + if (is_directory && (_path.delimiter.empty() || object.key == _path.prefix)) { continue; } - File value(DataTypeFile::FIELD_COUNT); - value[0] = Field::create_field<TYPE_STRING>("s3://" + _path.bucket + "/" + object.key); - value[2] = Field::create_field<TYPE_BIGINT>(object.size); - value[3] = Field::create_field<TYPE_STRING>( - std::string(infer_file_content_type_from_name(object.key))); - if (auto checksum = file_checksum_from_etag(object.etag)) { - value[4] = Field::create_field<TYPE_STRING>(std::move(*checksum)); + std::array<Field, 3> fields { + Field::create_field<TYPE_STRING>("s3://" + _path.bucket + "/" + object.key), + Field::create_field<TYPE_BIGINT>(is_directory ? 0 : object.size), Field()}; + if (!is_directory) { + File value(DataTypeFile::FIELD_COUNT); + value[0] = fields[0]; + value[2] = fields[1]; + value[3] = Field::create_field<TYPE_STRING>( + std::string(infer_file_content_type_from_name(object.key))); + if (auto checksum = file_checksum_from_etag(object.etag)) { + value[4] = Field::create_field<TYPE_STRING>(std::move(*checksum)); + } + RETURN_IF_ERROR(validate_file(value)); + fields[2] = Field::create_field<TYPE_FILE>(std::move(value)); } - RETURN_IF_ERROR(validate_file(value)); - std::array<Field, 2> fields {value[0], Field::create_field<TYPE_FILE>(std::move(value))}; for (size_t i = 0; i < columns.size(); ++i) { columns[i]->insert(fields[_column_indices[i]]); } diff --git a/be/src/io/fs/azure_obj_storage_client.cpp b/be/src/io/fs/azure_obj_storage_client.cpp index b955d0f36b8..2df2b2a6289 100644 --- a/be/src/io/fs/azure_obj_storage_client.cpp +++ b/be/src/io/fs/azure_obj_storage_client.cpp @@ -347,7 +347,9 @@ ObjectStorageListPageResponse AzureObjStorageClient::list_objects_page( if (opts.delimiter.empty()) { make_page(_client->ListBlobs(request)); } else { - make_page(_client->ListBlobsByHierarchy(opts.delimiter, request)); + auto response = _client->ListBlobsByHierarchy(opts.delimiter, request); + page.common_prefixes = std::move(response.BlobPrefixes); + make_page(std::move(response)); } }, opts, _tls_debug_context); diff --git a/be/src/io/fs/obj_storage_client.h b/be/src/io/fs/obj_storage_client.h index 686ac9c84f3..d7496bbd4b6 100644 --- a/be/src/io/fs/obj_storage_client.h +++ b/be/src/io/fs/obj_storage_client.h @@ -96,6 +96,7 @@ struct ObjectMeta { struct ObjectStorageListPageResponse { ObjectStorageResponse resp {}; std::vector<ObjectMeta> objects {}; + std::vector<std::string> common_prefixes {}; std::string continuation_token {}; bool has_more = false; }; diff --git a/be/src/io/fs/s3_obj_storage_client.cpp b/be/src/io/fs/s3_obj_storage_client.cpp index ea38f1998d5..4eddb523d7f 100644 --- a/be/src/io/fs/s3_obj_storage_client.cpp +++ b/be/src/io/fs/s3_obj_storage_client.cpp @@ -423,6 +423,10 @@ ObjectStorageListPageResponse S3ObjStorageClient::list_objects_page( .continuation_token = result.GetNextContinuationToken(), .has_more = result.GetIsTruncated(), }; + page.common_prefixes.reserve(result.GetCommonPrefixes().size()); + for (const auto& entry : result.GetCommonPrefixes()) { + page.common_prefixes.emplace_back(entry.GetPrefix()); + } page.objects.reserve(result.GetContents().size()); for (const auto& obj : result.GetContents()) { page.objects.push_back( diff --git a/be/test/exec/common/data_gen_functions/vlist_file_tvf_test.cpp b/be/test/exec/common/data_gen_functions/vlist_file_tvf_test.cpp index 3beb6907520..4d46b2318a6 100644 --- a/be/test/exec/common/data_gen_functions/vlist_file_tvf_test.cpp +++ b/be/test/exec/common/data_gen_functions/vlist_file_tvf_test.cpp @@ -29,6 +29,7 @@ #include "core/column/column_file.h" #include "core/data_type/data_type_file.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_string.h" #include "core/field.h" #include "exec/operator/datagen_operator.h" @@ -82,7 +83,7 @@ protected: void SetUp() override { descriptor = std::make_unique<MockRowDescriptor>( std::vector<DataTypePtr> {std::make_shared<DataTypeString>(), - std::make_shared<DataTypeFile>()}, + make_nullable(std::make_shared<DataTypeFile>())}, &pool); const std::array<std::string, 2> names {"path", "file"}; for (size_t i = 0; i < names.size(); ++i) { @@ -206,7 +207,7 @@ TEST_F(ListFileTVFTest, EmitsCompleteFilesAndSkipsDirectoryMarkers) { ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); ASSERT_EQ(block.rows(), 3); EXPECT_TRUE(eos); - EXPECT_FALSE(block.get_by_position(1).type->is_nullable()); + EXPECT_TRUE(block.get_by_position(1).type->is_nullable()); EXPECT_EQ((*block.get_by_position(0).column)[0].get<TYPE_STRING>(), "s3://bucket/dir/empty.txt"); const auto empty = row(block, 0); @@ -299,7 +300,7 @@ TEST_F(ListFileTVFTest, SplitsPagesIntoBatchesWithoutPrefetching) { } TEST_F(ListFileTVFTest, ContinuesPastEmptyAndMarkerOnlyIntermediatePages) { - ASSERT_TRUE(function->set_scan_ranges(ranges()).ok()); + ASSERT_TRUE(function->set_scan_ranges(ranges("s3://bucket/dir/", true)).ok()); testing::InSequence sequence; EXPECT_CALL(*s3, ListObjectsV2(testing::_)) .WillOnce(testing::Return(ListObjectsV2Outcome(page({}, "empty")))); diff --git a/be/test/exec/common/data_gen_functions/vlist_files_rows_test.cpp b/be/test/exec/common/data_gen_functions/vlist_files_rows_test.cpp new file mode 100644 index 00000000000..73eeec8d2e5 --- /dev/null +++ b/be/test/exec/common/data_gen_functions/vlist_files_rows_test.cpp @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <gen_cpp/Descriptors_types.h> +#include <gen_cpp/PlanNodes_types.h> +#include <gtest/gtest.h> + +#include <array> + +#include "core/data_type/data_type_file.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "exec/common/data_gen_functions/vlist_file_tvf.h" +#include "runtime/descriptors.h" +#include "runtime/runtime_state.h" + +namespace doris { + +TEST(ListFilesRowsTest, DirectoryNullsSizesAndProjectionsAcrossBatches) { + const std::array<std::string, 3> names {"path", "size", "file"}; + const std::array<DataTypePtr, 3> types {std::make_shared<DataTypeString>(), + std::make_shared<DataTypeInt64>(), + std::make_shared<DataTypeFile>()}; + const std::vector<std::vector<size_t>> projections {{0, 1, 2}, {0}, {1}, {2}, {2, 0}}; + for (bool recursive : {false, true}) { + for (const auto& projection : projections) { + TTupleDescriptor tuple_spec; + tuple_spec.__set_id(0); + TupleDescriptor tuple(tuple_spec, true); + for (const auto index : projection) { + TSlotDescriptor slot; + slot.__set_id(static_cast<int>(index)); + slot.__set_parent(0); + slot.__set_colName(names[index]); + slot.__set_slotType(types[index]->to_thrift()); + slot.__set_nullIndicatorBit(index == 2 ? 0 : -1); + tuple.add_slot(new SlotDescriptor(slot)); + } + VListFileTVF function(0, &tuple); + TTVFListFileScanRange params; + params.__set_uri("s3://bucket/a/"); + params.__set_recursive(recursive); + TScanRangeParams range; + range.scan_range.data_gen_scan_range.__set_list_file_params(params); + ASSERT_TRUE(function.set_scan_ranges({range}).ok()); + function._objects = {{.key = "a/"}, + {.key = "a/empty.txt"}, + {.key = "a/b/"}, + {.key = "a/c.png", .size = 3}, + {.key = "a/marker/", .size = 7}}; + function._has_more = false; + RuntimeState state; + state._query_options.__set_batch_size(1); + Block block; + bool eos = false; + size_t rows = 0; + while (!eos) { + ASSERT_TRUE(function.get_next(&state, &block, &eos).ok()); + for (size_t row = 0; row < block.rows(); ++row) { + const bool directory = !recursive && (rows == 1 || rows == 3); + const int64_t size = rows == (recursive ? 1U : 2U) ? 3 : 0; + for (size_t column = 0; column < projection.size(); ++column) { + const auto field = (*block.get_by_position(column).column)[row]; + switch (projection[column]) { + case 0: + EXPECT_EQ(field.get<TYPE_STRING>().ends_with('/'), directory); + break; + case 1: + EXPECT_EQ(field.get<TYPE_BIGINT>(), size); + break; + case 2: + EXPECT_EQ(field.is_null(), directory); + if (!directory) { + EXPECT_EQ(field.get<TYPE_FILE>()[2].get<TYPE_BIGINT>(), size); + } + break; + } + } + ++rows; + } + block.clear_column_data(); + } + EXPECT_EQ(rows, recursive ? 2 : 4); + } + } +} + +} // namespace doris diff --git a/be/test/io/fs/azure_obj_storage_client_test.cpp b/be/test/io/fs/azure_obj_storage_client_test.cpp index 2e5f94d9882..9f2fd1c8a5e 100644 --- a/be/test/io/fs/azure_obj_storage_client_test.cpp +++ b/be/test/io/fs/azure_obj_storage_client_test.cpp @@ -35,6 +35,7 @@ #include "core/block/block.h" #include "core/data_type/data_type_file.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_string.h" #include "core/field.h" #include "core/value/file_value.h" @@ -223,8 +224,9 @@ TEST_F(AzureFileMetadataTest, ListingEtagsReachFileChecksumsWithoutHeadRequests) for (const bool recursive : {false, true}) { auto transport = std::make_shared<AzureListTransport>(); ObjectPool pool; - MockRowDescriptor descriptor( - {std::make_shared<DataTypeString>(), std::make_shared<DataTypeFile>()}, &pool); + MockRowDescriptor descriptor({std::make_shared<DataTypeString>(), + make_nullable(std::make_shared<DataTypeFile>())}, + &pool); descriptor.tuple_desc_map[0]->slots()[0]->_col_name = "path"; descriptor.tuple_desc_map[0]->slots()[1]->_col_name = "file"; VListFileTVF listing(0, descriptor.tuple_desc_map[0]); @@ -242,14 +244,21 @@ TEST_F(AzureFileMetadataTest, ListingEtagsReachFileChecksumsWithoutHeadRequests) while (!eos) { ASSERT_TRUE(listing.get_next(&state, &block, &eos).ok()); for (size_t i = 0; i < block.rows(); ++i) { - const auto value = (*block.get_by_position(1).column)[i].get<TYPE_FILE>(); + const auto field = (*block.get_by_position(1).column)[i]; + if (field.is_null()) { + EXPECT_FALSE(recursive); + EXPECT_EQ((*block.get_by_position(0).column)[i].get<TYPE_STRING>(), + "s3://container/tree/sub/"); + continue; + } + const auto value = field.get<TYPE_FILE>(); ASSERT_FALSE(value[4].is_null()); EXPECT_EQ(value[4].get<TYPE_STRING>(), "ETAG:opaque-2"); } rows += block.rows(); block.clear_column_data(); } - EXPECT_EQ(rows, recursive ? 3 : 2); + EXPECT_EQ(rows, 3); EXPECT_EQ(transport->requests, 2); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java index ed124fb4fa5..4914ba4869f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java @@ -31,7 +31,7 @@ import org.apache.doris.nereids.trees.expressions.functions.table.HttpStream; import org.apache.doris.nereids.trees.expressions.functions.table.HudiMeta; import org.apache.doris.nereids.trees.expressions.functions.table.Jobs; import org.apache.doris.nereids.trees.expressions.functions.table.LanceIndexEntries; -import org.apache.doris.nereids.trees.expressions.functions.table.ListFile; +import org.apache.doris.nereids.trees.expressions.functions.table.ListFiles; import org.apache.doris.nereids.trees.expressions.functions.table.Local; import org.apache.doris.nereids.trees.expressions.functions.table.MvInfos; import org.apache.doris.nereids.trees.expressions.functions.table.Numbers; @@ -65,7 +65,7 @@ public class BuiltinTableValuedFunctions implements FunctionHelper { tableValued(HudiMeta.class, "hudi_meta"), tableValued(Hdfs.class, "hdfs"), tableValued(HttpStream.class, "http_stream"), - tableValued(ListFile.class, "list_file"), + tableValued(ListFiles.class, "list_files"), tableValued(Numbers.class, "numbers"), tableValued(S3.class, "s3"), tableValued(MvInfos.class, "mv_infos"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/ListFile.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/ListFiles.java similarity index 78% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/ListFile.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/ListFiles.java index 66ce307a8cf..6dcd00ff755 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/ListFile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/ListFiles.java @@ -23,17 +23,17 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Properties; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.FileType; -import org.apache.doris.tablefunction.ListFileTableValuedFunction; +import org.apache.doris.tablefunction.ListFilesTableValuedFunction; import org.apache.doris.tablefunction.TableValuedFunctionIf; import com.google.common.base.Preconditions; import java.util.List; -/** list_file */ -public class ListFile extends TableValuedFunction { - public ListFile(Properties properties) { - super(ListFileTableValuedFunction.NAME, properties); +/** list_files */ +public class ListFiles extends TableValuedFunction { + public ListFiles(Properties properties) { + super(ListFilesTableValuedFunction.NAME, properties); } @Override @@ -44,20 +44,20 @@ public class ListFile extends TableValuedFunction { @Override protected TableValuedFunctionIf toCatalogFunction() { try { - return new ListFileTableValuedFunction(getTVFProperties().getMap()); + return new ListFilesTableValuedFunction(getTVFProperties().getMap()); } catch (org.apache.doris.common.AnalysisException e) { - throw new AnalysisException("Can not build list_file(): " + e.getMessage(), e); + throw new AnalysisException("Can not build list_files(): " + e.getMessage(), e); } } @Override public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) { - return visitor.visitListFile(this, context); + return visitor.visitListFiles(this, context); } @Override - public ListFile withChildren(List<Expression> children) { + public ListFiles withChildren(List<Expression> children) { Preconditions.checkArgument(children.size() == 1 && children.get(0) instanceof Properties); - return new ListFile((Properties) children.get(0)); + return new ListFiles((Properties) children.get(0)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java index 3ad4e7c901b..43953eb6f3f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java @@ -29,7 +29,7 @@ import org.apache.doris.nereids.trees.expressions.functions.table.Http; import org.apache.doris.nereids.trees.expressions.functions.table.HttpStream; import org.apache.doris.nereids.trees.expressions.functions.table.HudiMeta; import org.apache.doris.nereids.trees.expressions.functions.table.Jobs; -import org.apache.doris.nereids.trees.expressions.functions.table.ListFile; +import org.apache.doris.nereids.trees.expressions.functions.table.ListFiles; import org.apache.doris.nereids.trees.expressions.functions.table.Local; import org.apache.doris.nereids.trees.expressions.functions.table.MvInfos; import org.apache.doris.nereids.trees.expressions.functions.table.Numbers; @@ -113,8 +113,8 @@ public interface TableValuedFunctionVisitor<R, C> { return visitTableValuedFunction(local, context); } - default R visitListFile(ListFile listFile, C context) { - return visitTableValuedFunction(listFile, context); + default R visitListFiles(ListFiles listFiles, C context) { + return visitTableValuedFunction(listFiles, context); } default R visitNumbers(Numbers numbers, C context) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ListFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ListFilesTableValuedFunction.java similarity index 87% rename from fe/fe-core/src/main/java/org/apache/doris/tablefunction/ListFileTableValuedFunction.java rename to fe/fe-core/src/main/java/org/apache/doris/tablefunction/ListFilesTableValuedFunction.java index ded4516d183..952d8c2174c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ListFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ListFilesTableValuedFunction.java @@ -44,15 +44,15 @@ import java.util.Locale; import java.util.Map; /** Resource-backed S3 directory listing, executed by one backend. */ -public class ListFileTableValuedFunction extends DataGenTableValuedFunction { - public static final String NAME = "list_file"; +public class ListFilesTableValuedFunction extends DataGenTableValuedFunction { + public static final String NAME = "list_files"; private static final ImmutableSet<String> PROPERTIES = ImmutableSet.of("resource", "uri", "recursive"); private final String resourceName; private final String uri; private final boolean recursive; - public ListFileTableValuedFunction(Map<String, String> params) throws AnalysisException { + public ListFilesTableValuedFunction(Map<String, String> params) throws AnalysisException { Map<String, String> properties = new HashMap<>(); for (Map.Entry<String, String> entry : params.entrySet()) { String key = entry.getKey().toLowerCase(Locale.ROOT); @@ -64,14 +64,14 @@ public class ListFileTableValuedFunction extends DataGenTableValuedFunction { resourceName = properties.get("resource"); uri = properties.get("uri"); if (resourceName == null || resourceName.isEmpty()) { - throw new AnalysisException("list_file resource is required"); + throw new AnalysisException("list_files resource is required"); } if (uri == null) { - throw new AnalysisException("list_file uri is required"); + throw new AnalysisException("list_files uri is required"); } - String recursiveValue = properties.getOrDefault("recursive", "false"); + String recursiveValue = properties.getOrDefault("recursive", "true"); if (!"true".equalsIgnoreCase(recursiveValue) && !"false".equalsIgnoreCase(recursiveValue)) { - throw new AnalysisException("list_file recursive must be true or false"); + throw new AnalysisException("list_files recursive must be true or false"); } recursive = Boolean.parseBoolean(recursiveValue); checkS3Resource(ConnectContext.get()); @@ -93,14 +93,15 @@ public class ListFileTableValuedFunction extends DataGenTableValuedFunction { @Override public String getTableName() { - return "ListFileTableValuedFunction"; + return "ListFilesTableValuedFunction"; } @Override public List<Column> getTableColumns() { return ImmutableList.of( new Column("path", Type.STRING, false), - new Column("file", FileType.create(), false)); + new Column("size", Type.BIGINT, false), + new Column("file", FileType.create(), true)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java index f9d81c04a5d..5f295ece185 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java @@ -47,8 +47,8 @@ public abstract class TableValuedFunctionIf { public static TableValuedFunctionIf getTableFunction(String funcName, Map<String, String> params) throws AnalysisException { switch (funcName.toLowerCase()) { - case ListFileTableValuedFunction.NAME: - return new ListFileTableValuedFunction(params); + case ListFilesTableValuedFunction.NAME: + return new ListFilesTableValuedFunction(params); case NumbersTableValuedFunction.NAME: return new NumbersTableValuedFunction(params); case S3TableValuedFunction.NAME: diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ListFileTableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ListFilesTableValuedFunctionTest.java similarity index 88% rename from fe/fe-core/src/test/java/org/apache/doris/tablefunction/ListFileTableValuedFunctionTest.java rename to fe/fe-core/src/test/java/org/apache/doris/tablefunction/ListFilesTableValuedFunctionTest.java index 7dcdc19a8ab..0036741ad6c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ListFileTableValuedFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ListFilesTableValuedFunctionTest.java @@ -27,7 +27,7 @@ import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.mysql.privilege.AccessControllerManager; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.trees.expressions.Properties; -import org.apache.doris.nereids.trees.expressions.functions.table.ListFile; +import org.apache.doris.nereids.trees.expressions.functions.table.ListFiles; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.system.Backend; @@ -45,15 +45,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -class ListFileTableValuedFunctionTest { +class ListFilesTableValuedFunctionTest { @Test - void exposesPathSizeNullableTimestampAndFileColumnsWithoutRemoteIo() throws Exception { + void exposesPathSizeAndNullableFileColumnsWithoutRemoteIo() throws Exception { try (Fixture fixture = new Fixture()) { - TableValuedFunctionIf function = TableValuedFunctionIf.getTableFunction("LIST_FILE", fixture.params()); + TableValuedFunctionIf function = TableValuedFunctionIf.getTableFunction("LIST_FILES", fixture.params()); List<Column> columns = function.getTableColumns(); assertSchema(columns); Assertions.assertInstanceOf(DataGenTableValuedFunction.class, function); - ListFile nereids = new ListFile(new Properties(fixture.params())); + ListFiles nereids = new ListFiles(new Properties(fixture.params())); assertSchema(nereids.getTableColumns()); } } @@ -73,10 +73,10 @@ class ListFileTableValuedFunctionTest { @Test void acceptsCaseInsensitiveKeysAndStrictBooleanValues() throws Exception { try (Fixture fixture = new Fixture()) { - ListFileTableValuedFunction function = new ListFileTableValuedFunction( + ListFilesTableValuedFunction function = new ListFilesTableValuedFunction( ImmutableMap.of("RESOURCE", "s3_resource", "URI", "s3://bucket/", "RECURSIVE", "TRUE")); Assertions.assertTrue(range(function).isRecursive()); - Assertions.assertFalse(range(new ListFileTableValuedFunction(fixture.params())).isRecursive()); + Assertions.assertTrue(range(new ListFilesTableValuedFunction(fixture.params())).isRecursive()); for (String value : new String[] {"yes", "1", "", " false "}) { Map<String, String> params = fixture.params(); params.put("recursive", value); @@ -91,7 +91,7 @@ class ListFileTableValuedFunctionTest { for (String uri : new String[] {"s3://bucket", "s3://bucket/", "s3://bucket/dir", "s3://bucket/dir/"}) { Map<String, String> params = fixture.params(); params.put("uri", uri); - Assertions.assertEquals(uri, range(new ListFileTableValuedFunction(params)).getUri()); + Assertions.assertEquals(uri, range(new ListFilesTableValuedFunction(params)).getUri()); } } } @@ -103,7 +103,7 @@ class ListFileTableValuedFunctionTest { "s3://another-bucket/a b+中文%ZZ#fragment"}) { Map<String, String> params = fixture.params(); params.put("uri", uri); - Assertions.assertEquals(uri, range(new ListFileTableValuedFunction(params)).getUri()); + Assertions.assertEquals(uri, range(new ListFilesTableValuedFunction(params)).getUri()); } } } @@ -125,7 +125,7 @@ class ListFileTableValuedFunctionTest { void queryConfigurationIsSharedWithToFileAndRefreshesNextExecute() throws Exception { try (Fixture fixture = new Fixture()) { fixture.properties.put("s3.secret_key", "prepare"); - ListFileTableValuedFunction function = new ListFileTableValuedFunction(fixture.params()); + ListFilesTableValuedFunction function = new ListFilesTableValuedFunction(fixture.params()); fixture.properties.put("s3.secret_key", "first execute"); fixture.context.getOrRegisterS3Resource("s3_resource"); fixture.properties.put("s3.secret_key", "second execute"); @@ -149,7 +149,7 @@ class ListFileTableValuedFunctionTest { Backend dead = Mockito.mock(Backend.class); Mockito.when(fixture.system.getBackendsByCurrentCluster()) .thenReturn(ImmutableMap.of(1L, dead, 2L, fixture.backend)); - ListFileTableValuedFunction function = new ListFileTableValuedFunction(fixture.params()); + ListFilesTableValuedFunction function = new ListFilesTableValuedFunction(fixture.params()); List<TableValuedFunctionTask> tasks = function.getTasks(); Assertions.assertEquals(1, tasks.size()); Assertions.assertSame(fixture.backend, tasks.get(0).getBackend()); @@ -161,22 +161,25 @@ class ListFileTableValuedFunctionTest { } private static void assertSchema(List<Column> columns) { - Assertions.assertEquals(2, columns.size()); + Assertions.assertEquals(3, columns.size()); Assertions.assertEquals("path", columns.get(0).getName()); Assertions.assertEquals(Type.STRING, columns.get(0).getType()); Assertions.assertFalse(columns.get(0).isAllowNull()); - Assertions.assertEquals("file", columns.get(1).getName()); - Assertions.assertTrue(columns.get(1).getType().isFileType()); + Assertions.assertEquals("size", columns.get(1).getName()); + Assertions.assertEquals(Type.BIGINT, columns.get(1).getType()); Assertions.assertFalse(columns.get(1).isAllowNull()); + Assertions.assertEquals("file", columns.get(2).getName()); + Assertions.assertTrue(columns.get(2).getType().isFileType()); + Assertions.assertTrue(columns.get(2).isAllowNull()); } - private static TTVFListFileScanRange range(ListFileTableValuedFunction function) throws Exception { + private static TTVFListFileScanRange range(ListFilesTableValuedFunction function) throws Exception { return function.getTasks().get(0).getExecParams().getDataGenScanRange().getListFileParams(); } private static void assertInvalid(Map<String, String> params, String message) { Exception error = Assertions.assertThrows(Exception.class, - () -> TableValuedFunctionIf.getTableFunction("list_file", params)); + () -> TableValuedFunctionIf.getTableFunction("list_files", params)); Assertions.assertTrue(error.getMessage().contains(message), error.getMessage()); } diff --git a/regression-test/data/datatype_p0/complex_types/test_list_file_resource_local.out b/regression-test/data/datatype_p0/complex_types/test_list_file_resource_local.out deleted file mode 100644 index 52112823787..00000000000 --- a/regression-test/data/datatype_p0/complex_types/test_list_file_resource_local.out +++ /dev/null @@ -1,165 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !creation_no_io -- -0 - --- !schema -- -path text No false \N NONE -file FILE No false \N NONE - --- !schema_no_io -- -0 - --- !default -- -s3://list-file-local/dir/a-empty.txt {"uri":"s3://list-file-local/dir/a-empty.txt","offset":null,"size":0,"content_type":"text/plain","checksum":null,"inline":null} -s3://list-file-local/dir/b.json {"uri":"s3://list-file-local/dir/b.json","offset":null,"size":7,"content_type":"application/json","checksum":"ETAG:list-7-2","inline":null} -s3://list-file-local/dir/c.PNG {"uri":"s3://list-file-local/dir/c.PNG","offset":null,"size":11,"content_type":"image/png","checksum":"ETAG:list-11-2","inline":null} -s3://list-file-local/dir/no_extension {"uri":"s3://list-file-local/dir/no_extension","offset":null,"size":13,"content_type":"application/octet-stream","checksum":null,"inline":null} - --- !default_protocol -- -4 true true - --- !listing_checksums -- -s3://list-file-local/dir/a-empty.txt \N -s3://list-file-local/dir/b.json ETAG:list-7-2 -s3://list-file-local/dir/c.PNG ETAG:list-11-2 -s3://list-file-local/dir/no_extension \N - --- !explicit_false -- -s3://list-file-local/dir/a-empty.txt {"uri":"s3://list-file-local/dir/a-empty.txt","offset":null,"size":0,"content_type":"text/plain","checksum":null,"inline":null} -s3://list-file-local/dir/b.json {"uri":"s3://list-file-local/dir/b.json","offset":null,"size":7,"content_type":"application/json","checksum":"ETAG:list-7-2","inline":null} -s3://list-file-local/dir/c.PNG {"uri":"s3://list-file-local/dir/c.PNG","offset":null,"size":11,"content_type":"image/png","checksum":"ETAG:list-11-2","inline":null} -s3://list-file-local/dir/no_extension {"uri":"s3://list-file-local/dir/no_extension","offset":null,"size":13,"content_type":"application/octet-stream","checksum":null,"inline":null} - --- !path_only -- -s3://list-file-local/dir/a-empty.txt -s3://list-file-local/dir/b.json -s3://list-file-local/dir/c.PNG -s3://list-file-local/dir/no_extension - --- !size_only -- -0 -7 -11 -13 - --- !file_only -- -{"uri":"s3://list-file-local/dir/a-empty.txt","offset":null,"size":0,"content_type":"text/plain","checksum":null,"inline":null} -{"uri":"s3://list-file-local/dir/b.json","offset":null,"size":7,"content_type":"application/json","checksum":"ETAG:list-7-2","inline":null} -{"uri":"s3://list-file-local/dir/c.PNG","offset":null,"size":11,"content_type":"image/png","checksum":"ETAG:list-11-2","inline":null} -{"uri":"s3://list-file-local/dir/no_extension","offset":null,"size":13,"content_type":"application/octet-stream","checksum":null,"inline":null} - --- !recursive_getters -- -s3://list-file-local/dir/a-empty.txt 0 true \N text/plain \N false -s3://list-file-local/dir/b.json 7 true \N application/json ETAG:list-7-2 false -s3://list-file-local/dir/c.PNG 11 true \N image/png ETAG:list-11-2 false -s3://list-file-local/dir/no_extension 13 true \N application/octet-stream \N false -s3://list-file-local/dir/sub/deep/payload.parquet 19 true \N application/x-parquet ETAG:list-19-2 false -s3://list-file-local/dir/sub/nested.csv 17 true \N text/csv ETAG:list-17-2 false - --- !recursive_protocol -- -6 true true - --- !root -- -s3://list-file-local/root.txt 23 - --- !root_slash -- -s3://list-file-local/root.txt 23 - --- !empty_directory -- - --- !count -- -6 6 67 - --- !parallel_count -- -6 6 67 - --- !parallel_requests -- -6 - --- !parallel_count_repeat -- -6 6 67 - --- !parallel_requests_repeat -- -6 - --- !parallel_paths -- -s3://list-file-local/dir/a-empty.txt -s3://list-file-local/dir/b.json -s3://list-file-local/dir/c.PNG -s3://list-file-local/dir/no_extension -s3://list-file-local/dir/sub/deep/payload.parquet -s3://list-file-local/dir/sub/nested.csv - --- !limit -- -s3://list-file-local/dir/a-empty.txt {"uri":"s3://list-file-local/dir/a-empty.txt","offset":null,"size":0,"content_type":"text/plain","checksum":null,"inline":null} - --- !limit_requests -- -1 - --- !limit_zero -- - --- !limit_zero_requests -- -0 - --- !canonical_list -- -s3://list-file-local/dir/a-empty.txt {"uri":"s3://list-file-local/dir/a-empty.txt","offset":null,"size":0,"content_type":"text/plain","checksum":null,"inline":null} -s3://list-file-local/dir/b.json {"uri":"s3://list-file-local/dir/b.json","offset":null,"size":7,"content_type":"application/json","checksum":"ETAG:list-7-2","inline":null} -s3://list-file-local/dir/c.PNG {"uri":"s3://list-file-local/dir/c.PNG","offset":null,"size":11,"content_type":"image/png","checksum":"ETAG:list-11-2","inline":null} -s3://list-file-local/dir/no_extension {"uri":"s3://list-file-local/dir/no_extension","offset":null,"size":13,"content_type":"application/octet-stream","checksum":null,"inline":null} - --- !canonical_list_protocol -- -true - --- !canonical_to_file_head -- -{"uri":"s3://list-file-local/dir/b.json","offset":null,"size":7,"content_type":"application/json","checksum":"ETAG:canonical-head-etag","inline":null} - --- !canonical_head_protocol -- -1 true - --- !canonical_mixed_buckets -- -1 {"uri":"s3://list-file-local/dir/b.json","offset":null,"size":7,"content_type":"application/json","checksum":"ETAG:canonical-head-etag","inline":null} -2 {"uri":"s3://list-file-other/dir/b.json","offset":null,"size":31,"content_type":"application/json","checksum":"ETAG:other-bucket-head-etag","inline":null} -3 {"uri":"s3://list-file-local/dir/b.json","offset":null,"size":7,"content_type":"application/json","checksum":"ETAG:canonical-head-etag","inline":null} - --- !list_uri_bucket -- -s3://other-bucket/dir/a-empty.txt 0 -s3://other-bucket/dir/b.json 7 -s3://other-bucket/dir/c.PNG 11 -s3://other-bucket/dir/no_extension 13 - --- !invalid_no_io -- -0 - --- !denied_no_io -- -0 - --- !granted_count -- -4 - --- !raw_uri_roundtrip -- -s3://list-file-local/roundtrip/a b.txt 101 s3://list-file-local/roundtrip/a b.txt 101 -s3://list-file-local/roundtrip/a%20b.txt 102 s3://list-file-local/roundtrip/a%20b.txt 102 -s3://list-file-local/roundtrip/a%2Fb.txt 105 s3://list-file-local/roundtrip/a%2Fb.txt 105 -s3://list-file-local/roundtrip/a+b.txt 103 s3://list-file-local/roundtrip/a+b.txt 103 -s3://list-file-local/roundtrip/a/b.txt 106 s3://list-file-local/roundtrip/a/b.txt 106 -s3://list-file-local/roundtrip/中文.txt 104 s3://list-file-local/roundtrip/中文.txt 104 - --- !raw_uri_request_protocol -- -6 true true - --- !literal_plus -- -{"uri":"s3://list-file-local/roundtrip/a+b.txt","offset":null,"size":103,"content_type":"application/json","checksum":"ETAG:roundtrip-103","inline":null} - --- !listing_head_checksum_consistency -- -s3://list-file-local/etag-consistency/absent.txt \N \N true -s3://list-file-local/etag-consistency/bare-multipart.txt ETAG:MiXeD:opaque-2 ETAG:MiXeD:opaque-2 true -s3://list-file-local/etag-consistency/empty.txt \N \N true -s3://list-file-local/etag-consistency/quoted-empty.txt \N \N true -s3://list-file-local/etag-consistency/quoted-multipart.txt ETAG:MiXeD:opaque-2 ETAG:MiXeD:opaque-2 true - --- !listing_head_checksum_protocol -- -5 true true - --- !handler_errors -- -0 - diff --git a/regression-test/data/datatype_p0/complex_types/test_list_files.out b/regression-test/data/datatype_p0/complex_types/test_list_files.out new file mode 100644 index 00000000000..2ca60063c11 --- /dev/null +++ b/regression-test/data/datatype_p0/complex_types/test_list_files.out @@ -0,0 +1,88 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !schema -- +path text No false \N NONE +size bigint No false \N NONE +file FILE Yes false \N NONE + +-- !direct -- +a.json 2 false 2 application/json +empty.txt 0 false 0 text/plain +empty/ 0 true \N \N +marker/ 0 true \N \N +sub/ 0 true \N \N + +-- !direct_counts -- +5 2 2 + +-- !recursive -- +a.json 2 false 2 application/json +empty.txt 0 false 0 text/plain +sub/c.png 3 false 3 image/png +sub/deep/d.txt 4 false 4 text/plain + +-- !recursive_counts -- +4 4 9 + +-- !defaults -- +a.json 2 false 2 application/json +empty.txt 0 false 0 text/plain +sub/c.png 3 false 3 image/png +sub/deep/d.txt 4 false 4 text/plain + +-- !defaults_counts -- +4 4 9 + +-- !size_only -- +2 +0 +0 +0 +0 + +-- !path_only -- +a.json +empty.txt +empty/ +marker/ +sub/ + +-- !file_only -- +false +false +true +true +true + +-- !limit -- +2 + +-- !roundtrip -- +a.json true true +empty.txt true true +sub/c.png true true +sub/deep/d.txt true true + +-- !no_trailing_slash -- +5 2 2 + +-- !empty_direct -- +empty/deep/ 0 true + +-- !empty_recursive -- +0 + +-- !missing -- +0 + +-- !directory_pages -- +1005 0 0 1005 + +-- !directory_pages_recursive -- +0 + +-- !file_pages -- +1005 1005 1005 1005 + +-- !file_pages_direct -- +1005 1005 1005 + diff --git a/regression-test/suites/datatype_p0/complex_types/test_list_file_resource_local.groovy b/regression-test/suites/datatype_p0/complex_types/test_list_file_resource_local.groovy deleted file mode 100644 index 097c4ab1870..00000000000 --- a/regression-test/suites/datatype_p0/complex_types/test_list_file_resource_local.groovy +++ /dev/null @@ -1,447 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import com.sun.net.httpserver.HttpExchange -import com.sun.net.httpserver.HttpHandler -import com.sun.net.httpserver.HttpServer -import java.net.InetAddress -import java.net.InetSocketAddress -import java.net.NetworkInterface -import java.net.URI -import java.net.URLDecoder -import java.util.concurrent.CopyOnWriteArrayList - -// The fixture runs in the regression JVM and requires a same-host FE/BE cluster. -suite("test_list_file_resource_local") { - if (!context.config.otherConfigs.get("enableFileTypeLocalResource")?.toString()?.toBoolean()) { - logger.info("Skip test_list_file_resource_local: enableFileTypeLocalResource requires a same-host cluster") - return - } - def backends = sql_return_maparray("SHOW BACKENDS").findAll { it.Alive.toString().toBoolean() } - if (URI.create(context.config.jdbcUrl.substring(5)).host != "127.0.0.1" - || backends.isEmpty() || backends.any { - NetworkInterface.getByInetAddress(InetAddress.getByName(it.Host.toString())) == null - }) { - throw new IllegalStateException("enableFileTypeLocalResource requires JDBC on 127.0.0.1 and all live BEs on local network interfaces") - } - - def requests = new CopyOnWriteArrayList<Map>() - def handlerErrors = new CopyOnWriteArrayList<String>() - def objects = [ - "dir/": 0, "dir/a-empty.txt": 0, "dir/b.json": 7, "dir/c.PNG": 11, - "dir/no_extension": 13, "dir/sub/": 0, "dir/sub/nested.csv": 17, - "dir/sub/deep/": 0, "dir/sub/deep/payload.parquet": 19, - "dir2/": 0, "dir2/neighbor.txt": 21, "root.txt": 23 - ] - def heads = [ - "/list-file-local/dir/b.json": [size: objects["dir/b.json"], etag: '"canonical-head-etag"'], - "/list-file-other/dir/b.json": [size: 31, etag: '"other-bucket-head-etag"'] - ] - def listingEtags = ["dir/a-empty.txt": "", "dir/no_extension": null] - HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0) - server.createContext("/", { HttpExchange exchange -> - try { - def query = [:] - exchange.requestURI.rawQuery?.split("&")?.each { part -> - def pair = part.split("=", 2) - query[URLDecoder.decode(pair[0], "UTF-8")] = - pair.length == 2 ? URLDecoder.decode(pair[1], "UTF-8") : "" - } - requests.add([method: exchange.requestMethod, path: exchange.requestURI.rawPath, - decodedPath: exchange.requestURI.path, query: query]) - exchange.responseHeaders.set("x-amz-request-id", "list-file-local-request") - if (exchange.requestMethod == "HEAD" - && heads.containsKey(exchange.requestURI.path) && query.isEmpty()) { - // The same key in different buckets has distinct metadata; routing must follow the URI. - def metadata = heads[exchange.requestURI.path] - exchange.responseHeaders.set("Content-Length", metadata.size.toString()) - exchange.responseHeaders.set("Content-Type", "application/json") - if (metadata.etag != null) { - exchange.responseHeaders.set("ETag", metadata.etag) - } - exchange.sendResponseHeaders(200, -1) - return - } - if (exchange.requestMethod != "GET" - || !(exchange.requestURI.rawPath in ["/list-file-local", "/list-file-local/", "/other-bucket", "/other-bucket/"]) - || query["list-type"] != "2") { - handlerErrors.add("Unexpected request: ${exchange.requestMethod} ${exchange.requestURI}".toString()) - exchange.sendResponseHeaders(400, -1) - return - } - - def prefix = query["prefix"] ?: "" - def delimiter = query["delimiter"] ?: "" - def entries = [:] - objects.each { key, size -> - if (key.startsWith(prefix)) { - def relative = key.substring(prefix.length()) - int separator = delimiter ? relative.indexOf(delimiter) : -1 - if (separator >= 0) { - def commonPrefix = prefix + relative.substring(0, separator + delimiter.length()) - entries[commonPrefix] = [key: commonPrefix, common: true] - } else { - entries[key] = [key: key, size: size, common: false] - } - } - } - def ordered = entries.values().sort { a, b -> a.key <=> b.key } - def token = query["continuation-token"] ?: "page:0" - int offset = token.substring(token.indexOf(":") + 1).toInteger() - int pageSize = Math.min(2, (query["max-keys"] ?: "1000").toInteger()) - // An empty truncated page proves callers follow tokens rather than stopping on zero objects. - def page = token.startsWith("empty:") ? [] : ordered.drop(offset).take(pageSize) - int nextOffset = offset + page.size() - boolean truncated = nextOffset < ordered.size() - def nextToken = offset == 0 && !page.isEmpty() ? "empty:${nextOffset}" : "page:${nextOffset}" - def xml = new StringBuilder('<?xml version="1.0" encoding="UTF-8"?>') - xml.append('<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/";>') - xml.append("<Name>list-file-local</Name><Prefix>${prefix}</Prefix>") - xml.append("<MaxKeys>${pageSize}</MaxKeys><KeyCount>${page.size()}</KeyCount>") - xml.append("<IsTruncated>${truncated}</IsTruncated>") - page.each { entry -> - if (entry.common) { - xml.append("<CommonPrefixes><Prefix>${entry.key}</Prefix></CommonPrefixes>") - } else { - xml.append("<Contents><Key>${entry.key}</Key><Size>${entry.size}</Size>") - // Exercise present, empty and absent ETags through real AWS SDK XML parsing. - def etag = listingEtags.containsKey(entry.key) ? listingEtags[entry.key] - : '"list-' + entry.size + '-2"' - if (etag != null) { - xml.append("<ETag>${etag}</ETag>") - } - xml.append('<StorageClass>STANDARD</StorageClass></Contents>') - } - } - if (truncated) { - xml.append("<NextContinuationToken>${nextToken}</NextContinuationToken>") - } - xml.append('</ListBucketResult>') - byte[] body = xml.toString().getBytes("UTF-8") - exchange.responseHeaders.set("Content-Type", "application/xml") - exchange.sendResponseHeaders(200, body.length) - exchange.responseBody.write(body) - } catch (Exception e) { - handlerErrors.add(e.toString()) - } finally { - exchange.close() - } - } as HttpHandler) - server.start() - - try { - sql "DROP RESOURCE IF EXISTS 'list_file_local_s3'" - sql "DROP RESOURCE IF EXISTS 'list_file_local_canonical_s3'" - sql "DROP RESOURCE IF EXISTS 'list_file_local_hdfs'" - sql """ - CREATE RESOURCE 'list_file_local_s3' PROPERTIES( - "type"="s3", "AWS_ENDPOINT"="http://127.0.0.1:${server.address.port}";, - "AWS_REGION"="us-east-1", "AWS_BUCKET"="list-file-local", "AWS_ROOT_PATH"="unused-root", - "AWS_ACCESS_KEY"="fake-list-file-key", "AWS_SECRET_KEY"="fake-list-file-secret", - "AWS_REQUEST_TIMEOUT_MS"="2000", "AWS_CONNECTION_TIMEOUT_MS"="500", - "use_path_style"="true", "s3_validity_check"="false") - """ - sql """ - CREATE RESOURCE 'list_file_local_canonical_s3' PROPERTIES( - "type"="s3", "s3.endpoint"="http://127.0.0.1:${server.address.port}";, - "s3.region"="us-east-1", "s3.bucket"="list-file-local", "s3.root.path"="unused-root", - "s3.access_key"="fake-list-file-key", "s3.secret_key"="fake-list-file-secret", - "s3.connection.request.timeout"="2000", "s3.connection.timeout"="500", - "use_path_style"="true", "s3_validity_check"="false") - """ - sql """ - CREATE RESOURCE 'list_file_local_hdfs' PROPERTIES( - "type"="hdfs", "fs.defaultFS"="hdfs://127.0.0.1:1", - "hadoop.username"="list_file_test", "ipc.client.connect.timeout"="100", - "ipc.client.connect.max.retries"="0") - """ - qt_creation_no_io "SELECT ${requests.size()}" - qt_schema 'DESC FUNCTION list_file("resource"="list_file_local_s3", "uri"="s3://list-file-local/dir/")' - qt_schema_no_io "SELECT ${requests.size()}" - - qt_default """ - SELECT * FROM list_file("resource"="list_file_local_s3", "uri"="s3://list-file-local/dir/") - ORDER BY path - """ - qt_default_protocol """ - SELECT ${requests.size()}, ${requests.every { it.method == "GET" - && it.path in ["/list-file-local", "/list-file-local/", "/other-bucket", "/other-bucket/"] - && it.query["list-type"] == "2" && it.query["prefix"] == "dir/" - && it.query["delimiter"] == "/" && it.query["max-keys"].toInteger() > 0 }}, - ${requests.any { it.query["continuation-token"]?.startsWith("empty:") }} - """ - - requests.clear() - qt_listing_checksums """ - SELECT path, ELEMENT_AT(file, 'checksum') - FROM list_file("resource"="list_file_local_s3", "uri"="s3://list-file-local/dir/") - ORDER BY path - """ - assertTrue(!requests.isEmpty() && requests.every { it.method == "GET" }, - "Listing ETags must not require per-object HEAD requests") - - qt_explicit_false """ - SELECT * FROM list_file("resource"="list_file_local_s3", "uri"="s3://list-file-local/dir", - "recursive"="false") ORDER BY path - """ - qt_path_only """ - SELECT path FROM list_file("resource"="list_file_local_s3", "uri"="s3://list-file-local/dir/") - ORDER BY path - """ - qt_size_only """ - SELECT ELEMENT_AT(file, 'size') AS size FROM list_file("resource"="list_file_local_s3", "uri"="s3://list-file-local/dir/") - ORDER BY size - """ - qt_file_only """ - SELECT file FROM list_file("resource"="list_file_local_s3", "uri"="s3://list-file-local/dir/") - ORDER BY ELEMENT_AT(file, 'uri') - """ - requests.clear() - qt_recursive_getters """ - SELECT path, ELEMENT_AT(file, 'size') AS size, path = ELEMENT_AT(file, 'uri'), - ELEMENT_AT(file, 'offset'), ELEMENT_AT(file, 'content_type'), ELEMENT_AT(file, 'checksum'), file IS NULL - FROM list_file("resource"="list_file_local_s3", "uri"="s3://list-file-local/dir/", "recursive"="true") - ORDER BY path - """ - qt_recursive_protocol """ - SELECT ${requests.size()}, ${requests.every { it.method == "GET" && it.query["prefix"] == "dir/" - && !it.query.containsKey("delimiter") }}, - ${requests.any { it.query["continuation-token"]?.startsWith("empty:") }} - """ - qt_root """ - SELECT path, ELEMENT_AT(file, 'size') AS size - FROM list_file("resource"="list_file_local_s3", "uri"="s3://list-file-local") - ORDER BY path - """ - qt_root_slash """ - SELECT path, ELEMENT_AT(file, 'size') AS size - FROM list_file("resource"="list_file_local_s3", "uri"="s3://list-file-local/") - ORDER BY path - """ - qt_empty_directory """ - SELECT * FROM list_file("resource"="list_file_local_s3", "uri"="s3://list-file-local/missing/") - ORDER BY path - """ - qt_count """ - SELECT COUNT(*), COUNT(file), SUM(ELEMENT_AT(file, 'size')) - FROM list_file("resource"="list_file_local_s3", "uri"="s3://list-file-local/dir", "recursive"="true") - """ - - def originalParallelSettings = sql("SELECT @@ignore_storage_data_distribution, @@parallel_pipeline_task_num")[0] - try { - sql "SET ignore_storage_data_distribution = true" - sql "SET parallel_pipeline_task_num = 2" - requests.clear() - qt_parallel_count """ - SELECT COUNT(*), COUNT(DISTINCT path), SUM(ELEMENT_AT(file, 'size')) - FROM list_file("resource"="list_file_local_s3", "uri"="s3://list-file-local/dir/", "recursive"="true") - """ - qt_parallel_requests "SELECT ${requests.size()}" - requests.clear() - qt_parallel_count_repeat """ - SELECT COUNT(*), COUNT(DISTINCT path), SUM(ELEMENT_AT(file, 'size')) - FROM list_file("resource"="list_file_local_s3", "uri"="s3://list-file-local/dir/", "recursive"="true") - """ - qt_parallel_requests_repeat "SELECT ${requests.size()}" - qt_parallel_paths """ - SELECT path FROM list_file("resource"="list_file_local_s3", "uri"="s3://list-file-local/dir/", "recursive"="true") - ORDER BY path - """ - } finally { - sql "SET ignore_storage_data_distribution = ${originalParallelSettings[0]}" - sql "SET parallel_pipeline_task_num = ${originalParallelSettings[1]}" - } - - requests.clear() - // The first page contains one marker and one empty file. LIMIT must stop before fetching the next page. - qt_limit """ - SELECT path, file - FROM list_file("resource"="list_file_local_s3", "uri"="s3://list-file-local/dir/", "recursive"="true") - LIMIT 1 - """ - qt_limit_requests "SELECT ${requests.size()}" - requests.clear() - qt_limit_zero """ - SELECT * FROM list_file("resource"="list_file_local_s3", "uri"="s3://list-file-local/dir/") - LIMIT 0 - """ - qt_limit_zero_requests "SELECT ${requests.size()}" - - sql "DROP TABLE IF EXISTS test_list_file_resource_values" - test { - sql """ - CREATE TABLE test_list_file_resource_values ( - path VARCHAR(65533) NOT NULL, f FILE NOT NULL) - DUPLICATE KEY(path) DISTRIBUTED BY HASH(path) BUCKETS 1 - PROPERTIES("replication_num"="1") - """ - exception "FILE columns are not supported in internal tables" - } - - requests.clear() - qt_canonical_list """ - SELECT path, file - FROM list_file("resource"="list_file_local_canonical_s3", "uri"="s3://list-file-local/dir/") - ORDER BY path - """ - qt_canonical_list_protocol "SELECT ${requests.every { it.method == 'GET' }}" - requests.clear() - qt_canonical_to_file_head """ - SELECT TO_FILE('list_file_local_canonical_s3', 's3://list-file-local/dir/b.json') - """ - qt_canonical_head_protocol """ - SELECT ${requests.size()}, ${requests.every { it.method == "HEAD" - && it.path == "/list-file-local/dir/b.json" && it.query.isEmpty() }} - """ - - sql "DROP TABLE IF EXISTS test_list_file_canonical_bucket_uris" - sql """ - CREATE TABLE test_list_file_canonical_bucket_uris (id INT NOT NULL, uri STRING NOT NULL) - DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 - PROPERTIES("replication_num"="1") - """ - sql """ - INSERT INTO test_list_file_canonical_bucket_uris VALUES - (1, 's3://list-file-local/dir/b.json'), - (2, 's3://list-file-other/dir/b.json'), - (3, 's3://list-file-local/dir/b.json') - """ - requests.clear() - // One TO_FILE expression reads primary -> other -> primary in the same batch. - // Its returned FILE rows must retain the corresponding 7 -> 31 -> 7 sizes. - qt_canonical_mixed_buckets """ - SELECT id, TO_FILE('list_file_local_canonical_s3', uri) - FROM test_list_file_canonical_bucket_uris ORDER BY id - """ - assertEquals(["/list-file-local/dir/b.json", "/list-file-other/dir/b.json", - "/list-file-local/dir/b.json"], requests.collect { it.path }, - "TO_FILE must HEAD each URI's bucket and safely return to the first bucket") - assertTrue(requests.every { it.method == "HEAD" && it.query.isEmpty() }, - "Mixed-bucket TO_FILE must use only HEAD requests") - - qt_list_uri_bucket """ - SELECT path, ELEMENT_AT(file, 'size') FROM list_file( - "resource"="list_file_local_s3", "uri"="s3://other-bucket/dir/") ORDER BY path - """ - requests.clear() - for (def invalid in [ - [properties: '"uri"="s3://list-file-local/dir/"', message: "resource"], - [properties: '"resource"="list_file_local_s3"', message: "uri"], - [properties: '"resource"="missing_list_file_resource", "uri"="s3://list-file-local/dir/"', message: "Can not find resource"], - [properties: '"resource"="list_file_local_hdfs", "uri"="s3://list-file-local/dir/"', message: "S3"], - [properties: '"resource"="list_file_local_s3", "uri"="s3://list-file-local/dir/", "unknown"="value"', message: "unknown"], - [properties: '"resource"="list_file_local_s3", "uri"="s3://list-file-local/dir/", "recursive"="yes"', message: "recursive"], - ]) { - test { - sql "SELECT * FROM list_file(${invalid.properties})" - exception invalid.message - } - } - for (String uri : ["NULL", "'hdfs://127.0.0.1:1/file'"]) { - test { - sql "SELECT TO_FILE('list_file_local_hdfs', ${uri})" - exception "S3 resource" - } - } - qt_invalid_no_io "SELECT ${requests.size()}" - - sql "DROP USER IF EXISTS 'list_file_local_user'" - sql "CREATE USER 'list_file_local_user' IDENTIFIED BY 'ListFile_123'" - sql "GRANT SELECT_PRIV ON ${context.dbName}.* TO 'list_file_local_user'" - def userJdbcUrl = context.connection.metaData.URL - connect("list_file_local_user", "ListFile_123", userJdbcUrl) { - test { - sql 'SELECT * FROM list_file("resource"="list_file_local_s3", "uri"="s3://list-file-local/dir/")' - exception "denied" - } - } - qt_denied_no_io "SELECT ${requests.size()}" - sql "GRANT USAGE_PRIV ON RESOURCE list_file_local_s3 TO 'list_file_local_user'" - connect("list_file_local_user", "ListFile_123", userJdbcUrl) { - qt_granted_count """ - SELECT COUNT(*) FROM list_file("resource"="list_file_local_s3", "uri"="s3://list-file-local/dir/") - """ - } - // Distinct raw keys have distinct sizes; neither LIST_FILE nor TO_FILE decodes them. - def rawObjects = [ - "roundtrip/a b.txt": 101, "roundtrip/a%20b.txt": 102, - "roundtrip/a+b.txt": 103, "roundtrip/中文.txt": 104, - "roundtrip/a%2Fb.txt": 105, "roundtrip/a/b.txt": 106 - ] - rawObjects.each { key, size -> - objects[key] = size - heads["/list-file-local/${key}".toString()] = [size: size, etag: '"roundtrip-' + size + '"'] - } - requests.clear() - qt_raw_uri_roundtrip """ - SELECT path, size, ELEMENT_AT(f, 'uri'), ELEMENT_AT(f, 'size') - FROM ( - SELECT path, ELEMENT_AT(file, 'size') AS size, TO_FILE('list_file_local_s3', path) AS f - FROM list_file("resource"="list_file_local_s3", - "uri"="s3://list-file-local/roundtrip/", "recursive"="true") - ) t ORDER BY path - """ - def headRequests = requests.findAll { it.method == "HEAD" } - def expectedPaths = rawObjects.keySet().collect { "/list-file-local/${it}".toString() }.sort() - qt_raw_uri_request_protocol """ - SELECT ${headRequests.size()}, - ${headRequests.collect { it.decodedPath }.sort() == expectedPaths}, - ${headRequests.every { it.query.isEmpty() }} - """ - qt_literal_plus """ - SELECT TO_FILE('list_file_local_s3', 's3://list-file-local/roundtrip/a+b.txt') - """ - // Feed identical provider bytes through LIST XML and HEAD headers for both FILE entry points. - def consistencyEtags = [ - "etag-consistency/absent.txt": null, - "etag-consistency/empty.txt": "", - "etag-consistency/quoted-empty.txt": '""', - "etag-consistency/bare-multipart.txt": "MiXeD:opaque-2", - "etag-consistency/quoted-multipart.txt": '"MiXeD:opaque-2"' - ] - consistencyEtags.each { key, etag -> - objects[key] = 7 - listingEtags[key] = etag - heads["/list-file-local/${key}".toString()] = [size: 7, etag: etag] - } - requests.clear() - qt_listing_head_checksum_consistency """ - SELECT path, listing_checksum, head_checksum, listing_checksum <=> head_checksum - FROM ( - SELECT path, ELEMENT_AT(file, 'checksum') AS listing_checksum, - ELEMENT_AT(TO_FILE('list_file_local_s3', path), 'checksum') AS head_checksum - FROM list_file("resource"="list_file_local_s3", "uri"="s3://list-file-local/etag-consistency/") - ) t ORDER BY path - """ - def consistencyHeads = requests.findAll { it.method == "HEAD" } - def consistencyPaths = consistencyEtags.keySet().collect { "/list-file-local/${it}".toString() }.sort() - qt_listing_head_checksum_protocol """ - SELECT ${consistencyHeads.size()}, - ${consistencyHeads.collect { it.decodedPath }.sort() == consistencyPaths}, - ${requests.every { it.method == "HEAD" ? it.query.isEmpty() - : it.method == "GET" && it.query["list-type"] == "2" - && it.query["prefix"] == "etag-consistency/" }} - """ - qt_handler_errors "SELECT ${handlerErrors.size()}" - } finally { - server.stop(0) - sql "DROP USER IF EXISTS 'list_file_local_user'" - sql "DROP RESOURCE IF EXISTS 'list_file_local_hdfs'" - sql "DROP RESOURCE IF EXISTS 'list_file_local_s3'" - sql "DROP RESOURCE IF EXISTS 'list_file_local_canonical_s3'" - } -} diff --git a/regression-test/suites/datatype_p0/complex_types/test_list_files.groovy b/regression-test/suites/datatype_p0/complex_types/test_list_files.groovy new file mode 100644 index 00000000000..fc963c96f6a --- /dev/null +++ b/regression-test/suites/datatype_p0/complex_types/test_list_files.groovy @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import com.amazonaws.auth.AWSStaticCredentialsProvider +import com.amazonaws.auth.BasicAWSCredentials +import com.amazonaws.client.builder.AwsClientBuilder +import com.amazonaws.services.s3.AmazonS3ClientBuilder +import com.amazonaws.services.s3.model.ObjectMetadata + +suite("test_list_files", "p0,external") { + def client = AmazonS3ClientBuilder.standard() + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(getS3Endpoint(), getS3Region())) + .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(getS3AK(), getS3SK()))) + .withPathStyleAccessEnabled(true).build() + String bucket = getS3BucketName() + String prefix = "list-files/${UUID.randomUUID()}/" + String root = "s3://${bucket}/${prefix}" + def keys = [] + def put = { String key, String contents -> + byte[] data = contents.getBytes("UTF-8") + def metadata = new ObjectMetadata() + metadata.setContentLength(data.length) + client.putObject(bucket, prefix + key, new ByteArrayInputStream(data), metadata) + keys.add(prefix + key) + } + try { + ["": "", "empty.txt": "", "a.json": "{}", "sub/": "", "sub/c.png": "abc", + "sub/deep/d.txt": "abcd", "empty/": "", "empty/deep/": "", "marker/": "marker"] + .each { key, contents -> put(key, contents) } + sql "DROP RESOURCE IF EXISTS list_files_s3" + sql """CREATE RESOURCE list_files_s3 PROPERTIES( + "type"="s3", "s3.endpoint"="${getS3Endpoint()}", "s3.region"="${getS3Region()}", + "s3.access_key"="${getS3AK()}", "s3.secret_key"="${getS3SK()}", + "s3.bucket"="${bucket}", "s3.root.path"="unused", "use_path_style"="true")""" + def listing = { String path, String recursive -> + """list_files("resource"="list_files_s3", "uri"="${path}"${recursive})""" + } + String direct = listing(root, ', "recursive"="false"') + String recursive = listing(root, ', "recursive"="true"') + String defaults = listing(root, '') + qt_schema "DESC FUNCTION ${direct}" + for (def entry : [direct: direct, recursive: recursive, defaults: defaults]) { + "qt_${entry.key}" """SELECT substring(path, ${root.length() + 1}), size, + file IS NULL, ELEMENT_AT(file, 'size'), ELEMENT_AT(file, 'content_type') + FROM ${entry.value} ORDER BY path""" + "qt_${entry.key}_counts" "SELECT count(*), count(file), sum(size) FROM ${entry.value}" + } + qt_size_only "SELECT size FROM ${direct} ORDER BY path" + qt_path_only "SELECT substring(path, ${root.length() + 1}) FROM ${direct} ORDER BY path" + qt_file_only "SELECT file IS NULL FROM ${direct} ORDER BY path" + qt_limit "SELECT count(*) FROM (SELECT * FROM ${direct} LIMIT 2) t" + qt_roundtrip """SELECT substring(path, ${root.length() + 1}), + ELEMENT_AT(TO_FILE('list_files_s3', path), 'size') = size, + ELEMENT_AT(file, 'checksum') = ELEMENT_AT(TO_FILE('list_files_s3', path), 'checksum') + FROM ${recursive} ORDER BY path""" + qt_no_trailing_slash """SELECT count(*), count(file), sum(size) + FROM ${listing(root.substring(0, root.length() - 1), ', "recursive"="false"')}""" + qt_empty_direct """SELECT substring(path, ${root.length() + 1}), size, file IS NULL + FROM ${listing(root + 'empty/', ', "recursive"="false"')} ORDER BY path""" + qt_empty_recursive "SELECT count(*) FROM ${listing(root + 'empty/', '')}" + qt_missing "SELECT count(*) FROM ${listing(root + 'missing/', '')}" + for (int i = 0; i < 1005; ++i) { + put(String.format("pages/d%04d/", i), "") + } + qt_directory_pages """SELECT count(*), count(file), sum(size), + count(distinct path) FROM ${listing(root + 'pages/', ', "recursive"="false"')}""" + qt_directory_pages_recursive "SELECT count(*) FROM ${listing(root + 'pages/', '')}" + for (int i = 0; i < 1005; ++i) { + put(String.format("files/f%04d.txt", i), "x") + } + qt_file_pages """SELECT count(*), count(file), sum(size), count(distinct path) + FROM ${listing(root + 'files/', '')}""" + qt_file_pages_direct """SELECT count(*), count(file), sum(size) + FROM ${listing(root + 'files/', ', "recursive"="false"')}""" + test { + sql """SELECT * FROM list_file("resource"="list_files_s3", "uri"="${root}")""" + exception "Can not found function" + } + test { + sql """SELECT * FROM list_files("resource"="list_files_s3", "uri"="${root}", "recursive"="yes")""" + exception "recursive must be true or false" + } + } finally { + keys.each { key -> client.deleteObject(bucket, key) } + client.shutdown() + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
