This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 156bae07933 [fix](be) Read dictionary-encoded Parquet BOOLEAN columns
(#69967)
156bae07933 is described below
commit 156bae0793386477fb98b08649a2d5090a7a5f0d
Author: Gabriel <[email protected]>
AuthorDate: Tue Nov 3 10:36:46 2026 +0800
[fix](be) Read dictionary-encoded Parquet BOOLEAN columns (#69967)
### What problem does this PR solve?
Issue Number: N/A
Problem Summary:
The native Parquet reader rejects every BOOLEAN dictionary page,
although PLAIN dictionaries with RLE_DICTIONARY or legacy
PLAIN_DICTIONARY data pages are valid. Reading a nullable BOOLEAN
interoperability fixture fails with `Boolean type cannot have a
dictionary page`.
Create the existing fixed-length dictionary decoder for BOOLEAN and
expand its LSB-first packed entries into tracked, contiguous one-byte
values for SerDe. Validate the packed extent before allocation and
preserve the common dictionary-index checks for reads, selections, and
skips.
The 257-byte interoperability fixture and synthetic tests cover byte
boundaries, invalid extents, dictionary replacement, both dictionary
encoding enums, V1/V2 pages, skipped/filtered values, and
dictionary-to-PLAIN transitions.
### Release note
Support reading dictionary-encoded BOOLEAN columns from Parquet files.
### Check List (For Author)
- Test
- [x] Regression test: S3 TVF reads compare against SQL literal rows,
covering predicates, aggregates, NULL alignment, and lazy
materialization on/off. Groovy compilation and a suite control-flow
check using the real FE variable manager pass, including settings
restoration after success or an injected failure; full SQL execution is
pending CI.
- [x] Unit Test: ASAN BE tests; all 199 tests in
`ParquetV2NativeDecoderTest.*:NewParquetReaderTest.*` pass. The five new
tests fail before the fix and pass after it.
- [ ] Manual test
- Behavior changed:
- [x] Yes. Valid BOOLEAN dictionaries are readable instead of being
rejected.
- Does this need documentation?
- [x] No. Compatibility fix for existing Parquet reads.
---
be/src/format_v2/parquet/reader/native/decoder.cpp | 3 +-
.../reader/native/fix_length_dict_decoder.hpp | 32 +++-
be/test/format_v2/parquet/native_decoder_test.cpp | 180 +++++++++++++++++++++
be/test/format_v2/parquet/parquet_reader_test.cpp | 53 ++++++
.../test_data/boolean_rle_dictionary.parquet | Bin 0 -> 257 bytes
.../tvf/test_data/boolean_rle_dictionary.parquet | Bin 0 -> 257 bytes
.../tvf/test_parquet_boolean_dictionary.groovy | 102 ++++++++++++
7 files changed, 362 insertions(+), 8 deletions(-)
diff --git a/be/src/format_v2/parquet/reader/native/decoder.cpp
b/be/src/format_v2/parquet/reader/native/decoder.cpp
index 845c9deacd7..68ec067104d 100644
--- a/be/src/format_v2/parquet/reader/native/decoder.cpp
+++ b/be/src/format_v2/parquet/reader/native/decoder.cpp
@@ -279,7 +279,8 @@ Status create_plain_decoder(tparquet::Type::type type,
std::unique_ptr<Decoder>&
Status create_dictionary_decoder(tparquet::Type::type type,
std::unique_ptr<Decoder>& decoder) {
switch (type) {
case tparquet::Type::BOOLEAN:
- return Status::InternalError("Boolean type cannot have a dictionary
page");
+ decoder =
std::make_unique<FixLengthDictDecoder<tparquet::Type::BOOLEAN>>();
+ return Status::OK();
case tparquet::Type::BYTE_ARRAY:
decoder = std::make_unique<ByteArrayDictDecoder>();
return Status::OK();
diff --git a/be/src/format_v2/parquet/reader/native/fix_length_dict_decoder.hpp
b/be/src/format_v2/parquet/reader/native/fix_length_dict_decoder.hpp
index 1f1166421a4..280b97cc853 100644
--- a/be/src/format_v2/parquet/reader/native/fix_length_dict_decoder.hpp
+++ b/be/src/format_v2/parquet/reader/native/fix_length_dict_decoder.hpp
@@ -18,6 +18,7 @@
#pragma once
#include <limits>
+#include <utility>
#include "format_v2/parquet/reader/native/decoder.h"
@@ -41,16 +42,33 @@ public:
Status set_dict(DorisUniqueBufferPtr<uint8_t>& dict, int32_t length,
size_t num_values) override {
- if (UNLIKELY(_type_length <= 0 || length < 0 ||
- num_values > std::numeric_limits<size_t>::max() /
- static_cast<size_t>(_type_length) ||
- num_values * static_cast<size_t>(_type_length) !=
- static_cast<size_t>(length))) {
- return Status::Corruption("Wrong dictionary data for fixed length
type");
- }
if (UNLIKELY(dict == nullptr)) {
return Status::Corruption("Fixed-length Parquet dictionary is
null");
}
+ if constexpr (PhysicalType == tparquet::Type::BOOLEAN) {
+ // A PLAIN BOOLEAN dictionary is LSB-first bit-packed, so nine
entries need two bytes.
+ // Validate that extent before expanding; division first also
avoids count + 7 overflow.
+ const size_t packed_bytes = num_values / 8 + (num_values % 8 !=
0U);
+ if (UNLIKELY(std::cmp_not_equal(packed_bytes, length))) {
+ return Status::Corruption("Wrong dictionary data for boolean
type");
+ }
+ // SerDe consumes contiguous one-byte values: e.g. 0x02 becomes
{false, true}.
+ auto unpacked = make_unique_buffer<uint8_t>(num_values);
+ for (size_t row = 0; row < num_values; ++row) {
+ unpacked[row] = static_cast<uint8_t>((dict[row / 8] >> (row %
8)) & 1);
+ }
+ dict = std::move(unpacked);
+ _type_length = sizeof(uint8_t);
+ } else {
+ // Other physical types store one complete fixed-width value per
dictionary entry.
+ if (UNLIKELY(_type_length <= 0 || length < 0 ||
+ num_values > std::numeric_limits<size_t>::max() /
+
static_cast<size_t>(_type_length) ||
+ num_values * static_cast<size_t>(_type_length) !=
+ static_cast<size_t>(length))) {
+ return Status::Corruption("Wrong dictionary data for fixed
length type");
+ }
+ }
_dict = std::move(dict);
_num_dictionary_values = num_values;
++_dictionary_generation;
diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp
b/be/test/format_v2/parquet/native_decoder_test.cpp
index 1dc88ace240..63f972a4946 100644
--- a/be/test/format_v2/parquet/native_decoder_test.cpp
+++ b/be/test/format_v2/parquet/native_decoder_test.cpp
@@ -295,6 +295,93 @@ public:
std::vector<uint32_t> indices;
};
+TEST(ParquetV2NativeDecoderTest, BooleanDictionaryUnpacksPlainEntries) {
+ for (const auto encoding :
+ {tparquet::Encoding::RLE_DICTIONARY,
tparquet::Encoding::PLAIN_DICTIONARY}) {
+ SCOPED_TRACE(encoding);
+ for (const size_t count : {1, 2, 8, 9, 129}) {
+ SCOPED_TRACE(count);
+ std::unique_ptr<Decoder> decoder;
+ ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BOOLEAN,
encoding, decoder).ok());
+ decoder->set_type_length(-1);
+ const size_t bytes = (count + 7) / 8;
+ auto dictionary = make_unique_buffer<uint8_t>(bytes);
+ memset(dictionary.get(), 0xaa, bytes);
+ ASSERT_TRUE(decoder->set_dict(dictionary, bytes, count).ok());
+ CaptureFixedConsumer values;
+ CaptureBinaryConsumer unused;
+ ASSERT_TRUE(decoder->decode_dictionary(values, unused).ok());
+ EXPECT_EQ(values.width, 1);
+ ASSERT_EQ(values.bytes.size(), count);
+ for (size_t row = 0; row < count; ++row) {
+ EXPECT_EQ(values.bytes[row], row % 2);
+ }
+ }
+ }
+}
+
+TEST(ParquetV2NativeDecoderTest, BooleanDictionaryReadSkipSelectionAndBounds) {
+ for (const auto encoding :
+ {tparquet::Encoding::RLE_DICTIONARY,
tparquet::Encoding::PLAIN_DICTIONARY}) {
+ SCOPED_TRACE(encoding);
+ std::unique_ptr<Decoder> decoder;
+ ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BOOLEAN, encoding,
decoder).ok());
+ auto dictionary = make_unique_buffer<uint8_t>(1);
+ dictionary[0] = 2;
+ ASSERT_TRUE(decoder->set_dict(dictionary, 1, 2).ok());
+ faststring encoded;
+ RleEncoder<uint32_t> encoder(&encoded, 2);
+ const std::vector<uint32_t> ids {1, 0, 1, 1, 0, 0, 1, 0};
+ for (const auto id : ids) {
+ encoder.Put(id);
+ }
+ encoder.Flush();
+ std::string payload(1, 2);
+ payload.append(reinterpret_cast<const char*>(encoded.data()),
encoded.size());
+ Slice data(payload);
+ ASSERT_TRUE(decoder->set_data(&data).ok());
+ std::vector<uint32_t> actual;
+ ASSERT_TRUE(decoder->decode_dictionary_indices(2, &actual).ok());
+ EXPECT_EQ(actual, (std::vector<uint32_t> {1, 0}));
+ ASSERT_TRUE(decoder->skip_values(2).ok());
+ ParquetSelection selection {
+ .total_values = 4, .selected_values = 2, .ranges = {{.first =
1, .count = 2}}};
+ actual.clear();
+ ASSERT_TRUE(decoder->decode_selected_dictionary_indices(selection,
&actual).ok());
+ EXPECT_EQ(actual, (std::vector<uint32_t> {0, 1}));
+
+ const auto generation = decoder->dictionary_generation();
+ auto replacement = make_unique_buffer<uint8_t>(1);
+ replacement[0] = 1;
+ ASSERT_TRUE(decoder->set_dict(replacement, 1, 1).ok());
+ EXPECT_GT(decoder->dictionary_generation(), generation);
+ CaptureFixedConsumer values;
+ CaptureBinaryConsumer unused;
+ ASSERT_TRUE(decoder->decode_dictionary(values, unused).ok());
+ EXPECT_EQ(values.bytes, (std::vector<uint8_t> {1}));
+ ASSERT_TRUE(decoder->set_data(&data).ok());
+ EXPECT_TRUE(decoder->skip_values(1).is<ErrorCode::CORRUPTION>());
+ ASSERT_TRUE(decoder->set_data(&data).ok());
+ EXPECT_TRUE(decoder->decode_dictionary_indices(1,
&actual).is<ErrorCode::CORRUPTION>());
+ }
+}
+
+TEST(ParquetV2NativeDecoderTest, BooleanDictionaryRejectsInvalidPackedExtent) {
+ std::unique_ptr<Decoder> decoder;
+ ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BOOLEAN,
tparquet::Encoding::RLE_DICTIONARY,
+ decoder)
+ .ok());
+ auto dictionary = make_unique_buffer<uint8_t>(2);
+ EXPECT_TRUE(decoder->set_dict(dictionary, 0,
1).is<ErrorCode::CORRUPTION>());
+ EXPECT_TRUE(decoder->set_dict(dictionary, 1,
9).is<ErrorCode::CORRUPTION>());
+ EXPECT_TRUE(decoder->set_dict(dictionary, 2,
2).is<ErrorCode::CORRUPTION>());
+ EXPECT_TRUE(decoder->set_dict(dictionary, -1,
2).is<ErrorCode::CORRUPTION>());
+ EXPECT_TRUE(decoder->set_dict(dictionary, 1,
std::numeric_limits<size_t>::max())
+ .is<ErrorCode::CORRUPTION>());
+ DorisUniqueBufferPtr<uint8_t> missing;
+ EXPECT_TRUE(decoder->set_dict(missing, 1, 2).is<ErrorCode::CORRUPTION>());
+}
+
TEST(ParquetV2NativeDecoderTest, FragmentedPlainSelectionUsesOneConsumerBatch)
{
std::array<int32_t, 32> input {};
std::iota(input.begin(), input.end(), 0);
@@ -609,6 +696,99 @@ std::vector<uint8_t> serialize_page(tparquet::PageHeader
header,
return bytes;
}
+TEST(ParquetV2NativeDecoderTest, BooleanDictionaryAcrossPagesAndPlainFallback)
{
+ for (const bool page_v2 : {false, true}) {
+ for (const auto encoding :
+ {tparquet::Encoding::RLE_DICTIONARY,
tparquet::Encoding::PLAIN_DICTIONARY}) {
+ SCOPED_TRACE(page_v2);
+ SCOPED_TRACE(encoding);
+ tparquet::PageHeader dict_header;
+ dict_header.__set_type(tparquet::PageType::DICTIONARY_PAGE);
+ dict_header.__set_compressed_page_size(1);
+ dict_header.__set_uncompressed_page_size(1);
+ tparquet::DictionaryPageHeader dict_info;
+ dict_info.__set_num_values(2);
+ dict_info.__set_encoding(tparquet::Encoding::PLAIN);
+ dict_header.__set_dictionary_page_header(dict_info);
+ // A nonzero dictionary offset distinguishes this chunk from one
without a dictionary.
+ std::vector<uint8_t> bytes(1, 0);
+ const auto dictionary = serialize_page(dict_header, {2});
+ bytes.insert(bytes.end(), dictionary.begin(), dictionary.end());
+ const auto data_offset = bytes.size();
+ for (size_t page = 0; page < 3; ++page) {
+ const auto page_encoding = page == 2 ?
tparquet::Encoding::PLAIN : encoding;
+ // Width 1 + a literal run of eight IDs (0x55); PLAIN uses the
opposite polarity.
+ const std::vector<uint8_t> payload =
+ page == 2 ? std::vector<uint8_t> {0xaa} :
std::vector<uint8_t> {1, 3, 0x55};
+ tparquet::PageHeader header;
+ header.__set_type(page_v2 ? tparquet::PageType::DATA_PAGE_V2
+ : tparquet::PageType::DATA_PAGE);
+ header.__set_compressed_page_size(payload.size());
+ header.__set_uncompressed_page_size(payload.size());
+ if (page_v2) {
+ tparquet::DataPageHeaderV2 info;
+ info.__set_num_values(8);
+ info.__set_num_rows(8);
+ info.__set_num_nulls(0);
+ info.__set_encoding(page_encoding);
+ info.__set_definition_levels_byte_length(0);
+ info.__set_repetition_levels_byte_length(0);
+ info.__set_is_compressed(false);
+ header.__set_data_page_header_v2(info);
+ } else {
+ tparquet::DataPageHeader info;
+ info.__set_num_values(8);
+ info.__set_encoding(page_encoding);
+
info.__set_definition_level_encoding(tparquet::Encoding::RLE);
+
info.__set_repetition_level_encoding(tparquet::Encoding::RLE);
+ header.__set_data_page_header(info);
+ }
+ const auto serialized = serialize_page(header, payload);
+ bytes.insert(bytes.end(), serialized.begin(),
serialized.end());
+ }
+ MemoryBufferedReader stream(bytes);
+ tparquet::ColumnChunk chunk;
+ chunk.meta_data.__set_type(tparquet::Type::BOOLEAN);
+
chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED);
+ chunk.meta_data.__set_num_values(24);
+ chunk.meta_data.__set_total_compressed_size(bytes.size() - 1);
+ chunk.meta_data.__set_dictionary_page_offset(1);
+ chunk.meta_data.__set_data_page_offset(data_offset);
+ NativeFieldSchema field;
+ field.physical_type = tparquet::Type::BOOLEAN;
+ ParquetPageReadContext page_context(false, "");
+ ColumnChunkReader<false, false> reader(&stream, &chunk, &field,
nullptr, 24, nullptr,
+ page_context);
+ ASSERT_TRUE(reader.init().ok());
+ DataTypeUInt8 type;
+ auto column = type.create_column();
+ ParquetDecodeContext context;
+ context.physical_type = ParquetPhysicalType::BOOLEAN;
+ ParquetMaterializationState state;
+ for (size_t page = 0; page < 3; ++page) {
+ ASSERT_TRUE(reader.parse_page_header().ok());
+ ASSERT_TRUE(reader.load_page_data().ok());
+ ASSERT_TRUE(reader.skip_values(2).ok());
+ const std::vector<uint8_t> selected {1, 0, 1, 1, 0, 1};
+ FilterMap filter;
+ ASSERT_TRUE(filter.init(selected.data(), selected.size(),
false).ok());
+ ColumnSelectVector selection;
+ ASSERT_TRUE(selection.init({6, 0}, 6, nullptr, &filter,
0).ok());
+ const auto status = reader.materialize_values(column,
*type.get_serde(), context,
+ state,
selection);
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_EQ(reader.remaining_num_values(), 0);
+ ASSERT_TRUE(reader.next_page().ok());
+ }
+ EXPECT_FALSE(reader.has_next_page());
+ const std::vector<uint8_t> expected {1, 1, 0, 0, 1, 1, 0, 0, 0, 0,
1, 1};
+ ASSERT_EQ(column->size(), expected.size());
+ const auto& values = assert_cast<const
ColumnUInt8&>(*column).get_data();
+ EXPECT_TRUE(std::equal(expected.begin(), expected.end(),
values.begin()));
+ }
+ }
+}
+
std::vector<uint8_t> serialize_plain_int32_page(const std::vector<int32_t>&
values) {
tparquet::PageHeader header;
header.type = tparquet::PageType::DATA_PAGE;
diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp
b/be/test/format_v2/parquet/parquet_reader_test.cpp
index 027f20ae89f..db173ec83ca 100644
--- a/be/test/format_v2/parquet/parquet_reader_test.cpp
+++ b/be/test/format_v2/parquet/parquet_reader_test.cpp
@@ -1973,6 +1973,59 @@ TEST_F(NewParquetReaderTest,
UuidPlainDictionaryNullableAndMappingMatrix) {
}
}
+TEST_F(NewParquetReaderTest, ReadsNullableBooleanDictionaryFromExternalWriter)
{
+ const char* source_root = std::getenv("ROOT");
+ ASSERT_NE(source_root, nullptr);
+ // Public interoperability fixture from StarRocks/starrocks PR #81788.
+ _file_path = std::string(source_root) +
+
"/be/test/format_v2/parquet/test_data/boolean_rle_dictionary.parquet";
+ for (const size_t batch_size : {1, 3, 1024}) {
+ SCOPED_TRACE(batch_size);
+ auto reader = create_reader();
+ reader->set_batch_size(batch_size);
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ ASSERT_EQ(schema.size(), 2);
+ auto request = std::make_shared<format::FileScanRequest>();
+ for (size_t column = 0; column < schema.size(); ++column) {
+
request->non_predicate_columns.push_back(field_projection(schema[column].local_id));
+
request->local_positions.emplace(format::LocalColumnId(schema[column].local_id),
+ format::LocalIndex(column));
+ }
+ ASSERT_TRUE(reader->open(request).ok());
+ const std::array<int, 10> expected {1, 0, -1, 1, 1, 0, -1, 0, 1, 1};
+ size_t total = 0;
+ bool eof = false;
+ while (!eof) {
+ Block block;
+ for (const auto& column : schema) {
+ block.insert({column.type->create_column(), column.type,
column.name});
+ }
+ size_t rows = 0;
+ const auto status = reader->get_block(&block, &rows, &eof);
+ ASSERT_TRUE(status.ok()) << status;
+ ASSERT_LE(total + rows, expected.size());
+ const auto& booleans =
+ assert_cast<const
ColumnNullable&>(*block.get_by_position(1).column);
+ for (size_t row = 0; row < rows; ++row) {
+
EXPECT_EQ(block.get_by_position(0).type->to_string(*block.get_by_position(0).column,
+ row),
+ std::to_string(total + row));
+ EXPECT_EQ(booleans.is_null_at(row), expected[total + row] ==
-1);
+ if (expected[total + row] != -1) {
+ EXPECT_EQ(assert_cast<const
ColumnUInt8&>(booleans.get_nested_column())
+ .get_element(row),
+ expected[total + row]);
+ }
+ }
+ total += rows;
+ }
+ EXPECT_EQ(total, expected.size());
+ }
+}
+
TEST_F(NewParquetReaderTest, InitializesProfileOnlyOncePerFile) {
class CountingParquetReader final : public format::parquet::ParquetReader {
public:
diff --git a/be/test/format_v2/parquet/test_data/boolean_rle_dictionary.parquet
b/be/test/format_v2/parquet/test_data/boolean_rle_dictionary.parquet
new file mode 100644
index 00000000000..f075f6e21ce
Binary files /dev/null and
b/be/test/format_v2/parquet/test_data/boolean_rle_dictionary.parquet differ
diff --git
a/regression-test/data/external_table_p0/tvf/test_data/boolean_rle_dictionary.parquet
b/regression-test/data/external_table_p0/tvf/test_data/boolean_rle_dictionary.parquet
new file mode 100644
index 00000000000..f075f6e21ce
Binary files /dev/null and
b/regression-test/data/external_table_p0/tvf/test_data/boolean_rle_dictionary.parquet
differ
diff --git
a/regression-test/suites/external_table_p0/tvf/test_parquet_boolean_dictionary.groovy
b/regression-test/suites/external_table_p0/tvf/test_parquet_boolean_dictionary.groovy
new file mode 100644
index 00000000000..d8cf489945a
--- /dev/null
+++
b/regression-test/suites/external_table_p0/tvf/test_parquet_boolean_dictionary.groovy
@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import com.azure.storage.blob.BlobClientBuilder
+import com.azure.storage.common.StorageSharedKeyCredential
+
+suite("test_parquet_boolean_dictionary", "p0,external") {
+ String provider = getS3Provider()
+ String endpoint = getS3Endpoint()
+ if (provider.equalsIgnoreCase("AZURE")) {
+ if (endpoint == "azure-endpoint") {
+ endpoint = "https://${getS3AK()}.blob.core.windows.net"
+ } else if (!endpoint.contains("://")) {
+ endpoint = "https://${endpoint}";
+ }
+ }
+ String bucket = context.config.otherConfigs.get('s3BucketName')
+ String key =
"parquet_boolean_dictionary/${UUID.randomUUID()}/values.parquet"
+ File fixture = new File(context.dataPath,
"test_data/boolean_rle_dictionary.parquet")
+ def blobClient = null
+ def s3Client = null
+ boolean uploaded = false
+ Map<String, String> originalSettings = [:]
+ // Use the public SQL name: SHOW VARIABLES silently returns no rows for
unknown names.
+ for (String variable : ['enable_sql_cache', 'enable_query_cache',
'enable_parquet_lazy_materialization']) {
+ originalSettings[variable] = sql("SHOW VARIABLES LIKE
'${variable}'")[0][1].toString()
+ }
+ try {
+ if (provider.equalsIgnoreCase("AZURE")) {
+ blobClient = new BlobClientBuilder()
+ .endpoint(endpoint)
+ .containerName(bucket)
+ .blobName(key)
+ .credential(new StorageSharedKeyCredential(getS3AK(),
getS3SK()))
+ .buildClient()
+ blobClient.uploadFromFile(fixture.absolutePath)
+ } else {
+ s3Client = getS3Client()
+ s3Client.putObject(bucket, key, fixture)
+ }
+ uploaded = true
+ String source = """S3('uri'='s3://${bucket}/${key}',
'format'='parquet',
+ 'provider'='${provider}', 's3.endpoint'='${endpoint}',
's3.region'='${getS3Region()}',
+ 's3.access_key'='${getS3AK()}',
's3.secret_key'='${getS3SK()}')"""
+ // OUTFILE writes PLAIN booleans, so use a fixture whose BOOLEAN
values require dictionary
+ // decoding. Compare with SQL literals to check NULL alignment without
hand-written output.
+ String expected = """(
+ SELECT 0 AS id, CAST(TRUE AS BOOLEAN) AS b
+ UNION ALL SELECT 1, FALSE
+ UNION ALL SELECT 2, CAST(NULL AS BOOLEAN)
+ UNION ALL SELECT 3, TRUE
+ UNION ALL SELECT 4, TRUE
+ UNION ALL SELECT 5, FALSE
+ UNION ALL SELECT 6, CAST(NULL AS BOOLEAN)
+ UNION ALL SELECT 7, FALSE
+ UNION ALL SELECT 8, TRUE
+ UNION ALL SELECT 9, TRUE
+ ) expected_boolean_dictionary"""
+ sql "SET enable_sql_cache = false"
+ sql "SET enable_query_cache = false"
+ for (boolean lazyMaterialization : [false, true]) {
+ sql "SET enable_parquet_lazy_materialization =
${lazyMaterialization}"
+ for (String predicate : ['', 'WHERE b = TRUE', 'WHERE b = FALSE',
'WHERE b IS NULL',
+ 'WHERE b <> TRUE OR b IS NULL', 'WHERE id IN (1, 2, 5, 6,
9)']) {
+ quickTest("boolean_dictionary_rows",
+ "SELECT id, b FROM ${source} ${predicate} ORDER BY id",
+ "SELECT id, b FROM ${expected} ${predicate} ORDER BY
id")
+ }
+ quickTest("boolean_dictionary_aggregates",
+ """SELECT COUNT(*), COUNT(b), SUM(IF(b, 1, 0)), SUM(IF(b
IS NULL, 1, 0))
+ FROM ${source}""",
+ """SELECT COUNT(*), COUNT(b), SUM(IF(b, 1, 0)), SUM(IF(b
IS NULL, 1, 0))
+ FROM ${expected}""")
+ }
+ } finally {
+ try {
+ originalSettings.each { variable, value -> sql "SET ${variable} =
${value}" }
+ } finally {
+ if (uploaded) {
+ if (blobClient != null) {
+ blobClient.delete()
+ } else {
+ s3Client.deleteObject(bucket, key)
+ }
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
