This is an automated email from the ASF dual-hosted git repository.
gabor pushed a change to branch encryption
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git.
omit 1ce6bcc PARQUET-1286: Crypto package (#614)
omit b6067ac PARQUET-1228: Format Structures encryption (#613)
add ce55c9b PARQUET-1703: Update API compatibility check (#709)
add e430527 PARQUET-1741: Restore APIs to keep backward compatibility
(#729)
add ac7840c PARQUET-1725: Replace Usage of Strings.join with JDK
Functionality in… (#713)
add 72738f5 PARQUET-1735: Clean Up parquet-columns Module (#723)
add 65eba55 PARQUET-1740: Make ParquetFileReader.getFilteredRecordCount
public (#728)
add f0fc29f PARQUET-1744: Some filters throws
ArrayIndexOutOfBoundsException (#732)
add 616e35f PARQUET-1593: Improve parquet-cli's example usage (#646)
add d85a8f5 PARQUET-1729: Avoid AutoBoxing in EncodingStats (#717)
add e083fbb Update current release version (#733)
add ad07d83 PARQUET-1738: Remove unused imports (#726)
add 8c1bc9b PARQUET-1765: Invalid filteredRowCount in
InternalParquetRecordReader (#747)
add d3e3118 PARQUET-1710: Use Objects.requireNonNull (#703)
add 438cb3c PARQUET-1749: Use Java 8 Streams for Empty PrimitiveIterator
(#734)
add 1849650 PARQUET-1737: Replace Test Class RandomStr with Apache
Commons Lang (#725)
add 1577037 PARQUET-1751: Fix Protobuf Build Warning (#736)
add 17bef40 PARQUET-1782: Use Switch Statement in AvroRecordConverter
(#752)
add 3bbf66c PARQUET-1622: Add implementation for BYTE_STREAM_SPLIT (#705)
add 57f6b46 PARQUET-1790: Add Api for writing DataPageV2 to
ParquetFileWriter class (#756)
add 474a2be PARQUET-1796: Bump Avro from 1.9.1 to 1.9.2 (#759)
add 1eaf16d PARQUET-1794: Random data generation may cause flaky tests
(#758)
add 583063b PARQUET-1802: Use job ClassLoader to load CompressionCodec
class (#760)
add a0c9e69 PARQUET-1791: Add 'prune' command to parquet-tools (#755)
add e3afb24 PARQUET-1759: InternalParquetRecordReader Use Singleton Set
(#743)
add 7469e87 PARQUET-1784: Column-wise configuration (#754)
add 806037c PARQUET-41: Add bloom filter (#757)
new b887d49 PARQUET-1228: Format Structures encryption (#613)
new 8b306bf PARQUET-1286: Crypto package (#614)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (1ce6bcc)
\
N -- N -- N refs/heads/encryption (8b306bf)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
README.md | 10 +-
dev/finalize-release | 1 +
.../apache/parquet/avro/AvroRecordConverter.java | 68 ++--
parquet-benchmarks/pom.xml | 5 -
.../parquet/cascading/ParquetTupleScheme.java | 7 +-
.../parquet/cascading/ParquetTupleScheme.java | 9 +-
.../parquet/cascading/ParquetValueScheme.java | 20 +-
parquet-cli/pom.xml | 5 -
.../src/main/java/org/apache/parquet/cli/Help.java | 11 +-
.../java/org/apache/parquet/cli/csv/AvroCSV.java | 8 +-
parquet-column/pom.xml | 11 +
.../org/apache/parquet/column/ColumnProperty.java | 137 ++++++++
.../java/org/apache/parquet/column/Encoding.java | 16 +
.../org/apache/parquet/column/EncodingStats.java | 35 +-
.../apache/parquet/column/ParquetProperties.java | 209 ++++++++---
.../parquet/column/impl/ColumnReaderBase.java | 38 +-
.../parquet/column/impl/ColumnWriteStoreBase.java | 43 ++-
.../parquet/column/impl/ColumnWriteStoreV1.java | 13 +-
.../parquet/column/impl/ColumnWriteStoreV2.java | 13 +-
.../parquet/column/impl/ColumnWriterBase.java | 80 +++++
.../apache/parquet/column/impl/ColumnWriterV1.java | 6 +
.../apache/parquet/column/impl/ColumnWriterV2.java | 12 +-
.../apache/parquet/column/page/DictionaryPage.java | 7 +-
.../bitpacking/ByteBitPackingValuesWriter.java | 1 -
.../values/bloomfilter/BlockSplitBloomFilter.java | 382 +++++++++++++++++++++
.../column/values/bloomfilter/BloomFilter.java | 171 +++++++++
.../bloomfilter/BloomFilterWriteStore.java} | 20 +-
.../values/bloomfilter/BloomFilterWriter.java | 19 +-
.../column/values/bloomfilter/HashFunction.java | 25 +-
.../parquet/column/values/bloomfilter/XxHash.java | 36 +-
.../ByteStreamSplitValuesReader.java | 100 ++++++
.../ByteStreamSplitValuesReaderForDouble.java} | 27 +-
.../ByteStreamSplitValuesReaderForFloat.java} | 27 +-
.../ByteStreamSplitValuesWriter.java | 142 ++++++++
.../delta/DeltaBinaryPackingValuesWriter.java | 3 -
.../DeltaBinaryPackingValuesWriterForLong.java | 1 -
.../values/deltastrings/DeltaByteArrayReader.java | 4 +-
.../values/deltastrings/DeltaByteArrayWriter.java | 1 -
.../values/dictionary/DictionaryValuesReader.java | 2 -
.../values/dictionary/DictionaryValuesWriter.java | 4 +-
.../parquet/column/values/dictionary/IntList.java | 2 +-
.../factory/DefaultV1ValuesWriterFactory.java | 15 +-
.../factory/DefaultV2ValuesWriterFactory.java | 15 +-
.../values/factory/DefaultValuesWriterFactory.java | 2 +-
.../values/fallback/FallbackValuesWriter.java | 10 +-
.../column/values/rle/ZeroIntegerValuesReader.java | 1 +
.../parquet/example/DummyRecordConverter.java | 3 +
.../org/apache/parquet/example/data/Group.java | 2 +
.../parquet/example/data/simple/SimpleGroup.java | 2 +-
.../org/apache/parquet/filter/AndRecordFilter.java | 6 +-
.../apache/parquet/filter/ColumnPredicates.java | 7 +-
.../apache/parquet/filter/ColumnRecordFilter.java | 6 +-
.../org/apache/parquet/filter/NotRecordFilter.java | 5 +-
.../org/apache/parquet/filter/OrRecordFilter.java | 7 +-
.../parquet/filter2/compat/FilterCompat.java | 9 +-
.../parquet/filter2/predicate/FilterApi.java | 16 +-
.../filter2/predicate/LogicalInverseRewriter.java | 5 +-
.../parquet/filter2/predicate/LogicalInverter.java | 18 +-
.../parquet/filter2/predicate/Operators.java | 33 +-
.../filter2/predicate/PrimitiveToBoxedClass.java | 2 +-
.../predicate/SchemaCompatibilityValidator.java | 12 +-
.../parquet/filter2/predicate/Statistics.java | 13 +-
.../parquet/filter2/predicate/ValidTypeMap.java | 8 +-
.../recordlevel/FilteringGroupConverter.java | 14 +-
.../recordlevel/FilteringPrimitiveConverter.java | 6 +-
.../recordlevel/FilteringRecordMaterializer.java | 27 +-
.../IncrementallyUpdatedFilterPredicate.java | 6 +-
...ementallyUpdatedFilterPredicateBuilderBase.java | 4 +-
...crementallyUpdatedFilterPredicateEvaluator.java | 5 +-
...ncrementallyUpdatedFilterPredicateResetter.java | 5 +-
.../columnindex/BinaryColumnIndexBuilder.java | 2 +-
.../columnindex/BooleanColumnIndexBuilder.java | 2 +-
.../internal/column/columnindex/BoundaryOrder.java | 32 ++
.../columnindex/DoubleColumnIndexBuilder.java | 2 +-
.../columnindex/FloatColumnIndexBuilder.java | 2 +-
.../internal/column/columnindex/IndexIterator.java | 13 +-
.../column/columnindex/IntColumnIndexBuilder.java | 2 +-
.../column/columnindex/LongColumnIndexBuilder.java | 2 +-
.../org/apache/parquet/io/ColumnIOFactory.java | 2 +-
.../java/org/apache/parquet/io/GroupColumnIO.java | 12 +-
.../org/apache/parquet/io/MessageColumnIO.java | 24 +-
.../parquet/io/RecordReaderImplementation.java | 10 +-
.../parquet/io/ValidatingRecordConsumer.java | 20 +-
.../java/org/apache/parquet/io/api/Binary.java | 1 +
.../org/apache/parquet/schema/ColumnOrder.java | 4 +-
.../java/org/apache/parquet/schema/GroupType.java | 12 +-
.../org/apache/parquet/schema/MessageType.java | 4 +-
.../org/apache/parquet/schema/PrimitiveType.java | 1 -
.../main/java/org/apache/parquet/schema/Type.java | 14 +-
.../main/java/org/apache/parquet/schema/Types.java | 102 +++---
.../org/apache/parquet/FixedBinaryTestUtils.java | 96 ++++++
.../apache/parquet/column/TestEncodingStats.java | 36 +-
.../parquet/column/impl/TestColumnReaderImpl.java | 4 +-
.../column/impl/TestCorruptDeltaByteArrays.java | 4 +-
.../apache/parquet/column/mem/TestMemColumn.java | 3 +-
.../parquet/column/page/mem/MemPageReader.java | 6 +-
.../parquet/column/page/mem/MemPageStore.java | 4 +-
.../parquet/column/page/mem/MemPageWriter.java | 2 +-
.../parquet/column/statistics/TestStatistics.java | 4 +-
.../apache/parquet/column/values/RandomStr.java | 51 ---
.../column/values/TestValuesReaderImpl.java | 2 -
.../org/apache/parquet/column/values/Utils.java | 13 +-
.../values/bitpacking/TestBitPackingColumn.java | 4 +
.../bloomfilter/TestBlockSplitBloomFilter.java | 229 ++++++++++++
.../ByteStreamSplitValuesEndToEndTest.java | 111 ++++++
.../ByteStreamSplitValuesReaderTest.java | 193 +++++++++++
.../ByteStreamSplitValuesWriterTest.java | 189 ++++++++++
.../values/delta/benchmark/BenchMarkTest.java | 2 -
.../benchmark/BenchmarkDeltaLengthByteArray.java | 4 +-
.../values/deltastrings/TestDeltaByteArray.java | 1 -
.../benchmark/BenchmarkDeltaByteArray.java | 8 +-
.../factory/DefaultValuesWriterFactoryTest.java | 230 ++++++++++++-
.../RunLengthBitPackingHybridIntegrationTest.java | 12 -
.../rle/TestRunLengthBitPackingHybridEncoder.java | 2 +-
.../predicate/TestLogicalInverseRewriter.java | 4 +-
.../filter2/predicate/TestLogicalInverter.java | 6 +-
.../column/columnindex/TestBoundaryOrder.java | 56 +++
.../column/columnindex/TestOffsetIndexBuilder.java | 2 -
.../filter2/columnindex/TestColumnIndexFilter.java | 106 ++++--
.../org/apache/parquet/io/ConverterConsumer.java | 4 +-
.../parquet/io/ExpectationValidatingConverter.java | 2 +-
.../java/org/apache/parquet/io/TestColumnIO.java | 18 +-
.../java/org/apache/parquet/io/TestFiltered.java | 6 +-
.../apache/parquet/schema/TestTypeBuilders.java | 96 +++---
.../java/org/apache/parquet/Preconditions.java | 4 +
.../src/main/java/org/apache/parquet/Strings.java | 6 +
.../java/org/apache/parquet/glob/WildcardPath.java | 7 +-
.../apache/parquet/hadoop/metadata/ColumnPath.java | 9 +-
.../main/java/org/apache/parquet/format/Util.java | 8 +
parquet-hadoop/pom.xml | 7 +
.../java/org/apache/parquet/HadoopReadOptions.java | 9 +-
.../org/apache/parquet/ParquetReadOptions.java | 21 +-
.../filter2/bloomfilterlevel/BloomFilterImpl.java | 150 ++++++++
.../parquet/filter2/compat/RowGroupFilter.java | 31 +-
.../filter2/dictionarylevel/DictionaryFilter.java | 8 +-
.../filter2/statisticslevel/StatisticsFilter.java | 7 +-
.../format/converter/ParquetMetadataConverter.java | 39 ++-
.../apache/parquet/hadoop/BloomFilterReader.java | 70 ++++
.../org/apache/parquet/hadoop/CodecFactory.java | 8 +-
.../parquet/hadoop/ColumnChunkPageWriteStore.java | 18 +-
.../apache/parquet/hadoop/ColumnConfigParser.java | 72 ++++
.../parquet/hadoop/DictionaryPageReader.java | 3 +-
.../apache/parquet/hadoop/DirectCodecFactory.java | 7 +-
.../hadoop/InternalParquetRecordReader.java | 19 +-
.../hadoop/InternalParquetRecordWriter.java | 15 +-
.../org/apache/parquet/hadoop/MemoryManager.java | 6 +-
.../apache/parquet/hadoop/ParquetFileReader.java | 52 ++-
.../apache/parquet/hadoop/ParquetFileWriter.java | 143 +++++++-
.../apache/parquet/hadoop/ParquetInputFormat.java | 11 +-
.../apache/parquet/hadoop/ParquetOutputFormat.java | 81 ++++-
.../org/apache/parquet/hadoop/ParquetReader.java | 31 +-
.../apache/parquet/hadoop/ParquetRecordWriter.java | 7 +-
.../org/apache/parquet/hadoop/ParquetWriter.java | 53 ++-
.../apache/parquet/hadoop/api/WriteSupport.java | 15 +-
.../parquet/hadoop/example/GroupWriteSupport.java | 6 +-
.../hadoop/metadata/ColumnChunkMetaData.java | 25 +-
.../parquet/hadoop/metadata/FileMetaData.java | 9 +-
.../parquet/hadoop/metadata/GlobalMetaData.java | 9 +-
.../apache/parquet/hadoop/util/HadoopStreams.java | 6 +-
.../dictionarylevel/DictionaryFilterTest.java | 9 +-
.../filter2/recordlevel/PhoneBookWriter.java | 8 +
.../converter/TestParquetMetadataConverter.java | 5 +-
.../apache/parquet/hadoop/TestBloomFiltering.java | 257 ++++++++++++++
.../hadoop/TestColumnChunkPageWriteStore.java | 1 +
.../parquet/hadoop/TestColumnIndexFiltering.java | 50 +++
.../parquet/hadoop/TestParquetFileWriter.java | 125 ++++++-
.../apache/parquet/hadoop/TestParquetWriter.java | 54 ++-
.../hadoop/example/TestInputOutputFormat.java | 3 +-
.../apache/parquet/statistics/RandomValues.java | 3 +-
.../src/test/resources/TestProto3.proto | 4 +-
.../src/test/resources/TestProtobuf.proto | 2 +
.../org/apache/parquet/thrift/ConvertedField.java | 10 +-
.../apache/parquet/thrift/ThriftParquetReader.java | 11 +-
.../parquet/thrift/ThriftSchemaConvertVisitor.java | 5 +-
.../DeprecatedFieldProjectionFilter.java | 4 +-
.../thrift/struct/CompatibilityChecker.java | 4 +-
.../parquet/thrift/TestThriftRecordConverter.java | 5 +-
parquet-tools/pom.xml | 1 -
.../parquet/tools/command/PruneColumnsCommand.java | 168 +++++++++
.../org/apache/parquet/tools/command/Registry.java | 1 +
.../tools/command/TestPruneColumnsCommand.java | 255 ++++++++++++++
pom.xml | 105 +++---
182 files changed, 4910 insertions(+), 920 deletions(-)
create mode 100644
parquet-column/src/main/java/org/apache/parquet/column/ColumnProperty.java
create mode 100644
parquet-column/src/main/java/org/apache/parquet/column/values/bloomfilter/BlockSplitBloomFilter.java
create mode 100644
parquet-column/src/main/java/org/apache/parquet/column/values/bloomfilter/BloomFilter.java
copy
parquet-column/src/main/java/org/apache/parquet/column/{page/PageWriteStore.java
=> values/bloomfilter/BloomFilterWriteStore.java} (74%)
copy parquet-generator/src/main/java/org/apache/parquet/version/Generator.java
=>
parquet-column/src/main/java/org/apache/parquet/column/values/bloomfilter/BloomFilterWriter.java
(77%)
copy
parquet-common/src/main/java/org/apache/parquet/bytes/ByteBufferAllocator.java
=>
parquet-column/src/main/java/org/apache/parquet/column/values/bloomfilter/HashFunction.java
(62%)
copy
parquet-common/src/main/java/org/apache/parquet/bytes/HeapByteBufferAllocator.java
=>
parquet-column/src/main/java/org/apache/parquet/column/values/bloomfilter/XxHash.java
(64%)
create mode 100644
parquet-column/src/main/java/org/apache/parquet/column/values/bytestreamsplit/ByteStreamSplitValuesReader.java
copy
parquet-column/src/main/java/org/apache/parquet/{example/data/simple/BooleanValue.java
=> column/values/bytestreamsplit/ByteStreamSplitValuesReaderForDouble.java}
(62%)
copy
parquet-column/src/main/java/org/apache/parquet/{example/data/simple/BooleanValue.java
=> column/values/bytestreamsplit/ByteStreamSplitValuesReaderForFloat.java}
(62%)
create mode 100644
parquet-column/src/main/java/org/apache/parquet/column/values/bytestreamsplit/ByteStreamSplitValuesWriter.java
create mode 100644
parquet-column/src/test/java/org/apache/parquet/FixedBinaryTestUtils.java
delete mode 100644
parquet-column/src/test/java/org/apache/parquet/column/values/RandomStr.java
create mode 100644
parquet-column/src/test/java/org/apache/parquet/column/values/bloomfilter/TestBlockSplitBloomFilter.java
create mode 100644
parquet-column/src/test/java/org/apache/parquet/column/values/bytestreamsplit/ByteStreamSplitValuesEndToEndTest.java
create mode 100644
parquet-column/src/test/java/org/apache/parquet/column/values/bytestreamsplit/ByteStreamSplitValuesReaderTest.java
create mode 100644
parquet-column/src/test/java/org/apache/parquet/column/values/bytestreamsplit/ByteStreamSplitValuesWriterTest.java
create mode 100644
parquet-hadoop/src/main/java/org/apache/parquet/filter2/bloomfilterlevel/BloomFilterImpl.java
create mode 100644
parquet-hadoop/src/main/java/org/apache/parquet/hadoop/BloomFilterReader.java
create mode 100644
parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnConfigParser.java
create mode 100644
parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestBloomFiltering.java
create mode 100644
parquet-tools/src/main/java/org/apache/parquet/tools/command/PruneColumnsCommand.java
create mode 100644
parquet-tools/src/test/java/org/apache/parquet/tools/command/TestPruneColumnsCommand.java
