This is an automated email from the ASF dual-hosted git repository. JingsongLi pushed a commit to branch codex/sync-orc-198-copy-fixes in repository https://gitbox.apache.org/repos/asf/paimon.git
commit c45e67a525f4325d40af3220b0123eb53766f604 Author: JingsongLi <[email protected]> AuthorDate: Mon Jun 29 19:39:38 2026 +0800 [format] Sync ORC copy fixes for 1.9.8 --- .../java/org/apache/orc/impl/PhysicalFsWriter.java | 4 +- .../java/org/apache/orc/impl/RecordReaderImpl.java | 16 ++++- .../org/apache/orc/impl/RecordReaderUtils.java | 29 ++++++-- .../main/java/org/apache/orc/impl/WriterImpl.java | 2 +- .../org/apache/orc/impl/RecordReaderImplTest.java | 80 ++++++++++++++++++++++ 5 files changed, 120 insertions(+), 11 deletions(-) diff --git a/paimon-format/src/main/java/org/apache/orc/impl/PhysicalFsWriter.java b/paimon-format/src/main/java/org/apache/orc/impl/PhysicalFsWriter.java index aab908fc71..868c577bb9 100644 --- a/paimon-format/src/main/java/org/apache/orc/impl/PhysicalFsWriter.java +++ b/paimon-format/src/main/java/org/apache/orc/impl/PhysicalFsWriter.java @@ -110,7 +110,7 @@ public class PhysicalFsWriter implements PhysicalWriter { opts, encryption); this.path = path; - LOG.info( + LOG.debug( "ORC writer created for path: {} with stripeSize: {} blockSize: {}" + " compression: {}", path, @@ -539,7 +539,7 @@ public class PhysicalFsWriter implements PhysicalWriter { // space in the block if (length < blockSize && length > availBlockSpace && addBlockPadding) { byte[] pad = new byte[(int) Math.min(HDFS_BUFFER_SIZE, availBlockSpace)]; - LOG.info("Padding ORC by {} bytes while merging", availBlockSpace); + LOG.debug("Padding ORC by {} bytes while merging", availBlockSpace); start += availBlockSpace; while (availBlockSpace > 0) { int writeLen = (int) Math.min(availBlockSpace, pad.length); diff --git a/paimon-format/src/main/java/org/apache/orc/impl/RecordReaderImpl.java b/paimon-format/src/main/java/org/apache/orc/impl/RecordReaderImpl.java index 4c821185d7..ab91bb1946 100644 --- a/paimon-format/src/main/java/org/apache/orc/impl/RecordReaderImpl.java +++ b/paimon-format/src/main/java/org/apache/orc/impl/RecordReaderImpl.java @@ -747,6 +747,12 @@ public class RecordReaderImpl implements RecordReader { TypeDescription type, boolean writerUsedProlepticGregorian, boolean useUTCTimestamp) { + // When statsProto is EMPTY_COLUMN_STATISTICS, this column does not actually provide + // statistics, so we cannot make any assumptions. + if (statsProto == EMPTY_COLUMN_STATISTICS) { + return SearchArgument.TruthValue.YES_NO_NULL; + } + ColumnStatistics cs = ColumnStatisticsImpl.deserialize( null, statsProto, writerUsedProlepticGregorian, true); @@ -840,14 +846,22 @@ public class RecordReaderImpl implements RecordReader { ValueRange range, BloomFilter bloomFilter, boolean useUTCTimestamp) { + // An invalid range means no value, including null, is written to this column. if (!range.isValid()) { - return SearchArgument.TruthValue.YES_NO_NULL; + return SearchArgument.TruthValue.NO; } // if we didn't have any values, everything must have been null if (!range.hasValues()) { if (predicate.getOperator() == PredicateLeaf.Operator.IS_NULL) { return SearchArgument.TruthValue.YES; + } else if (predicate.getOperator() == PredicateLeaf.Operator.NULL_SAFE_EQUALS) { + Object literal = predicate.getLiteral(); + if (literal == null) { + return SearchArgument.TruthValue.YES; + } else { + return SearchArgument.TruthValue.NO; + } } else { return SearchArgument.TruthValue.NULL; } diff --git a/paimon-format/src/main/java/org/apache/orc/impl/RecordReaderUtils.java b/paimon-format/src/main/java/org/apache/orc/impl/RecordReaderUtils.java index 0ceaadb449..5546117744 100644 --- a/paimon-format/src/main/java/org/apache/orc/impl/RecordReaderUtils.java +++ b/paimon-format/src/main/java/org/apache/orc/impl/RecordReaderUtils.java @@ -653,18 +653,18 @@ public class RecordReaderUtils { } static ChunkReader create(BufferChunk from, BufferChunk to) { - long f = Integer.MAX_VALUE; - long e = Integer.MIN_VALUE; + long f = Long.MAX_VALUE; + long e = Long.MIN_VALUE; - long cf = Integer.MAX_VALUE; - long ef = Integer.MIN_VALUE; - int reqBytes = 0; + long cf = Long.MAX_VALUE; + long ef = Long.MIN_VALUE; + long reqBytes = 0L; BufferChunk current = from; while (current != to.next) { f = Math.min(f, current.getOffset()); e = Math.max(e, current.getEnd()); - if (ef == Integer.MIN_VALUE || current.getOffset() <= ef) { + if (ef == Long.MIN_VALUE || current.getOffset() <= ef) { cf = Math.min(cf, current.getOffset()); ef = Math.max(ef, current.getEnd()); } else { @@ -675,7 +675,22 @@ public class RecordReaderUtils { current = (BufferChunk) current.next; } reqBytes += ef - cf; - return new ChunkReader(from, to, (int) (e - f), reqBytes); + if (reqBytes > IOUtils.MAX_ARRAY_SIZE) { + throw new IllegalArgumentException( + "invalid reqBytes value " + + reqBytes + + ",out of bounds " + + IOUtils.MAX_ARRAY_SIZE); + } + long readBytes = e - f; + if (readBytes > IOUtils.MAX_ARRAY_SIZE) { + throw new IllegalArgumentException( + "invalid readBytes value " + + readBytes + + ",out of bounds " + + IOUtils.MAX_ARRAY_SIZE); + } + return new ChunkReader(from, to, (int) readBytes, (int) reqBytes); } static ChunkReader create(BufferChunk from, int minSeekSize) { diff --git a/paimon-format/src/main/java/org/apache/orc/impl/WriterImpl.java b/paimon-format/src/main/java/org/apache/orc/impl/WriterImpl.java index 4249c8bd35..c6b027bfa5 100644 --- a/paimon-format/src/main/java/org/apache/orc/impl/WriterImpl.java +++ b/paimon-format/src/main/java/org/apache/orc/impl/WriterImpl.java @@ -235,7 +235,7 @@ public class WriterImpl implements WriterInternal, MemoryManager.Callback { treeWriter = TreeWriter.Factory.create(schema, null, new StreamFactory()); - LOG.info( + LOG.debug( "ORC writer created for path: {} with stripeSize: {} options: {}", path, stripeSize, diff --git a/paimon-format/src/test/java/org/apache/orc/impl/RecordReaderImplTest.java b/paimon-format/src/test/java/org/apache/orc/impl/RecordReaderImplTest.java new file mode 100644 index 0000000000..aff77b867e --- /dev/null +++ b/paimon-format/src/test/java/org/apache/orc/impl/RecordReaderImplTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.orc.impl; + +import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; +import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; +import org.apache.orc.TypeDescription; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link RecordReaderImpl}. */ +public class RecordReaderImplTest { + + @Test + public void testNullSafeEqualsWithAllNullStatistics() { + ColumnStatisticsImpl stats = ColumnStatisticsImpl.create(TypeDescription.createString()); + stats.setNull(); + + assertThat( + RecordReaderImpl.evaluatePredicate( + stats, nullSafeEqualsPredicate("value"), null)) + .isEqualTo(SearchArgument.TruthValue.NO); + assertThat(RecordReaderImpl.evaluatePredicate(stats, nullSafeEqualsPredicate(null), null)) + .isEqualTo(SearchArgument.TruthValue.YES); + } + + private static PredicateLeaf nullSafeEqualsPredicate(Object literal) { + return new PredicateLeaf() { + @Override + public Operator getOperator() { + return Operator.NULL_SAFE_EQUALS; + } + + @Override + public Type getType() { + return Type.STRING; + } + + @Override + public String getColumnName() { + return "field"; + } + + @Override + public Object getLiteral() { + return literal; + } + + @Override + public List<Object> getLiteralList() { + return Collections.emptyList(); + } + + @Override + public int getId() { + return 0; + } + }; + } +}
