This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/orc.git
The following commit(s) were added to refs/heads/main by this push:
new 5099aa47d ORC-1986: Trigger flush stripe for large input rows
5099aa47d is described below
commit 5099aa47d50845eff6d79d5dcc9ed0784a0e297d
Author: WanKun <[email protected]>
AuthorDate: Thu Sep 25 15:37:38 2025 -0700
ORC-1986: Trigger flush stripe for large input rows
### What changes were proposed in this pull request?
For large input rows, the stripe may excessively large , requiring more
memory for both reading and writing one strip. We can check the tree write size
in bytes and flush the strip even when the input rows count is less than 5000.
```log
Stripes:
Stripe: offset: 3 data: 347494188 rows: 5120 tail: 244 index: 2304
Stream: column 0 section ROW_INDEX start: 3 length 12
Stream: column 1 section ROW_INDEX start: 15 length 110
Stream: column 2 section ROW_INDEX start: 125 length 893
Stream: column 3 section ROW_INDEX start: 1018 length 31
Stream: column 4 section ROW_INDEX start: 1049 length 65
Stream: column 5 section ROW_INDEX start: 1114 length 923
Stream: column 6 section ROW_INDEX start: 2037 length 25
Stream: column 7 section ROW_INDEX start: 2062 length 155
Stream: column 8 section ROW_INDEX start: 2217 length 28
Stream: column 9 section ROW_INDEX start: 2245 length 31
Stream: column 10 section ROW_INDEX start: 2276 length 31
Stream: column 1 section DATA start: 2307 length 81853
Stream: column 1 section LENGTH start: 84160 length 2191
Stream: column 2 section DATA start: 86351 length 345862763
Stream: column 2 section LENGTH start: 345949114 length 13736
Stream: column 3 section DATA start: 345962850 length 22
Stream: column 3 section LENGTH start: 345962872 length 6
Stream: column 3 section DICTIONARY_DATA start: 345962878 length 5
Stream: column 4 section PRESENT start: 345962883 length 200
Stream: column 4 section DATA start: 345963083 length 6322
Stream: column 4 section LENGTH start: 345969405 length 495
Stream: column 4 section DICTIONARY_DATA start: 345969900 length 2919
Stream: column 5 section DATA start: 345972819 length 1507883
Stream: column 5 section LENGTH start: 347480702 length 7346
Stream: column 6 section DATA start: 347488048 length 22
Stream: column 6 section LENGTH start: 347488070 length 6
Stream: column 6 section DICTIONARY_DATA start: 347488076 length 0
Stream: column 7 section DATA start: 347488076 length 5795
Stream: column 7 section LENGTH start: 347493871 length 301
Stream: column 7 section DICTIONARY_DATA start: 347494172 length 2187
Stream: column 8 section DATA start: 347496359 length 22
Stream: column 8 section LENGTH start: 347496381 length 6
Stream: column 8 section DICTIONARY_DATA start: 347496387 length 4
Stream: column 9 section DATA start: 347496391 length 58
Stream: column 9 section LENGTH start: 347496449 length 6
Stream: column 9 section DICTIONARY_DATA start: 347496455 length 7
Stream: column 10 section DATA start: 347496462 length 22
Stream: column 10 section LENGTH start: 347496484 length 6
Stream: column 10 section DICTIONARY_DATA start: 347496490 length 5
Encoding column 0: DIRECT
Encoding column 1: DIRECT_V2
Encoding column 2: DIRECT_V2
Encoding column 3: DICTIONARY_V2[1]
Encoding column 4: DICTIONARY_V2[661]
Encoding column 5: DIRECT_V2
Encoding column 6: DICTIONARY_V2[1]
Encoding column 7: DICTIONARY_V2[682]
Encoding column 8: DICTIONARY_V2[1]
Encoding column 9: DICTIONARY_V2[2]
Encoding column 10: DICTIONARY_V2[1]
```
### Why are the changes needed?
To optimize the memory usage.
### How was this patch tested?
Local test
Stripe with this change:
```log
Stripe: offset: 3 data: 69573620 rows: 1024 tail: 227 index: 2245
Stream: column 0 section ROW_INDEX start: 3 length 12
Stream: column 1 section ROW_INDEX start: 15 length 111
Stream: column 2 section ROW_INDEX start: 126 length 914
Stream: column 3 section ROW_INDEX start: 1040 length 30
Stream: column 4 section ROW_INDEX start: 1070 length 62
Stream: column 5 section ROW_INDEX start: 1132 length 848
Stream: column 6 section ROW_INDEX start: 1980 length 25
Stream: column 7 section ROW_INDEX start: 2005 length 155
Stream: column 8 section ROW_INDEX start: 2160 length 28
Stream: column 9 section ROW_INDEX start: 2188 length 30
Stream: column 10 section ROW_INDEX start: 2218 length 30
Stream: column 1 section DATA start: 2248 length 15899
Stream: column 1 section LENGTH start: 18147 length 478
Stream: column 2 section DATA start: 18625 length 69245402
Stream: column 2 section LENGTH start: 69264027 length 2795
Stream: column 3 section DATA start: 69266822 length 11
Stream: column 3 section LENGTH start: 69266833 length 6
Stream: column 3 section DICTIONARY_DATA start: 69266839 length 5
Stream: column 4 section PRESENT start: 69266844 length 55
Stream: column 4 section DATA start: 69266899 length 1269
Stream: column 4 section LENGTH start: 69268168 length 231
Stream: column 4 section DICTIONARY_DATA start: 69268399 length 1261
Stream: column 5 section DATA start: 69269660 length 302251
Stream: column 5 section LENGTH start: 69571911 length 1548
Stream: column 6 section DATA start: 69573459 length 11
Stream: column 6 section LENGTH start: 69573470 length 6
Stream: column 6 section DICTIONARY_DATA start: 69573476 length 0
Stream: column 7 section DATA start: 69573476 length 1129
Stream: column 7 section LENGTH start: 69574605 length 168
Stream: column 7 section DICTIONARY_DATA start: 69574773 length 1030
Stream: column 8 section DATA start: 69575803 length 11
Stream: column 8 section LENGTH start: 69575814 length 6
Stream: column 8 section DICTIONARY_DATA start: 69575820 length 4
Stream: column 9 section DATA start: 69575824 length 11
Stream: column 9 section LENGTH start: 69575835 length 6
Stream: column 9 section DICTIONARY_DATA start: 69575841 length 5
Stream: column 10 section DATA start: 69575846 length 11
Stream: column 10 section LENGTH start: 69575857 length 6
Stream: column 10 section DICTIONARY_DATA start: 69575863 length 5
Encoding column 0: DIRECT
Encoding column 1: DIRECT_V2
Encoding column 2: DIRECT_V2
Encoding column 3: DICTIONARY_V2[1]
Encoding column 4: DICTIONARY_V2[266]
Encoding column 5: DIRECT_V2
Encoding column 6: DICTIONARY_V2[1]
Encoding column 7: DICTIONARY_V2[297]
Encoding column 8: DICTIONARY_V2[1]
Encoding column 9: DICTIONARY_V2[1]
Encoding column 10: DICTIONARY_V2[1]
```
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #2371 from wankunde/force_spill_stripe.
Lead-authored-by: WanKun <[email protected]>
Co-authored-by: wankun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
java/core/src/java/org/apache/orc/OrcConf.java | 10 ++++++++++
java/core/src/java/org/apache/orc/impl/WriterImpl.java | 8 ++++++--
.../java/org/apache/orc/impl/writer/StringBaseTreeWriter.java | 6 +++++-
java/core/src/test/org/apache/orc/TestVectorOrcFile.java | 1 +
java/tools/src/test/org/apache/orc/tools/TestFileDump.java | 1 +
5 files changed, 23 insertions(+), 3 deletions(-)
diff --git a/java/core/src/java/org/apache/orc/OrcConf.java
b/java/core/src/java/org/apache/orc/OrcConf.java
index 26d1b7881..18609204f 100644
--- a/java/core/src/java/org/apache/orc/OrcConf.java
+++ b/java/core/src/java/org/apache/orc/OrcConf.java
@@ -118,6 +118,11 @@ public enum OrcConf {
"If the number of distinct keys in a dictionary is greater than this\n" +
"fraction of the total number of non-null rows, turn off \n" +
"dictionary encoding. Use 1 to always use dictionary encoding."),
+ DICTIONARY_MAX_SIZE_IN_BYTES("orc.dictionary.maxSizeInBytes",
+ "orc.dictionary.maxSizeInBytes",
+ 16 * 1024 * 1024,
+ "If the total size of the dictionary is greater than this\n" +
+ ", turn off dictionary encoding. Use 0 to disable this check."),
ROW_INDEX_STRIDE_DICTIONARY_CHECK("orc.dictionary.early.check",
"hive.orc.row.index.stride.dictionary.check",
true,
@@ -182,6 +187,11 @@ public enum OrcConf {
"added to all of the writers. Valid range is [1,10000] and is primarily
meant for" +
"testing. Setting this too low may negatively affect performance."
+ " Use orc.stripe.row.count instead if the value larger than
orc.stripe.row.count."),
+ STRIPE_SIZE_CHECKRATIO("orc.stripe.size.checkRatio",
+ "orc.stripe.size.checkRatio",
+ 2.0,
+ "Flush stripe if the tree writer size in bytes is larger than (this *
orc.stripe.size). " +
+ "Use 0 to disable this check."),
OVERWRITE_OUTPUT_FILE("orc.overwrite.output.file",
"orc.overwrite.output.file", false,
"A boolean flag to enable overwriting of the output file if it already
exists.\n"),
IS_SCHEMA_EVOLUTION_CASE_SENSITIVE("orc.schema.evolution.case.sensitive",
diff --git a/java/core/src/java/org/apache/orc/impl/WriterImpl.java
b/java/core/src/java/org/apache/orc/impl/WriterImpl.java
index bd1e6afad..56c7b20d6 100644
--- a/java/core/src/java/org/apache/orc/impl/WriterImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/WriterImpl.java
@@ -112,6 +112,7 @@ public class WriterImpl implements WriterInternal,
MemoryManager.Callback {
private long previousAllocation = -1;
private long memoryLimit;
private final long ROWS_PER_CHECK;
+ private final double STRIPE_SIZE_PER_CHECK;
private long rowsSinceCheck = 0;
private final OrcFile.Version version;
private final Configuration conf;
@@ -224,6 +225,8 @@ public class WriterImpl implements WriterInternal,
MemoryManager.Callback {
this.stripeRowCount= opts.getStripeRowCountValue();
this.stripeSize = opts.getStripeSize();
memoryLimit = stripeSize;
+ double stripeSizeCheckRatio =
OrcConf.STRIPE_SIZE_CHECKRATIO.getDouble(conf);
+ STRIPE_SIZE_PER_CHECK = stripeSizeCheckRatio <= 0 ? 0 :
stripeSizeCheckRatio * stripeSize;
memoryManager = opts.getMemoryManager();
memoryManager.addWriter(path, stripeSize, this);
@@ -325,9 +328,10 @@ public class WriterImpl implements WriterInternal,
MemoryManager.Callback {
}
private boolean checkMemory() throws IOException {
- if (rowsSinceCheck >= ROWS_PER_CHECK) {
+ long size = rowsSinceCheck < ROWS_PER_CHECK && STRIPE_SIZE_PER_CHECK == 0
+ ? 0 : treeWriter.estimateMemory();
+ if (rowsSinceCheck >= ROWS_PER_CHECK || size > STRIPE_SIZE_PER_CHECK) {
rowsSinceCheck = 0;
- long size = treeWriter.estimateMemory();
if (LOG.isDebugEnabled()) {
LOG.debug("ORC writer " + physicalWriter + " size = " + size +
" memoryLimit = " + memoryLimit + " rowsInStripe = " +
rowsInStripe +
diff --git
a/java/core/src/java/org/apache/orc/impl/writer/StringBaseTreeWriter.java
b/java/core/src/java/org/apache/orc/impl/writer/StringBaseTreeWriter.java
index 9d87873e7..1afa6e73e 100644
--- a/java/core/src/java/org/apache/orc/impl/writer/StringBaseTreeWriter.java
+++ b/java/core/src/java/org/apache/orc/impl/writer/StringBaseTreeWriter.java
@@ -59,6 +59,7 @@ public abstract class StringBaseTreeWriter extends
TreeWriterBase {
// If the number of keys in a dictionary is greater than this fraction of
//the total number of non-null rows, turn off dictionary encoding
private final double dictionaryKeySizeThreshold;
+ private final long dictionaryMaxSizeInBytes;
protected Dictionary dictionary;
protected boolean useDictionaryEncoding = true;
private boolean isDirectV2 = true;
@@ -101,6 +102,7 @@ public abstract class StringBaseTreeWriter extends
TreeWriterBase {
rowIndexValueCount.add(0L);
buildIndex = context.buildIndex();
dictionaryKeySizeThreshold = context.getDictionaryKeySizeThreshold(id);
+ dictionaryMaxSizeInBytes =
OrcConf.DICTIONARY_MAX_SIZE_IN_BYTES.getLong(conf);
strideDictionaryCheck =
OrcConf.ROW_INDEX_STRIDE_DICTIONARY_CHECK.getBoolean(conf);
if (dictionaryKeySizeThreshold <= 0.0) {
@@ -118,7 +120,9 @@ public abstract class StringBaseTreeWriter extends
TreeWriterBase {
// based on whether or not the fraction of distinct keys over number of
// non-null rows is less than the configured threshold
float ratio = rows.size() > 0 ? (float) (dictionary.size()) /
rows.size() : 0.0f;
- useDictionaryEncoding = !isDirectV2 || ratio <=
dictionaryKeySizeThreshold;
+ useDictionaryEncoding = !isDirectV2 || (ratio <=
dictionaryKeySizeThreshold &&
+ (dictionaryMaxSizeInBytes <= 0 ||
+ dictionary.getSizeInBytes() <= dictionaryMaxSizeInBytes));
doneDictionaryCheck = true;
}
}
diff --git a/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
b/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
index 76681f462..be024dd9e 100644
--- a/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
+++ b/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
@@ -197,6 +197,7 @@ public class TestVectorOrcFile implements TestConf {
@BeforeEach
public void openFileSystem(TestInfo testInfo) throws Exception {
+ conf.set("orc.stripe.size.checkRatio", "0");
fs = FileSystem.getLocal(conf);
testFilePath = new Path(workDir, "TestVectorOrcFile." +
testInfo.getTestMethod().get().getName().replaceFirst("\\[[0-9]+\\]",
"")
diff --git a/java/tools/src/test/org/apache/orc/tools/TestFileDump.java
b/java/tools/src/test/org/apache/orc/tools/TestFileDump.java
index fc4a90c8e..e5228a99d 100644
--- a/java/tools/src/test/org/apache/orc/tools/TestFileDump.java
+++ b/java/tools/src/test/org/apache/orc/tools/TestFileDump.java
@@ -83,6 +83,7 @@ public class TestFileDump implements TestConf {
@BeforeEach
public void openFileSystem () throws Exception {
+ conf.set("orc.stripe.size.checkRatio", "0");
fs = FileSystem.getLocal(conf);
testFilePath = new Path(workDir + File.separator +
"TestFileDump.testDump.orc");
fs.delete(testFilePath, false);
