This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/orc.git
The following commit(s) were added to refs/heads/main by this push:
new 3365300cb ORC-2014: Rename variables and configurations for periodic
stripe size and dictionary size checks
3365300cb is described below
commit 3365300cb2fa0b6304bc4b49623ac91fcf7c444c
Author: Laszlo Bodor <[email protected]>
AuthorDate: Tue Oct 7 23:31:27 2025 -0700
ORC-2014: Rename variables and configurations for periodic stripe size and
dictionary size checks
### What changes were proposed in this pull request?
Rename variables and configurations for periodic stripe size and dictionary
size checks.
### Why are the changes needed?
Code and OrcConf API quality.
### How was this patch tested?
Existing unit tests.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #2429 from abstractdog/ORC-2014.
Authored-by: Laszlo Bodor <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
java/core/src/java/org/apache/orc/OrcConf.java | 8 ++++----
java/core/src/java/org/apache/orc/impl/WriterImpl.java | 12 ++++++------
java/core/src/test/org/apache/orc/TestVectorOrcFile.java | 2 +-
java/tools/src/test/org/apache/orc/tools/TestFileDump.java | 2 +-
4 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/java/core/src/java/org/apache/orc/OrcConf.java
b/java/core/src/java/org/apache/orc/OrcConf.java
index 18609204f..89dfc47e9 100644
--- a/java/core/src/java/org/apache/orc/OrcConf.java
+++ b/java/core/src/java/org/apache/orc/OrcConf.java
@@ -118,8 +118,8 @@ public enum OrcConf {
"If the number of distinct keys in a dictionary is greater than this\n" +
"fraction of the total number of non-null rows, turn off \n" +
"dictionary encoding. Use 1 to always use dictionary encoding."),
- DICTIONARY_MAX_SIZE_IN_BYTES("orc.dictionary.maxSizeInBytes",
- "orc.dictionary.maxSizeInBytes",
+ DICTIONARY_MAX_SIZE_IN_BYTES("orc.dictionary.max.size.bytes",
+ "orc.dictionary.max.size.bytes",
16 * 1024 * 1024,
"If the total size of the dictionary is greater than this\n" +
", turn off dictionary encoding. Use 0 to disable this check."),
@@ -187,8 +187,8 @@ public enum OrcConf {
"added to all of the writers. Valid range is [1,10000] and is primarily
meant for" +
"testing. Setting this too low may negatively affect performance."
+ " Use orc.stripe.row.count instead if the value larger than
orc.stripe.row.count."),
- STRIPE_SIZE_CHECKRATIO("orc.stripe.size.checkRatio",
- "orc.stripe.size.checkRatio",
+ STRIPE_SIZE_CHECKRATIO("orc.stripe.size.check.ratio",
+ "orc.stripe.size.check.ratio",
2.0,
"Flush stripe if the tree writer size in bytes is larger than (this *
orc.stripe.size). " +
"Use 0 to disable this check."),
diff --git a/java/core/src/java/org/apache/orc/impl/WriterImpl.java
b/java/core/src/java/org/apache/orc/impl/WriterImpl.java
index 56c7b20d6..f6b08cde9 100644
--- a/java/core/src/java/org/apache/orc/impl/WriterImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/WriterImpl.java
@@ -111,8 +111,8 @@ public class WriterImpl implements WriterInternal,
MemoryManager.Callback {
private final MemoryManager memoryManager;
private long previousAllocation = -1;
private long memoryLimit;
- private final long ROWS_PER_CHECK;
- private final double STRIPE_SIZE_PER_CHECK;
+ private final long rowsPerCheck;
+ private final double stripeSizePerCheck;
private long rowsSinceCheck = 0;
private final OrcFile.Version version;
private final Configuration conf;
@@ -220,13 +220,13 @@ public class WriterImpl implements WriterInternal,
MemoryManager.Callback {
}
// ensure that we are able to handle callbacks before we register ourselves
- ROWS_PER_CHECK = Math.min(opts.getStripeRowCountValue(),
+ rowsPerCheck = Math.min(opts.getStripeRowCountValue(),
OrcConf.ROWS_BETWEEN_CHECKS.getLong(conf));
this.stripeRowCount= opts.getStripeRowCountValue();
this.stripeSize = opts.getStripeSize();
memoryLimit = stripeSize;
double stripeSizeCheckRatio =
OrcConf.STRIPE_SIZE_CHECKRATIO.getDouble(conf);
- STRIPE_SIZE_PER_CHECK = stripeSizeCheckRatio <= 0 ? 0 :
stripeSizeCheckRatio * stripeSize;
+ stripeSizePerCheck = stripeSizeCheckRatio <= 0 ? 0 : stripeSizeCheckRatio
* stripeSize;
memoryManager = opts.getMemoryManager();
memoryManager.addWriter(path, stripeSize, this);
@@ -328,9 +328,9 @@ public class WriterImpl implements WriterInternal,
MemoryManager.Callback {
}
private boolean checkMemory() throws IOException {
- long size = rowsSinceCheck < ROWS_PER_CHECK && STRIPE_SIZE_PER_CHECK == 0
+ long size = rowsSinceCheck < rowsPerCheck && stripeSizePerCheck == 0
? 0 : treeWriter.estimateMemory();
- if (rowsSinceCheck >= ROWS_PER_CHECK || size > STRIPE_SIZE_PER_CHECK) {
+ if (rowsSinceCheck >= rowsPerCheck || size > stripeSizePerCheck) {
rowsSinceCheck = 0;
if (LOG.isDebugEnabled()) {
LOG.debug("ORC writer " + physicalWriter + " size = " + size +
diff --git a/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
b/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
index be024dd9e..53262d0aa 100644
--- a/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
+++ b/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
@@ -197,7 +197,7 @@ public class TestVectorOrcFile implements TestConf {
@BeforeEach
public void openFileSystem(TestInfo testInfo) throws Exception {
- conf.set("orc.stripe.size.checkRatio", "0");
+ conf.setFloat(OrcConf.STRIPE_SIZE_CHECKRATIO.getAttribute(), 0.0f);
fs = FileSystem.getLocal(conf);
testFilePath = new Path(workDir, "TestVectorOrcFile." +
testInfo.getTestMethod().get().getName().replaceFirst("\\[[0-9]+\\]",
"")
diff --git a/java/tools/src/test/org/apache/orc/tools/TestFileDump.java
b/java/tools/src/test/org/apache/orc/tools/TestFileDump.java
index e5228a99d..315e53446 100644
--- a/java/tools/src/test/org/apache/orc/tools/TestFileDump.java
+++ b/java/tools/src/test/org/apache/orc/tools/TestFileDump.java
@@ -83,7 +83,7 @@ public class TestFileDump implements TestConf {
@BeforeEach
public void openFileSystem () throws Exception {
- conf.set("orc.stripe.size.checkRatio", "0");
+ conf.setFloat(OrcConf.STRIPE_SIZE_CHECKRATIO.getAttribute(), 0.0f);
fs = FileSystem.getLocal(conf);
testFilePath = new Path(workDir + File.separator +
"TestFileDump.testDump.orc");
fs.delete(testFilePath, false);
