This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 0287593797 [core] Update row-count defaults and rename sorted index
file option (#10035)
0287593797 is described below
commit 0287593797fa05ecc4f8af04b663ed8fef57b6f0
Author: YeJunHao <[email protected]>
AuthorDate: Tue Sep 22 13:49:39 2026 +0800
[core] Update row-count defaults and rename sorted index file option
(#10035)
---
docs/docs/flink/procedures/indexes.md | 2 +-
docs/docs/multimodal-table/global-index/bitmap.mdx | 6 +--
docs/docs/multimodal-table/global-index/btree.mdx | 4 +-
.../global-index/manage-indexes.mdx | 4 +-
.../multimodal-table/global-index/multivalue.mdx | 6 +--
docs/docs/spark/procedures/indexes.md | 2 +-
docs/generated/core_configuration.html | 2 +-
.../main/java/org/apache/paimon/CoreOptions.java | 3 +-
.../globalindex/sorted/SortedIndexOptions.java | 9 ++--
.../globalindex/sorted/SortedIndexOptionsTest.java | 28 ++++++++++--
.../sorted/SortedGlobalIndexWriter.java | 2 +-
.../pksorted/PrimaryKeySortedIndexOptionsTest.java | 2 +
.../flink/globalindex/SortedIndexTopoBuilder.java | 2 +-
.../procedure/CreateGlobalIndexProcedureTest.java | 4 +-
.../pypaimon/common/options/core_options.py | 17 +++++---
.../pypaimon/globalindex/create_global_index.py | 6 +--
.../pypaimon/tests/daft/daft_integration_test.py | 3 +-
.../pypaimon/tests/global_index_build_test.py | 4 +-
.../pypaimon/tests/sorted_index_options_test.py | 51 ++++++++++++++++++++++
.../globalindex/sorted/SortedIndexTopoBuilder.java | 2 +-
.../procedure/CreateGlobalIndexProcedureTest.java | 4 +-
21 files changed, 122 insertions(+), 41 deletions(-)
diff --git a/docs/docs/flink/procedures/indexes.md
b/docs/docs/flink/procedures/indexes.md
index e16bc1ef33..5b9715d248 100644
--- a/docs/docs/flink/procedures/indexes.md
+++ b/docs/docs/flink/procedures/indexes.md
@@ -67,7 +67,7 @@ CALL sys.create_global_index(
`table` => 'default.T',
`index_column` => 'tag',
`index_type` => 'bitmap',
- `options` => 'sorted-index.records-per-range=1000000');
+ `options` => 'sorted-index.records-per-file=1000000');
-- Create exact FM contains index
CALL sys.create_global_index(
diff --git a/docs/docs/multimodal-table/global-index/bitmap.mdx
b/docs/docs/multimodal-table/global-index/bitmap.mdx
index 94a1868da0..4648bfcf03 100644
--- a/docs/docs/multimodal-table/global-index/bitmap.mdx
+++ b/docs/docs/multimodal-table/global-index/bitmap.mdx
@@ -111,7 +111,7 @@ added_files = table.create_global_index(
index_type="bitmap",
partitions=[{"dt": "2026-06-18"}, {"dt": "2026-06-19"}],
options={
- "sorted-index.records-per-range": "10000000",
+ "sorted-index.records-per-file": "25000000",
"bitmap-index.dictionary-block-size": "16 kb",
},
)
@@ -123,7 +123,7 @@ print(added_files)
</Tabs>
Bitmap indexes share the sorted index build path with BTree indexes. Use
-`sorted-index.records-per-range` to control the expected records per generated
index
+`sorted-index.records-per-file` to control the expected records per generated
index
file, and `sorted-index.build.max-parallelism` to cap Flink or Spark build
parallelism. The legacy `btree-index.records-per-range` and
`btree-index.build.max-parallelism` keys are still recognized as fallback keys.
@@ -179,7 +179,7 @@ query unions multiple bitmap index files.
| Option | Default | Description |
|---|---|---|
-| `sorted-index.records-per-range` | `10000000` | Expected number of records
per sorted global index file for BTree, Bitmap, and Multivalue builds. |
+| `sorted-index.records-per-file` | `25000000` | Expected number of records
per sorted global index file for BTree, Bitmap, and Multivalue builds. |
| `sorted-index.build.max-parallelism` | `4096` | Maximum Flink or Spark
parallelism for building sorted global indexes. |
| `bitmap-index.dictionary-block-size` | `16 kb` | Target size of dictionary
blocks in bitmap global index files. Smaller blocks reduce dictionary read
amplification for high-cardinality columns; larger blocks reduce dictionary
block index size. |
| `bitmap-index.compression` | `none` | Compression algorithm for bitmap
dictionary blocks and the dictionary block index. Supported values are the same
block codecs as BTree index, such as `none`, `lz4`, `lzo`, and `zstd`. |
diff --git a/docs/docs/multimodal-table/global-index/btree.mdx
b/docs/docs/multimodal-table/global-index/btree.mdx
index c019630fab..d73fdbfc9b 100644
--- a/docs/docs/multimodal-table/global-index/btree.mdx
+++ b/docs/docs/multimodal-table/global-index/btree.mdx
@@ -99,7 +99,7 @@ added_files = table.create_global_index(
"name",
index_type="btree",
partitions=[{"dt": "2026-06-18"}, {"dt": "2026-06-19"}],
- options={"sorted-index.records-per-range": "10000000"},
+ options={"sorted-index.records-per-file": "25000000"},
)
print(added_files)
```
@@ -149,7 +149,7 @@ print(pa_table)
| Option | Default | Description |
|---|---|---|
-| `sorted-index.records-per-range` | `10000000` | Expected number of records
per sorted global index file for BTree, Bitmap, and Multivalue builds. |
+| `sorted-index.records-per-file` | `25000000` | Expected number of records
per sorted global index file for BTree, Bitmap, and Multivalue builds. |
| `sorted-index.build.max-parallelism` | `4096` | Maximum Flink or Spark
parallelism for building sorted global indexes. |
| `btree-index.block-size` | `64 kb` | Block size used by BTree index files. |
| `btree-index.bloom-filter.enabled` | `false` | Whether to write a Bloom
filter to accelerate BTree equality and `IN` lookups. |
diff --git a/docs/docs/multimodal-table/global-index/manage-indexes.mdx
b/docs/docs/multimodal-table/global-index/manage-indexes.mdx
index 24f2e70a54..07cd302010 100644
--- a/docs/docs/multimodal-table/global-index/manage-indexes.mdx
+++ b/docs/docs/multimodal-table/global-index/manage-indexes.mdx
@@ -79,7 +79,7 @@ added_files = table.create_global_index(
"name",
index_type="btree",
partitions=[{"dt": "2026-06-18"}, {"dt": "2026-06-19"}],
- options={"sorted-index.records-per-range": "10000000"},
+ options={"sorted-index.records-per-file": "25000000"},
)
```
@@ -451,7 +451,7 @@ These table options affect global index build and read
behavior:
| `full-text-index.search-mode` | `fast` | Search mode for full-text queries. |
| `global-index.external-path` | Not set | Root directory for global index
files. If not set, files are stored under the table index directory. |
| `global-index.column-update-action` | `THROW_ERROR` | Update policy:
`THROW_ERROR`, `DROP_PARTITION_INDEX`, or `IGNORE`. See [Update Indexed
Columns](#update-indexed-columns) for refresh requirements and engine
differences. |
-| `sorted-index.records-per-range` | `10000000` | Expected number of records
per sorted global index file for BTree, Bitmap, and Multivalue builds. |
+| `sorted-index.records-per-file` | `25000000` | Expected number of records
per sorted global index file for BTree, Bitmap, and Multivalue builds. Falls
back to `sorted-index.records-per-range`, then `btree-index.records-per-range`.
|
| `sorted-index.build.max-parallelism` | `4096` | Maximum Flink or Spark
parallelism for building sorted global indexes. |
| `global-index.row-count-per-shard` | `100000` | Target row count per shard
for non-sorted global index builds such as vector and full-text indexes. |
| `global-index.build.parallelism` | `1` | Number of shards built concurrently
by the local PyPaimon builder. Each shard may use native worker threads, so
increase this value conservatively. |
diff --git a/docs/docs/multimodal-table/global-index/multivalue.mdx
b/docs/docs/multimodal-table/global-index/multivalue.mdx
index 83606e1192..c64cf89764 100644
--- a/docs/docs/multimodal-table/global-index/multivalue.mdx
+++ b/docs/docs/multimodal-table/global-index/multivalue.mdx
@@ -57,7 +57,7 @@ CALL sys.create_global_index(
table => 'db.my_table',
index_column => 'tags',
index_type => 'multivalue',
- options => 'sorted-index.records-per-range=10000000'
+ options => 'sorted-index.records-per-file=25000000'
);
```
@@ -110,12 +110,12 @@ CALL sys.drop_global_index(
| Option | Default | Description |
|---|---|---|
-| `sorted-index.records-per-range` | `10000000` | Expected number of source
records per generated Multivalue index file. |
+| `sorted-index.records-per-file` | `25000000` | Expected number of source
records per generated Multivalue index file. |
| `sorted-index.build.max-parallelism` | `4096` | Maximum Flink or Spark
parallelism for building the index. |
| `multivalue-index.dictionary-block-size` | `16 kb` | Target size of the
block-indexed element dictionary. |
| `multivalue-index.compression` | `none` | Compression algorithm for
dictionary blocks. |
| `multivalue-index.compression-level` | `1` | Compression level used by
codecs which support levels. |
The expanded entries use the engine's spillable sort, and the writer keeps
only the current
-element's posting list in memory. `sorted-index.records-per-range` bounds the
source-row range—and
+element's posting list in memory. `sorted-index.records-per-file` bounds the
source-row range—and
therefore the largest posting list—owned by one Data Evolution index file.
diff --git a/docs/docs/spark/procedures/indexes.md
b/docs/docs/spark/procedures/indexes.md
index ded7675e9c..ea4032bf23 100644
--- a/docs/docs/spark/procedures/indexes.md
+++ b/docs/docs/spark/procedures/indexes.md
@@ -63,7 +63,7 @@ CALL sys.create_global_index(
table => 'default.T',
index_column => 'tag',
index_type => 'bitmap',
- options => 'sorted-index.records-per-range=1000000'
+ options => 'sorted-index.records-per-file=1000000'
);
CALL sys.create_global_index(table => 'default.T', index_column => 'content',
index_type => 'fm');
diff --git a/docs/generated/core_configuration.html
b/docs/generated/core_configuration.html
index 231c032471..750f6131fa 100644
--- a/docs/generated/core_configuration.html
+++ b/docs/generated/core_configuration.html
@@ -556,7 +556,7 @@ under the License.
</tr>
<tr>
<td><h5>data-evolution.reassign.skip-contiguous-row-count</h5></td>
- <td style="word-wrap: break-word;">1000000000</td>
+ <td style="word-wrap: break-word;">20000000000</td>
<td>Long</td>
<td>Strictly contiguous same-partition logical row-id runs
containing more than this number of rows are excluded from row-id reassignment.
Set to 0 to disable this filtering.</td>
</tr>
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
index e1ccb5ed2c..9818e68d9c 100644
--- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
@@ -2618,7 +2618,7 @@ public class CoreOptions implements Serializable {
public static final ConfigOption<Long>
DATA_EVOLUTION_REASSIGN_SKIP_CONTIGUOUS_ROW_COUNT =
key("data-evolution.reassign.skip-contiguous-row-count")
.longType()
- .defaultValue(1_000_000_000L)
+ .defaultValue(20_000_000_000L)
.withDescription(
"Strictly contiguous same-partition logical row-id
runs containing "
+ "more than this number of rows are
excluded from row-id "
@@ -5015,6 +5015,7 @@ public class CoreOptions implements Serializable {
private Options primaryKeySortedIndexOptions(
String column, String optionFamily, String algorithmPrefix) {
Options resolved = new Options(toConfiguration().toMap());
+ resolved.remove("sorted-index.records-per-file");
resolved.remove("sorted-index.records-per-range");
String optionKey = "fields." + column + "." + optionFamily +
".index.options";
String serialized = options.get(optionKey);
diff --git
a/paimon-common/src/main/java/org/apache/paimon/globalindex/sorted/SortedIndexOptions.java
b/paimon-common/src/main/java/org/apache/paimon/globalindex/sorted/SortedIndexOptions.java
index 4c437d890a..2c3691f5a6 100644
---
a/paimon-common/src/main/java/org/apache/paimon/globalindex/sorted/SortedIndexOptions.java
+++
b/paimon-common/src/main/java/org/apache/paimon/globalindex/sorted/SortedIndexOptions.java
@@ -24,11 +24,12 @@ import org.apache.paimon.options.ConfigOptions;
/** Options for sorted global index build. */
public class SortedIndexOptions {
- public static final ConfigOption<Long> SORTED_INDEX_RECORDS_PER_RANGE =
- ConfigOptions.key("sorted-index.records-per-range")
+ public static final ConfigOption<Long> SORTED_INDEX_RECORDS_PER_FILE =
+ ConfigOptions.key("sorted-index.records-per-file")
.longType()
- .defaultValue(10_000_000L)
- .withFallbackKeys("btree-index.records-per-range")
+ .defaultValue(25_000_000L)
+ .withFallbackKeys(
+ "sorted-index.records-per-range",
"btree-index.records-per-range")
.withDescription("The expected number of records per
sorted index file.");
public static final ConfigOption<Integer>
SORTED_INDEX_BUILD_MAX_PARALLELISM =
diff --git
a/paimon-common/src/test/java/org/apache/paimon/globalindex/sorted/SortedIndexOptionsTest.java
b/paimon-common/src/test/java/org/apache/paimon/globalindex/sorted/SortedIndexOptionsTest.java
index 8ad1c55793..f01a11c537 100644
---
a/paimon-common/src/test/java/org/apache/paimon/globalindex/sorted/SortedIndexOptionsTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/globalindex/sorted/SortedIndexOptionsTest.java
@@ -28,9 +28,29 @@ import static org.assertj.core.api.Assertions.assertThat;
class SortedIndexOptionsTest {
@Test
- void testDefaultRecordsPerRange() {
-
assertThat(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_RANGE.defaultValue())
- .isEqualTo(10_000_000L);
+ void testDefaultRecordsPerFile() {
+
assertThat(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_FILE.defaultValue())
+ .isEqualTo(25_000_000L);
+ }
+
+ @Test
+ void testRecordsPerFileOption() {
+ Options options = new Options();
+ options.setString("sorted-index.records-per-file", "300");
+
+
assertThat(options.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_FILE)).isEqualTo(300L);
+ }
+
+ @Test
+ void testRecordsPerFileFallbackPriority() {
+ Options options = new Options();
+ options.setString("btree-index.records-per-range", "100");
+ options.setString("sorted-index.records-per-range", "200");
+
+
assertThat(options.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_FILE)).isEqualTo(200L);
+
+ options.setString("sorted-index.records-per-file", "300");
+
assertThat(options.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_FILE)).isEqualTo(300L);
}
@Test
@@ -39,7 +59,7 @@ class SortedIndexOptionsTest {
options.setString("btree-index.records-per-range", "100");
options.setString("btree-index.build.max-parallelism", "8");
-
assertThat(options.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_RANGE)).isEqualTo(100L);
+
assertThat(options.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_FILE)).isEqualTo(100L);
assertThat(options.get(SortedIndexOptions.SORTED_INDEX_BUILD_MAX_PARALLELISM)).isEqualTo(8);
}
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriter.java
b/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriter.java
index 52cfd6479f..b0bd408d87 100644
---
a/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriter.java
+++
b/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriter.java
@@ -76,7 +76,7 @@ public class SortedGlobalIndexWriter implements Serializable {
this.rowType = this.table.rowType();
this.options = options;
this.recordsPerRange =
- (long)
(options.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_RANGE) * FLOATING);
+ (long)
(options.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_FILE) * FLOATING);
}
public SortedGlobalIndexWriter withIndexField(String indexField) {
diff --git
a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexOptionsTest.java
b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexOptionsTest.java
index dc78a087df..2035fb7805 100644
---
a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexOptionsTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexOptionsTest.java
@@ -81,6 +81,7 @@ class PrimaryKeySortedIndexOptionsTest {
@Test
void testResolvesBTreeIndexAndSortOptions() {
Map<String, String> values = new HashMap<>();
+ values.put("sorted-index.records-per-file", "20");
values.put("sorted-index.records-per-range", "10");
values.put("write-buffer-size", "8 mb");
values.put("page-size", "32 kb");
@@ -97,6 +98,7 @@ class PrimaryKeySortedIndexOptionsTest {
assertThat(options.get("local-sort.max-num-file-handles")).isEqualTo("16");
assertThat(options.get("spill-compression")).isEqualTo("lz4");
assertThat(options.get("write-buffer-spill.max-disk-size")).isEqualTo("1 gb");
+ assertThat(options.get("sorted-index.records-per-file")).isNull();
assertThat(options.get("sorted-index.records-per-range")).isNull();
}
diff --git
a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/SortedIndexTopoBuilder.java
b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/SortedIndexTopoBuilder.java
index c5bcb89b0a..67a73da986 100644
---
a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/SortedIndexTopoBuilder.java
+++
b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/SortedIndexTopoBuilder.java
@@ -200,7 +200,7 @@ public class SortedIndexTopoBuilder {
// 3. Calculate maximum parallelism bound
long recordsPerRange =
-
userOptions.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_RANGE);
+
userOptions.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_FILE);
int maxParallelism =
userOptions.get(SortedIndexOptions.SORTED_INDEX_BUILD_MAX_PARALLELISM);
diff --git
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/CreateGlobalIndexProcedureTest.java
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/CreateGlobalIndexProcedureTest.java
index 025623bcec..bdd6f2e9a1 100644
---
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/CreateGlobalIndexProcedureTest.java
+++
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/CreateGlobalIndexProcedureTest.java
@@ -42,11 +42,11 @@ public class CreateGlobalIndexProcedureTest {
Options userOptions =
CreateGlobalIndexProcedure.createUserOptions(
tableOptions,
- SortedIndexOptions.SORTED_INDEX_RECORDS_PER_RANGE.key()
+ SortedIndexOptions.SORTED_INDEX_RECORDS_PER_FILE.key()
+ "=200;procedure-only=procedure-value");
assertThat(userOptions.get(BTreeIndexOptions.BTREE_INDEX_COMPRESSION)).isEqualTo("zstd");
-
assertThat(userOptions.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_RANGE))
+
assertThat(userOptions.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_FILE))
.isEqualTo(200L);
assertThat(userOptions.get("unrelated-table-option")).isEqualTo("table-value");
assertThat(userOptions.get("procedure-only")).isEqualTo("procedure-value");
diff --git a/paimon-python/pypaimon/common/options/core_options.py
b/paimon-python/pypaimon/common/options/core_options.py
index 79b0202125..674cc1dcf6 100644
--- a/paimon-python/pypaimon/common/options/core_options.py
+++ b/paimon-python/pypaimon/common/options/core_options.py
@@ -1057,10 +1057,10 @@ class CoreOptions:
)
)
- SORTED_INDEX_RECORDS_PER_RANGE: ConfigOption[int] = (
- ConfigOptions.key("sorted-index.records-per-range")
+ SORTED_INDEX_RECORDS_PER_FILE: ConfigOption[int] = (
+ ConfigOptions.key("sorted-index.records-per-file")
.long_type()
- .default_value(10_000_000)
+ .default_value(25_000_000)
.with_description("The expected number of records per sorted global
index file.")
)
@@ -1835,6 +1835,7 @@ class CoreOptions:
def _primary_key_sorted_index_options(
self, column: str, option_family: str, algorithm_prefix: str) ->
Options:
resolved = dict(self.options.to_map())
+ resolved.pop("sorted-index.records-per-file", None)
resolved.pop("sorted-index.records-per-range", None)
option_key = "fields.%s.%s.index.options" % (column, option_family)
serialized = self.options.to_map().get(option_key)
@@ -1892,9 +1893,13 @@ class CoreOptions:
return self.options.get(CoreOptions.BTREE_INDEX_BLOOM_FILTER_ENABLED)
def sorted_index_records_per_range(self) -> int:
- if self.options.contains(CoreOptions.SORTED_INDEX_RECORDS_PER_RANGE):
- return self.options.get(CoreOptions.SORTED_INDEX_RECORDS_PER_RANGE)
- return self.options.get(CoreOptions.BTREE_INDEX_RECORDS_PER_RANGE)
+ option = CoreOptions.SORTED_INDEX_RECORDS_PER_FILE
+ for key in (option.key(), "sorted-index.records-per-range",
+ CoreOptions.BTREE_INDEX_RECORDS_PER_RANGE.key()):
+ value = self.options.to_map().get(key)
+ if value is not None:
+ return OptionsUtils.convert_to_long(value)
+ return option.default_value()
def bitmap_index_fallback_scan_max_size(self) -> int:
return self.options.get(
diff --git a/paimon-python/pypaimon/globalindex/create_global_index.py
b/paimon-python/pypaimon/globalindex/create_global_index.py
index 60caf4e8bd..0671ab09e1 100644
--- a/paimon-python/pypaimon/globalindex/create_global_index.py
+++ b/paimon-python/pypaimon/globalindex/create_global_index.py
@@ -103,7 +103,7 @@ _SORTED_INDEX_IDENTIFIERS = (BTREE_IDENTIFIER,
BITMAP_IDENTIFIER)
_GENERIC_INDEX_IDENTIFIERS = tuple(VINDEX_IDENTIFIERS) + (
FULL_TEXT_IDENTIFIER,
)
-_SORTED_INDEX_RECORDS_PER_RANGE_FLOATING = 1.2
+_SORTED_INDEX_RECORDS_PER_FILE_FLOATING = 1.2
class GlobalIndexBuilder:
@@ -218,10 +218,10 @@ class GlobalIndexBuilder:
configured_records_per_range = (
self._core_options.sorted_index_records_per_range())
if configured_records_per_range <= 0:
- raise ValueError("sorted-index.records-per-range must be
positive.")
+ raise ValueError("sorted-index.records-per-file must be positive.")
records_per_range = int(
configured_records_per_range
- * _SORTED_INDEX_RECORDS_PER_RANGE_FLOATING
+ * _SORTED_INDEX_RECORDS_PER_FILE_FLOATING
)
messages = []
diff --git a/paimon-python/pypaimon/tests/daft/daft_integration_test.py
b/paimon-python/pypaimon/tests/daft/daft_integration_test.py
index 9e9ea55a06..3320c032a4 100644
--- a/paimon-python/pypaimon/tests/daft/daft_integration_test.py
+++ b/paimon-python/pypaimon/tests/daft/daft_integration_test.py
@@ -426,7 +426,8 @@ def
test_native_read_handles_nested_type_widening(catalog_options):
pa.decimal128(10, 2),
AtomicType("DECIMAL(10, 2)"),
[decimal.Decimal("1.2355"), decimal.Decimal("-4.5678")],
- [decimal.Decimal("1.23"), decimal.Decimal("-4.56")],
+ # PyPaimon follows Java's HALF_UP rounding when reducing scale.
+ [decimal.Decimal("1.24"), decimal.Decimal("-4.57")],
id="decimal-scale-down",
),
pytest.param(
diff --git a/paimon-python/pypaimon/tests/global_index_build_test.py
b/paimon-python/pypaimon/tests/global_index_build_test.py
index 9735d5b1c0..9876d486fd 100644
--- a/paimon-python/pypaimon/tests/global_index_build_test.py
+++ b/paimon-python/pypaimon/tests/global_index_build_test.py
@@ -454,7 +454,7 @@ class GlobalIndexBuildTest(
options={'bitmap-index.compression': 'lz4'},
)
- def test_sorted_index_records_per_range_matches_java_floating_factor(self):
+ def test_sorted_index_records_per_file_matches_java_floating_factor(self):
table = self._create_table()
rows = list(range(12))
self._write_arrow(table, pa.table(
@@ -469,7 +469,7 @@ class GlobalIndexBuildTest(
added = table.create_global_index(
'id',
- options={'sorted-index.records-per-range': '10'},
+ options={'sorted-index.records-per-file': '10'},
)
self.assertEqual(1, added)
diff --git a/paimon-python/pypaimon/tests/sorted_index_options_test.py
b/paimon-python/pypaimon/tests/sorted_index_options_test.py
new file mode 100644
index 0000000000..611a92eaa6
--- /dev/null
+++ b/paimon-python/pypaimon/tests/sorted_index_options_test.py
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pytest
+
+from pypaimon.common.options.core_options import CoreOptions
+
+
[email protected]("options, expected", [
+ ({}, 25_000_000),
+ ({"sorted-index.records-per-file": "300"}, 300),
+ ({"sorted-index.records-per-range": "200"}, 200),
+ ({"btree-index.records-per-range": "100"}, 100),
+ ({"sorted-index.records-per-range": "200",
+ "btree-index.records-per-range": "100"}, 200),
+ ({"sorted-index.records-per-file": "300",
+ "sorted-index.records-per-range": "200",
+ "btree-index.records-per-range": "100"}, 300),
+])
+def test_sorted_index_records_per_file_default_and_fallback(options, expected):
+ assert CoreOptions.from_dict(options).sorted_index_records_per_range() ==
expected
+
+
[email protected]("index_type", ["btree", "bitmap"])
+def test_primary_key_options_exclude_global_index_file_size(index_type):
+ core_options = CoreOptions.from_dict({
+ "sorted-index.records-per-file": "300",
+ "sorted-index.records-per-range": "200",
+ "write-buffer-size": "8 mb",
+ })
+
+ resolve_options = getattr(core_options, "primary_key_%s_index_options" %
index_type)
+ options = resolve_options("name").to_map()
+
+ assert "sorted-index.records-per-file" not in options
+ assert "sorted-index.records-per-range" not in options
+ assert options["write-buffer-size"] == "8 mb"
diff --git
a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/sorted/SortedIndexTopoBuilder.java
b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/sorted/SortedIndexTopoBuilder.java
index 0ee778de3e..7431d062f8 100644
---
a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/sorted/SortedIndexTopoBuilder.java
+++
b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/sorted/SortedIndexTopoBuilder.java
@@ -127,7 +127,7 @@ public class SortedIndexTopoBuilder implements
GlobalIndexTopologyBuilder {
}
// Calculate maximum parallelism bound
- long recordsPerRange =
options.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_RANGE);
+ long recordsPerRange =
options.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_FILE);
int maxParallelism =
options.get(SortedIndexOptions.SORTED_INDEX_BUILD_MAX_PARALLELISM);
List<CommitMessage> allMessages = new ArrayList<>();
diff --git
a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/procedure/CreateGlobalIndexProcedureTest.java
b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/procedure/CreateGlobalIndexProcedureTest.java
index f25bde2ba5..e076869220 100644
---
a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/procedure/CreateGlobalIndexProcedureTest.java
+++
b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/procedure/CreateGlobalIndexProcedureTest.java
@@ -64,11 +64,11 @@ public class CreateGlobalIndexProcedureTest {
Options userOptions =
CreateGlobalIndexProcedure.createUserOptions(
tableOptions,
- SortedIndexOptions.SORTED_INDEX_RECORDS_PER_RANGE.key()
+ SortedIndexOptions.SORTED_INDEX_RECORDS_PER_FILE.key()
+ "=200, procedure-only=procedure-value");
assertThat(userOptions.get(BTreeIndexOptions.BTREE_INDEX_COMPRESSION)).isEqualTo("zstd");
-
assertThat(userOptions.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_RANGE))
+
assertThat(userOptions.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_FILE))
.isEqualTo(200L);
assertThat(userOptions.get("unrelated-table-option")).isEqualTo("table-value");
assertThat(userOptions.get("procedure-only")).isEqualTo("procedure-value");
