This is an automated email from the ASF dual-hosted git repository.
mblow pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
from 6bf5bf5974 [ASTERIXDB-3637][COMP] Simplify Index-only plan
new 7a764591f0 [NO ISSUE][*DB][TEST] avoid exceptions on duplicate s3mock
shutdowns
new 4fb5fa4aac [NO ISSUE][HYR][MISC] Update commons-lang3 to 3.18.0 for
CVE-2025-48924
new 031f238a1f [NO ISSUE][OTH] Update parquet version
new 37fd8f74f2 Merge branch 'gerrit/neo' into 'gerrit/trinity'
new c0cd3f51d3 [NO ISSUE][HYR] Update Jackson to 2.19.2 to address CVEs
new 8b342cc119 Merge branch 'gerrit/neo' into 'gerrit/trinity'
new 13f95fc15f Merge branch 'gerrit/trinity' into 'gerrit/ionic', update
aws hadoop dep
new 62e5f251af Merge branch 'gerrit/ionic' into 'gerrit/phoenix'
new 17ca94a0ee [NO ISSUE] Reset operators after join selectivity estimation
new bfe4b3a7a2 [NO ISSUE][EXT] Upgrage Delta Kernel API to v4.0.0
new 2753cf0fdb [NO ISSUE][MISC] Update nimbus-jose-jwt to 10.0.2 to
address CVE
new 4565c0d72f Merge branch 'gerrit/neo' into 'gerrit/trinity'
new 51aada131b Merge branch 'gerrit/trinity' into 'gerrit/ionic'
new ca6f48c659 Merge branch 'gerrit/ionic' into 'gerrit/phoenix'
new 3311364a51 [ASTERIXDB-3633][EXT] Support file splits while reading
delta tables
new 43137d598a [ASTERIXDB-3636][STO] Add ColumnBufferPool to prevent OOM
during high-volume column ingestion
new 49c645601e [NO ISSUE][OTH] Add extensions for drop transaction
pre-commit
new fc7842ec1f [NO ISSUE][STO] Delete masked files before reading resource
files
new d244eb2ac9 Merge branch 'gerrit/ionic' into 'gerrit/phoenix'
new e9fb62dbdd [ASTERIXDB-3638][COMP] Consolidate same scan operators
new 94d33a63dc Merge branch 'gerrit/trinity' into 'gerrit/phoenix'
new 2eb5dc5ace [ASTERIXDB-3639]: add datasetName for index
new eb82df5088 Merge branch 'gerrit/ionic' into 'gerrit/phoenix'
new 4f01464b0e [NO ISSUE] Add missing licenses
new f244d4e0f8 Merge branch 'gerrit/ionic' into 'gerrit/phoenix'
new 4e72caf4fc Merge branch 'gerrit/phoenix' into 'master'
The 26 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../provider/SqlppCompilationProvider.java | 3 +-
.../apache/asterix/optimizer/rules/cbo/Stats.java | 6 +-
.../optimizer/rules/pushdown/PushdownContext.java | 1 -
...ateProjectionAndFilterExpressionsProcessor.java | 11 +
asterixdb/asterix-app/pom.xml | 7 +-
.../apache/asterix/app/nc/NCAppRuntimeContext.java | 22 +
.../app/nc/task/LocalStorageCleanupTask.java | 2 +-
.../asterix/app/translator/QueryTranslator.java | 18 +-
.../apache/asterix/api/common/LocalCloudUtil.java | 9 +-
.../asterix/common/config/ConfigUsageTest.java | 2 +-
.../deltalake/DeltaTableGenerator.java | 38 +-
.../same-datasource-diff-scan.000.ddl.sqlpp} | 36 +-
.../same-datasource-diff-scan.001.update.sqlpp} | 38 +-
.../same-datasource-diff-scan.002.query.sqlpp} | 29 +-
.../same-datasource-diff-scan.003.query.sqlpp} | 29 +-
.../same-datasource-diff-scan.004.query.sqlpp} | 31 +-
.../same-datasource-diff-scan.005.query.sqlpp} | 33 +-
.../same-datasource-diff-scan.006.query.sqlpp} | 34 +-
.../same-datasource-diff-scan.007.query.sqlpp} | 31 +-
.../same-datasource-diff-scan.999.ddl.sqlpp} | 0
.../deltalake-file-nine.02.query.sqlpp} | 23 +-
.../deltalake-multiple_file_read.02.query.sqlpp} | 23 +-
...deltalake-partitioned-file-read.03.query.sqlpp} | 23 +-
.../api/cluster_state_1/cluster_state_1.1.regexadm | 11 +
.../cluster_state_1_full.1.regexadm | 11 +
.../cluster_state_1_less.1.regexadm | 11 +
.../same-datasource-diff-scan.002.adm | 5 +
.../same-datasource-diff-scan.003.adm | 3 +
.../same-datasource-diff-scan.004.plan | 64 +++
.../same-datasource-diff-scan.005.plan | 94 ++++
.../same-datasource-diff-scan.006.plan | 96 ++++
.../same-datasource-diff-scan.007.plan | 64 +++
...file-nine.01.adm => deltalake-file-nine.02.adm} | 0
.../{read-data.2.adm => read-data.3.adm} | 0
.../{read-data.2.adm => read-data.3.adm} | 0
.../same-datasource-diff-scan.004.plan | 64 +++
.../same-datasource-diff-scan.005.plan | 94 ++++
.../same-datasource-diff-scan.006.plan | 94 ++++
.../same-datasource-diff-scan.007.plan | 74 +++
.../same-datasource-diff-scan.004.plan | 72 +++
.../same-datasource-diff-scan.005.plan | 106 +++++
.../same-datasource-diff-scan.006.plan | 108 +++++
.../same-datasource-diff-scan.007.plan | 64 +++
.../same-datasource-diff-scan.004.plan | 72 +++
.../same-datasource-diff-scan.005.plan | 106 +++++
.../same-datasource-diff-scan.006.plan | 108 +++++
.../same-datasource-diff-scan.007.plan | 64 +++
.../src/test/resources/runtimets/sqlpp_queries.xml | 7 +-
asterixdb/asterix-cloud/pom.xml | 2 +-
.../asterix/cloud/AbstractCloudIOManager.java | 1 +
.../apache/asterix/cloud/clients/CloudFile.java | 7 +-
.../apache/asterix/cloud/util/CloudFileUtil.java | 6 +-
.../out/AbstractMultiBufferBytesOutputStream.java | 10 +-
.../out/MultiTemporaryBufferBytesOutputStream.java | 57 ++-
.../operation/lsm/flush/FlushColumnMetadata.java | 6 +-
.../lsm/flush/FlushColumnTupleWriter.java | 6 +
.../lsm/merge/MergeColumnTupleWriter.java | 6 +
.../lsm/merge/MergeColumnWriteMetadata.java | 2 +-
.../column/common/buffer/NoOpWriteMultiPageOp.java | 10 +
.../column/common/buffer/TestWriteMultiPageOp.java | 10 +
.../asterix/common/api/INcApplicationContext.java | 3 +
.../asterix/common/api/IPropertiesFactory.java | 3 +
.../asterix/common/config/AsterixProperties.java | 10 +-
.../asterix/common/config/CompilerProperties.java | 8 +-
.../asterix/common/config/JacksonProperties.java | 138 ++++++
.../asterix/common/config/PropertiesFactory.java | 5 +
.../asterix/common/config/StorageProperties.java | 42 +-
asterixdb/asterix-external-data/pom.xml | 13 +-
.../reader/aws/delta/DefaultExpressionUtils.java | 366 ---------------
.../input/record/reader/aws/delta/DeltaEngine.java | 23 +-
.../reader/aws/delta/DeltaExpressionEvaluator.java | 339 --------------
.../reader/aws/delta/DeltaFileRecordReader.java | 286 +++---------
.../reader/aws/delta/DeltaParquetFileReader.java | 187 ++++++++
.../reader/aws/delta/DeltaParquetHandler.java | 80 ++++
.../reader/aws/delta/DeltaPredicateEvaluator.java | 66 ---
.../reader/aws/delta/DeltaReaderFactory.java | 110 ++++-
.../reader/aws/delta/ElementAtEvaluator.java | 151 ------
.../record/reader/aws/delta/ExpressionVisitor.java | 136 ------
.../reader/aws/delta/ImplicitCastExpression.java | 267 -----------
.../reader/aws/delta/PartitionValueEvaluator.java | 136 ------
...sionHandler.java => SerializableFileSplit.java} | 33 +-
.../metadata/declared/DatasetDataSource.java | 11 +
.../asterix/metadata/entities/EntityDetails.java | 22 +-
.../ColumnDatasetProjectionFiltrationInfo.java | 2 +-
.../runtime/utils/RuntimeComponentsProvider.java | 6 +
asterixdb/asterix-server/pom.xml | 8 +-
asterixdb/pom.xml | 105 ++++-
...icrosoft.azure--msal4j--1.22.0_MIT_License.txt} | 0
...4j-persistence-extension--1.3.0_MIT_License.txt | 3 +-
...content.com_luben_zstd-jni_v1.5.6-6_LICENSE.txt | 26 ++
.../content/www.bouncycastle.org_licence.html.txt | 19 +-
.../org/apache/hyracks/control/cc/CCDriver.java | 5 +-
.../control/common/config/ConfigManager.java | 7 +-
.../hyracks/control/common/config/OptionTypes.java | 94 +++-
.../control/common/controllers/NCConfig.java | 3 +-
.../btree/helper/BTreeHelperStorageManager.java | 6 +
.../examples/btree/helper/RuntimeContext.java | 9 +
hyracks-fullstack/hyracks/hyracks-hdfs/pom.xml | 8 -
.../column/api/AbstractColumnTupleWriter.java | 5 +
.../btree/column/api/IColumnWriteMultiPageOp.java | 11 +
.../dataflow/LSMColumnBTreeLocalResource.java | 5 +-
.../lsm/btree/column/impls/btree/ColumnBTree.java | 10 +-
.../column/impls/btree/ColumnBTreeBulkloader.java | 315 +++++++++----
.../column/impls/btree/ColumnBTreeFactory.java | 10 +-
.../lsm/btree/column/impls/lsm/LSMColumnBTree.java | 22 +-
.../lsm/btree/column/utils/LSMColumnBTreeUtil.java | 33 +-
.../hyracks/storage/common/IStorageManager.java | 6 +
.../common/buffercache/ColumnBufferPool.java | 243 ++++++++++
.../common/buffercache/FreeColumnBufferPool.java | 36 +-
.../common/buffercache/IColumnBufferPool.java | 78 ++++
.../hyracks/test/support/TestStorageManager.java | 6 +
.../support/TestStorageManagerComponentHolder.java | 21 +
.../storage/common/ColumnBufferPoolTest.java | 514 +++++++++++++++++++++
.../java/org/apache/hyracks/util/StorageUtil.java | 23 +-
.../org/apache/hyracks/util/StorageUnitTest.java | 17 +-
hyracks-fullstack/pom.xml | 122 ++++-
116 files changed, 3946 insertions(+), 2231 deletions(-)
copy
asterixdb/{asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaEngine.java
=>
asterix-app/src/test/resources/runtimets/queries_sqlpp/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.000.ddl.sqlpp}
(57%)
copy
asterixdb/{asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaEngine.java
=>
asterix-app/src/test/resources/runtimets/queries_sqlpp/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.001.update.sqlpp}
(57%)
copy
asterixdb/{asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaEngine.java
=>
asterix-app/src/test/resources/runtimets/queries_sqlpp/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.002.query.sqlpp}
(57%)
copy
asterixdb/{asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaEngine.java
=>
asterix-app/src/test/resources/runtimets/queries_sqlpp/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.003.query.sqlpp}
(57%)
copy
asterixdb/{asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaEngine.java
=>
asterix-app/src/test/resources/runtimets/queries_sqlpp/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.004.query.sqlpp}
(57%)
copy
asterixdb/{asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaEngine.java
=>
asterix-app/src/test/resources/runtimets/queries_sqlpp/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.005.query.sqlpp}
(57%)
copy
asterixdb/{asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaEngine.java
=>
asterix-app/src/test/resources/runtimets/queries_sqlpp/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.006.query.sqlpp}
(57%)
copy
asterixdb/{asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaEngine.java
=>
asterix-app/src/test/resources/runtimets/queries_sqlpp/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.007.query.sqlpp}
(57%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{aggregate-sql/median/median.99.ddl.sqlpp
=>
column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.999.ddl.sqlpp}
(100%)
copy
asterixdb/{asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaEngine.java
=>
asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-file-nine/deltalake-file-nine.02.query.sqlpp}
(57%)
copy
asterixdb/{asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaEngine.java
=>
asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.02.query.sqlpp}
(57%)
copy
asterixdb/{asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaEngine.java
=>
asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-timestamp-partitioned-file-read/deltalake-partitioned-file-read.03.query.sqlpp}
(57%)
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.002.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.003.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.004.plan
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.005.plan
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.006.plan
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.007.plan
copy
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-file-nine/{deltalake-file-nine.01.adm
=> deltalake-file-nine.02.adm} (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-multiple-file-read/{read-data.2.adm
=> read-data.3.adm} (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-timestamp-partitioned-file-read/{read-data.2.adm
=> read-data.3.adm} (100%)
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.004.plan
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.005.plan
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.006.plan
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.007.plan
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results_full_parallelism/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.004.plan
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results_full_parallelism/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.005.plan
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results_full_parallelism/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.006.plan
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results_full_parallelism/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.007.plan
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results_less_parallelism/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.004.plan
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results_less_parallelism/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.005.plan
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results_less_parallelism/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.006.plan
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results_less_parallelism/column/pushdown/same-datasource-diff-scan/same-datasource-diff-scan.007.plan
create mode 100644
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/JacksonProperties.java
delete mode 100644
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DefaultExpressionUtils.java
delete mode 100644
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaExpressionEvaluator.java
create mode 100644
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaParquetFileReader.java
create mode 100644
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaParquetHandler.java
delete mode 100644
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaPredicateEvaluator.java
delete mode 100644
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/ElementAtEvaluator.java
delete mode 100644
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/ExpressionVisitor.java
delete mode 100644
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/ImplicitCastExpression.java
delete mode 100644
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/PartitionValueEvaluator.java
rename
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/{DeltaExpressionHandler.java
=> SerializableFileSplit.java} (52%)
copy
asterixdb/src/main/licenses/content/{raw.githubusercontent.com_AzureAD_microsoft-authentication-library-for-java_refs_tags_v1.17.1_LICENSE.txt
=> com.microsoft.azure--msal4j--1.22.0_MIT_License.txt} (100%)
create mode 100644
asterixdb/src/main/licenses/content/raw.githubusercontent.com_luben_zstd-jni_v1.5.6-6_LICENSE.txt
create mode 100644
hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ColumnBufferPool.java
copy
asterixdb/asterix-column/src/test/java/org/apache/asterix/column/common/buffer/NoOpWriteMultiPageOp.java
=>
hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/FreeColumnBufferPool.java
(55%)
create mode 100644
hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IColumnBufferPool.java
create mode 100644
hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/ColumnBufferPoolTest.java
