This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 4e72caf4fc1b3e37c07e4afa538116136828da37 Merge: 6bf5bf5974 f244d4e0f8 Author: Michael Blow <[email protected]> AuthorDate: Mon Sep 8 14:01:14 2025 -0400 Merge branch 'gerrit/phoenix' into 'master' * [NO ISSUE][HYR][MISC] Update commons-lang3 to 3.18.0 for CVE-2025-48924 * [NO ISSUE][OTH] Update parquet version * [NO ISSUE][HYR] Update Jackson to 2.19.2 to address CVEs * [NO ISSUE][*DB][TEST] avoid exceptions on duplicate s3mock shutdowns * [NO ISSUE][MISC] Update nimbus-jose-jwt to 10.0.2 to address CVE * [NO ISSUE] Reset operators after join selectivity estimation * [NO ISSUE][EXT] Upgrage Delta Kernel API to v4.0.0 * [ASTERIXDB-3633][EXT] Support file splits while reading delta tables * [ASTERIXDB-3636][STO] Add ColumnBufferPool to prevent OOM during high-volume column ingestion * [NO ISSUE][OTH] Add extensions for drop transaction pre-commit * [NO ISSUE][STO] Delete masked files before reading resource files * [ASTERIXDB-3638][COMP] Consolidate same scan operators * [ASTERIXDB-3639]: add datasetName for index * [NO ISSUE] Add missing licenses Change-Id: I97f224b07f35e671e16547b6b89521044ea25545 .../provider/SqlppCompilationProvider.java | 3 +- .../apache/asterix/optimizer/rules/cbo/Stats.java | 6 +- .../optimizer/rules/pushdown/PushdownContext.java | 1 - ...ateProjectionAndFilterExpressionsProcessor.java | 11 + asterixdb/asterix-app/pom.xml | 7 +- .../apache/asterix/app/nc/NCAppRuntimeContext.java | 22 + .../app/nc/task/LocalStorageCleanupTask.java | 2 +- .../asterix/app/translator/QueryTranslator.java | 18 +- .../apache/asterix/api/common/LocalCloudUtil.java | 9 +- .../asterix/common/config/ConfigUsageTest.java | 2 +- .../deltalake/DeltaTableGenerator.java | 38 +- .../same-datasource-diff-scan.000.ddl.sqlpp} | 36 +- .../same-datasource-diff-scan.001.update.sqlpp} | 38 +- .../same-datasource-diff-scan.002.query.sqlpp} | 29 +- .../same-datasource-diff-scan.003.query.sqlpp} | 29 +- .../same-datasource-diff-scan.004.query.sqlpp} | 31 +- .../same-datasource-diff-scan.005.query.sqlpp} | 33 +- .../same-datasource-diff-scan.006.query.sqlpp} | 34 +- .../same-datasource-diff-scan.007.query.sqlpp} | 31 +- .../same-datasource-diff-scan.999.ddl.sqlpp} | 22 +- .../deltalake-file-nine.02.query.sqlpp} | 23 +- .../deltalake-multiple_file_read.02.query.sqlpp} | 23 +- ...deltalake-partitioned-file-read.03.query.sqlpp} | 23 +- .../api/cluster_state_1/cluster_state_1.1.regexadm | 11 + .../cluster_state_1_full.1.regexadm | 11 + .../cluster_state_1_less.1.regexadm | 11 + .../same-datasource-diff-scan.002.adm | 5 + .../same-datasource-diff-scan.003.adm | 3 + .../same-datasource-diff-scan.004.plan | 64 +++ .../same-datasource-diff-scan.005.plan | 94 ++++ .../same-datasource-diff-scan.006.plan | 96 ++++ .../same-datasource-diff-scan.007.plan | 64 +++ .../deltalake-file-nine/deltalake-file-nine.02.adm | 9 + .../deltalake-multiple-file-read/read-data.3.adm | 4 + .../read-data.3.adm | 3 + .../same-datasource-diff-scan.004.plan | 64 +++ .../same-datasource-diff-scan.005.plan | 94 ++++ .../same-datasource-diff-scan.006.plan | 94 ++++ .../same-datasource-diff-scan.007.plan | 74 +++ .../same-datasource-diff-scan.004.plan | 72 +++ .../same-datasource-diff-scan.005.plan | 106 +++++ .../same-datasource-diff-scan.006.plan | 108 +++++ .../same-datasource-diff-scan.007.plan | 64 +++ .../same-datasource-diff-scan.004.plan | 72 +++ .../same-datasource-diff-scan.005.plan | 106 +++++ .../same-datasource-diff-scan.006.plan | 108 +++++ .../same-datasource-diff-scan.007.plan | 64 +++ .../src/test/resources/runtimets/sqlpp_queries.xml | 7 +- asterixdb/asterix-cloud/pom.xml | 2 +- .../asterix/cloud/AbstractCloudIOManager.java | 1 + .../apache/asterix/cloud/clients/CloudFile.java | 7 +- .../apache/asterix/cloud/util/CloudFileUtil.java | 6 +- .../out/AbstractMultiBufferBytesOutputStream.java | 10 +- .../out/MultiTemporaryBufferBytesOutputStream.java | 57 ++- .../operation/lsm/flush/FlushColumnMetadata.java | 6 +- .../lsm/flush/FlushColumnTupleWriter.java | 6 + .../lsm/merge/MergeColumnTupleWriter.java | 6 + .../lsm/merge/MergeColumnWriteMetadata.java | 2 +- .../column/common/buffer/NoOpWriteMultiPageOp.java | 10 + .../column/common/buffer/TestWriteMultiPageOp.java | 10 + .../asterix/common/api/INcApplicationContext.java | 3 + .../asterix/common/api/IPropertiesFactory.java | 3 + .../asterix/common/config/AsterixProperties.java | 10 +- .../asterix/common/config/CompilerProperties.java | 8 +- .../asterix/common/config/JacksonProperties.java | 138 ++++++ .../asterix/common/config/PropertiesFactory.java | 5 + .../asterix/common/config/StorageProperties.java | 42 +- asterixdb/asterix-external-data/pom.xml | 13 +- .../reader/aws/delta/DefaultExpressionUtils.java | 366 --------------- .../input/record/reader/aws/delta/DeltaEngine.java | 23 +- .../reader/aws/delta/DeltaExpressionEvaluator.java | 339 -------------- .../reader/aws/delta/DeltaFileRecordReader.java | 286 +++--------- .../reader/aws/delta/DeltaParquetFileReader.java | 187 ++++++++ .../reader/aws/delta/DeltaParquetHandler.java | 80 ++++ .../reader/aws/delta/DeltaPredicateEvaluator.java | 66 --- .../reader/aws/delta/DeltaReaderFactory.java | 110 ++++- .../reader/aws/delta/ElementAtEvaluator.java | 151 ------ .../record/reader/aws/delta/ExpressionVisitor.java | 136 ------ .../reader/aws/delta/ImplicitCastExpression.java | 267 ----------- .../reader/aws/delta/PartitionValueEvaluator.java | 136 ------ ...sionHandler.java => SerializableFileSplit.java} | 33 +- .../metadata/declared/DatasetDataSource.java | 11 + .../asterix/metadata/entities/EntityDetails.java | 22 +- .../ColumnDatasetProjectionFiltrationInfo.java | 2 +- .../runtime/utils/RuntimeComponentsProvider.java | 6 + asterixdb/asterix-server/pom.xml | 8 +- asterixdb/pom.xml | 105 ++++- ...microsoft.azure--msal4j--1.22.0_MIT_License.txt | 21 + ...4j-persistence-extension--1.3.0_MIT_License.txt | 3 +- ...content.com_luben_zstd-jni_v1.5.6-6_LICENSE.txt | 26 ++ .../content/www.bouncycastle.org_licence.html.txt | 19 +- .../org/apache/hyracks/control/cc/CCDriver.java | 5 +- .../control/common/config/ConfigManager.java | 7 +- .../hyracks/control/common/config/OptionTypes.java | 94 +++- .../control/common/controllers/NCConfig.java | 3 +- .../btree/helper/BTreeHelperStorageManager.java | 6 + .../examples/btree/helper/RuntimeContext.java | 9 + hyracks-fullstack/hyracks/hyracks-hdfs/pom.xml | 8 - .../column/api/AbstractColumnTupleWriter.java | 5 + .../btree/column/api/IColumnWriteMultiPageOp.java | 11 + .../dataflow/LSMColumnBTreeLocalResource.java | 5 +- .../lsm/btree/column/impls/btree/ColumnBTree.java | 10 +- .../column/impls/btree/ColumnBTreeBulkloader.java | 315 +++++++++---- .../column/impls/btree/ColumnBTreeFactory.java | 10 +- .../lsm/btree/column/impls/lsm/LSMColumnBTree.java | 22 +- .../lsm/btree/column/utils/LSMColumnBTreeUtil.java | 33 +- .../hyracks/storage/common/IStorageManager.java | 6 + .../common/buffercache/ColumnBufferPool.java | 243 ++++++++++ .../common/buffercache/FreeColumnBufferPool.java | 36 +- .../common/buffercache/IColumnBufferPool.java | 78 ++++ .../hyracks/test/support/TestStorageManager.java | 6 + .../support/TestStorageManagerComponentHolder.java | 21 + .../storage/common/ColumnBufferPoolTest.java | 514 +++++++++++++++++++++ .../java/org/apache/hyracks/util/StorageUtil.java | 23 +- .../org/apache/hyracks/util/StorageUnitTest.java | 17 +- hyracks-fullstack/pom.xml | 122 ++++- 116 files changed, 3984 insertions(+), 2252 deletions(-) diff --cc asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/Stats.java index deac2feff8,0d0558e1a7..7ffffd53b7 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/Stats.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/Stats.java @@@ -276,12 -254,11 +276,11 @@@ public class Stats private double findJoinSelFromSamples(ILogicalOperator left, ILogicalOperator right, Index index1, Index index2, AbstractFunctionCallExpression joinExpr, JoinOperator join) throws AlgebricksException { AbstractBinaryJoinOperator abjoin = join.getAbstractJoinOp(); - Pair<ILogicalOperator, Double> leftOutput = replaceDataSourceWithSample(left, index1); + Pair<ILogicalOperator, Double> leftOutput = replaceDataSourceWithSample(left, index1, joinExpr); - ILogicalOperator originalLeft, originalRight; - originalLeft = abjoin.getInputs().get(0).getValue(); - originalRight = abjoin.getInputs().get(1).getValue(); + ILogicalOperator originalLeft = abjoin.getInputs().get(0).getValue(); + ILogicalOperator originalRight = abjoin.getInputs().get(1).getValue(); abjoin.getInputs().get(0).setValue(leftOutput.getFirst()); - Pair<ILogicalOperator, Double> rightOutput = replaceDataSourceWithSample(right, index2); + Pair<ILogicalOperator, Double> rightOutput = replaceDataSourceWithSample(right, index2, joinExpr); abjoin.getInputs().get(1).setValue(rightOutput.getFirst()); abjoin.getCondition().setValue(joinExpr); List<List<IAObject>> result = runSamplingQuery(optCtx, abjoin);
