This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit d244eb2ac93995afb6083a13eee4a0a5151995f1 Merge: ca6f48c659 fc7842ec1f Author: Michael Blow <[email protected]> AuthorDate: Thu Sep 4 10:15:24 2025 -0400 Merge branch 'gerrit/ionic' into 'gerrit/phoenix' * [ASTERIXDB-3633][EXT] Support file splits while reading delta tables * [ASTERIXDB-3636][STO] Add ColumnBufferPool to prevent OOM during high-volume column ingestion * [NO ISSUE][OTH] Add extensions for drop transaction pre-commit * [NO ISSUE][STO] Delete masked files before reading resource files Change-Id: I89d309e1fdb05536a8410cdfd3b2447cde64ba06 Ext-ref: MB-67819 .../provider/SqlppCompilationProvider.java | 3 +- .../apache/asterix/app/nc/NCAppRuntimeContext.java | 22 + .../app/nc/task/LocalStorageCleanupTask.java | 2 +- .../asterix/app/translator/QueryTranslator.java | 16 + .../deltalake-file-nine.02.query.sqlpp | 23 + .../deltalake-multiple_file_read.02.query.sqlpp | 23 + .../deltalake-partitioned-file-read.03.query.sqlpp | 23 + .../api/cluster_state_1/cluster_state_1.1.regexadm | 5 + .../cluster_state_1_full.1.regexadm | 5 + .../cluster_state_1_less.1.regexadm | 5 + .../deltalake-file-nine/deltalake-file-nine.02.adm | 9 + .../deltalake-multiple-file-read/read-data.3.adm | 4 + .../read-data.3.adm | 3 + .../out/AbstractMultiBufferBytesOutputStream.java | 10 +- .../out/MultiTemporaryBufferBytesOutputStream.java | 57 ++- .../operation/lsm/flush/FlushColumnMetadata.java | 6 +- .../lsm/flush/FlushColumnTupleWriter.java | 6 + .../lsm/merge/MergeColumnTupleWriter.java | 6 + .../lsm/merge/MergeColumnWriteMetadata.java | 2 +- .../column/common/buffer/NoOpWriteMultiPageOp.java | 10 + .../column/common/buffer/TestWriteMultiPageOp.java | 10 + .../asterix/common/api/INcApplicationContext.java | 3 + .../asterix/common/config/CompilerProperties.java | 8 +- .../asterix/common/config/StorageProperties.java | 42 +- .../input/record/reader/aws/delta/DeltaEngine.java | 47 ++ .../reader/aws/delta/DeltaFileRecordReader.java | 75 ++- .../reader/aws/delta/DeltaParquetFileReader.java | 187 ++++++++ .../reader/aws/delta/DeltaParquetHandler.java | 80 ++++ .../reader/aws/delta/DeltaReaderFactory.java | 95 +++- .../reader/aws/delta/SerializableFileSplit.java | 45 ++ .../metadata/declared/DatasetDataSource.java | 11 + .../runtime/utils/RuntimeComponentsProvider.java | 6 + .../hyracks/control/common/config/OptionTypes.java | 94 +++- .../btree/helper/BTreeHelperStorageManager.java | 6 + .../examples/btree/helper/RuntimeContext.java | 9 + .../column/api/AbstractColumnTupleWriter.java | 5 + .../btree/column/api/IColumnWriteMultiPageOp.java | 11 + .../dataflow/LSMColumnBTreeLocalResource.java | 5 +- .../lsm/btree/column/impls/btree/ColumnBTree.java | 10 +- .../column/impls/btree/ColumnBTreeBulkloader.java | 315 +++++++++---- .../column/impls/btree/ColumnBTreeFactory.java | 10 +- .../lsm/btree/column/impls/lsm/LSMColumnBTree.java | 22 +- .../lsm/btree/column/utils/LSMColumnBTreeUtil.java | 33 +- .../hyracks/storage/common/IStorageManager.java | 6 + .../common/buffercache/ColumnBufferPool.java | 243 ++++++++++ .../common/buffercache/FreeColumnBufferPool.java | 36 +- .../common/buffercache/IColumnBufferPool.java | 78 ++++ .../hyracks/test/support/TestStorageManager.java | 6 + .../support/TestStorageManagerComponentHolder.java | 21 + .../storage/common/ColumnBufferPoolTest.java | 514 +++++++++++++++++++++ 50 files changed, 2079 insertions(+), 194 deletions(-) diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index bdbcbfb39a,2b75477c6e..b27d791d70 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@@ -3600,6 -3608,8 +3608,7 @@@ public class QueryTranslator extends Ab MetadataManager.INSTANCE.dropDatatype(mdTxnCtx, inlineType.getDatabaseName(), inlineType.getDataverseName(), inlineType.getName()); } - beforeDropTxnCommit(metadataProvider, mdTxnCtx, - EntityDetails.newFunction(databaseName, dataverseName, signature.getName(), signature.getArity())); ++ beforeDropTxnCommit(metadataProvider, mdTxnCtx, EntityDetails.newFunction(signature)); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); return true; } catch (Exception e) {
