This is an automated email from the ASF dual-hosted git repository.
mblow pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
from d21a38bf55 [ASTERIXDB-3514][EXT]: Assume role only when temporary
credentials expire
new 6249ff58f3 [ASTERIXDB-3503][EXT] Add column filter for Delta Reader.
new 49929014b3 [ASTERIXDB-3503][EXT] Improve logic of distributing files
to partitions
new 994f901c8d [ASTERIXDB-3503][EXT] Tests for Field pushdown and delta
all type
new f11758e0d8 [NO ISSUE][HYR][STO,MISC] LSM enhancements, misc utilities
new 096385e346 [NO ISSUE][HYR][STO] Add pre-exit hook to
IFrameOperationCallback
new 33db60a5ae Merge branch 'gerrit/neo' into 'gerrit/trinity'
new 1b975b5599 Merge branch 'gerrit/trinity' into 'gerrit/goldfish'
new 9f75957c6b Merge "Merge branch 'gerrit/trinity' into
'gerrit/goldfish'" into goldfish
new e9772ce511 [NO ISSUE]: Support Reading credentials from String
new 5ddcc8f276 [ASTERIXDB-3503][EXT] Fixing Internal Error issue when
Delta table does not exists.
new 4dde600f6c [NO ISSUE][COMP] Exponential recursion in
OperatorManipulationUtil.substituteVarRec method
new 52b8df6313 Merge branch 'gerrit/neo' into 'gerrit/trinity'
new bdaf4e97e4 Merge branch 'gerrit/trinity' into 'gerrit/goldfish'
new 7026faf391 Merge "Merge branch 'gerrit/trinity' into
'gerrit/goldfish'" into goldfish
new dec9f981e4 Merge branch 'gerrit/goldfish' into 'master'
The 15 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../rules/PushAggregateIntoNestedSubplanRule.java | 6 +-
.../optimizer/rules/PushGroupByThroughProduct.java | 4 +-
.../external_dataset/ExternalDatasetTestUtils.java | 4 +
.../deltalake/DeltaAllTypeGenerator.java | 268 +++++++++++++++++++++
.../deltalake-all-type.00.ddl.sqlpp | 52 ++--
.../deltalake-all-type.01.query.sqlpp | 30 +--
.../deltalake-all-type.02.query.sqlpp | 30 +--
.../deltalake-field-access-pushdown.00.ddl.sqlpp | 53 ++--
.../deltalake-field-access-pushdown.01.query.sqlpp | 30 +--
.../deltalake-field-access-pushdown.02.query.sqlpp | 33 +--
.../deltalake-field-access-pushdown.03.query.sqlpp | 32 +--
.../deltalake-field-access-pushdown.04.query.sqlpp | 33 +--
.../deltalake-field-access-pushdown.05.query.sqlpp | 32 +--
.../deltalake-field-access-pushdown.06.query.sqlpp | 34 +--
.../deltalake-field-access-pushdown.07.query.sqlpp | 33 +--
.../deltalake-field-access-pushdown.08.query.sqlpp | 34 +--
.../deltalake-field-access-pushdown.09.query.sqlpp | 33 +--
.../deltalake-field-access-pushdown.10.query.sqlpp | 34 +--
.../deltalake-table-not-exists.00.ddl.sqlpp | 37 +--
.../deltalake-all-type/deltalake-all-type.01.adm | 5 +
.../deltalake-all-type/deltalake-all-type.02.adm | 5 +
.../deltalake-field-access-pushdown.01.adm | 5 +
.../deltalake-field-access-pushdown.02.plan | 22 ++
.../deltalake-field-access-pushdown.03.adm | 5 +
.../deltalake-field-access-pushdown.04.plan | 22 ++
.../deltalake-field-access-pushdown.05.adm | 5 +
.../deltalake-field-access-pushdown.06.plan | 50 ++++
.../deltalake-field-access-pushdown.07.adm | 5 +
.../deltalake-field-access-pushdown.08.plan | 50 ++++
.../deltalake-field-access-pushdown.09.adm | 5 +
.../deltalake-field-access-pushdown.10.adm | 46 ++++
.../runtimets/testsuite_external_dataset_s3.xml | 19 ++
.../NoOpFrameOperationCallbackFactory.java | 5 +
.../aws/delta/AsterixDeltaRuntimeException.java | 34 +--
.../aws/delta/AsterixTypeToDeltaTypeVisitor.java | 145 +++++++++++
.../reader/aws/delta/AwsS3DeltaReaderFactory.java | 150 +++++++++---
.../reader/aws/delta/DeltaFileRecordReader.java | 10 +-
.../aws/delta/converter/DeltaConverterContext.java | 10 +-
.../asterix/external/parser/DeltaDataParser.java | 13 +-
.../asterix/external/util/ExternalDataUtils.java | 34 ++-
.../asterix/external/util/aws/s3/S3AuthUtils.java | 4 +
.../external/util/google/gcs/GCSConstants.java | 12 +
.../asterix/external/util/google/gcs/GCSUtils.java | 35 ++-
.../LSMPrimaryInsertOperatorNodePushable.java | 10 +-
.../LSMPrimaryUpsertOperatorNodePushable.java | 16 +-
.../runtime/operators/StandardBatchController.java | 45 ++++
.../algebra/util/OperatorManipulationUtil.java | 19 +-
.../rewriter/rules/AbstractDecorrelationRule.java | 4 +-
.../subplan/IntroduceGroupByForSubplanRule.java | 7 +-
.../apache/hyracks/api/comm/IFrameAppender.java | 2 +-
.../hyracks/api/util/HyracksThrowingAction.java} | 30 +--
.../org/apache/hyracks/api/util/InvokeUtil.java | 36 +++
.../common/io/MessagingFrameTupleAppender.java | 9 -
.../hyracks/dataflow/common/utils/TaskUtil.java | 14 ++
...perationCallback.java => IBatchController.java} | 29 +--
.../am/lsm/common/api/IFrameOperationCallback.java | 14 +-
.../storage/am/lsm/common/api/ILSMHarness.java | 9 +-
.../storage/am/lsm/common/impls/LSMHarness.java | 31 ++-
.../am/lsm/common/impls/LSMTreeIndexAccessor.java | 6 +-
59 files changed, 1232 insertions(+), 557 deletions(-)
create mode 100644
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/deltalake/DeltaAllTypeGenerator.java
copy
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
=>
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-all-type/deltalake-all-type.00.ddl.sqlpp
(51%)
copy
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
=>
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-all-type/deltalake-all-type.01.query.sqlpp
(51%)
copy
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
=>
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-all-type/deltalake-all-type.02.query.sqlpp
(51%)
copy
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
=>
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.00.ddl.sqlpp
(50%)
copy
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
=>
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.01.query.sqlpp
(52%)
copy
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
=>
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.02.query.sqlpp
(51%)
copy
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
=>
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.03.query.sqlpp
(51%)
copy
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
=>
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.04.query.sqlpp
(51%)
copy
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
=>
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.05.query.sqlpp
(51%)
copy
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
=>
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.06.query.sqlpp
(51%)
copy
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
=>
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.07.query.sqlpp
(51%)
copy
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
=>
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.08.query.sqlpp
(51%)
copy
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
=>
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.09.query.sqlpp
(51%)
copy
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
=>
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.10.query.sqlpp
(51%)
copy
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
=>
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-table-not-exists/deltalake-table-not-exists.00.ddl.sqlpp
(52%)
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-all-type/deltalake-all-type.01.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-all-type/deltalake-all-type.02.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.01.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.02.plan
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.03.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.04.plan
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.05.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.06.plan
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.07.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.08.plan
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.09.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.10.adm
copy
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
=>
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AsterixDeltaRuntimeException.java
(56%)
create mode 100644
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AsterixTypeToDeltaTypeVisitor.java
create mode 100644
asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/StandardBatchController.java
copy
hyracks-fullstack/hyracks/{hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
=>
hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksThrowingAction.java}
(55%)
copy
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/{IFrameOperationCallback.java
=> IBatchController.java} (59%)
