This is an automated email from the ASF dual-hosted git repository.
htowaileb pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git.
from 64ae834 [ASTERIXDB-2730][STO] Optimize flush in GVBC
new 0163af6 [ASTERIXDB-2741][EXT] Support include/exclude filters for S3
external datasets
new 7e74c73 Merge commit '0163af68d0' from 'gerrit/mad-hatter'
new 51c5b05 Merge "Merge commit '0163af68d0' from 'gerrit/mad-hatter'"
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../asterix/app/translator/QueryTranslator.java | 14 +-
.../aws/AwsS3ExternalDatasetOnePartitionTest.java | 6 +-
.../aws/AwsS3ExternalDatasetTest.java | 154 ++++++++++++++----
.../external_dataset.000.ddl.sqlpp | 2 +-
.../bad-name-1/test.000.ddl.sqlpp} | 8 +-
.../bad-name-1}/test.099.ddl.sqlpp | 0
.../bad-name-2/test.000.ddl.sqlpp} | 8 +-
.../bad-name-2}/test.099.ddl.sqlpp | 0
.../bad-name-3/test.000.ddl.sqlpp} | 8 +-
.../bad-name-3}/test.099.ddl.sqlpp | 0
.../both/test.000.ddl.sqlpp} | 8 +-
.../both}/test.099.ddl.sqlpp | 0
.../exclude-1/test.000.ddl.sqlpp} | 7 +-
.../exclude-1/test.001.query.sqlpp} | 18 +--
.../exclude-1}/test.099.ddl.sqlpp | 0
.../exclude-2/test.000.ddl.sqlpp} | 9 +-
.../exclude-2/test.001.query.sqlpp} | 18 +--
.../exclude-2}/test.099.ddl.sqlpp | 0
.../exclude-3/test.000.ddl.sqlpp} | 8 +-
.../exclude-3/test.001.query.sqlpp} | 18 +--
.../exclude-3}/test.099.ddl.sqlpp | 0
.../exclude-4/test.000.ddl.sqlpp} | 8 +-
.../exclude-4/test.001.query.sqlpp} | 18 +--
.../exclude-4}/test.099.ddl.sqlpp | 0
.../exclude-5/test.000.ddl.sqlpp} | 8 +-
.../exclude-5/test.001.query.sqlpp} | 18 +--
.../exclude-5}/test.099.ddl.sqlpp | 0
.../exclude-6/test.000.ddl.sqlpp} | 7 +-
.../exclude-6/test.001.query.sqlpp} | 18 +--
.../exclude-6}/test.099.ddl.sqlpp | 0
.../exclude-all/test.000.ddl.sqlpp} | 7 +-
.../exclude-all/test.001.query.sqlpp} | 18 +--
.../exclude-all}/test.099.ddl.sqlpp | 0
.../include-1/test.000.ddl.sqlpp} | 7 +-
.../include-1/test.001.query.sqlpp} | 18 +--
.../include-1}/test.099.ddl.sqlpp | 0
.../include-10/test.000.ddl.sqlpp} | 8 +-
.../include-10/test.001.query.sqlpp} | 18 +--
.../include-10}/test.099.ddl.sqlpp | 0
.../include-2/test.000.ddl.sqlpp} | 7 +-
.../include-2/test.001.query.sqlpp} | 18 +--
.../include-2/test.002.query.sqlpp} | 18 +--
.../include-2}/test.099.ddl.sqlpp | 0
.../include-3/test.000.ddl.sqlpp} | 10 +-
.../include-3/test.001.query.sqlpp} | 18 +--
.../include-3}/test.099.ddl.sqlpp | 0
.../include-4/test.000.ddl.sqlpp} | 10 +-
.../include-4/test.001.query.sqlpp} | 18 +--
.../include-4}/test.099.ddl.sqlpp | 0
.../include-5/test.000.ddl.sqlpp} | 7 +-
.../include-5/test.001.query.sqlpp} | 18 +--
.../include-5}/test.099.ddl.sqlpp | 0
.../include-6/test.000.ddl.sqlpp} | 7 +-
.../include-6/test.001.query.sqlpp} | 18 +--
.../include-6}/test.099.ddl.sqlpp | 0
.../include-7/test.000.ddl.sqlpp} | 8 +-
.../include-7/test.001.query.sqlpp} | 18 +--
.../include-7}/test.099.ddl.sqlpp | 0
.../include-8/test.000.ddl.sqlpp} | 8 +-
.../include-8/test.001.query.sqlpp} | 18 +--
.../include-8}/test.099.ddl.sqlpp | 0
.../include-9/test.000.ddl.sqlpp} | 8 +-
.../include-9/test.001.query.sqlpp} | 18 +--
.../include-9}/test.099.ddl.sqlpp | 0
.../include-all/test.000.ddl.sqlpp} | 7 +-
.../include-all/test.001.query.sqlpp} | 18 +--
.../include-all}/test.099.ddl.sqlpp | 0
.../s3/include-exclude/exclude-1/result.001.adm | 1 +
.../s3/include-exclude/exclude-2/result.001.adm | 1 +
.../s3/include-exclude/exclude-3/result.001.adm | 1 +
.../s3/include-exclude/exclude-4/result.001.adm | 1 +
.../s3/include-exclude/exclude-5/result.001.adm | 1 +
.../s3/include-exclude/exclude-6/result.001.adm | 1 +
.../exclude-all}/result.001.adm | 0
.../s3/include-exclude/include-1/result.001.adm | 1 +
.../s3/include-exclude/include-2/result.001.adm | 1 +
.../s3/include-exclude/include-2/result.002.adm | 4 +
.../s3/include-exclude/include-3/result.001.adm | 1 +
.../s3/include-exclude/include-4/result.001.adm | 1 +
.../s3/include-exclude/include-5/result.001.adm | 1 +
.../s3/include-exclude/include-6/result.001.adm | 1 +
.../s3/include-exclude/include-7/result.001.adm | 1 +
.../s3/include-exclude/include-8/result.001.adm | 1 +
.../s3/include-exclude/include-9/result.001.adm | 1 +
.../runtimets/testsuite_external_dataset.xml | 118 ++++++++++++++
.../test/resources/runtimets/testsuite_sqlpp.xml | 22 +--
.../asterix/common/exceptions/ErrorCode.java | 18 ++-
.../src/main/resources/asx_errormsg/en.properties | 18 ++-
.../record/reader/aws/AwsS3InputStreamFactory.java | 113 ++++++-------
.../external/util/ExternalDataConstants.java | 2 +
.../asterix/external/util/ExternalDataUtils.java | 174 ++++++++++++++++++++-
91 files changed, 658 insertions(+), 517 deletions(-)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/bad-name-1/test.000.ddl.sqlpp} (89%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/bad-name-1}/test.099.ddl.sqlpp (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/bad-name-2/test.000.ddl.sqlpp} (89%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/bad-name-2}/test.099.ddl.sqlpp (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/bad-name-3/test.000.ddl.sqlpp} (89%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/bad-name-3}/test.099.ddl.sqlpp (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/both/test.000.ddl.sqlpp} (90%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/both}/test.099.ddl.sqlpp (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/exclude-1/test.000.ddl.sqlpp} (91%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/exclude-1/test.001.query.sqlpp} (67%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/exclude-1}/test.099.ddl.sqlpp (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/exclude-2/test.000.ddl.sqlpp} (84%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/exclude-2/test.001.query.sqlpp} (67%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/exclude-2}/test.099.ddl.sqlpp (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/exclude-3/test.000.ddl.sqlpp} (88%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/exclude-3/test.001.query.sqlpp} (67%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/exclude-3}/test.099.ddl.sqlpp (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/exclude-4/test.000.ddl.sqlpp} (88%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/exclude-4/test.001.query.sqlpp} (67%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/exclude-4}/test.099.ddl.sqlpp (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/exclude-5/test.000.ddl.sqlpp} (86%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/exclude-5/test.001.query.sqlpp} (67%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/exclude-5}/test.099.ddl.sqlpp (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/exclude-6/test.000.ddl.sqlpp} (90%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/exclude-6/test.001.query.sqlpp} (67%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/exclude-6}/test.099.ddl.sqlpp (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/exclude-all/test.000.ddl.sqlpp} (91%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/exclude-all/test.001.query.sqlpp} (67%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/exclude-all}/test.099.ddl.sqlpp (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-1/test.000.ddl.sqlpp} (91%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-1/test.001.query.sqlpp} (67%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/include-1}/test.099.ddl.sqlpp (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-10/test.000.ddl.sqlpp} (90%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-10/test.001.query.sqlpp} (67%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/include-10}/test.099.ddl.sqlpp (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-2/test.000.ddl.sqlpp} (91%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-2/test.001.query.sqlpp} (67%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-2/test.002.query.sqlpp} (67%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/include-2}/test.099.ddl.sqlpp (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-3/test.000.ddl.sqlpp} (88%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-3/test.001.query.sqlpp} (67%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/include-3}/test.099.ddl.sqlpp (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-4/test.000.ddl.sqlpp} (88%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-4/test.001.query.sqlpp} (67%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/include-4}/test.099.ddl.sqlpp (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-5/test.000.ddl.sqlpp} (91%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-5/test.001.query.sqlpp} (67%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/include-5}/test.099.ddl.sqlpp (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-6/test.000.ddl.sqlpp} (90%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-6/test.001.query.sqlpp} (67%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/include-6}/test.099.ddl.sqlpp (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-7/test.000.ddl.sqlpp} (86%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-7/test.001.query.sqlpp} (67%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/include-7}/test.099.ddl.sqlpp (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-8/test.000.ddl.sqlpp} (89%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-8/test.001.query.sqlpp} (67%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/include-8}/test.099.ddl.sqlpp (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-9/test.000.ddl.sqlpp} (89%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-9/test.001.query.sqlpp} (67%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/include-9}/test.099.ddl.sqlpp (100%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-all/test.000.ddl.sqlpp} (91%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp
=> include-exclude/include-all/test.001.query.sqlpp} (67%)
copy
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint
=> include-exclude/include-all}/test.099.ddl.sqlpp (100%)
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-1/result.001.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-2/result.001.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-3/result.001.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-4/result.001.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-5/result.001.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-6/result.001.adm
copy
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/{definition-does-not-exist
=> include-exclude/exclude-all}/result.001.adm (100%)
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-1/result.001.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-2/result.001.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-2/result.002.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-3/result.001.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-4/result.001.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-5/result.001.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-6/result.001.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-7/result.001.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-8/result.001.adm
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-9/result.001.adm
