This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 3340ccdd258dd0e11cb98605f6b723d88b2b26c9 Merge: 8490b92 88c0096 Author: Michael Blow <[email protected]> AuthorDate: Tue Apr 14 12:00:01 2020 -0400 Merge branch 'gerrit/march-hare' Change-Id: I183476cb9e32d7f300fd310afee0126eb9c6af31 .gitignore | 2 - asterixdb/asterix-app/data/csv/01.csv | 3 + asterixdb/asterix-app/data/csv/02.csv | 3 + asterixdb/asterix-app/data/csv/sample_09.csv | 17 ++ asterixdb/asterix-app/data/csv/sample_10.csv | 39 ++++ asterixdb/asterix-app/data/csv/sample_11.csv | 4 + asterixdb/asterix-app/data/csv/sample_12.csv | 15 ++ asterixdb/asterix-app/data/tsv/01.tsv | 3 + asterixdb/asterix-app/data/tsv/02.tsv | 3 + asterixdb/asterix-app/data/tsv/sample_01.tsv | 28 +++ asterixdb/asterix-app/pom.xml | 33 +++ .../asterix/app/external/ExternalLibraryUtils.java | 2 +- .../asterix/app/translator/QueryTranslator.java | 177 +++++++++++--- .../asterix/hyracks/bootstrap/CCApplication.java | 10 +- .../org/apache/asterix/utils/FeedOperations.java | 12 +- .../aws/AwsS3ExternalDatasetTest.java | 242 ++++++++++++++++++++ .../resources/runtimets/only_external_dataset.xml | 23 ++ .../avg_mixed/avg_mixed.3.query.sqlpp | 3 +- .../serial_avg_double.1.ddl.sqlpp} | 19 +- .../serial_avg_double.2.update.sqlpp} | 18 +- .../serial_avg_double.3.query.sqlpp} | 11 +- .../serial_avg_double_null.1.ddl.sqlpp} | 19 +- .../serial_avg_double_null.2.update.sqlpp} | 18 +- .../serial_avg_double_null.3.query.sqlpp} | 6 + .../serial_avg_empty.1.ddl.sqlpp} | 19 +- .../serial_avg_empty.2.update.sqlpp} | 1 - .../serial_avg_empty.3.query.sqlpp} | 6 + .../serial_avg_float.1.ddl.sqlpp} | 19 +- .../serial_avg_float.2.update.sqlpp} | 18 +- .../serial_avg_float.3.query.sqlpp} | 6 + .../serial_avg_float_null.1.ddl.sqlpp} | 19 +- .../serial_avg_float_null.2.update.sqlpp} | 18 +- .../serial_avg_float_null.3.query.sqlpp} | 6 + .../serial_avg_int16.1.ddl.sqlpp} | 19 +- .../serial_avg_int16.2.update.sqlpp} | 18 +- .../serial_avg_int16.3.query.sqlpp} | 6 + .../serial_avg_int16_null.1.ddl.sqlpp} | 19 +- .../serial_avg_int16_null.2.update.sqlpp} | 18 +- .../serial_avg_int16_null.3.query.sqlpp} | 6 + .../serial_avg_int32.1.ddl.sqlpp} | 19 +- .../serial_avg_int32.2.update.sqlpp} | 18 +- .../serial_avg_int32.3.query.sqlpp} | 6 + .../serial_avg_int32_null.1.ddl.sqlpp} | 19 +- .../serial_avg_int32_null.2.update.sqlpp} | 18 +- .../serial_avg_int32_null.3.query.sqlpp} | 6 + .../serial_avg_int64.1.ddl.sqlpp} | 19 +- .../serial_avg_int64.2.update.sqlpp} | 18 +- .../serial_avg_int64.3.query.sqlpp} | 6 + .../serial_avg_int64_null.1.ddl.sqlpp} | 19 +- .../serial_avg_int64_null.2.update.sqlpp} | 18 +- .../serial_avg_int64_null.3.query.sqlpp} | 6 + .../serial_avg_int8.1.ddl.sqlpp} | 19 +- .../serial_avg_int8.2.update.sqlpp} | 18 +- .../serial_avg_int8.3.query.sqlpp} | 6 + .../serial_avg_int8_null.1.ddl.sqlpp} | 19 +- .../serial_avg_int8_null.2.update.sqlpp} | 18 +- .../serial_avg_int8_null.3.query.sqlpp} | 6 + .../serial_avg_mixed.1.ddl.sqlpp} | 19 +- .../serial_avg_mixed.2.update.sqlpp} | 16 +- .../serial_avg_mixed.3.query.sqlpp} | 18 +- .../aggregate/avg_mixed/avg_mixed.3.query.sqlpp | 1 + .../serial_avg_double.1.ddl.sqlpp} | 19 +- .../serial_avg_double.2.update.sqlpp} | 18 +- .../serial_avg_double.3.query.sqlpp} | 11 +- .../serial_avg_double_null.1.ddl.sqlpp} | 19 +- .../serial_avg_double_null.2.update.sqlpp} | 18 +- .../serial_avg_double_null.3.query.sqlpp} | 11 +- .../serial_avg_empty/serial_avg_empty.1.ddl.sqlpp} | 19 +- .../serial_avg_empty.2.update.sqlpp} | 1 - .../serial_avg_empty.3.query.sqlpp} | 11 +- .../serial_avg_float/serial_avg_float.1.ddl.sqlpp} | 19 +- .../serial_avg_float.2.update.sqlpp} | 18 +- .../serial_avg_float.3.query.sqlpp} | 11 +- .../serial_avg_float_null.1.ddl.sqlpp} | 19 +- .../serial_avg_float_null.2.update.sqlpp} | 18 +- .../serial_avg_float_null.3.query.sqlpp} | 11 +- .../serial_avg_int16/serial_avg_int16.1.ddl.sqlpp} | 19 +- .../serial_avg_int16.2.update.sqlpp} | 18 +- .../serial_avg_int16.3.query.sqlpp} | 11 +- .../serial_avg_int16_null.1.ddl.sqlpp} | 19 +- .../serial_avg_int16_null.2.update.sqlpp} | 18 +- .../serial_avg_int16_null.3.query.sqlpp} | 11 +- .../serial_avg_int32/serial_avg_int32.1.ddl.sqlpp} | 19 +- .../serial_avg_int32.2.update.sqlpp} | 18 +- .../serial_avg_int32.3.query.sqlpp} | 11 +- .../serial_avg_int32_null.1.ddl.sqlpp} | 19 +- .../serial_avg_int32_null.2.update.sqlpp} | 18 +- .../serial_avg_int32_null.3.query.sqlpp} | 11 +- .../serial_avg_int64/serial_avg_int64.1.ddl.sqlpp} | 19 +- .../serial_avg_int64.2.update.sqlpp} | 18 +- .../serial_avg_int64.3.query.sqlpp} | 11 +- .../serial_avg_int64_null.1.ddl.sqlpp} | 19 +- .../serial_avg_int64_null.2.update.sqlpp} | 18 +- .../serial_avg_int64_null.3.query.sqlpp} | 11 +- .../serial_avg_int8/serial_avg_int8.1.ddl.sqlpp} | 19 +- .../serial_avg_int8.2.update.sqlpp} | 18 +- .../serial_avg_int8/serial_avg_int8.3.query.sqlpp} | 11 +- .../serial_avg_int8_null.1.ddl.sqlpp} | 19 +- .../serial_avg_int8_null.2.update.sqlpp} | 18 +- .../serial_avg_int8_null.3.query.sqlpp} | 11 +- .../serial_avg_mixed/serial_avg_mixed.1.ddl.sqlpp} | 19 +- .../serial_avg_mixed.2.update.sqlpp} | 16 +- .../serial_avg_mixed.3.query.sqlpp} | 17 +- .../csv-parser-001/csv-parser-001.1.ddl.sqlpp} | 26 +-- .../csv-parser-001/csv-parser-001.2.query.sqlpp} | 3 + .../csv-parser-001/csv-parser-001.3.query.sqlpp} | 3 + .../csv-parser-001/csv-parser-001.4.query.sqlpp} | 3 + .../csv-parser-001/csv-parser-001.5.query.sqlpp} | 3 + .../csv-parser-001/csv-parser-001.6.ddl.sqlpp} | 1 + .../tsv-parser-001/tsv-parser-002.1.ddl.sqlpp} | 17 +- .../tsv-parser-001/tsv-parser-002.2.query.sqlpp} | 3 + .../tsv-parser-001/tsv-parser-002.3.ddl.sqlpp} | 1 + .../create-dataset-inline-type-1.1.ddl.sqlpp | 141 ++++++++++++ .../create-dataset-inline-type-1.2.query.sqlpp} | 3 + .../create-dataset-inline-type-1.3.ddl.sqlpp} | 17 +- .../create-dataset-inline-type-1.4.query.sqlpp} | 10 +- .../create-dataset-inline-type-2.1.ddl.sqlpp} | 21 +- .../create-dataset-inline-type-2.2.query.sqlpp} | 11 +- .../create-dataset-inline-type-2.3.ddl.sqlpp} | 14 +- .../create-dataset-inline-type-2.4.ddl.sqlpp} | 19 +- .../aws/s3/000/external_dataset.000.ddl.sqlpp} | 23 +- .../aws/s3/000/external_dataset.001.query.sqlpp} | 6 + .../aws/s3/000/external_dataset.002.ddl.sqlpp} | 1 + .../aws/s3/001/query-dataset.000.ddl.sqlpp} | 21 +- .../aws/s3/001/query-dataset.002.query.sqlpp} | 3 + .../aws/s3/001/query-dataset.003.ddl.sqlpp} | 1 + .../aws/s3/002/query-dataset.000.ddl.sqlpp} | 21 +- .../aws/s3/002/query-dataset.002.query.sqlpp} | 3 + .../aws/s3/002/query-dataset.003.ddl.sqlpp} | 1 + .../aggregate-sql/avg_mixed/avg_mixed.1.adm | 2 +- .../serial_avg_double/serial_avg_double.1.adm | 1 + .../serial_avg_double_null.1.adm | 1 + .../serial_avg_empty/serial_sum_empty.1.adm} | 0 .../serial_avg_float/serial_avg_float.1.adm | 1 + .../serial_avg_float_null.1.adm | 1 + .../serial_avg_int16/serial_avg_int16.1.adm | 1 + .../serial_avg_int16_null.1.adm | 1 + .../serial_avg_int32/serial_avg_int32.1.adm | 1 + .../serial_avg_int32_null.1.adm | 1 + .../serial_avg_int64/serial_avg_int64.1.adm | 1 + .../serial_avg_int64_null.1.adm | 1 + .../serial_avg_int8/serial_avg_int8.1.adm | 1 + .../serial_avg_int8_null.1.adm | 1 + .../serial_avg_mixed/serial_avg_mixed.1.adm | 1 + .../results/aggregate/avg_mixed/avg_mixed.1.adm | 2 +- .../serial_avg_double/serial_avg_double.1.adm | 1 + .../serial_avg_double_null.1.adm | 1 + .../serial_avg_empty/serial_sum_empty.1.adm} | 0 .../serial_avg_float/serial_avg_float.1.adm | 1 + .../serial_avg_float_null.1.adm | 1 + .../serial_avg_int16/serial_avg_int16.1.adm | 1 + .../serial_avg_int16_null.1.adm | 1 + .../serial_avg_int32/serial_avg_int32.1.adm | 1 + .../serial_avg_int32_null.1.adm | 1 + .../serial_avg_int64/serial_avg_int64.1.adm | 1 + .../serial_avg_int64_null.1.adm | 1 + .../serial_avg_int8/serial_avg_int8.1.adm | 1 + .../serial_avg_int8_null.1.adm | 1 + .../serial_avg_mixed/serial_avg_mixed.1.adm | 1 + .../csv-parser-001/csv-parser-001.2.adm | 15 ++ .../csv-parser-001/csv-parser-001.3.adm | 32 +++ .../csv-parser-001/csv-parser-001.4.adm | 4 + .../csv-parser-001/csv-parser-001.5.adm | 13 ++ .../tsv-parser-001/tsv-parser-001.2.adm | 28 +++ .../create-dataset-inline-type-1.2.adm | 21 ++ .../create-dataset-inline-type-1.4.adm | 1 + .../create-dataset-inline-type-2.2.adm | 1 + .../aws/s3/000/external_dataset.001.adm | 14 ++ .../aws/s3/001/external_dataset.001.adm | 6 + .../aws/s3/002/external_dataset.001.adm | 6 + .../results/feeds/feeds_01/feeds_01.1.adm | 2 +- .../aggregate-sql/avg_mixed/avg_mixed.1.ast | 0 .../aggregate-sql/avg_mixed/avg_mixed.2.ast | 0 .../aggregate-sql/avg_mixed/avg_mixed.3.ast | 2 +- .../cross-dataverse/cross-dv01/cross-dv01.1.ast | 6 +- .../runtimets/testsuite_external_dataset.xml | 38 ++++ .../test/resources/runtimets/testsuite_sqlpp.xml | 174 +++++++++++++- .../asterix/common/api/ICoordinationService.java | 12 + .../common/dataflow/ICcApplicationContext.java | 16 ++ .../asterix/common/external}/IAdapterFactory.java | 31 +-- .../common/external/IAdapterFactoryService.java} | 16 +- .../common/external}/IDataSourceAdapter.java | 2 +- asterixdb/asterix-external-data/pom.xml | 12 + .../factory/AdapterFactoryService.java} | 18 +- .../adapter/factory/GenericAdapterFactory.java | 26 ++- .../external/api/IIndexingAdapterFactory.java | 2 +- .../external/api/ITypedAdapterFactory.java} | 43 ++-- .../external/dataset/adapter/FeedAdapter.java | 2 +- .../external/dataset/adapter/GenericAdapter.java | 2 +- .../CSVToRecordWithMetadataAndPKConverter.java | 13 +- .../input/record/reader/aws/AwsS3InputStream.java | 163 +++++++++++++ .../record/reader/aws/AwsS3InputStreamFactory.java | 253 +++++++++++++++++++++ .../record/reader/aws/AwsS3ReaderFactory.java | 90 ++++++++ .../record/reader/stream/LineRecordReader.java | 5 +- .../reader/stream/QuotedLineRecordReader.java | 67 +++--- .../operators/ExternalScanOperatorDescriptor.java | 8 +- .../operators/FeedIntakeOperatorDescriptor.java | 16 +- .../operators/FeedIntakeOperatorNodePushable.java | 4 +- .../external/parser/DelimitedDataParser.java | 31 ++- .../parser/factory/DelimitedDataParserFactory.java | 44 +--- .../external/provider/AdapterFactoryProvider.java | 24 +- .../provider/DatasourceFactoryProvider.java | 1 + .../util/ExternalDataCompatibilityUtils.java | 17 +- .../external/util/ExternalDataConstants.java | 18 ++ .../asterix/external/util/ExternalDataUtils.java | 88 ++++--- ...pache.asterix.external.api.IRecordReaderFactory | 1 + .../library/adapter/TestTypedAdapterFactory.java | 6 +- asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj | 14 +- .../asterix/lang/common/statement/DatasetDecl.java | 68 ++---- .../common/util/DatasetDeclParametersUtil.java | 18 +- .../lang/common/visitor/FormatPrintVisitor.java | 12 +- .../lang/common/visitor/QueryPrintVisitor.java | 15 +- .../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 128 +++++++++-- .../metadata/bootstrap/MetadataBootstrap.java | 7 +- .../metadata/bootstrap/MetadataRecordTypes.java | 2 +- .../metadata/declared/DatasetDataSource.java | 4 +- .../metadata/declared/LoadableDataSource.java | 4 +- .../metadata/declared/MetadataManagerUtil.java | 12 +- .../metadata/declared/MetadataProvider.java | 23 +- .../metadata/entities/DatasourceAdapter.java | 2 +- .../DatasetTupleTranslator.java | 111 +++++---- .../DatasourceAdapterTupleTranslator.java | 2 +- .../asterix/metadata/feeds/FeedMetadataUtil.java | 22 +- .../apache/asterix/metadata/utils/DatasetUtil.java | 21 +- .../metadata/utils/ExternalIndexingOperations.java | 4 +- .../asterix/metadata/utils/MetadataUtil.java | 1 + .../AbstractSerializableAvgAggregateFunction.java | 36 ++- .../std/AbstractAvgAggregateFunction.java | 35 ++- .../runtime/utils/CcApplicationContext.java | 12 +- .../runtime/utils/NoOpCoordinationService.java | 5 + asterixdb/asterix-server/pom.xml | 6 + asterixdb/asterix-spidersilk/config/cc.conf | 35 --- asterixdb/asterix-spidersilk/docker/Dockerfile | 22 -- asterixdb/asterix-spidersilk/pom.xml | 87 ------- .../apache/asterix/spidersilk/SampleTestIT.java | 87 ------- .../org/apache/asterix/spidersilk/TestUtil.java | 74 ------ .../src/test/resources/logback.xml | 35 --- asterixdb/pom.xml | 80 ++++++- ...streams_reactive-streams-jvm_v1.0.2_COPYING.txt | 121 ++++++++++ ...streams_reactive-streams-jvm_v1.0.2_LICENSE.txt | 8 + .../common/data/parsers/BooleanParserFactory.java | 62 +++++ .../std/file/DelimitedDataTupleParserFactory.java | 9 +- .../file/FieldCursorForDelimitedDataParser.java | 179 ++++++++------- .../hyracks/dataflow/std/file/CursorTest.java | 5 +- 244 files changed, 3647 insertions(+), 1400 deletions(-) diff --cc asterixdb/asterix-server/pom.xml index 421dab8,7acfc04..a542c70 --- a/asterixdb/asterix-server/pom.xml +++ b/asterixdb/asterix-server/pom.xml @@@ -169,9 -169,14 +169,14 @@@ <url>https://raw.githubusercontent.com/mojohaus/appassembler/appassembler-2.0.0/LICENSE.txt</url> </override> <override> - <gav>io.netty:netty-all:4.1.46.Final</gav> - <noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.46.Final/NOTICE.txt</noticeUrl> + <gav>io.netty:netty-all:4.1.48.Final</gav> + <noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.48.Final/NOTICE.txt</noticeUrl> </override> + <override> + <gav>org.reactivestreams:reactive-streams:1.0.2</gav> + <noticeUrl>https://raw.githubusercontent.com/reactive-streams/reactive-streams-jvm/v1.0.2/COPYING.txt</noticeUrl> + <url>https://raw.githubusercontent.com/reactive-streams/reactive-streams-jvm/v1.0.2/LICENSE.txt</url> + </override> </overrides> <licenses> <license>
