This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 722db2097c2ad4e39b9bffc0d615df1f6e25af90 Author: Hussain Towaileb <[email protected]> AuthorDate: Tue Jun 23 03:33:39 2020 +0300 [ASTERIXDB-2751][EXT] Issue a warning if external dataset configuration returns no files - user model changes: no - storage format changes: no - interface changes: no Details: - Issue a warning when creating an external dataset if the dataset configuration returns no files. - Issue a warning when querying an external dataset if no files are returned. Change-Id: I0b2be49bf3c69c3e055d942a5c3f0106b3069d52 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/6843 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Hussain Towaileb <[email protected]> Reviewed-by: Dmitry Lychagin <[email protected]> --- .../optimizer/rules/ConstantFoldingRule.java | 4 ++- .../asterix/app/translator/QueryTranslator.java | 4 +-- .../apache/asterix/test/common/TestExecutor.java | 1 - .../test.000.ddl.sqlpp} | 35 ++++++++++---------- .../test.001.query.sqlpp} | 22 ++----------- .../test.099.ddl.sqlpp} | 20 +----------- .../exclude-all-files/test.000.ddl.sqlpp} | 35 ++++++++++---------- .../exclude-all-files/test.001.query.sqlpp} | 22 ++----------- .../exclude-all-files/test.099.ddl.sqlpp} | 20 +----------- .../include-no-files/test.000.ddl.sqlpp} | 35 ++++++++++---------- .../include-no-files/test.001.query.sqlpp} | 22 ++----------- .../include-no-files/test.099.ddl.sqlpp} | 20 +----------- .../definition-points-to-nothing/result.001.adm | 1 + .../exclude-all-files/result.001.adm | 1 + .../include-no-files/result.001.adm | 1 + .../runtimets/testsuite_external_dataset.xml | 18 ++++++++++ .../asterix/common/exceptions/ErrorCode.java | 1 + .../common/exceptions/NoOpWarningCollector.java} | 28 ++++++++++------ .../asterix/common/external/IAdapterFactory.java | 6 ++-- .../src/main/resources/asx_errormsg/en.properties | 1 + .../adapter/factory/GenericAdapterFactory.java | 15 +++++---- .../adapter/factory/LookupAdapterFactory.java | 10 +++--- .../external/api/IExternalDataSourceFactory.java | 3 +- .../external/input/HDFSDataSourceFactory.java | 4 ++- .../record/reader/aws/AwsS3InputStreamFactory.java | 17 +++++++--- .../record/reader/aws/AwsS3ReaderFactory.java | 5 +-- .../reader/hdfs/HDFSLookupReaderFactory.java | 4 ++- .../reader/http/HttpServerRecordReaderFactory.java | 4 ++- .../record/reader/rss/RSSRecordReaderFactory.java | 4 ++- .../reader/stream/StreamRecordReaderFactory.java | 7 ++-- .../reader/twitter/TwitterRecordReaderFactory.java | 4 ++- .../stream/factory/LocalFSInputStreamFactory.java | 4 ++- .../factory/SocketClientInputStreamFactory.java | 4 ++- .../factory/SocketServerInputStreamFactory.java | 4 ++- .../factory/TwitterFirehoseStreamFactory.java | 4 ++- .../operators/FeedIntakeOperatorDescriptor.java | 3 +- .../external/provider/AdapterFactoryProvider.java | 15 +++++---- .../provider/LookupReaderFactoryProvider.java | 5 +-- .../asterix/external/util/ExternalDataUtils.java | 38 ++++++++++++++++++---- .../apache/asterix/external/util/ParseUtil.java | 5 +-- .../reader/RecordWithPKTestReaderFactory.java | 4 ++- .../record/reader/kv/KVTestReaderFactory.java | 4 ++- .../library/adapter/TestTypedAdapterFactory.java | 4 ++- .../metadata/declared/DatasetDataSource.java | 5 +-- .../declared/FunctionDataSourceFactory.java | 3 +- .../metadata/declared/LoadableDataSource.java | 2 +- .../metadata/declared/MetadataProvider.java | 11 ++++--- .../asterix/metadata/feeds/FeedMetadataUtil.java | 12 ++++--- .../metadata/utils/ExternalIndexingOperations.java | 3 +- .../api/HeuristicCompilerFactoryBuilder.java | 2 +- .../algebricks/core/jobgen/impl/JobGenContext.java | 10 +++++- .../hyracks/api/exceptions/SourceLocation.java | 18 +++++++--- .../org/apache/hyracks/api/exceptions/Warning.java | 3 +- .../file/FieldCursorForDelimitedDataParser.java | 4 +-- 54 files changed, 279 insertions(+), 262 deletions(-) diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java index d8fa3d6..494b1b4 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java @@ -29,6 +29,7 @@ import org.apache.asterix.common.config.GlobalConfig; import org.apache.asterix.common.dataflow.ICcApplicationContext; import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.common.exceptions.ErrorCode; +import org.apache.asterix.common.exceptions.NoOpWarningCollector; import org.apache.asterix.common.exceptions.WarningCollector; import org.apache.asterix.common.exceptions.WarningUtil; import org.apache.asterix.dataflow.data.common.ExpressionTypeComputer; @@ -143,7 +144,8 @@ public class ConstantFoldingRule implements IAlgebraicRewriteRule { BinaryComparatorFactoryProvider.INSTANCE, TypeTraitProvider.INSTANCE, BinaryBooleanInspector.FACTORY, BinaryIntegerInspector.FACTORY, ADMPrinterFactoryProvider.INSTANCE, MissingWriterFactory.INSTANCE, null, new ExpressionRuntimeProvider(new QueryLogicalExpressionJobGen(metadataProvider.getFunctionManager())), - ExpressionTypeComputer.INSTANCE, null, null, null, null, GlobalConfig.DEFAULT_FRAME_SIZE, null, 0); + ExpressionTypeComputer.INSTANCE, null, null, null, null, GlobalConfig.DEFAULT_FRAME_SIZE, null, + NoOpWarningCollector.INSTANCE, 0); } @Override diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index f6536ac..ab2cf61 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -2232,7 +2232,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen ExternalDataUtils.normalize(configuration); ExternalDataUtils.validate(configuration); feed = new Feed(dataverseName, feedName, configuration); - FeedMetadataUtil.validateFeed(feed, mdTxnCtx, appCtx); + FeedMetadataUtil.validateFeed(feed, mdTxnCtx, appCtx, warningCollector); MetadataManager.INSTANCE.addFeed(metadataProvider.getMetadataTxnContext(), feed); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); } catch (Exception e) { @@ -3256,6 +3256,6 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen */ protected void validateAdapterSpecificProperties(Map<String, String> configuration, SourceLocation srcLoc) throws CompilationException { - ExternalDataUtils.validateAdapterSpecificProperties(configuration, srcLoc); + ExternalDataUtils.validateAdapterSpecificProperties(configuration, srcLoc, warningCollector); } } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java index 6bae914..5d71a26 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java @@ -1808,7 +1808,6 @@ public class TestExecutor { List<CompilationUnit> cUnits = testCaseCtx.getTestCase().getCompilationUnit(); for (CompilationUnit cUnit : cUnits) { List<String> expectedErrors = cUnit.getExpectedError(); - int expectedWarnCount = cUnit.getExpectedWarn().size(); BitSet expectedWarnings = new BitSet(cUnit.getExpectedWarn().size()); expectedWarnings.set(0, cUnit.getExpectedWarn().size()); LOGGER.info( diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/test.000.ddl.sqlpp similarity index 53% copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/test.000.ddl.sqlpp index 598d9ff..b51a33f 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/test.000.ddl.sqlpp @@ -16,23 +16,24 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.asterix.external.util; +-- param max-warnings:json=1000 -import org.apache.hyracks.api.exceptions.ErrorCode; -import org.apache.hyracks.api.exceptions.IWarningCollector; -import org.apache.hyracks.api.exceptions.SourceLocation; -import org.apache.hyracks.api.exceptions.Warning; +drop dataverse test if exists; +create dataverse test; +use test; -public class ParseUtil { +drop type test if exists; +create type test as open { +}; - private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1); - - private ParseUtil() { - } - - public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum, - String warnMessage) { - warningCollector.warn( - Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage)); - } -} +drop dataset test if exists; +create external dataset test(test) using S3 ( +("accessKeyId"="dummyAccessKey"), +("secretAccessKey"="dummySecretKey"), +("region"="us-west-2"), +("serviceEndpoint"="http://localhost:8001";), +("container"="include-exclude"), +("definition"="point-to-nothing"), +("format"="json"), +("include"="*") +); \ No newline at end of file diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/test.001.query.sqlpp similarity index 53% copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/test.001.query.sqlpp index 598d9ff..284eb58 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/test.001.query.sqlpp @@ -16,23 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.asterix.external.util; +-- param max-warnings:json=1000 -import org.apache.hyracks.api.exceptions.ErrorCode; -import org.apache.hyracks.api.exceptions.IWarningCollector; -import org.apache.hyracks.api.exceptions.SourceLocation; -import org.apache.hyracks.api.exceptions.Warning; - -public class ParseUtil { - - private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1); - - private ParseUtil() { - } - - public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum, - String warnMessage) { - warningCollector.warn( - Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage)); - } -} +use test; +select count(*) as `count` from test; \ No newline at end of file diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/test.099.ddl.sqlpp similarity index 53% copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/test.099.ddl.sqlpp index 598d9ff..548e632 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/test.099.ddl.sqlpp @@ -16,23 +16,5 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.asterix.external.util; -import org.apache.hyracks.api.exceptions.ErrorCode; -import org.apache.hyracks.api.exceptions.IWarningCollector; -import org.apache.hyracks.api.exceptions.SourceLocation; -import org.apache.hyracks.api.exceptions.Warning; - -public class ParseUtil { - - private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1); - - private ParseUtil() { - } - - public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum, - String warnMessage) { - warningCollector.warn( - Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage)); - } -} +drop dataverse test if exists; \ No newline at end of file diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/exclude-all-files/test.000.ddl.sqlpp similarity index 53% copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/exclude-all-files/test.000.ddl.sqlpp index 598d9ff..2e3dc68 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/exclude-all-files/test.000.ddl.sqlpp @@ -16,23 +16,24 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.asterix.external.util; +-- param max-warnings:json=1000 -import org.apache.hyracks.api.exceptions.ErrorCode; -import org.apache.hyracks.api.exceptions.IWarningCollector; -import org.apache.hyracks.api.exceptions.SourceLocation; -import org.apache.hyracks.api.exceptions.Warning; +drop dataverse test if exists; +create dataverse test; +use test; -public class ParseUtil { +drop type test if exists; +create type test as open { +}; - private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1); - - private ParseUtil() { - } - - public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum, - String warnMessage) { - warningCollector.warn( - Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage)); - } -} +drop dataset test if exists; +create external dataset test(test) using S3 ( +("accessKeyId"="dummyAccessKey"), +("secretAccessKey"="dummySecretKey"), +("region"="us-west-2"), +("serviceEndpoint"="http://localhost:8001";), +("container"="include-exclude"), +("definition"="data/mixed/"), +("format"="json"), +("exclude"="*") +); \ No newline at end of file diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/exclude-all-files/test.001.query.sqlpp similarity index 53% copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/exclude-all-files/test.001.query.sqlpp index 598d9ff..284eb58 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/exclude-all-files/test.001.query.sqlpp @@ -16,23 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.asterix.external.util; +-- param max-warnings:json=1000 -import org.apache.hyracks.api.exceptions.ErrorCode; -import org.apache.hyracks.api.exceptions.IWarningCollector; -import org.apache.hyracks.api.exceptions.SourceLocation; -import org.apache.hyracks.api.exceptions.Warning; - -public class ParseUtil { - - private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1); - - private ParseUtil() { - } - - public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum, - String warnMessage) { - warningCollector.warn( - Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage)); - } -} +use test; +select count(*) as `count` from test; \ No newline at end of file diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/exclude-all-files/test.099.ddl.sqlpp similarity index 53% copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/exclude-all-files/test.099.ddl.sqlpp index 598d9ff..548e632 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/exclude-all-files/test.099.ddl.sqlpp @@ -16,23 +16,5 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.asterix.external.util; -import org.apache.hyracks.api.exceptions.ErrorCode; -import org.apache.hyracks.api.exceptions.IWarningCollector; -import org.apache.hyracks.api.exceptions.SourceLocation; -import org.apache.hyracks.api.exceptions.Warning; - -public class ParseUtil { - - private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1); - - private ParseUtil() { - } - - public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum, - String warnMessage) { - warningCollector.warn( - Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage)); - } -} +drop dataverse test if exists; \ No newline at end of file diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/include-no-files/test.000.ddl.sqlpp similarity index 53% copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/include-no-files/test.000.ddl.sqlpp index 598d9ff..9abdbde 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/include-no-files/test.000.ddl.sqlpp @@ -16,23 +16,24 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.asterix.external.util; +-- param max-warnings:json=1000 -import org.apache.hyracks.api.exceptions.ErrorCode; -import org.apache.hyracks.api.exceptions.IWarningCollector; -import org.apache.hyracks.api.exceptions.SourceLocation; -import org.apache.hyracks.api.exceptions.Warning; +drop dataverse test if exists; +create dataverse test; +use test; -public class ParseUtil { +drop type test if exists; +create type test as open { +}; - private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1); - - private ParseUtil() { - } - - public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum, - String warnMessage) { - warningCollector.warn( - Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage)); - } -} +drop dataset test if exists; +create external dataset test(test) using S3 ( +("accessKeyId"="dummyAccessKey"), +("secretAccessKey"="dummySecretKey"), +("region"="us-west-2"), +("serviceEndpoint"="http://localhost:8001";), +("container"="include-exclude"), +("definition"="data/mixed/"), +("format"="json"), +("include"="*.notRealExtension") +); \ No newline at end of file diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/include-no-files/test.001.query.sqlpp similarity index 53% copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/include-no-files/test.001.query.sqlpp index 598d9ff..284eb58 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/include-no-files/test.001.query.sqlpp @@ -16,23 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.asterix.external.util; +-- param max-warnings:json=1000 -import org.apache.hyracks.api.exceptions.ErrorCode; -import org.apache.hyracks.api.exceptions.IWarningCollector; -import org.apache.hyracks.api.exceptions.SourceLocation; -import org.apache.hyracks.api.exceptions.Warning; - -public class ParseUtil { - - private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1); - - private ParseUtil() { - } - - public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum, - String warnMessage) { - warningCollector.warn( - Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage)); - } -} +use test; +select count(*) as `count` from test; \ No newline at end of file diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/include-no-files/test.099.ddl.sqlpp similarity index 53% copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/include-no-files/test.099.ddl.sqlpp index 598d9ff..548e632 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/include-no-files/test.099.ddl.sqlpp @@ -16,23 +16,5 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.asterix.external.util; -import org.apache.hyracks.api.exceptions.ErrorCode; -import org.apache.hyracks.api.exceptions.IWarningCollector; -import org.apache.hyracks.api.exceptions.SourceLocation; -import org.apache.hyracks.api.exceptions.Warning; - -public class ParseUtil { - - private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1); - - private ParseUtil() { - } - - public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum, - String warnMessage) { - warningCollector.warn( - Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage)); - } -} +drop dataverse test if exists; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/result.001.adm new file mode 100644 index 0000000..c1a0ea2 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/result.001.adm @@ -0,0 +1 @@ +{ "count": 0 } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/no-files-returned/exclude-all-files/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/no-files-returned/exclude-all-files/result.001.adm new file mode 100644 index 0000000..c1a0ea2 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/no-files-returned/exclude-all-files/result.001.adm @@ -0,0 +1 @@ +{ "count": 0 } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/no-files-returned/include-no-files/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/no-files-returned/include-no-files/result.001.adm new file mode 100644 index 0000000..c1a0ea2 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/no-files-returned/include-no-files/result.001.adm @@ -0,0 +1 @@ +{ "count": 0 } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml index 7c4cf37..ad17afe 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml @@ -99,6 +99,24 @@ <expected-error>External source error. The specified bucket does not exist (Service: S3, Status Code: 404, Request ID: null)</expected-error> </compilation-unit> </test-case> + <test-case FilePath="external-dataset" check-warnings="true"> + <compilation-unit name="aws/s3/no-files-returned/definition-points-to-nothing"> + <output-dir compare="Text">aws/s3/no-files-returned/definition-points-to-nothing</output-dir> + <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn> + </compilation-unit> + </test-case> + <test-case FilePath="external-dataset" check-warnings="true"> + <compilation-unit name="aws/s3/no-files-returned/exclude-all-files"> + <output-dir compare="Text">aws/s3/no-files-returned/exclude-all-files</output-dir> + <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn> + </compilation-unit> + </test-case> + <test-case FilePath="external-dataset" check-warnings="true"> + <compilation-unit name="aws/s3/no-files-returned/include-no-files"> + <output-dir compare="Text">aws/s3/no-files-returned/include-no-files</output-dir> + <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn> + </compilation-unit> + </test-case> </test-group> <test-group name="include-exclude"> <test-case FilePath="external-dataset"> diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java index 46a6228..d57c433 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java @@ -202,6 +202,7 @@ public class ErrorCode { public static final int PROPERTY_INVALID_VALUE_TYPE = 1111; public static final int INVALID_PROPERTY_FORMAT = 1112; public static final int INVALID_REGEX_PATTERN = 1113; + public static final int EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES = 1114; // Feed errors public static final int DATAFLOW_ILLEGAL_STATE = 3001; diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/NoOpWarningCollector.java similarity index 61% copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java copy to asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/NoOpWarningCollector.java index 598d9ff..a036b7e 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/NoOpWarningCollector.java @@ -16,23 +16,31 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.asterix.external.util; -import org.apache.hyracks.api.exceptions.ErrorCode; +package org.apache.asterix.common.exceptions; + import org.apache.hyracks.api.exceptions.IWarningCollector; -import org.apache.hyracks.api.exceptions.SourceLocation; import org.apache.hyracks.api.exceptions.Warning; -public class ParseUtil { +public final class NoOpWarningCollector implements IWarningCollector { + + public static final IWarningCollector INSTANCE = new NoOpWarningCollector(); - private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1); + private NoOpWarningCollector() { + } + + @Override + public void warn(Warning warning) { + // no-op + } - private ParseUtil() { + @Override + public boolean shouldWarn() { + return false; } - public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum, - String warnMessage) { - warningCollector.warn( - Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage)); + @Override + public long getTotalWarningsCount() { + return 0; } } diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/IAdapterFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/IAdapterFactory.java index e2e7e3e..6d2df37 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/IAdapterFactory.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/IAdapterFactory.java @@ -26,6 +26,7 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.IWarningCollector; /** * Base interface for IGenericDatasetAdapterFactory and ITypedDatasetAdapterFactory. @@ -71,9 +72,10 @@ public interface IAdapterFactory extends Serializable { * * @param serviceContext * @param configuration + * @param warningCollector warning collector * @throws AlgebricksException * @throws HyracksDataException */ - void configure(IServiceContext serviceContext, Map<String, String> configuration) - throws HyracksDataException, AlgebricksException; + void configure(IServiceContext serviceContext, Map<String, String> configuration, + IWarningCollector warningCollector) throws HyracksDataException, AlgebricksException; } diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties index b070552..5cf3968 100644 --- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties +++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties @@ -197,6 +197,7 @@ 1111 = Property \"%1$s\" expects value(s) of type %2$s 1112 = Invalid format for property \"%1$s\" 1113 = Invalid pattern %1$s +1114 = The provided external dataset configuration returned no files from the external source # Feed Errors 3001 = Illegal state. diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java index 078316c..badf105 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java @@ -55,6 +55,7 @@ import org.apache.hyracks.api.application.INCServiceContext; import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.IWarningCollector; import org.apache.hyracks.api.io.FileSplit; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; @@ -100,7 +101,7 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, ITypedAda INCServiceContext serviceCtx = ctx.getJobletContext().getServiceContext(); INcApplicationContext appCtx = (INcApplicationContext) serviceCtx.getApplicationContext(); try { - restoreExternalObjects(serviceCtx, appCtx.getLibraryManager()); + restoreExternalObjects(serviceCtx, appCtx.getLibraryManager(), ctx.getWarningCollector()); } catch (Exception e) { LOGGER.log(Level.INFO, "Failure restoring external objects", e); throw HyracksDataException.create(e); @@ -120,15 +121,15 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, ITypedAda } } - private void restoreExternalObjects(IServiceContext serviceContext, ILibraryManager libraryManager) - throws HyracksDataException, AlgebricksException { + private void restoreExternalObjects(IServiceContext serviceContext, ILibraryManager libraryManager, + IWarningCollector warningCollector) throws HyracksDataException, AlgebricksException { if (dataSourceFactory == null) { dataSourceFactory = createExternalDataSourceFactory(configuration, libraryManager); // create and configure parser factory if (dataSourceFactory.isIndexible() && (files != null)) { ((IIndexibleExternalDataSource) dataSourceFactory).setSnapshot(files, indexingOp); } - dataSourceFactory.configure(serviceContext, configuration); + dataSourceFactory.configure(serviceContext, configuration, warningCollector); } if (dataParserFactory == null) { // create and configure parser factory @@ -140,8 +141,8 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, ITypedAda } @Override - public void configure(IServiceContext serviceContext, Map<String, String> configuration) - throws HyracksDataException, AlgebricksException { + public void configure(IServiceContext serviceContext, Map<String, String> configuration, + IWarningCollector warningCollector) throws HyracksDataException, AlgebricksException { this.configuration = configuration; IApplicationContext appCtx = (IApplicationContext) serviceContext.getApplicationContext(); ExternalDataUtils.validateDataSourceParameters(configuration); @@ -149,7 +150,7 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, ITypedAda if (dataSourceFactory.isIndexible() && (files != null)) { ((IIndexibleExternalDataSource) dataSourceFactory).setSnapshot(files, indexingOp); } - dataSourceFactory.configure(serviceContext, configuration); + dataSourceFactory.configure(serviceContext, configuration, warningCollector); ExternalDataUtils.validateDataParserParameters(configuration); dataParserFactory = createDataParserFactory(configuration, appCtx.getLibraryManager()); dataParserFactory.setRecordType(recordType); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/LookupAdapterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/LookupAdapterFactory.java index b51416a..b543804 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/LookupAdapterFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/LookupAdapterFactory.java @@ -40,6 +40,7 @@ import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory; import org.apache.hyracks.api.dataflow.value.RecordDescriptor; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.IWarningCollector; @SuppressWarnings({ "rawtypes", "unchecked" }) public class LookupAdapterFactory<T> implements Serializable { @@ -78,15 +79,16 @@ public class LookupAdapterFactory<T> implements Serializable { } } - public void configure(IServiceContext serviceContext, Map<String, String> configuration) - throws HyracksDataException, AlgebricksException { + public void configure(IServiceContext serviceContext, Map<String, String> configuration, + IWarningCollector warningCollector) throws HyracksDataException, AlgebricksException { this.configuration = configuration; IApplicationContext appCtx = (IApplicationContext) serviceContext.getApplicationContext(); - readerFactory = LookupReaderFactoryProvider.getLookupReaderFactory(serviceContext, configuration); + readerFactory = + LookupReaderFactoryProvider.getLookupReaderFactory(serviceContext, configuration, warningCollector); dataParserFactory = (IRecordDataParserFactory<T>) ParserFactoryProvider .getDataParserFactory(appCtx.getLibraryManager(), configuration); dataParserFactory.setRecordType(recordType); - readerFactory.configure(serviceContext, configuration); + readerFactory.configure(serviceContext, configuration, warningCollector); dataParserFactory.configure(configuration); } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java index efa12ef..17cbff4 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java @@ -31,6 +31,7 @@ import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartit import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.IWarningCollector; public interface IExternalDataSourceFactory extends Serializable { @@ -63,7 +64,7 @@ public interface IExternalDataSourceFactory extends Serializable { * @param configuration * @throws AsterixException */ - void configure(IServiceContext ctx, Map<String, String> configuration) + void configure(IServiceContext ctx, Map<String, String> configuration, IWarningCollector warningCollector) throws AlgebricksException, HyracksDataException; /** diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java index 7702dde..dba0f27 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java @@ -51,6 +51,7 @@ import org.apache.hyracks.api.application.ICCServiceContext; import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.IWarningCollector; import org.apache.hyracks.hdfs.dataflow.ConfFactory; import org.apache.hyracks.hdfs.dataflow.InputSplitsFactory; import org.apache.hyracks.hdfs.scheduler.Scheduler; @@ -80,7 +81,8 @@ public class HDFSDataSourceFactory implements IRecordReaderFactory<Object>, IInd private static final List<String> recordReaderNames = Collections.unmodifiableList(Arrays.asList("hdfs")); @Override - public void configure(IServiceContext serviceCtx, Map<String, String> configuration) throws AsterixException { + public void configure(IServiceContext serviceCtx, Map<String, String> configuration, + IWarningCollector warningCollector) throws AsterixException { try { this.serviceCtx = serviceCtx; this.configuration = configuration; diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java index 6ef5a23..67a7c93 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java @@ -35,6 +35,7 @@ import java.util.regex.PatternSyntaxException; import org.apache.asterix.common.dataflow.ICcApplicationContext; import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.common.exceptions.ErrorCode; +import org.apache.asterix.common.exceptions.WarningUtil; import org.apache.asterix.external.api.AsterixInputStream; import org.apache.asterix.external.api.IInputStreamFactory; import org.apache.asterix.external.util.ExternalDataUtils; @@ -43,6 +44,8 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.IWarningCollector; +import org.apache.hyracks.api.exceptions.Warning; import org.apache.hyracks.api.util.CleanupUtils; import software.amazon.awssdk.core.exception.SdkException; @@ -80,7 +83,8 @@ public class AwsS3InputStreamFactory implements IInputStreamFactory { } @Override - public void configure(IServiceContext ctx, Map<String, String> configuration) throws AlgebricksException { + public void configure(IServiceContext ctx, Map<String, String> configuration, IWarningCollector warningCollector) + throws AlgebricksException { this.configuration = configuration; ICcApplicationContext ccApplicationContext = (ICcApplicationContext) ctx.getApplicationContext(); @@ -126,10 +130,7 @@ public class AwsS3InputStreamFactory implements IInputStreamFactory { // Get all objects in a bucket and extract the paths to files ListObjectsV2Request.Builder listObjectsBuilder = ListObjectsV2Request.builder().bucket(container); - String path = configuration.get(AwsS3.DEFINITION_FIELD_NAME); - if (path != null) { - listObjectsBuilder.prefix(path + (!path.isEmpty() && !path.endsWith("/") ? "/" : "")); - } + ExternalDataUtils.AwsS3.setPrefix(configuration, listObjectsBuilder); ListObjectsV2Response listObjectsResponse; boolean done = false; @@ -163,6 +164,12 @@ public class AwsS3InputStreamFactory implements IInputStreamFactory { } } + // Warn if no files are returned + if (filesOnly.isEmpty() && warningCollector.shouldWarn()) { + Warning warning = WarningUtil.forAsterix(null, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES); + warningCollector.warn(warning); + } + // Partition constraints partitionConstraint = ccApplicationContext.getClusterStateManager().getClusterLocations(); int partitionsCount = partitionConstraint.getLocations().length; diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3ReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3ReaderFactory.java index 6484d4e..1dd0c8b 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3ReaderFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3ReaderFactory.java @@ -29,6 +29,7 @@ import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartit import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.IWarningCollector; public class AwsS3ReaderFactory extends StreamRecordReaderFactory { @@ -58,13 +59,13 @@ public class AwsS3ReaderFactory extends StreamRecordReaderFactory { } @Override - public void configure(IServiceContext ctx, Map<String, String> configuration) + public void configure(IServiceContext ctx, Map<String, String> configuration, IWarningCollector warningCollector) throws AlgebricksException, HyracksDataException { this.configuration = configuration; // Stream factory streamFactory = new AwsS3InputStreamFactory(); - streamFactory.configure(ctx, configuration); + streamFactory.configure(ctx, configuration, warningCollector); // record reader recordReaderClazz = StreamRecordReaderProvider.getRecordReaderClazz(configuration); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSLookupReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSLookupReaderFactory.java index 98f78cc..8b258e4 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSLookupReaderFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSLookupReaderFactory.java @@ -34,6 +34,7 @@ import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartit import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.IWarningCollector; import org.apache.hyracks.hdfs.dataflow.ConfFactory; public class HDFSLookupReaderFactory<T> implements ILookupReaderFactory<T> { @@ -60,7 +61,8 @@ public class HDFSLookupReaderFactory<T> implements ILookupReaderFactory<T> { } @Override - public void configure(IServiceContext serviceCtx, Map<String, String> configuration) throws AsterixException { + public void configure(IServiceContext serviceCtx, Map<String, String> configuration, + IWarningCollector warningCollector) throws AsterixException { this.serviceCtx = serviceCtx; this.configuration = configuration; JobConf conf = HDFSUtils.configureHDFSJobConf(configuration); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReaderFactory.java index d14071f..5954d74 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReaderFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReaderFactory.java @@ -33,6 +33,7 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.IWarningCollector; import org.apache.hyracks.http.server.HttpServerConfigBuilder; public class HttpServerRecordReaderFactory implements IRecordReaderFactory<char[]> { @@ -86,7 +87,8 @@ public class HttpServerRecordReaderFactory implements IRecordReaderFactory<char[ } @Override - public void configure(IServiceContext ctx, Map<String, String> configuration) throws AlgebricksException { + public void configure(IServiceContext ctx, Map<String, String> configuration, IWarningCollector warningCollector) + throws AlgebricksException { this.configurations = configuration; // necessary configs addrValue = getConfigurationValue(KEY_CONFIGURATION_ADDRESSES, true); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReaderFactory.java index 2c07c35..b52636a 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReaderFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReaderFactory.java @@ -35,6 +35,7 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.IWarningCollector; import com.rometools.rome.feed.synd.SyndEntry; @@ -60,7 +61,8 @@ public class RSSRecordReaderFactory implements IRecordReaderFactory<SyndEntry> { } @Override - public void configure(IServiceContext serviceContext, Map<String, String> configuration) { + public void configure(IServiceContext serviceContext, Map<String, String> configuration, + IWarningCollector warningCollector) { this.serviceContext = serviceContext; String url = configuration.get(ExternalDataConstants.KEY_RSS_URL); if (url == null) { diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java index 9bb50f6..c0a1b38 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java @@ -39,6 +39,7 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.IWarningCollector; public class StreamRecordReaderFactory implements IRecordReaderFactory<char[]> { @@ -81,11 +82,11 @@ public class StreamRecordReaderFactory implements IRecordReaderFactory<char[]> { } @Override - public void configure(IServiceContext serviceCtx, Map<String, String> configuration) - throws HyracksDataException, AlgebricksException { + public void configure(IServiceContext serviceCtx, Map<String, String> configuration, + IWarningCollector warningCollector) throws HyracksDataException, AlgebricksException { this.configuration = configuration; configureInputStreamFactory(configuration); - streamFactory.configure(serviceCtx, configuration); + streamFactory.configure(serviceCtx, configuration, warningCollector); recordReaderClazz = StreamRecordReaderProvider.getRecordReaderClazz(configuration); } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java index 43a3816..0fd0b95 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java @@ -38,6 +38,7 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.IWarningCollector; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -78,7 +79,8 @@ public class TwitterRecordReaderFactory implements IRecordReaderFactory<char[]> } @Override - public void configure(IServiceContext serviceCtx, Map<String, String> configuration) throws AsterixException { + public void configure(IServiceContext serviceCtx, Map<String, String> configuration, + IWarningCollector warningCollector) throws AsterixException { try { Class.forName("twitter4j.Twitter"); } catch (ClassNotFoundException e) { diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java index 6d91067..58ef2a4 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java @@ -43,6 +43,7 @@ import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartit import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.IWarningCollector; import org.apache.hyracks.api.io.UnmanagedFileSplit; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; @@ -90,7 +91,8 @@ public class LocalFSInputStreamFactory implements IInputStreamFactory { } @Override - public void configure(IServiceContext serviceCtx, Map<String, String> configuration) throws AsterixException { + public void configure(IServiceContext serviceCtx, Map<String, String> configuration, + IWarningCollector warningCollector) throws AsterixException { this.configuration = configuration; String[] splits = configuration.get(ExternalDataConstants.KEY_PATH).split(","); if (inputFileSplits == null) { diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java index bfc2cb7..5d2b2a6 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java @@ -39,6 +39,7 @@ import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.IWarningCollector; public class SocketClientInputStreamFactory implements IInputStreamFactory { @@ -55,7 +56,8 @@ public class SocketClientInputStreamFactory implements IInputStreamFactory { } @Override - public void configure(IServiceContext serviceCtx, Map<String, String> configuration) throws AsterixException { + public void configure(IServiceContext serviceCtx, Map<String, String> configuration, + IWarningCollector warningCollector) throws AsterixException { try { this.serviceCtx = serviceCtx; this.sockets = new ArrayList<>(); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java index ac3ac42..1bd08d9 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java @@ -36,6 +36,7 @@ import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartit import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.IWarningCollector; public class SocketServerInputStreamFactory implements IInputStreamFactory { @@ -43,7 +44,8 @@ public class SocketServerInputStreamFactory implements IInputStreamFactory { private List<Pair<String, Integer>> sockets; @Override - public void configure(IServiceContext serviceCtx, Map<String, String> configuration) throws CompilationException { + public void configure(IServiceContext serviceCtx, Map<String, String> configuration, + IWarningCollector warningCollector) throws CompilationException { try { sockets = FeedUtils.extractHostsPorts(configuration.get(ExternalDataConstants.KEY_MODE), serviceCtx, configuration.get(ExternalDataConstants.KEY_SOCKETS)); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java index c7b8633..2b0bb55 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java @@ -31,6 +31,7 @@ import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartit import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.IWarningCollector; /** * Factory class for creating @see{TwitterFirehoseFeedAdapter}. The adapter @@ -84,7 +85,8 @@ public class TwitterFirehoseStreamFactory implements IInputStreamFactory { } @Override - public void configure(IServiceContext serviceCtx, Map<String, String> configuration) { + public void configure(IServiceContext serviceCtx, Map<String, String> configuration, + IWarningCollector warningCollector) { this.serviceCtx = serviceCtx; this.configuration = configuration; } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java index d63e8a8..7c709fb 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java @@ -110,7 +110,8 @@ public class FeedIntakeOperatorDescriptor extends AbstractSingleActivityOperator try { adapterFactory = (ITypedAdapterFactory) (classLoader.loadClass(adaptorFactoryClassName).newInstance()); adapterFactory.setOutputType(adapterOutputType); - adapterFactory.configure(ctx.getJobletContext().getServiceContext(), adaptorConfiguration); + adapterFactory.configure(ctx.getJobletContext().getServiceContext(), adaptorConfiguration, + ctx.getWarningCollector()); } catch (Exception e) { throw HyracksDataException.create(e); } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java index 27ac10e..96f06f4 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java @@ -33,6 +33,7 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.IWarningCollector; /** * This class represents the entry point to all things adapters @@ -44,8 +45,8 @@ public class AdapterFactoryProvider { // get adapter factory. this method has the side effect of modifying the configuration as necessary public static ITypedAdapterFactory getAdapterFactory(IServiceContext serviceCtx, String adapterName, - Map<String, String> configuration, ARecordType itemType, ARecordType metaType) - throws HyracksDataException, AlgebricksException { + Map<String, String> configuration, ARecordType itemType, ARecordType metaType, + IWarningCollector warningCollector) throws HyracksDataException, AlgebricksException { ExternalDataUtils.defaultConfiguration(configuration); ExternalDataUtils.prepare(adapterName, configuration); ICcApplicationContext context = (ICcApplicationContext) serviceCtx.getApplicationContext(); @@ -53,32 +54,32 @@ public class AdapterFactoryProvider { (ITypedAdapterFactory) context.getAdapterFactoryService().createAdapterFactory(); adapterFactory.setOutputType(itemType); adapterFactory.setMetaType(metaType); - adapterFactory.configure(serviceCtx, configuration); + adapterFactory.configure(serviceCtx, configuration, warningCollector); return adapterFactory; } // get indexing adapter factory. this method has the side effect of modifying the configuration as necessary public static IIndexingAdapterFactory getIndexingAdapterFactory(IServiceContext serviceCtx, String adapterName, Map<String, String> configuration, ARecordType itemType, List<ExternalFile> snapshot, boolean indexingOp, - ARecordType metaType) throws HyracksDataException, AlgebricksException { + ARecordType metaType, IWarningCollector warningCollector) throws HyracksDataException, AlgebricksException { ExternalDataUtils.defaultConfiguration(configuration); ExternalDataUtils.prepare(adapterName, configuration); GenericAdapterFactory adapterFactory = new GenericAdapterFactory(); adapterFactory.setOutputType(itemType); adapterFactory.setMetaType(metaType); adapterFactory.setSnapshot(snapshot, indexingOp); - adapterFactory.configure(serviceCtx, configuration); + adapterFactory.configure(serviceCtx, configuration, warningCollector); return adapterFactory; } // Lookup Adapters public static LookupAdapterFactory<?> getLookupAdapterFactory(IServiceContext serviceCtx, Map<String, String> configuration, ARecordType recordType, int[] ridFields, boolean retainInput, - boolean retainMissing, IMissingWriterFactory missingWriterFactory) + boolean retainMissing, IMissingWriterFactory missingWriterFactory, IWarningCollector warningCollector) throws HyracksDataException, AlgebricksException { LookupAdapterFactory<?> adapterFactory = new LookupAdapterFactory<>(recordType, ridFields, retainInput, retainMissing, missingWriterFactory); - adapterFactory.configure(serviceCtx, configuration); + adapterFactory.configure(serviceCtx, configuration, warningCollector); return adapterFactory; } } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/LookupReaderFactoryProvider.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/LookupReaderFactoryProvider.java index 69f619f..db80c2a 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/LookupReaderFactoryProvider.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/LookupReaderFactoryProvider.java @@ -26,18 +26,19 @@ import org.apache.asterix.external.input.record.reader.hdfs.HDFSLookupReaderFact import org.apache.asterix.external.util.ExternalDataConstants; import org.apache.asterix.external.util.HDFSUtils; import org.apache.hyracks.api.application.IServiceContext; +import org.apache.hyracks.api.exceptions.IWarningCollector; public class LookupReaderFactoryProvider { @SuppressWarnings("rawtypes") public static ILookupReaderFactory getLookupReaderFactory(IServiceContext serviceCtx, - Map<String, String> configuration) throws AsterixException { + Map<String, String> configuration, IWarningCollector warningCollector) throws AsterixException { String inputFormat = HDFSUtils.getInputFormatClassName(configuration); if (inputFormat.equals(ExternalDataConstants.CLASS_NAME_TEXT_INPUT_FORMAT) || inputFormat.equals(ExternalDataConstants.CLASS_NAME_SEQUENCE_INPUT_FORMAT) || inputFormat.equals(ExternalDataConstants.CLASS_NAME_RC_INPUT_FORMAT)) { HDFSLookupReaderFactory<Object> readerFactory = new HDFSLookupReaderFactory<>(); - readerFactory.configure(serviceCtx, configuration); + readerFactory.configure(serviceCtx, configuration, warningCollector); return readerFactory; } else { throw new AsterixException("Unrecognized external format"); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java index 8496761..1300ac3 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java @@ -37,6 +37,7 @@ import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.common.exceptions.ErrorCode; import org.apache.asterix.common.exceptions.RuntimeDataException; +import org.apache.asterix.common.exceptions.WarningUtil; import org.apache.asterix.common.library.ILibraryManager; import org.apache.asterix.external.api.IDataParserFactory; import org.apache.asterix.external.api.IExternalDataSourceFactory.DataSourceType; @@ -48,7 +49,9 @@ import org.apache.asterix.om.types.AUnionType; import org.apache.asterix.runtime.evaluators.common.NumberUtils; import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.IWarningCollector; import org.apache.hyracks.api.exceptions.SourceLocation; +import org.apache.hyracks.api.exceptions.Warning; import org.apache.hyracks.api.util.CleanupUtils; import org.apache.hyracks.dataflow.common.data.parsers.BooleanParserFactory; import org.apache.hyracks.dataflow.common.data.parsers.DoubleParserFactory; @@ -462,13 +465,13 @@ public class ExternalDataUtils { * * @param configuration properties */ - public static void validateAdapterSpecificProperties(Map<String, String> configuration, SourceLocation srcLoc) - throws CompilationException { + public static void validateAdapterSpecificProperties(Map<String, String> configuration, SourceLocation srcLoc, + IWarningCollector collector) throws CompilationException { String type = configuration.get(ExternalDataConstants.KEY_EXTERNAL_SOURCE_TYPE); switch (type) { case ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3: - ExternalDataUtils.AwsS3.validateProperties(configuration, srcLoc); + ExternalDataUtils.AwsS3.validateProperties(configuration, srcLoc, collector); break; default: // Nothing needs to be done @@ -622,14 +625,27 @@ public class ExternalDataUtils { } /** + * Sets the prefix for the list objects builder if it is available + * + * @param configuration configuration + * @param builder builder + */ + public static void setPrefix(Map<String, String> configuration, ListObjectsV2Request.Builder builder) { + String definition = configuration.get(ExternalDataConstants.AwsS3.DEFINITION_FIELD_NAME); + if (definition != null) { + builder.prefix(definition + (!definition.isEmpty() && !definition.endsWith("/") ? "/" : "")); + } + } + + /** * Validate external dataset properties * * @param configuration properties * * @throws CompilationException Compilation exception */ - public static void validateProperties(Map<String, String> configuration, SourceLocation srcLoc) - throws CompilationException { + public static void validateProperties(Map<String, String> configuration, SourceLocation srcLoc, + IWarningCollector collector) throws CompilationException { // check if the format property is present if (configuration.get(ExternalDataConstants.KEY_FORMAT) == null) { @@ -643,8 +659,17 @@ public class ExternalDataUtils { try { String container = configuration.get(ExternalDataConstants.AwsS3.CONTAINER_NAME_FIELD_NAME); s3Client = buildAwsS3Client(configuration); + ListObjectsV2Request.Builder listObjectsBuilder = ListObjectsV2Request.builder(); + setPrefix(configuration, listObjectsBuilder); + ListObjectsV2Response response = - s3Client.listObjectsV2(ListObjectsV2Request.builder().bucket(container).maxKeys(1).build()); + s3Client.listObjectsV2(listObjectsBuilder.bucket(container).maxKeys(1).build()); + + if (response.contents().isEmpty() && collector.shouldWarn()) { + Warning warning = + WarningUtil.forAsterix(srcLoc, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES); + collector.warn(warning); + } // Returns 200 only in case the bucket exists, however, otherwise, throws an exception. However, to // ensure coverage, check if the result is successful as well and not only catch exceptions @@ -661,7 +686,6 @@ public class ExternalDataUtils { } /** - * TODO(Hussain) * @param configuration * @throws CompilationException */ diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java index 598d9ff..7afc592 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java @@ -20,19 +20,16 @@ package org.apache.asterix.external.util; import org.apache.hyracks.api.exceptions.ErrorCode; import org.apache.hyracks.api.exceptions.IWarningCollector; -import org.apache.hyracks.api.exceptions.SourceLocation; import org.apache.hyracks.api.exceptions.Warning; public class ParseUtil { - private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1); - private ParseUtil() { } public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum, String warnMessage) { warningCollector.warn( - Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage)); + Warning.forHyracks(null, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage)); } } diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RecordWithPKTestReaderFactory.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RecordWithPKTestReaderFactory.java index 715ad02..c65e00d 100644 --- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RecordWithPKTestReaderFactory.java +++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RecordWithPKTestReaderFactory.java @@ -32,6 +32,7 @@ import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartit import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.context.IHyracksTaskContext; +import org.apache.hyracks.api.exceptions.IWarningCollector; public class RecordWithPKTestReaderFactory implements IRecordReaderFactory<RecordWithPK<char[]>> { @@ -48,7 +49,8 @@ public class RecordWithPKTestReaderFactory implements IRecordReaderFactory<Recor } @Override - public void configure(IServiceContext serviceCtx, final Map<String, String> configuration) { + public void configure(IServiceContext serviceCtx, final Map<String, String> configuration, + IWarningCollector warningCollector) { this.serviceCtx = serviceCtx; } diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java index c09b9eb..49ca23e 100644 --- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java +++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java @@ -29,6 +29,7 @@ import org.apache.asterix.external.api.IRecordReaderFactory; import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.context.IHyracksTaskContext; +import org.apache.hyracks.api.exceptions.IWarningCollector; import com.couchbase.client.core.message.dcp.DCPRequest; @@ -58,7 +59,8 @@ public class KVTestReaderFactory implements IRecordReaderFactory<DCPRequest> { } @Override - public void configure(IServiceContext serviceCtx, final Map<String, String> configuration) { + public void configure(IServiceContext serviceCtx, final Map<String, String> configuration, + IWarningCollector warningCollector) { this.serviceCtx = serviceCtx; if (configuration.containsKey("num-of-records")) { numOfRecords = Integer.parseInt(configuration.get("num-of-records")); diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java index a947c7e..0bd601b 100644 --- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java +++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java @@ -37,6 +37,7 @@ import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.comm.IFrameWriter; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.IWarningCollector; import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder; import org.apache.hyracks.dataflow.std.file.ITupleParser; import org.apache.hyracks.dataflow.std.file.ITupleParserFactory; @@ -113,7 +114,8 @@ public class TestTypedAdapterFactory implements ITypedAdapterFactory { } @Override - public void configure(IServiceContext serviceContext, Map<String, String> configuration) { + public void configure(IServiceContext serviceContext, Map<String, String> configuration, + IWarningCollector warningCollector) { this.serviceContext = serviceContext; this.configuration = configuration; } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java index 07bbc57..71d389e 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java @@ -111,8 +111,9 @@ public class DatasetDataSource extends DataSource { externalDataset.getItemTypeDataverseName(), itemTypeName).getDatatype(); ExternalDatasetDetails edd = (ExternalDatasetDetails) externalDataset.getDatasetDetails(); - ITypedAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(externalDataset, - edd.getAdapter(), edd.getProperties(), (ARecordType) itemType, null); + ITypedAdapterFactory adapterFactory = + metadataProvider.getConfiguredAdapterFactory(externalDataset, edd.getAdapter(), + edd.getProperties(), (ARecordType) itemType, null, context.getWarningCollector()); return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory); case INTERNAL: DataSourceId id = getId(); diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSourceFactory.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSourceFactory.java index 1e0ddbf..5503cb0 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSourceFactory.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSourceFactory.java @@ -30,6 +30,7 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.IWarningCollector; public class FunctionDataSourceFactory implements IRecordReaderFactory<char[]> { @@ -51,7 +52,7 @@ public class FunctionDataSourceFactory implements IRecordReaderFactory<char[]> { } @Override - public void configure(IServiceContext ctx, Map<String, String> configuration) + public void configure(IServiceContext ctx, Map<String, String> configuration, IWarningCollector warningCollector) throws AlgebricksException, HyracksDataException { // No Op } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java index c2983af..e25ea22 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java @@ -138,7 +138,7 @@ public class LoadableDataSource extends DataSource { LoadableDataSource alds = (LoadableDataSource) dataSource; ARecordType itemType = (ARecordType) alds.getLoadedType(); ITypedAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(alds.getTargetDataset(), - alds.getAdapter(), alds.getAdapterProperties(), itemType, null); + alds.getAdapter(), alds.getAdapterProperties(), itemType, null, context.getWarningCollector()); RecordDescriptor rDesc = JobGenHelper.mkRecordDescriptor(typeEnv, opSchema, context); return metadataProvider.buildLoadableDatasetScan(jobSpec, adapterFactory, rDesc); } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java index 6b6cc78..c5109b7 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java @@ -130,6 +130,7 @@ import org.apache.hyracks.api.dataflow.value.IResultSerializerFactory; import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.dataflow.value.RecordDescriptor; +import org.apache.hyracks.api.exceptions.IWarningCollector; import org.apache.hyracks.api.io.FileSplit; import org.apache.hyracks.api.job.JobSpecification; import org.apache.hyracks.api.result.IResultMetadata; @@ -781,11 +782,13 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> } protected ITypedAdapterFactory getConfiguredAdapterFactory(Dataset dataset, String adapterName, - Map<String, String> configuration, ARecordType itemType, ARecordType metaType) throws AlgebricksException { + Map<String, String> configuration, ARecordType itemType, ARecordType metaType, + IWarningCollector warningCollector) throws AlgebricksException { try { configuration.put(ExternalDataConstants.KEY_DATAVERSE, dataset.getDataverseName()); - ITypedAdapterFactory adapterFactory = AdapterFactoryProvider.getAdapterFactory( - getApplicationContext().getServiceContext(), adapterName, configuration, itemType, metaType); + ITypedAdapterFactory adapterFactory = + AdapterFactoryProvider.getAdapterFactory(getApplicationContext().getServiceContext(), adapterName, + configuration, itemType, metaType, warningCollector); // check to see if dataset is indexed Index filesIndex = @@ -858,7 +861,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> ExternalDatasetDetails datasetDetails = (ExternalDatasetDetails) dataset.getDatasetDetails(); LookupAdapterFactory<?> adapterFactory = AdapterFactoryProvider.getLookupAdapterFactory( getApplicationContext().getServiceContext(), datasetDetails.getProperties(), itemType, ridIndexes, - retainInput, retainMissing, context.getMissingWriterFactory()); + retainInput, retainMissing, context.getMissingWriterFactory(), context.getWarningCollector()); String fileIndexName = IndexingConstants.getFilesIndexName(dataset.getDatasetName()); Pair<IFileSplitProvider, AlgebricksPartitionConstraint> spPc = metadataProvider.getSplitProviderAndConstraints(dataset, fileIndexName); diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java index 071fd1c..792501b 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java @@ -28,6 +28,7 @@ import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.common.exceptions.ErrorCode; import org.apache.asterix.common.exceptions.MetadataException; +import org.apache.asterix.common.exceptions.NoOpWarningCollector; import org.apache.asterix.common.external.IDataSourceAdapter; import org.apache.asterix.common.external.IDataSourceAdapter.AdapterType; import org.apache.asterix.external.api.ITypedAdapterFactory; @@ -52,6 +53,7 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.Triple; import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; import org.apache.hyracks.api.dataflow.value.RecordDescriptor; +import org.apache.hyracks.api.exceptions.IWarningCollector; /** * A utility class for providing helper functions for feeds TODO: Refactor this @@ -98,8 +100,8 @@ public class FeedMetadataUtil { return feedPolicy; } - public static void validateFeed(Feed feed, MetadataTransactionContext mdTxnCtx, ICcApplicationContext appCtx) - throws AlgebricksException { + public static void validateFeed(Feed feed, MetadataTransactionContext mdTxnCtx, ICcApplicationContext appCtx, + IWarningCollector warningCollector) throws AlgebricksException { try { Map<String, String> configuration = feed.getConfiguration(); ARecordType adapterOutputType = getOutputType(feed, configuration.get(ExternalDataConstants.KEY_TYPE_NAME)); @@ -141,7 +143,7 @@ public class FeedMetadataUtil { } adapterFactory.setOutputType(adapterOutputType); adapterFactory.setMetaType(metaType); - adapterFactory.configure(appCtx.getServiceContext(), configuration); + adapterFactory.configure(appCtx.getServiceContext(), configuration, warningCollector); if (metaType == null && configuration.containsKey(ExternalDataConstants.KEY_META_TYPE_NAME)) { metaType = getOutputType(feed, configuration.get(ExternalDataConstants.KEY_META_TYPE_NAME)); if (metaType == null) { @@ -210,10 +212,10 @@ public class FeedMetadataUtil { } adapterFactory.setOutputType(adapterOutputType); adapterFactory.setMetaType(metaType); - adapterFactory.configure(appCtx.getServiceContext(), configuration); + adapterFactory.configure(appCtx.getServiceContext(), configuration, NoOpWarningCollector.INSTANCE); } else { adapterFactory = AdapterFactoryProvider.getAdapterFactory(appCtx.getServiceContext(), adapterName, - configuration, adapterOutputType, metaType); + configuration, adapterOutputType, metaType, NoOpWarningCollector.INSTANCE); adapterType = IDataSourceAdapter.AdapterType.INTERNAL; } if (metaType == null) { diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ExternalIndexingOperations.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ExternalIndexingOperations.java index c1d8f42..9847fe4 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ExternalIndexingOperations.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ExternalIndexingOperations.java @@ -30,6 +30,7 @@ import org.apache.asterix.common.config.DatasetConfig.DatasetType; import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp; import org.apache.asterix.common.config.DatasetConfig.TransactionState; import org.apache.asterix.common.context.IStorageComponentProvider; +import org.apache.asterix.common.exceptions.NoOpWarningCollector; import org.apache.asterix.external.api.ITypedAdapterFactory; import org.apache.asterix.external.indexing.ExternalFile; import org.apache.asterix.external.indexing.IndexingConstants; @@ -256,7 +257,7 @@ public class ExternalIndexingOperations { Map<String, String> configuration = externalDatasetDetails.getProperties(); ITypedAdapterFactory adapterFactory = AdapterFactoryProvider.getIndexingAdapterFactory( metadataProvider.getApplicationContext().getServiceContext(), externalDatasetDetails.getAdapter(), - configuration, (ARecordType) itemType, files, true, null); + configuration, (ARecordType) itemType, files, true, null, NoOpWarningCollector.INSTANCE); ExternalScanOperatorDescriptor scanOp = new ExternalScanOperatorDescriptor(jobSpec, indexerDesc, adapterFactory); scanOp.setSourceLocation(sourceLoc); diff --git a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java index ab68701..0fd107f 100644 --- a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java +++ b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java @@ -105,7 +105,7 @@ public class HeuristicCompilerFactoryBuilder extends AbstractCompilerFactoryBuil normalizedKeyComputerFactoryProvider, expressionRuntimeProvider, expressionTypeComputer, oc, expressionEvalSizeComputer, partialAggregationTypeComputer, predEvaluatorFactoryProvider, physicalOptimizationConfig.getFrameSize(), - clusterLocations, maxWarnings); + clusterLocations, warningCollector, maxWarnings); PlanCompiler pc = new PlanCompiler(context); return pc.compilePlan(plan, jobEventListenerFactory); diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenContext.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenContext.java index ae333e2..4bc5689 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenContext.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenContext.java @@ -45,6 +45,7 @@ import org.apache.hyracks.algebricks.data.ISerializerDeserializerProvider; import org.apache.hyracks.algebricks.data.ITypeTraitProvider; import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory; import org.apache.hyracks.api.dataflow.value.IPredicateEvaluatorFactoryProvider; +import org.apache.hyracks.api.exceptions.IWarningCollector; public class JobGenContext { private final IOperatorSchema outerFlowSchema; @@ -70,6 +71,7 @@ public class JobGenContext { private AlgebricksAbsolutePartitionConstraint clusterLocations; private int varCounter; private final ITypingContext typingContext; + private final IWarningCollector warningCollector; private final long maxWarnings; public JobGenContext(IOperatorSchema outerFlowSchema, IMetadataProvider<?, ?> metadataProvider, Object appContext, @@ -85,7 +87,8 @@ public class JobGenContext { ITypingContext typingContext, IExpressionEvalSizeComputer expressionEvalSizeComputer, IPartialAggregationTypeComputer partialAggregationTypeComputer, IPredicateEvaluatorFactoryProvider predEvaluatorFactoryProvider, int frameSize, - AlgebricksAbsolutePartitionConstraint clusterLocations, long maxWarnings) { + AlgebricksAbsolutePartitionConstraint clusterLocations, IWarningCollector warningCollector, + long maxWarnings) { this.outerFlowSchema = outerFlowSchema; this.metadataProvider = metadataProvider; this.appContext = appContext; @@ -108,6 +111,7 @@ public class JobGenContext { this.predEvaluatorFactoryProvider = predEvaluatorFactoryProvider; this.frameSize = frameSize; this.varCounter = 0; + this.warningCollector = warningCollector; this.maxWarnings = maxWarnings; } @@ -209,6 +213,10 @@ public class JobGenContext { return typingContext.getOutputTypeEnvironment(op); } + public IWarningCollector getWarningCollector() { + return warningCollector; + } + public long getMaxWarnings() { return maxWarnings; } diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/SourceLocation.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/SourceLocation.java index 3accc38..2a8636a 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/SourceLocation.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/SourceLocation.java @@ -63,12 +63,20 @@ public final class SourceLocation implements Serializable { return column; } - public void writeFields(DataOutput output) throws IOException { - output.writeInt(line); - output.writeInt(column); + public static void writeFields(SourceLocation sourceLocation, DataOutput output) throws IOException { + if (sourceLocation == null) { + output.writeInt(-1); + } else { + output.writeInt(sourceLocation.getLine()); + output.writeInt(sourceLocation.getColumn()); + } } public static SourceLocation create(DataInput dataInput) throws IOException { - return new SourceLocation(dataInput.readInt(), dataInput.readInt()); + int row = dataInput.readInt(); + if (row == -1) { + return null; + } + return new SourceLocation(row, dataInput.readInt()); } -} \ No newline at end of file +} diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/Warning.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/Warning.java index 78fe3d8..3face66 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/Warning.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/Warning.java @@ -42,7 +42,6 @@ public class Warning implements Serializable { } public static Warning of(String component, SourceLocation srcLocation, int code, String message) { - Objects.requireNonNull(srcLocation, "warnings must have source location"); return new Warning(component, srcLocation, code, message); } @@ -88,7 +87,7 @@ public class Warning implements Serializable { output.writeUTF(component); output.writeInt(code); output.writeUTF(message); - srcLocation.writeFields(output); + SourceLocation.writeFields(srcLocation, output); } public static Warning create(DataInput input) throws IOException { diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FieldCursorForDelimitedDataParser.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FieldCursorForDelimitedDataParser.java index 48cfca6..e401efc 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FieldCursorForDelimitedDataParser.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FieldCursorForDelimitedDataParser.java @@ -25,7 +25,6 @@ import java.util.function.Supplier; import org.apache.hyracks.api.exceptions.ErrorCode; import org.apache.hyracks.api.exceptions.IWarningCollector; -import org.apache.hyracks.api.exceptions.SourceLocation; import org.apache.hyracks.api.exceptions.Warning; public class FieldCursorForDelimitedDataParser { @@ -45,7 +44,6 @@ public class FieldCursorForDelimitedDataParser { END } - private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1); private static final String CLOSING_Q = "missing a closing quote"; private static final String OPENING_Q = "a quote should be in the beginning"; private static final String DELIMITER_AFTER_Q = "a quote enclosing a field needs to be followed by the delimiter"; @@ -450,7 +448,7 @@ public class FieldCursorForDelimitedDataParser { } private void warn(String message) { - warnings.warn(Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName.get(), lineCount, fieldCount, + warnings.warn(Warning.forHyracks(null, ErrorCode.PARSING_ERROR, dataSourceName.get(), lineCount, fieldCount, message)); } }
