This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 74cecd932af78831c7fe74a52658ae6844606abe Author: Dmitry Lychagin <[email protected]> AuthorDate: Tue Mar 31 10:35:40 2020 -0700 [NO ISSUE] Minor cleanup in external datasets - user model changes: no - storage format changes: no - interface changes: no Details: - Introduce createExternalDataSourceFactory() and createDataParserFactory() in GenericAdapterFactory for better extensibility by products - Cleanup ExternalDataConstants Change-Id: Ifb56ffce34a0b62ece3340582331bb78f2f5c5ca Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/5525 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Dmitry Lychagin <[email protected]> Reviewed-by: Hussain Towaileb <[email protected]> --- .../runtimets/results/feeds/feeds_01/feeds_01.1.adm | 2 +- .../adapter/factory/GenericAdapterFactory.java | 20 +++++++++++++++----- .../util/ExternalDataCompatibilityUtils.java | 7 ++----- .../asterix/external/util/ExternalDataConstants.java | 10 ---------- 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feeds_01/feeds_01.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feeds_01/feeds_01.1.adm index 1dc31dc..3ab469e 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feeds_01/feeds_01.1.adm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feeds_01/feeds_01.1.adm @@ -1 +1 @@ -{ "DataverseName": "feeds", "FeedName": "TweetFeed", "AdapterConfiguration": {{ { "Name": "path", "Value": "asterix_nc1://data/twitter/obamatweets.adm" }, { "Name": "feed", "Value": "TweetFeed" }, { "Name": "adapter-name", "Value": "localfs" }, { "Name": "is-feed", "Value": "true" }, { "Name": "parser", "Value": "adm" }, { "Name": "reader", "Value": "localfs" }, { "Name": "format", "Value": "adm" }, { "Name": "tuple-interval", "Value": "10" }, { "Name": "linkName", "Value": "localfs" }, [...] \ No newline at end of file +{ "DataverseName": "feeds", "FeedName": "TweetFeed", "AdapterConfiguration": {{ { "Name": "path", "Value": "asterix_nc1://data/twitter/obamatweets.adm" }, { "Name": "feed", "Value": "TweetFeed" }, { "Name": "adapter-name", "Value": "localfs" }, { "Name": "is-feed", "Value": "true" }, { "Name": "parser", "Value": "adm" }, { "Name": "reader", "Value": "localfs" }, { "Name": "format", "Value": "adm" }, { "Name": "tuple-interval", "Value": "10" }, { "Name": "type-name", "Value": "TweetType" [...] \ No newline at end of file diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java index d081e56..078316c 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java @@ -25,6 +25,7 @@ import java.util.Map; import org.apache.asterix.common.api.IApplicationContext; import org.apache.asterix.common.api.INcApplicationContext; import org.apache.asterix.common.dataflow.ICcApplicationContext; +import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.common.external.IDataSourceAdapter; import org.apache.asterix.common.library.ILibraryManager; import org.apache.asterix.external.api.IDataFlowController; @@ -122,7 +123,7 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, ITypedAda private void restoreExternalObjects(IServiceContext serviceContext, ILibraryManager libraryManager) throws HyracksDataException, AlgebricksException { if (dataSourceFactory == null) { - dataSourceFactory = DatasourceFactoryProvider.getExternalDataSourceFactory(libraryManager, configuration); + dataSourceFactory = createExternalDataSourceFactory(configuration, libraryManager); // create and configure parser factory if (dataSourceFactory.isIndexible() && (files != null)) { ((IIndexibleExternalDataSource) dataSourceFactory).setSnapshot(files, indexingOp); @@ -131,7 +132,7 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, ITypedAda } if (dataParserFactory == null) { // create and configure parser factory - dataParserFactory = ParserFactoryProvider.getDataParserFactory(libraryManager, configuration); + dataParserFactory = createDataParserFactory(configuration, libraryManager); dataParserFactory.setRecordType(recordType); dataParserFactory.setMetaType(metaType); dataParserFactory.configure(configuration); @@ -144,14 +145,13 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, ITypedAda this.configuration = configuration; IApplicationContext appCtx = (IApplicationContext) serviceContext.getApplicationContext(); ExternalDataUtils.validateDataSourceParameters(configuration); - dataSourceFactory = - DatasourceFactoryProvider.getExternalDataSourceFactory(appCtx.getLibraryManager(), configuration); + dataSourceFactory = createExternalDataSourceFactory(configuration, appCtx.getLibraryManager()); if (dataSourceFactory.isIndexible() && (files != null)) { ((IIndexibleExternalDataSource) dataSourceFactory).setSnapshot(files, indexingOp); } dataSourceFactory.configure(serviceContext, configuration); ExternalDataUtils.validateDataParserParameters(configuration); - dataParserFactory = ParserFactoryProvider.getDataParserFactory(appCtx.getLibraryManager(), configuration); + dataParserFactory = createDataParserFactory(configuration, appCtx.getLibraryManager()); dataParserFactory.setRecordType(recordType); dataParserFactory.setMetaType(metaType); dataParserFactory.configure(configuration); @@ -222,4 +222,14 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, ITypedAda dataParserFactory.configure(Collections.emptyMap()); configuration = Collections.emptyMap(); } + + protected IExternalDataSourceFactory createExternalDataSourceFactory(Map<String, String> configuration, + ILibraryManager libraryManager) throws HyracksDataException, AsterixException { + return DatasourceFactoryProvider.getExternalDataSourceFactory(libraryManager, configuration); + } + + protected IDataParserFactory createDataParserFactory(Map<String, String> configuration, + ILibraryManager libraryManager) throws AsterixException { + return ParserFactoryProvider.getDataParserFactory(libraryManager, configuration); + } } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataCompatibilityUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataCompatibilityUtils.java index 77cbb96..e222e99 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataCompatibilityUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataCompatibilityUtils.java @@ -60,11 +60,8 @@ public class ExternalDataCompatibilityUtils { } public static void prepare(String adapterName, Map<String, String> configuration) { - // Adapter name in some cases can carry the link name for external datasets, always add it to configuration - configuration.put(ExternalDataConstants.KEY_LINK_NAME, adapterName); - - if (!configuration.containsKey(ExternalDataConstants.KEY_READER)) { // SThree - configuration.put(ExternalDataConstants.KEY_READER, adapterName); // myAwsLink + if (!configuration.containsKey(ExternalDataConstants.KEY_READER)) { + configuration.put(ExternalDataConstants.KEY_READER, adapterName); } if (!configuration.containsKey(ExternalDataConstants.KEY_PARSER)) { if (configuration.containsKey(ExternalDataConstants.KEY_FORMAT)) { diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java index e44144a..26f5402 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java @@ -231,16 +231,6 @@ public class ExternalDataConstants { public static final String ERROR_PARSE_RECORD = "Parser failed to parse record"; - // TODO(Hussain): Move link related items to a different place - /** - * Common external link fields - */ - public static final String KEY_DATAVERSE_NAME = "dataverseName"; - public static final String KEY_LINK_NAME = "linkName"; - public static final String KEY_LINK_TYPE = "linkType"; - public static final String[] KEY_EXTERNAL_DATASET_REQUIRED_CONNECTION_PARAMETERS = - new String[] { KEY_DATAVERSE_NAME, KEY_LINK_NAME, KEY_LINK_TYPE }; - public static class AwsS3Constants { public static final String REGION_FIELD_NAME = "region"; public static final String ACCESS_KEY_FIELD_NAME = "accessKey";
