abdullah alamoudi has uploaded a new change for review. https://asterix-gerrit.ics.uci.edu/906
Change subject: PLEASE EDIT to provide a meaningful commit message! ...................................................................... PLEASE EDIT to provide a meaningful commit message! The following commits from your working branch will be included: commit ebce45c396bebb79dcc1eb122af62e2b29026429 Author: Abdullah Alamoudi <[email protected]> Date: Sat Jun 4 04:09:34 2016 +0300 Improve External Data Change-Id: Iae2560c73fa63e9454f731b8e893ae779a2ac7d9 --- M .gitignore M asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta/change-feed-with-meta-pk-in-meta.1.ddl.aql M asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/ParserFactoryProvider.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java 10 files changed, 45 insertions(+), 31 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/06/906/1 diff --git a/.gitignore b/.gitignore index ced139e..636fb2f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ asterix_logs build bin +asterixdb/asterix-opt/ asterixdb/asterix-app/src/test/resources/externallib/ asterixdb/asterix-app/rttest/ asterixdb/asterix-app/mdtest/ diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta/change-feed-with-meta-pk-in-meta.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta/change-feed-with-meta-pk-in-meta.1.ddl.aql index 31d6ea8..2c74ac7 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta/change-feed-with-meta-pk-in-meta.1.ddl.aql +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta/change-feed-with-meta-pk-in-meta.1.ddl.aql @@ -31,11 +31,9 @@ create type KVMetaType as open{ "key":string, -bucket:string, vbucket:int32, seq:int64, cas:int64, -creationTime:int64, expiration:int32, flags:int32, revSeq:int64, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml index 0b19e22..ccaf86a 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml @@ -6336,13 +6336,13 @@ <test-case FilePath="load"> <compilation-unit name="issue14_query"> <output-dir compare="Text">issue14_query</output-dir> - <expected-error>The parameter format must be specified</expected-error> + <expected-error>The parameter format or parser-factory must be specified</expected-error> </compilation-unit> </test-case> <test-case FilePath="load"> <compilation-unit name="issue315_query"> <output-dir compare="Text">none</output-dir> - <expected-error>The parameter format must be specified</expected-error> + <expected-error>The parameter format or parser-factory must be specified</expected-error> </compilation-unit> </test-case> <test-case FilePath="load"> diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml index 3080027..3091284 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml @@ -6330,13 +6330,13 @@ <test-case FilePath="load"> <compilation-unit name="issue14_query"> <output-dir compare="Text">issue14_query</output-dir> - <expected-error>The parameter format must be specified</expected-error> + <expected-error>The parameter format or parser-factory must be specified</expected-error> </compilation-unit> </test-case> <test-case FilePath="load"> <compilation-unit name="issue315_query"> <output-dir compare="Text">none</output-dir> - <expected-error>The parameter format must be specified</expected-error> + <expected-error>The parameter format or parser-factory must be specified</expected-error> </compilation-unit> </test-case> <test-case FilePath="load"> diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java index d3abd50..5274655 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java @@ -127,12 +127,14 @@ this.recordType = outputType; this.metaType = metaType; this.configuration = configuration; + ExternalDataUtils.validateDataSourceParameters(configuration); dataSourceFactory = DatasourceFactoryProvider.getExternalDataSourceFactory(configuration); - dataParserFactory = ParserFactoryProvider.getDataParserFactory(configuration); if (dataSourceFactory.isIndexible() && (files != null)) { ((IIndexibleExternalDataSource) dataSourceFactory).setSnapshot(files, indexingOp); } dataSourceFactory.configure(configuration); + ExternalDataUtils.validateDataParserParameters(configuration); + dataParserFactory = ParserFactoryProvider.getDataParserFactory(configuration); dataParserFactory.setRecordType(recordType); dataParserFactory.setMetaType(metaType); dataParserFactory.configure(configuration); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java index 724699c..4aa6dd9 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java @@ -47,10 +47,10 @@ private final CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder(); private final ByteBuffer bytes = ByteBuffer.allocateDirect(ExternalDataConstants.DEFAULT_BUFFER_SIZE); private final CharBuffer chars = CharBuffer.allocate(ExternalDataConstants.DEFAULT_BUFFER_SIZE); - // metaTypes = {key(string), bucket(string), vbucket(int32), seq(long), cas(long), - // creationTime(long),expiration(int32),flags(int32),revSeqNumber(long),lockTime(int32)} - private static final IAType[] CB_META_TYPES = new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING, - BuiltinType.AINT32, BuiltinType.AINT64, BuiltinType.AINT64, BuiltinType.AINT64, BuiltinType.AINT32, + // metaTypes = {key(string), vbucket(int32), seq(long), cas(long), + //expiration(int32),flags(int32),revSeqNumber(long),lockTime(int32)} + private static final IAType[] CB_META_TYPES = new IAType[] { BuiltinType.ASTRING, + BuiltinType.AINT32, BuiltinType.AINT64, BuiltinType.AINT64, BuiltinType.AINT32, BuiltinType.AINT32, BuiltinType.AINT64, BuiltinType.AINT32 }; private static final int[] PK_INDICATOR = { 1 }; private static final int[] PK_INDEXES = { 0 }; @@ -70,24 +70,21 @@ final String key = message.key(); final int vbucket = message.partition(); final long seq = message.bySequenceNumber(); - final String bucket = message.bucket(); final long cas = message.cas(); - final long creationTime = message.creationTime(); final int expiration = message.expiration(); final int flags = message.flags(); final long revSeqNumber = message.revisionSequenceNumber(); final int lockTime = message.lockTime(); + int i = 0; recordWithMetadata.reset(); - recordWithMetadata.setMetadata(0, key); - recordWithMetadata.setMetadata(1, bucket); - recordWithMetadata.setMetadata(2, vbucket); - recordWithMetadata.setMetadata(3, seq); - recordWithMetadata.setMetadata(4, cas); - recordWithMetadata.setMetadata(5, creationTime); - recordWithMetadata.setMetadata(6, expiration); - recordWithMetadata.setMetadata(7, flags); - recordWithMetadata.setMetadata(8, revSeqNumber); - recordWithMetadata.setMetadata(9, lockTime); + recordWithMetadata.setMetadata(i++, key); + recordWithMetadata.setMetadata(i++, vbucket); + recordWithMetadata.setMetadata(i++, seq); + recordWithMetadata.setMetadata(i++, cas); + recordWithMetadata.setMetadata(i++, expiration); + recordWithMetadata.setMetadata(i++, flags); + recordWithMetadata.setMetadata(i++, revSeqNumber); + recordWithMetadata.setMetadata(i++, lockTime); DCPRequestToRecordWithMetadataAndPKConverter.set(message.content(), decoder, bytes, chars, value); ReferenceCountUtil.release(message.content()); } else if (dcpRequest instanceof RemoveMessage) { diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java index e39e3bd..bf86ef3 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java @@ -28,7 +28,6 @@ import org.apache.asterix.external.api.IIndexingAdapterFactory; import org.apache.asterix.external.indexing.ExternalFile; import org.apache.asterix.external.util.ExternalDataCompatibilityUtils; -import org.apache.asterix.external.util.ExternalDataUtils; import org.apache.asterix.om.types.ARecordType; import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory; @@ -37,11 +36,10 @@ */ public class AdapterFactoryProvider { - // Internal Adapters + // Adapters public static IAdapterFactory getAdapterFactory(String adapterName, Map<String, String> configuration, ARecordType itemType, ARecordType metaType) throws AsterixException { ExternalDataCompatibilityUtils.prepare(adapterName, configuration); - ExternalDataUtils.validateParameters(configuration); GenericAdapterFactory adapterFactory = new GenericAdapterFactory(); adapterFactory.configure(configuration, itemType, metaType); return adapterFactory; @@ -52,7 +50,6 @@ Map<String, String> configuration, ARecordType itemType, List<ExternalFile> snapshot, boolean indexingOp, ARecordType metaType) throws AsterixException { ExternalDataCompatibilityUtils.prepare(adapterName, configuration); - ExternalDataUtils.validateParameters(configuration); GenericAdapterFactory adapterFactory = new GenericAdapterFactory(); adapterFactory.setSnapshot(snapshot, indexingOp); adapterFactory.configure(configuration, itemType, metaType); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/ParserFactoryProvider.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/ParserFactoryProvider.java index 2e1a5a7..b37198a 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/ParserFactoryProvider.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/ParserFactoryProvider.java @@ -45,8 +45,11 @@ return ExternalDataUtils.createExternalParserFactory(ExternalDataUtils.getDataverse(configuration), parserFactoryName); } else { - parserFactory = ParserFactoryProvider - .getDataParserFactory(ExternalDataUtils.getRecordFormat(configuration)); + String parserFactoryKey = ExternalDataUtils.getRecordFormat(configuration); + if (parserFactoryKey == null) { + parserFactoryKey = configuration.get(ExternalDataConstants.KEY_PARSER_FACTORY); + } + parserFactory = ParserFactoryProvider.getDataParserFactory(parserFactoryKey); } return parserFactory; } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java index c992723..55dee04 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java @@ -55,7 +55,7 @@ public static final String KEY_HADOOP_BUFFER_SIZE = "io.file.buffer.size"; public static final String KEY_SOURCE_DATATYPE = "type-name"; public static final String KEY_DELIMITER = "delimiter"; - public static final String KEY_PARSER_FACTORY = "tuple-parser"; + public static final String KEY_PARSER_FACTORY = "parser-factory"; public static final String KEY_DATA_PARSER = "parser"; public static final String KEY_HEADER = "header"; public static final String KEY_READER = "reader"; diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java index 1df4e42..cc66560 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java @@ -78,13 +78,29 @@ } public static void validateParameters(Map<String, String> configuration) throws AsterixException { + validateDataSourceParameters(configuration); + validateDataParserParameters(configuration); + } + + public static void validateDataParserParameters(Map<String, String> configuration) throws AsterixException { String reader = configuration.get(ExternalDataConstants.KEY_READER); if (reader == null) { throw new AsterixException("The parameter " + ExternalDataConstants.KEY_READER + " must be specified."); } String parser = configuration.get(ExternalDataConstants.KEY_FORMAT); if (parser == null) { - throw new AsterixException("The parameter " + ExternalDataConstants.KEY_FORMAT + " must be specified."); + String parserFactory = configuration.get(ExternalDataConstants.KEY_PARSER_FACTORY); + if (parserFactory == null) { + throw new AsterixException("The parameter " + ExternalDataConstants.KEY_FORMAT + " or " + + ExternalDataConstants.KEY_PARSER_FACTORY + " must be specified."); + } + } + } + + public static void validateDataSourceParameters(Map<String, String> configuration) throws AsterixException { + String reader = configuration.get(ExternalDataConstants.KEY_READER); + if (reader == null) { + throw new AsterixException("The parameter " + ExternalDataConstants.KEY_READER + " must be specified."); } } -- To view, visit https://asterix-gerrit.ics.uci.edu/906 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Iae2560c73fa63e9454f731b8e893ae779a2ac7d9 Gerrit-PatchSet: 1 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: abdullah alamoudi <[email protected]>
