This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 54bff8184513edbe511f4e0c73ec6f52b10ee59e Merge: b564661 f1cd417 Author: Michael Blow <[email protected]> AuthorDate: Wed May 13 15:19:24 2020 -0400 Merge commit 'f1cd417' from 'gerrit/mad-hatter' Change-Id: I0f30399a6485708f4b154af09ea36377fe4d3dbf asterixdb/asterix-app/data/csv/01.csv | 3 + asterixdb/asterix-app/data/csv/02.csv | 3 + asterixdb/asterix-app/data/csv/empty.csv | 5 + asterixdb/asterix-app/data/csv/sample_09.csv | 17 ++ asterixdb/asterix-app/data/csv/sample_10.csv | 39 ++++ asterixdb/asterix-app/data/csv/sample_11.csv | 4 + asterixdb/asterix-app/data/csv/sample_12.csv | 16 ++ asterixdb/asterix-app/data/csv/sample_13.csv | 11 ++ asterixdb/asterix-app/data/tsv/01.tsv | 3 + asterixdb/asterix-app/data/tsv/02.tsv | 3 + asterixdb/asterix-app/data/tsv/sample_01.tsv | 28 +++ .../asterix/app/translator/QueryTranslator.java | 54 +++-- .../aws/AwsS3ExternalDatasetTest.java | 27 +++ .../test/runtime/ResultStreamingFailureTest.java | 83 ++++++++ .../csv-parser-001/csv-parser-001.1.ddl.sqlpp | 35 ++++ .../csv-parser-001/csv-parser-001.2.query.sqlpp} | 8 +- .../csv-parser-001/csv-parser-001.3.query.sqlpp} | 8 +- .../csv-parser-001/csv-parser-001.4.query.sqlpp} | 8 +- .../csv-parser-001/csv-parser-001.5.query.sqlpp} | 8 +- .../csv-parser-001/csv-parser-001.6.query.sqlpp} | 8 +- .../csv-parser-001/csv-parser-001.7.query.sqlpp} | 8 +- .../csv-parser-001/csv-parser-001.8.ddl.sqlpp} | 8 +- .../tsv-parser-001/tsv-parser-002.1.ddl.sqlpp} | 14 +- .../tsv-parser-001/tsv-parser-002.2.query.sqlpp} | 8 +- .../tsv-parser-001/tsv-parser-002.3.ddl.sqlpp} | 8 +- .../aws/s3/001/query-dataset.000.ddl.sqlpp} | 24 ++- .../aws/s3/001/query-dataset.002.query.sqlpp} | 8 +- .../aws/s3/001/query-dataset.003.ddl.sqlpp} | 8 +- .../aws/s3/002/query-dataset.000.ddl.sqlpp} | 24 ++- .../aws/s3/002/query-dataset.002.query.sqlpp} | 8 +- .../aws/s3/002/query-dataset.003.ddl.sqlpp} | 8 +- .../aws/s3/negative/negative.000.ddl.sqlpp} | 23 ++- .../get-item_03/get-item_03.4.query.sqlpp} | 19 +- .../misc/case_01/case_01.1.query.sqlpp | 2 +- .../misc/case_02/case_02.1.query.sqlpp | 2 +- .../misc/case_03/case_03.1.query.sqlpp | 2 +- .../misc/case_04/case_04.1.query.sqlpp | 2 +- .../misc/case_05/case_05.1.query.sqlpp | 2 +- .../misc/case_06/case_06.1.query.sqlpp | 2 +- .../misc/case_07/case_07.1.query.sqlpp | 2 +- .../numeric/add_double/add_double.1.query.sqlpp | 2 +- .../union/union_nested/union_nested.1.ddl.sqlpp | 48 +++-- .../union_nested/union_nested.2.update.sqlpp} | 8 +- .../union/union_nested/union_nested.3.query.sqlpp | 93 +++++++++ .../csv-parser-001/csv-parser-001.2.adm | 15 ++ .../csv-parser-001/csv-parser-001.3.adm | 32 +++ .../csv-parser-001/csv-parser-001.4.adm | 4 + .../csv-parser-001/csv-parser-001.5.adm | 13 ++ .../csv-parser-001/csv-parser-001.6.adm | 3 + .../csv-parser-001/csv-parser-001.7.adm | 0 .../tsv-parser-001/tsv-parser-001.2.adm | 28 +++ .../aws/s3/001/external_dataset.001.adm | 6 + .../aws/s3/002/external_dataset.001.adm | 6 + .../{get-item_03.1.adm => get-item_03.3.adm} | 0 .../results/list/get-item_03/get-item_03.4.adm | 1 + .../results/union/union_nested/union_nested.3.adm | 138 +++++++++++++ .../numeric/add_double/add_double.1.ast | 7 +- .../runtimets/testsuite_external_dataset.xml | 18 +- .../test/resources/runtimets/testsuite_sqlpp.xml | 17 ++ .../asterix/common/exceptions/ErrorCode.java | 6 +- .../src/main/resources/asx_errormsg/en.properties | 6 +- .../asterix-doc/src/main/markdown/sqlpp/2_expr.md | 9 +- .../external/input/HDFSDataSourceFactory.java | 2 +- .../external/input/record/CharArrayRecord.java | 6 + .../CSVToRecordWithMetadataAndPKConverter.java | 13 +- .../record/reader/aws/AwsS3InputStreamFactory.java | 17 +- .../record/reader/aws/AwsS3ReaderFactory.java | 18 -- .../stream/EmptyLineSeparatedRecordReader.java | 3 +- .../record/reader/stream/LineRecordReader.java | 28 ++- .../reader/stream/QuotedLineRecordReader.java | 87 ++++---- .../reader/stream/SemiStructuredRecordReader.java | 4 +- .../record/reader/stream/StreamRecordReader.java | 3 +- .../reader/stream/StreamRecordReaderFactory.java | 2 +- .../input/stream/AsterixInputStreamReader.java | 4 +- .../input/stream/SocketServerInputStream.java | 5 + .../external/parser/DelimitedDataParser.java | 65 +++--- .../parser/factory/DelimitedDataParserFactory.java | 52 +---- .../factory/RecordWithMetadataParserFactory.java | 2 +- .../external/provider/AdapterFactoryProvider.java | 15 +- .../external/provider/ParserFactoryProvider.java | 7 +- .../util/ExternalDataCompatibilityUtils.java | 17 +- .../external/util/ExternalDataConstants.java | 35 +++- .../asterix/external/util/ExternalDataUtils.java | 220 +++++++++++++++------ .../parser/test/ByteBufUTF8DecodeTest.java | 5 +- .../external/parser/test/RecordWithMetaTest.java | 5 +- .../DatasetTupleTranslator.java | 103 ++++++---- .../asterix/metadata/feeds/FeedMetadataUtil.java | 10 +- .../evaluators/functions/GetItemDescriptor.java | 4 + asterixdb/asterix-server/pom.xml | 4 +- .../appended-resources/supplemental-models.xml | 6 +- ....com_netty_netty_netty-4.1.48.Final_NOTICE.txt} | 0 .../apache/hyracks/api/channels/IInputChannel.java | 5 + .../src/main/resources/errormsg/en.properties | 2 +- .../hyracks/client/result/ResultSetReader.java | 65 +++--- .../hyracks/comm/channels/NetworkInputChannel.java | 5 + .../comm/channels/ResultNetworkInputChannel.java | 6 + .../MaterializedPartitionInputChannel.java | 5 + .../common/data/parsers/BooleanParserFactory.java | 62 ++++++ .../std/file/DelimitedDataTupleParserFactory.java | 9 +- .../file/FieldCursorForDelimitedDataParser.java | 179 ++++++++++------- .../hyracks/dataflow/std/file/CursorTest.java | 5 +- .../muxdemux/AbstractChannelWriteInterface.java | 10 +- .../muxdemux/FullFrameChannelWriteInterface.java | 18 +- .../storage/am/common/impls/AbstractTreeIndex.java | 12 +- .../storage/common/compression/file/LAFWriter.java | 23 ++- .../org/apache/hyracks/test/support/TestUtils.java | 5 + .../java/org/apache/hyracks/util/NetworkUtil.java | 11 ++ .../org/apache/hyracks/util/NetworkUtilTest.java | 42 ++++ hyracks-fullstack/pom.xml | 27 ++- 109 files changed, 1611 insertions(+), 635 deletions(-) diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index f83e21e,3906bd5..ea2ce40 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@@ -681,9 -641,10 +683,10 @@@ public class QueryTranslator extends Ab case RECORD: itemTypeDataverseName = dataverseName; itemTypeName = DatasetUtil.createInlineTypeName(datasetName, false); - MetadataLockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(), itemTypeDataverseName, - itemTypeDataverseName + "." + itemTypeName); + lockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(), itemTypeDataverseName, + itemTypeName); itemType = translateType(itemTypeDataverseName, itemTypeName, itemTypeExpr, mdTxnCtx); + validateDatasetItemType(dsType, itemType, false, sourceLoc); MetadataManager.INSTANCE.addDatatype(mdTxnCtx, new Datatype(itemTypeDataverseName, itemTypeName, itemType, true)); break; @@@ -728,10 -681,11 +723,11 @@@ case RECORD: metaItemTypeDataverseName = dataverseName; metaItemTypeName = DatasetUtil.createInlineTypeName(datasetName, true); - MetadataLockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(), - metaItemTypeDataverseName, metaItemTypeDataverseName + "." + metaItemTypeName); + lockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(), + metaItemTypeDataverseName, metaItemTypeName); metaItemType = translateType(metaItemTypeDataverseName, metaItemTypeName, metaItemTypeExpr, mdTxnCtx); + validateDatasetItemType(dsType, metaItemType, true, sourceLoc); MetadataManager.INSTANCE.addDatatype(mdTxnCtx, new Datatype(metaItemTypeDataverseName, metaItemTypeName, metaItemType, true)); break; @@@ -2216,11 -1969,15 +2223,13 @@@ MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); boolean bActiveTxn = true; metadataProvider.setMetadataTxnContext(mdTxnCtx); - MetadataLockUtil.modifyDatasetBegin(lockManager, metadataProvider.getLocks(), dataverseName, - dataverseName + "." + datasetName); + lockUtil.modifyDatasetBegin(lockManager, metadataProvider.getLocks(), dataverseName, datasetName); try { - CompiledLoadFromFileStatement cls = - new CompiledLoadFromFileStatement(dataverseName, loadStmt.getDatasetName(), loadStmt.getAdapter(), - loadStmt.getProperties(), loadStmt.dataIsAlreadySorted()); + Map<String, String> properties = loadStmt.getProperties(); + ExternalDataUtils.normalize(properties); + ExternalDataUtils.validate(properties); - CompiledLoadFromFileStatement cls = - new CompiledLoadFromFileStatement(dataverseName, loadStmt.getDatasetName().getValue(), - loadStmt.getAdapter(), properties, loadStmt.dataIsAlreadySorted()); ++ CompiledLoadFromFileStatement cls = new CompiledLoadFromFileStatement(dataverseName, ++ loadStmt.getDatasetName(), loadStmt.getAdapter(), properties, loadStmt.dataIsAlreadySorted()); cls.setSourceLocation(stmt.getSourceLocation()); JobSpecification spec = apiFramework.compileQuery(hcc, metadataProvider, null, 0, null, sessionOutput, cls, null, responsePrinter, warningCollector); diff --cc asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java index 7fef69e,14f7c83..4e10498 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java @@@ -314,7 -307,8 +314,9 @@@ public class ErrorCode public static final int FAILED_TO_PARSE_METADATA = 3115; public static final int INPUT_DECODE_FAILURE = 3116; public static final int FAILED_TO_PARSE_MALFORMED_LOG_RECORD = 3117; - public static final int METADATA_DROP_LIBRARY_IN_USE = 3118; + public static final int PARAMETERS_REQUIRED = 3118; + public static final int MALFORMED_RECORD = 3119; ++ public static final int METADATA_DROP_LIBRARY_IN_USE = 3120; // Lifecycle management errors public static final int DUPLICATE_PARTITION_ID = 4000; diff --cc asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties index 4461ff1,98ae1a7..48d849f --- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties +++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties @@@ -312,7 -305,8 +312,9 @@@ 3115 = Failed to parse record metadata 3116 = Failed to decode input 3117 = Failed to parse record, malformed log record - 3118 = Library %1$s is being used. It cannot be dropped + 3118 = Parameter(s) %1$s must be specified + 3119 = Record number %1$s is malformed ++3120 = Library %1$s is being used. It cannot be dropped # Lifecycle management errors 4000 = Partition id %1$s for node %2$s already in use by node %3$s diff --cc asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java index 8aebd90,b7b441b..652eae5 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java @@@ -145,34 -145,17 +149,17 @@@ public class ExternalDataUtils } } - public static String getDataverse(Map<String, String> configuration) { - return configuration.get(ExternalDataConstants.KEY_DATAVERSE); + public static DataverseName getDataverse(Map<String, String> configuration) { + return DataverseName.createFromCanonicalForm(configuration.get(ExternalDataConstants.KEY_DATAVERSE)); } - public static String getRecordFormat(Map<String, String> configuration) { - String parserFormat = configuration.get(ExternalDataConstants.KEY_DATA_PARSER); - return parserFormat != null ? parserFormat : configuration.get(ExternalDataConstants.KEY_FORMAT); - } - - public static void setRecordFormat(Map<String, String> configuration, String format) { - if (!configuration.containsKey(ExternalDataConstants.KEY_DATA_PARSER)) { - configuration.put(ExternalDataConstants.KEY_DATA_PARSER, format); - } - if (!configuration.containsKey(ExternalDataConstants.KEY_FORMAT)) { - configuration.put(ExternalDataConstants.KEY_FORMAT, format); + public static String getParserFactory(Map<String, String> configuration) { + String parserFactory = configuration.get(ExternalDataConstants.KEY_PARSER); + if (parserFactory != null) { + return parserFactory; } - } - - private static Map<ATypeTag, IValueParserFactory> valueParserFactoryMap = initializeValueParserFactoryMap(); - - private static Map<ATypeTag, IValueParserFactory> initializeValueParserFactoryMap() { - Map<ATypeTag, IValueParserFactory> m = new EnumMap<>(ATypeTag.class); - m.put(ATypeTag.INTEGER, IntegerParserFactory.INSTANCE); - m.put(ATypeTag.FLOAT, FloatParserFactory.INSTANCE); - m.put(ATypeTag.DOUBLE, DoubleParserFactory.INSTANCE); - m.put(ATypeTag.BIGINT, LongParserFactory.INSTANCE); - m.put(ATypeTag.STRING, UTF8StringParserFactory.INSTANCE); - return m; + parserFactory = configuration.get(ExternalDataConstants.KEY_FORMAT); + return parserFactory != null ? parserFactory : configuration.get(ExternalDataConstants.KEY_PARSER_FACTORY); } public static IValueParserFactory[] getValueParserFactories(ARecordType recordType) { diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java index 64b703b,74f5076..f0567a4 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java @@@ -64,6 -68,9 +65,7 @@@ import org.apache.asterix.om.types.AUno import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.om.types.IAType; import org.apache.asterix.runtime.compression.CompressionManager; -import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; + import org.apache.hyracks.algebricks.common.utils.Pair; -import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; @@@ -395,8 -409,39 +390,36 @@@ public class DatasetTupleTranslator ext return tuple; } + protected void writeCompactionPolicy(DatasetType datasetType, String compactionPolicy, + Map<String, String> compactionPolicyProperties, OrderedListBuilder listBuilder, + ArrayBackedValueStorage itemValue) throws HyracksDataException { + // write field 6 + fieldValue.reset(); + aString.setValue(compactionPolicy); + stringSerde.serialize(aString, fieldValue.getDataOutput()); + recordBuilder.addField(MetadataRecordTypes.DATASET_ARECORD_COMPACTION_POLICY_FIELD_INDEX, fieldValue); + + // write field 7 + listBuilder.reset((AOrderedListType) MetadataRecordTypes.DATASET_RECORDTYPE + .getFieldTypes()[MetadataRecordTypes.DATASET_ARECORD_COMPACTION_POLICY_PROPERTIES_FIELD_INDEX]); + if (compactionPolicyProperties != null && !compactionPolicyProperties.isEmpty()) { + for (Map.Entry<String, String> property : compactionPolicyProperties.entrySet()) { + String name = property.getKey(); + String value = property.getValue(); + itemValue.reset(); + DatasetUtil.writePropertyTypeRecord(name, value, itemValue.getDataOutput(), + MetadataRecordTypes.COMPACTION_POLICY_PROPERTIES_RECORDTYPE); + listBuilder.addItem(itemValue); + } + } + fieldValue.reset(); + listBuilder.write(fieldValue.getDataOutput(), true); + recordBuilder.addField(MetadataRecordTypes.DATASET_ARECORD_COMPACTION_POLICY_PROPERTIES_FIELD_INDEX, + fieldValue); + } + /** * Keep protected to allow other extensions to add additional fields - * - * @param dataset - * @throws HyracksDataException */ protected void writeOpenFields(Dataset dataset) throws HyracksDataException { writeMetaPart(dataset);
