>From Ali Alsuliman <[email protected]>: Ali Alsuliman has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18407 )
Change subject: [ASTERIXDB-3446][RT] Remove unnecessary tags ...................................................................... [ASTERIXDB-3446][RT] Remove unnecessary tags - user model changes: no - storage format changes: no - interface changes: no Details: Tags are already part of the values returned. There is no need for a separate list containing IVisitablePointable holding the tag. Change-Id: I89952db7ad9e1e941b3f6c4a06c01c1a2dbd7b72 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18407 Reviewed-by: Ali Alsuliman <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> --- M asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/AListVisitablePointable.java M asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/ARecordVisitablePointable.java M asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/cast/ARecordCaster.java M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/ListDeepEqualityChecker.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/msgpack/MsgPackAccessors.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectAccessors.java A asterixdb/asterix-om/src/main/java/org/apache/asterix/om/utils/PointableHelper.java M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordMergeEvaluator.java M asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/cast/AListCaster.java M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/RecordDeepEqualityChecker.java M asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/printer/AListPrinter.java M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRemoveFieldsEvalFactory.java 12 files changed, 131 insertions(+), 174 deletions(-) Approvals: Murtadha Hubail: Looks good to me, approved Ali Alsuliman: Looks good to me, but someone else must approve Jenkins: Verified; Verified Objections: Anon. E. Moose #1000171: Violations found diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectAccessors.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectAccessors.java index 5479b56a..f239ba6 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectAccessors.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectAccessors.java @@ -69,10 +69,10 @@ import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.AbstractCollectionType; import org.apache.asterix.om.types.BuiltinType; -import org.apache.asterix.om.types.EnumDeserializer; import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.types.TypeTagUtil; import org.apache.asterix.om.util.container.IObjectPool; +import org.apache.asterix.runtime.evaluators.functions.PointableHelper; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.api.IPointable; import org.apache.hyracks.util.string.UTF8StringReader; @@ -375,7 +375,6 @@ jRecord.reset(); ARecordVisitablePointable recordPointable = pointable; List<IVisitablePointable> fieldPointables = recordPointable.getFieldValues(); - List<IVisitablePointable> fieldTypeTags = recordPointable.getFieldTypeTags(); List<IVisitablePointable> fieldNames = recordPointable.getFieldNames(); int index = 0; boolean closedPart; @@ -383,9 +382,7 @@ IJObject fieldObject = null; for (IPointable fieldPointable : fieldPointables) { closedPart = index < recordType.getFieldTypes().length; - IPointable tt = fieldTypeTags.get(index); - ATypeTag typeTag = - EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(tt.getByteArray()[tt.getStartOffset()]); + ATypeTag typeTag = PointableHelper.getTypeTag(fieldPointables.get(index)); IAType fieldType; fieldType = closedPart ? recordType.getFieldTypes()[index] : TypeTagUtil.getBuiltinTypeByTag(typeTag); @@ -449,7 +446,6 @@ public IJObject access(AListVisitablePointable pointable, IObjectPool<IJObject, IAType> objectPool, IAType listType, JObjectPointableVisitor pointableVisitor) throws HyracksDataException { List<IVisitablePointable> items = pointable.getItems(); - List<IVisitablePointable> itemTags = pointable.getItemTags(); JList list = pointable.ordered() ? new JOrderedList(listType) : new JUnorderedList(listType); IJObject listItem; for (int iter1 = 0; iter1 < items.size(); iter1++) { @@ -458,10 +454,7 @@ IAType fieldType = ((AbstractCollectionType) listType).getItemType(); if (fieldType.getTypeTag() == ATypeTag.ANY) { // Second, if defined type is not available, try to infer it from data - IVisitablePointable itemTagPointable = itemTags.get(iter1); - ATypeTag itemTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER - .deserialize(itemTagPointable.getByteArray()[itemTagPointable.getStartOffset()]); - fieldType = TypeTagUtil.getBuiltinTypeByTag(itemTypeTag); + fieldType = TypeTagUtil.getBuiltinTypeByTag(PointableHelper.getTypeTag(itemPointable)); } typeInfo.reset(fieldType, fieldType.getTypeTag()); switch (typeInfo.getTypeTag()) { diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/msgpack/MsgPackAccessors.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/msgpack/MsgPackAccessors.java index 6efbb6e..6f097d0 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/msgpack/MsgPackAccessors.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/msgpack/MsgPackAccessors.java @@ -54,9 +54,9 @@ import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.AbstractCollectionType; import org.apache.asterix.om.types.BuiltinType; -import org.apache.asterix.om.types.EnumDeserializer; import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.types.TypeTagUtil; +import org.apache.asterix.runtime.evaluators.functions.PointableHelper; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.api.IPointable; import org.apache.hyracks.util.encoding.VarLenIntEncoderDecoder; @@ -207,7 +207,6 @@ public static Void access(ARecordVisitablePointable pointable, PyTypeInfo arg, MsgPackPointableVisitor pointableVisitor) throws HyracksDataException { List<IVisitablePointable> fieldPointables = pointable.getFieldValues(); - List<IVisitablePointable> fieldTypeTags = pointable.getFieldTypeTags(); List<IVisitablePointable> fieldNames = pointable.getFieldNames(); boolean closedPart; int index = 0; @@ -222,9 +221,7 @@ try { for (IVisitablePointable fieldPointable : fieldPointables) { closedPart = index < recordType.getFieldTypes().length; - IVisitablePointable tt = fieldTypeTags.get(index); - ATypeTag typeTag = - EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(tt.getByteArray()[tt.getStartOffset()]); + ATypeTag typeTag = PointableHelper.getTypeTag(fieldPointables.get(index)); IAType fieldType; fieldType = closedPart ? recordType.getFieldTypes()[index] : TypeTagUtil.getBuiltinTypeByTag(typeTag); @@ -247,7 +244,6 @@ public static Void access(AListVisitablePointable pointable, PyTypeInfo arg, MsgPackPointableVisitor pointableVisitor) throws HyracksDataException { List<IVisitablePointable> items = pointable.getItems(); - List<IVisitablePointable> itemTags = pointable.getItemTags(); DataOutput out = arg.getDataOutput(); try { out.writeByte(ARRAY32); @@ -261,10 +257,7 @@ IAType fieldType = ((AbstractCollectionType) arg.getType()).getItemType(); if (fieldType.getTypeTag() == ATypeTag.ANY) { // Second, if defined type is not available, try to infer it from data - IVisitablePointable itemTagPointable = itemTags.get(iter1); - ATypeTag itemTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER - .deserialize(itemTagPointable.getByteArray()[itemTagPointable.getStartOffset()]); - fieldType = TypeTagUtil.getBuiltinTypeByTag(itemTypeTag); + fieldType = TypeTagUtil.getBuiltinTypeByTag(PointableHelper.getTypeTag(itemPointable)); } PyTypeInfo fieldTypeInfo = pointableVisitor.getTypeInfo(fieldType, out); itemPointable.accept(pointableVisitor, fieldTypeInfo); diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/AListVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/AListVisitablePointable.java index 15766b3..88a81d4 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/AListVisitablePointable.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/AListVisitablePointable.java @@ -50,8 +50,7 @@ static IObjectFactory<AListVisitablePointable, IAType> FACTORY = type -> new AListVisitablePointable((AbstractCollectionType) type); - private final List<IVisitablePointable> items = new ArrayList<IVisitablePointable>(); - private final List<IVisitablePointable> itemTags = new ArrayList<IVisitablePointable>(); + private final List<IVisitablePointable> items = new ArrayList<>(); private final PointableAllocator allocator = new PointableAllocator(); private final ResettableByteArrayOutputStream dataBos = new ResettableByteArrayOutputStream(); @@ -87,7 +86,6 @@ private void reset() { allocator.reset(); items.clear(); - itemTags.clear(); dataBos.reset(); } @@ -109,21 +107,13 @@ if (typedItemList) { for (int i = 0; i < numberOfitems; i++) { itemLength = NonTaggedFormatUtil.getFieldValueLength(b, itemOffset, itemTag, false); - IVisitablePointable tag = allocator.allocateEmpty(); IVisitablePointable item = allocator.allocateFieldValue(itemType); - // set item type tag + // set item value including the tag int start = dataBos.size(); dataDos.writeByte(itemTag.serialize()); - int end = dataBos.size(); - tag.set(dataBos.getByteArray(), start, end - start); - itemTags.add(tag); - - // set item value - start = dataBos.size(); - dataDos.writeByte(itemTag.serialize()); dataDos.write(b, itemOffset, itemLength); - end = dataBos.size(); + int end = dataBos.size(); item.set(dataBos.getByteArray(), start, end - start); itemOffset += itemLength; items.add(item); @@ -132,16 +122,8 @@ for (int i = 0; i < numberOfitems; i++) { itemTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(b[itemOffset]); itemLength = NonTaggedFormatUtil.getFieldValueLength(b, itemOffset, itemTag, true) + 1; - IVisitablePointable tag = allocator.allocateEmpty(); IVisitablePointable item = allocator.allocateFieldValue(itemTag, b, itemOffset + 1); - // set item type tag - int start = dataBos.size(); - dataDos.writeByte(itemTag.serialize()); - int end = dataBos.size(); - tag.set(dataBos.getByteArray(), start, end - start); - itemTags.add(tag); - // open part field already include the type tag item.set(b, itemOffset, itemLength); itemOffset += itemLength; @@ -154,18 +136,14 @@ } @Override - public <R, T> R accept(IVisitablePointableVisitor<R, T> vistor, T tag) throws HyracksDataException { - return vistor.visit(this, tag); + public <R, T> R accept(IVisitablePointableVisitor<R, T> visitor, T tag) throws HyracksDataException { + return visitor.visit(this, tag); } public List<IVisitablePointable> getItems() { return items; } - public List<IVisitablePointable> getItemTags() { - return itemTags; - } - public boolean ordered() { return ordered; } diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/ARecordVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/ARecordVisitablePointable.java index 193ff7d..6637af8 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/ARecordVisitablePointable.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/ARecordVisitablePointable.java @@ -55,7 +55,6 @@ // access results: field names, field types, and field values private final List<IVisitablePointable> fieldNames = new ArrayList<>(); - private final List<IVisitablePointable> fieldTypeTags = new ArrayList<>(); private final List<IVisitablePointable> fieldValues = new ArrayList<>(); // pointable allocator @@ -88,28 +87,12 @@ String[] fieldNameStrs = inputType.getFieldNames(); numberOfSchemaFields = fieldTypes.length; - // initialize the buffer for closed parts(fieldName bytes+ type bytes) + - // constant(null bytes) + // initialize the buffer for closed parts(fieldName bytes+ type bytes) + constant(null bytes) try { final DataOutputStream typeDos = new DataOutputStream(typeBos); final UTF8StringWriter utf8Writer = new UTF8StringWriter(); for (int i = 0; i < numberOfSchemaFields; i++) { - ATypeTag ftypeTag = fieldTypes[i].getTypeTag(); - - if (NonTaggedFormatUtil.isOptional(fieldTypes[i])) { - // optional field: add the embedded non-null type tag - ftypeTag = ((AUnionType) fieldTypes[i]).getActualType().getTypeTag(); - } - - // add type tag Reference - int tagStart = typeBos.size(); - typeDos.writeByte(ftypeTag.serialize()); - int tagEnd = typeBos.size(); - IVisitablePointable typeTagReference = AFlatValuePointable.FACTORY.create(null); - typeTagReference.set(typeBos.getByteArray(), tagStart, tagEnd - tagStart); - fieldTypeTags.add(typeTagReference); - - // add type name Reference (including a astring type tag) + // add type name Reference (including a string type tag) int nameStart = typeBos.size(); typeDos.writeByte(ATypeTag.SERIALIZED_STRING_TYPE_TAG); utf8Writer.writeUTF8(fieldNameStrs[i], typeDos); @@ -147,9 +130,6 @@ for (int i = fieldNames.size() - 1; i >= numberOfSchemaFields; i--) { fieldNames.remove(i); } - for (int i = fieldTypeTags.size() - 1; i >= numberOfSchemaFields; i--) { - fieldTypeTags.remove(i); - } fieldValues.clear(); } @@ -239,8 +219,7 @@ int numberOfOpenFields = AInt32SerializerDeserializer.getInt(b, openPartOffset); int fieldOffset = openPartOffset + 4 + (8 * numberOfOpenFields); for (int i = 0; i < numberOfOpenFields; i++) { - // set the field name (including a type tag, which is - // astring) + // set the field name (including a type tag, which is a string) int fieldValueLength = NonTaggedFormatUtil.getFieldValueLength(b, fieldOffset, ATypeTag.STRING, false); int fnstart = dataBos.size(); @@ -252,12 +231,7 @@ fieldNames.add(fieldName); fieldOffset += fieldValueLength; - // set the field type tag - IVisitablePointable fieldTypeTag = allocator.allocateEmpty(); - fieldTypeTag.set(b, fieldOffset, 1); - fieldTypeTags.add(fieldTypeTag); typeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(b[fieldOffset]); - // set the field value (already including type tag) fieldValueLength = NonTaggedFormatUtil.getFieldValueLength(b, fieldOffset, typeTag, true) + 1; @@ -277,10 +251,6 @@ return fieldNames; } - public List<IVisitablePointable> getFieldTypeTags() { - return fieldTypeTags; - } - public List<IVisitablePointable> getFieldValues() { return fieldValues; } diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/cast/AListCaster.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/cast/AListCaster.java index 96c7db1..c0e59de 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/cast/AListCaster.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/cast/AListCaster.java @@ -30,8 +30,8 @@ import org.apache.asterix.om.pointables.base.IVisitablePointable; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.AbstractCollectionType; -import org.apache.asterix.om.types.EnumDeserializer; import org.apache.asterix.om.types.IAType; +import org.apache.asterix.om.utils.PointableHelper; import org.apache.asterix.om.utils.ResettableByteArrayOutputStream; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.api.IPointable; @@ -61,16 +61,11 @@ reqItemType = reqType.getItemType(); } dataBos.reset(); - - List<IVisitablePointable> itemTags = listAccessor.getItemTags(); List<IVisitablePointable> items = listAccessor.getItems(); - int start = dataBos.size(); for (int i = 0; i < items.size(); i++) { - IVisitablePointable itemTypeTag = itemTags.get(i); IVisitablePointable item = items.get(i); - ATypeTag typeTag = EnumDeserializer.ATYPETAGDESERIALIZER - .deserialize(itemTypeTag.getByteArray()[itemTypeTag.getStartOffset()]); + ATypeTag typeTag = PointableHelper.getTypeTag(item); if (reqItemType == null || reqItemType.getTypeTag().equals(ATypeTag.ANY)) { itemCastResult.setOutType(DefaultOpenFieldType.getDefaultOpenFieldType(typeTag)); } else { diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/cast/ARecordCaster.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/cast/ARecordCaster.java index 6dda563..64a7965 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/cast/ARecordCaster.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/cast/ARecordCaster.java @@ -39,10 +39,10 @@ import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.AUnionType; import org.apache.asterix.om.types.BuiltinType; -import org.apache.asterix.om.types.EnumDeserializer; import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.types.hierachy.ATypeHierarchy; import org.apache.asterix.om.utils.NonTaggedFormatUtil; +import org.apache.asterix.om.utils.PointableHelper; import org.apache.asterix.om.utils.ResettableByteArrayOutputStream; import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.api.dataflow.value.IBinaryComparator; @@ -63,7 +63,7 @@ // pointable allocator private final PointableAllocator allocator = new PointableAllocator(); private final List<IVisitablePointable> reqFieldNames = new ArrayList<>(); - private final List<IVisitablePointable> reqFieldTypeTags = new ArrayList<>(); + private final List<ATypeTag> reqFieldTypeTags = new ArrayList<>(); private ARecordType cachedReqType = null; private final ResettableByteArrayOutputStream bos = new ResettableByteArrayOutputStream(); private final DataOutputStream dos = new DataOutputStream(bos); @@ -104,7 +104,6 @@ public void castRecord(ARecordVisitablePointable recordAccessor, IPointable castOutResult, ARecordType reqType, ACastVisitor visitor) throws HyracksDataException { List<IVisitablePointable> fieldNames = recordAccessor.getFieldNames(); - List<IVisitablePointable> fieldTypeTags = recordAccessor.getFieldTypeTags(); List<IVisitablePointable> fieldValues = recordAccessor.getFieldValues(); numInputFields = fieldNames.size(); @@ -122,8 +121,8 @@ // clear the previous states reset(); - matchClosedPart(fieldNames, fieldTypeTags); - writeOutput(fieldNames, fieldTypeTags, fieldValues, outputDos, visitor); + matchClosedPart(fieldNames, fieldValues); + writeOutput(fieldNames, fieldValues, outputDos, visitor); castOutResult.set(outputBos.getByteArray(), 0, outputBos.size()); } @@ -166,12 +165,7 @@ ftypeTag = ((AUnionType) fieldTypes[i]).getActualType().getTypeTag(); optionalFields[i] = true; } - int tagStart = bos.size(); - dos.writeByte(ftypeTag.serialize()); - int tagEnd = bos.size(); - IVisitablePointable typeTagPointable = allocator.allocateEmpty(); - typeTagPointable.set(bos.getByteArray(), tagStart, tagEnd - tagStart); - reqFieldTypeTags.add(typeTagPointable); + reqFieldTypeTags.add(ftypeTag); // add type name pointable (including a string type tag) int nameStart = bos.size(); @@ -191,7 +185,7 @@ quickSort(reqFieldNamesSortedIndex, reqFieldNames, 0, reqFieldNamesSortedIndex.length - 1); } - private void matchClosedPart(List<IVisitablePointable> fieldNames, List<IVisitablePointable> fieldTypeTags) + private void matchClosedPart(List<IVisitablePointable> fieldNames, List<IVisitablePointable> fieldValues) throws HyracksDataException { int fnStart = 0; int reqFnStart = 0; @@ -204,20 +198,18 @@ int reqFnPos = reqFieldNamesSortedIndex[reqFnStart]; int c = compare(fieldNames.get(fnPos), reqFieldNames.get(reqFnPos)); if (c == 0) { - IVisitablePointable fieldTypeTag = fieldTypeTags.get(fnPos); - IVisitablePointable reqFieldTypeTag = reqFieldTypeTags.get(reqFnPos); + ATypeTag fieldTypeTag = PointableHelper.getTypeTag(fieldValues.get(fnPos)); + ATypeTag reqFieldTypeTag = reqFieldTypeTags.get(reqFnPos); + //TODO(ali): check the last || condition if (fieldTypeTag.equals(reqFieldTypeTag) || ( // match the null type of optional field - optionalFields[reqFnPos] && (fieldTypeTag.equals(nullTypeTag)) - || fieldTypeTag.equals(missingTypeTag))) { + optionalFields[reqFnPos] && ((fieldTypeTag == ATypeTag.NULL) || fieldTypeTag == ATypeTag.MISSING))) { fieldPermutation[reqFnPos] = fnPos; openFields[fnPos] = false; } else { // if mismatch, check whether input type can be promoted to the required type - ATypeTag inputTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER - .deserialize(fieldTypeTag.getByteArray()[fieldTypeTag.getStartOffset()]); - ATypeTag requiredTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER - .deserialize(reqFieldTypeTag.getByteArray()[reqFieldTypeTag.getStartOffset()]); + ATypeTag inputTypeTag = fieldTypeTag; + ATypeTag requiredTypeTag = reqFieldTypeTag; if (ATypeHierarchy.canPromote(inputTypeTag, requiredTypeTag) || ATypeHierarchy.canDemote(inputTypeTag, requiredTypeTag)) { @@ -253,9 +245,7 @@ ps.print(":"); //print the field type - IVisitablePointable fieldType = fieldTypeTags.get(i); - ATypeTag typeTag = EnumDeserializer.ATYPETAGDESERIALIZER - .deserialize(fieldType.getByteArray()[fieldType.getStartOffset()]); + ATypeTag typeTag = PointableHelper.getTypeTag(fieldValues.get(i)); ps.print(typeTag); //collect the output message and throw the exception @@ -276,9 +266,8 @@ } } - private void writeOutput(List<IVisitablePointable> fieldNames, List<IVisitablePointable> fieldTypeTags, - List<IVisitablePointable> fieldValues, DataOutput output, ACastVisitor visitor) - throws HyracksDataException { + private void writeOutput(List<IVisitablePointable> fieldNames, List<IVisitablePointable> fieldValues, + DataOutput output, ACastVisitor visitor) throws HyracksDataException { // reset the states of the record builder recBuilder.reset(cachedReqType); recBuilder.init(); @@ -294,10 +283,10 @@ // as flat if (optionalFields[i]) { //the field is optional in the input record - IVisitablePointable fieldTypeTag = pos >= 0 ? fieldTypeTags.get(pos) : null; - if (fieldTypeTag == null || fieldTypeTag.equals(missingTypeTag)) { + ATypeTag fieldTypeTag = pos >= 0 ? PointableHelper.getTypeTag(fieldValues.get(pos)) : null; + if (fieldTypeTag == null || fieldTypeTag == ATypeTag.MISSING) { fieldCastResult.setOutType(BuiltinType.AMISSING); - } else if (fieldTypeTag.equals(nullTypeTag)) { + } else if (fieldTypeTag == ATypeTag.NULL) { fieldCastResult.setOutType(BuiltinType.ANULL); } else { fieldCastResult.setOutType(((AUnionType) fType).getActualType()); @@ -312,10 +301,7 @@ if (openFields[i]) { IVisitablePointable name = fieldNames.get(i); IVisitablePointable field = fieldValues.get(i); - IVisitablePointable fieldTypeTag = fieldTypeTags.get(i); - - ATypeTag typeTag = EnumDeserializer.ATYPETAGDESERIALIZER - .deserialize(fieldTypeTag.getByteArray()[fieldTypeTag.getStartOffset()]); + ATypeTag typeTag = PointableHelper.getTypeTag(field); fieldCastResult.setOutType(DefaultOpenFieldType.getDefaultOpenFieldType(typeTag)); field.accept(visitor, fieldCastResult); recBuilder.addField(name, fieldCastResult.getOutPointable()); diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/printer/AListPrinter.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/printer/AListPrinter.java index 3fd23f7..e5114ec 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/printer/AListPrinter.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/printer/AListPrinter.java @@ -25,7 +25,7 @@ import org.apache.asterix.om.pointables.AListVisitablePointable; import org.apache.asterix.om.pointables.base.IVisitablePointable; import org.apache.asterix.om.types.ATypeTag; -import org.apache.asterix.om.types.EnumDeserializer; +import org.apache.asterix.om.utils.PointableHelper; import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.api.exceptions.HyracksDataException; @@ -47,33 +47,28 @@ public void printList(AListVisitablePointable listAccessor, PrintStream ps, IPrintVisitor visitor) throws HyracksDataException { - List<IVisitablePointable> itemTags = listAccessor.getItemTags(); List<IVisitablePointable> items = listAccessor.getItems(); itemVisitorArg.first = ps; - ps.print(startList); // print item 0 to n-2 final int size = items.size(); for (int i = 0; i < size - 1; i++) { - printItem(visitor, itemTags, items, i); + printItem(visitor, items, i); ps.print(separator); } // print item n-1 if (size > 0) { - printItem(visitor, itemTags, items, size - 1); + printItem(visitor, items, size - 1); } ps.print(endList); } - private void printItem(IPrintVisitor visitor, List<IVisitablePointable> itemTags, List<IVisitablePointable> items, - int i) throws HyracksDataException { - IVisitablePointable itemTypeTag = itemTags.get(i); + private void printItem(IPrintVisitor visitor, List<IVisitablePointable> items, int i) throws HyracksDataException { IVisitablePointable item = items.get(i); - ATypeTag typeTag = EnumDeserializer.ATYPETAGDESERIALIZER - .deserialize(itemTypeTag.getByteArray()[itemTypeTag.getStartOffset()]); + ATypeTag typeTag = PointableHelper.getTypeTag(item); itemVisitorArg.second = getItemTypeTag(item, typeTag); item.accept(visitor, itemVisitorArg); } diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/utils/PointableHelper.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/utils/PointableHelper.java new file mode 100644 index 0000000..547c289 --- /dev/null +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/utils/PointableHelper.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.om.utils; + +import org.apache.asterix.om.types.ATypeTag; +import org.apache.asterix.om.types.EnumDeserializer; +import org.apache.hyracks.data.std.api.IValueReference; + +public class PointableHelper { + + //TODO(ali): should be refactored with the other PointableHelper + public static ATypeTag getTypeTag(IValueReference val) { + return EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(val.getByteArray()[val.getStartOffset()]); + } +} diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordMergeEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordMergeEvaluator.java index 95003a3..86f74af 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordMergeEvaluator.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordMergeEvaluator.java @@ -83,8 +83,8 @@ private final RuntimeRecordTypeInfo runtimeRecordTypeInfo = new RuntimeRecordTypeInfo(); private final DeepEqualAssessor deepEqualAssessor = new DeepEqualAssessor(); - private ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage(); - private DataOutput out = resultStorage.getDataOutput(); + private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage(); + private final DataOutput out = resultStorage.getDataOutput(); RecordMergeEvaluator(IEvaluatorContext ctx, IScalarEvaluatorFactory[] args, IAType[] argTypes, SourceLocation sourceLocation, FunctionIdentifier identifier, boolean isIgnoreDuplicates) @@ -143,22 +143,21 @@ for (int i = 0; i < leftRecord.getFieldNames().size(); i++) { IVisitablePointable leftName = leftRecord.getFieldNames().get(i); IVisitablePointable leftValue = leftRecord.getFieldValues().get(i); - IVisitablePointable leftType = leftRecord.getFieldTypeTags().get(i); + ATypeTag leftType = PointableHelper.getTypeTag(leftValue); // Check if a match for the left record exists on the right record boolean foundMatch = false; for (int j = 0; j < rightRecord.getFieldNames().size(); j++) { IVisitablePointable rightName = rightRecord.getFieldNames().get(j); IVisitablePointable rightValue = rightRecord.getFieldValues().get(j); - IVisitablePointable rightType = rightRecord.getFieldTypeTags().get(j); + ATypeTag rightType = PointableHelper.getTypeTag(rightValue); // Check if same field name and not same value exists (same name and value, just take the left one) if (PointableHelper.isEqual(leftName, rightName, stringBinaryComparator) && !deepEqualAssessor.isEqual(leftValue, rightValue)) { // Same name, different value, both of type Record, do nested join - if (PointableHelper.sameType(ATypeTag.OBJECT, rightType) - && PointableHelper.sameType(ATypeTag.OBJECT, leftType)) { + if (ATypeTag.OBJECT == rightType && ATypeTag.OBJECT == leftType) { // We are merging two sub records addFieldToSubRecord(combinedType, leftName, leftValue, rightValue, nestedLevel); foundMatch = true; diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRemoveFieldsEvalFactory.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRemoveFieldsEvalFactory.java index 1a453a7..7ff0dae 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRemoveFieldsEvalFactory.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordRemoveFieldsEvalFactory.java @@ -145,7 +145,6 @@ List<IVisitablePointable> fieldNames = srp.getFieldNames(); List<IVisitablePointable> fieldValues = srp.getFieldValues(); - List<IVisitablePointable> fieldTypes = srp.getFieldTypeTags(); for (int i = 0; i < fieldNames.size(); i++) { IVisitablePointable subRecFieldName = fieldNames.get(i); @@ -153,10 +152,11 @@ if (isValidPath(inputList)) { if (requiredType != null && requiredType.getTypeTag() != ATypeTag.ANY) { addKeptFieldToSubRecord(requiredType, subRecFieldName, fieldValues.get(i), - fieldTypes.get(i), inputList, nestedLevel); + PointableHelper.getTypeTag(fieldValues.get(i)), inputList, nestedLevel); } else { addKeptFieldToSubRecord(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE, subRecFieldName, - fieldValues.get(i), fieldTypes.get(i), inputList, nestedLevel); + fieldValues.get(i), PointableHelper.getTypeTag(fieldValues.get(i)), inputList, + nestedLevel); } } recordPath.pop(); @@ -164,14 +164,14 @@ } private void addKeptFieldToSubRecord(ARecordType requiredType, IVisitablePointable fieldNamePointable, - IVisitablePointable fieldValuePointable, IVisitablePointable fieldTypePointable, + IVisitablePointable fieldValuePointable, ATypeTag fieldTypePointable, AListVisitablePointable inputList, int nestedLevel) throws IOException { runtimeRecordTypeInfo.reset(requiredType); int pos = runtimeRecordTypeInfo.getFieldIndex(fieldNamePointable.getByteArray(), fieldNamePointable.getStartOffset() + 1, fieldNamePointable.getLength() - 1); if (pos >= 0) { // Closed field - if (PointableHelper.sameType(ATypeTag.OBJECT, fieldTypePointable) + if (ATypeTag.OBJECT == fieldTypePointable && PointableHelper.sameType(ATypeTag.OBJECT, fieldValuePointable)) { processRecord((ARecordType) TypeComputeUtils.getActualType(requiredType.getFieldTypes()[pos]), (ARecordVisitablePointable) fieldValuePointable, inputList, nestedLevel + 1); @@ -182,7 +182,7 @@ rbStack.get(nestedLevel).addField(pos, fieldValuePointable); } } else { // Open field - if (PointableHelper.sameType(ATypeTag.OBJECT, fieldTypePointable)) { + if (ATypeTag.OBJECT == fieldTypePointable) { processRecord(null, (ARecordVisitablePointable) fieldValuePointable, inputList, nestedLevel + 1); tabvs.reset(); @@ -196,12 +196,10 @@ private boolean isValidPath(AListVisitablePointable inputList) throws HyracksDataException { List<IVisitablePointable> items = inputList.getItems(); - List<IVisitablePointable> typeTags = inputList.getItemTags(); - int pathLen = recordPath.size(); for (int i = 0; i < items.size(); i++) { IVisitablePointable item = items.get(i); - if (PointableHelper.sameType(ATypeTag.ARRAY, typeTags.get(i))) { + if (PointableHelper.sameType(ATypeTag.ARRAY, item)) { List<IVisitablePointable> inputPathItems = ((AListVisitablePointable) item).getItems(); if (pathLen == inputPathItems.size()) { diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/ListDeepEqualityChecker.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/ListDeepEqualityChecker.java index e510441..e9dbe30 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/ListDeepEqualityChecker.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/ListDeepEqualityChecker.java @@ -51,11 +51,9 @@ AListVisitablePointable listLeft = (AListVisitablePointable) listPointableLeft; List<IVisitablePointable> itemsLeft = listLeft.getItems(); - List<IVisitablePointable> itemTagTypesLeft = listLeft.getItemTags(); AListVisitablePointable listRight = (AListVisitablePointable) listPointableRight; List<IVisitablePointable> itemsRight = listRight.getItems(); - List<IVisitablePointable> itemTagTypesRight = listRight.getItemTags(); if (itemsLeft.size() != itemsRight.size()) return false; @@ -65,23 +63,23 @@ return false; if (isOrderedRight) { - return processOrderedList(itemsLeft, itemTagTypesLeft, itemsRight, itemTagTypesRight); + return processOrderedList(itemsLeft, itemsRight); } else { - return processUnorderedList(itemsLeft, itemTagTypesLeft, itemsRight, itemTagTypesRight); + return processUnorderedList(itemsLeft, itemsRight); } } - private boolean processOrderedList(List<IVisitablePointable> itemsLeft, List<IVisitablePointable> itemTagTypesLeft, - List<IVisitablePointable> itemsRight, List<IVisitablePointable> itemTagTypesRight) + private boolean processOrderedList(List<IVisitablePointable> itemsLeft, List<IVisitablePointable> itemsRight) throws HyracksDataException { for (int i = 0; i < itemsLeft.size(); i++) { - ATypeTag fieldTypeLeft = PointableHelper.getTypeTag(itemTagTypesLeft.get(i)); - if (fieldTypeLeft.isDerivedType() - && fieldTypeLeft != PointableHelper.getTypeTag(itemTagTypesRight.get(i))) { + IVisitablePointable itemLeft = itemsLeft.get(i); + IVisitablePointable itemRight = itemsRight.get(i); + ATypeTag fieldTypeLeft = PointableHelper.getTypeTag(itemLeft); + if (fieldTypeLeft.isDerivedType() && fieldTypeLeft != PointableHelper.getTypeTag(itemRight)) { return false; } - itemVisitorArg.first = itemsRight.get(i); - itemsLeft.get(i).accept(visitor, itemVisitorArg); + itemVisitorArg.first = itemRight; + itemLeft.accept(visitor, itemVisitorArg); if (itemVisitorArg.second == false) return false; } @@ -89,9 +87,8 @@ return true; } - private boolean processUnorderedList(List<IVisitablePointable> itemsLeft, - List<IVisitablePointable> itemTagTypesLeft, List<IVisitablePointable> itemsRight, - List<IVisitablePointable> itemTagTypesRight) throws HyracksDataException { + private boolean processUnorderedList(List<IVisitablePointable> itemsLeft, List<IVisitablePointable> itemsRight) + throws HyracksDataException { hashMap.clear(); // Build phase: Add items into hash map, starting with first list. @@ -105,11 +102,10 @@ hashMap.put(keyEntry, valEntry); } - return probeHashMap(itemsLeft, itemTagTypesLeft, itemsRight, itemTagTypesRight); + return probeHashMap(itemsLeft, itemsRight); } - private boolean probeHashMap(List<IVisitablePointable> itemsLeft, List<IVisitablePointable> itemTagTypesLeft, - List<IVisitablePointable> itemsRight, List<IVisitablePointable> itemTagTypesRight) + private boolean probeHashMap(List<IVisitablePointable> itemsLeft, List<IVisitablePointable> itemsRight) throws HyracksDataException { // Probe phase: Probe items from second list for (int indexRight = 0; indexRight < itemsRight.size(); indexRight++) { @@ -126,14 +122,14 @@ } int indexLeft = IntegerPointable.getInteger(entry.getBuf(), entry.getOffset()); - ATypeTag fieldTypeLeft = PointableHelper.getTypeTag(itemTagTypesLeft.get(indexLeft)); - if (fieldTypeLeft.isDerivedType() - && fieldTypeLeft != PointableHelper.getTypeTag(itemTagTypesRight.get(indexRight))) { + IVisitablePointable itemLeft = itemsLeft.get(indexLeft); + ATypeTag fieldTypeLeft = PointableHelper.getTypeTag(itemLeft); + if (fieldTypeLeft.isDerivedType() && fieldTypeLeft != PointableHelper.getTypeTag(itemRight)) { return false; } itemVisitorArg.first = itemRight; - itemsLeft.get(indexLeft).accept(visitor, itemVisitorArg); + itemLeft.accept(visitor, itemVisitorArg); if (itemVisitorArg.second == false) return false; } diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/RecordDeepEqualityChecker.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/RecordDeepEqualityChecker.java index 1c033e9..f2261e2 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/RecordDeepEqualityChecker.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/RecordDeepEqualityChecker.java @@ -79,12 +79,10 @@ hashMap.put(keyEntry, valEntry); } - return compareValues(recLeft.getFieldTypeTags(), recLeft.getFieldValues(), fieldNamesRight, - recRight.getFieldTypeTags(), recRight.getFieldValues()); + return compareValues(recLeft.getFieldValues(), fieldNamesRight, recRight.getFieldValues()); } - private boolean compareValues(List<IVisitablePointable> fieldTypesLeft, List<IVisitablePointable> fieldValuesLeft, - List<IVisitablePointable> fieldNamesRight, List<IVisitablePointable> fieldTypesRight, + private boolean compareValues(List<IVisitablePointable> fieldValuesLeft, List<IVisitablePointable> fieldNamesRight, List<IVisitablePointable> fieldValuesRight) throws HyracksDataException { // Probe phase: Probe items from second record @@ -97,12 +95,14 @@ } int fieldIdLeft = AInt32SerializerDeserializer.getInt(entry.getBuf(), entry.getOffset()); - ATypeTag fieldTypeLeft = PointableHelper.getTypeTag(fieldTypesLeft.get(fieldIdLeft)); - if (fieldTypeLeft.isDerivedType() && fieldTypeLeft != PointableHelper.getTypeTag(fieldTypesRight.get(i))) { + IVisitablePointable fieldValLeft = fieldValuesLeft.get(fieldIdLeft); + IVisitablePointable fieldValRight = fieldValuesRight.get(i); + ATypeTag fieldTypeLeft = PointableHelper.getTypeTag(fieldValLeft); + if (fieldTypeLeft.isDerivedType() && fieldTypeLeft != PointableHelper.getTypeTag(fieldValRight)) { return false; } - nestedVisitorArg.first = fieldValuesRight.get(i); - fieldValuesLeft.get(fieldIdLeft).accept(visitor, nestedVisitorArg); + nestedVisitorArg.first = fieldValRight; + fieldValLeft.accept(visitor, nestedVisitorArg); if (nestedVisitorArg.second == false) { return false; } -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18407 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I89952db7ad9e1e941b3f6c4a06c01c1a2dbd7b72 Gerrit-Change-Number: 18407 Gerrit-PatchSet: 2 Gerrit-Owner: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Wail Alkowaileet <[email protected]> Gerrit-MessageType: merged
