Heri Ramampiaro has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/523
Change subject: ASTERIXDB-944: added an AQL function that prints internal data
format
......................................................................
ASTERIXDB-944: added an AQL function that prints internal data format
The adm-to-bytes implemented here is for debugging/documentation. It prints an
accurate picture of the internal representation of an ADM instance, i.e., a
slightly marked up version of the bytes used to represent the instance.
Change-Id: I31d0b2ec2d8686531833811937596c3dca660b1e
---
A asterix-doc/src/site/markdown/aql/admdebuggerfunctions.md
M
asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
A
asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/AdmToBytesTypeComputer.java
A
asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/TypeComputerUtils.java
A
asterix-om/src/main/java/org/apache/asterix/om/util/admdebugger/AnnotatedFieldNameComputerUtil.java
A
asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AdmToBytesDescriptor.java
A
asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AdmToBytesFactory.java
A
asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AdmToBytesHelper.java
A
asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PointableUtils.java
A
asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/admdebugging/AdmToBytesVisitor.java
A
asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/admdebugging/ListBytesProcessor.java
A
asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/admdebugging/RecordBytesProcessor.java
M
asterix-runtime/src/main/java/org/apache/asterix/runtime/formats/NonTaggedDataFormat.java
13 files changed, 1,683 insertions(+), 13 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/23/523/1
diff --git a/asterix-doc/src/site/markdown/aql/admdebuggerfunctions.md
b/asterix-doc/src/site/markdown/aql/admdebuggerfunctions.md
new file mode 100644
index 0000000..097403f
--- /dev/null
+++ b/asterix-doc/src/site/markdown/aql/admdebuggerfunctions.md
@@ -0,0 +1,68 @@
+### adm-to-bytes ###
+* Syntax:
+
+ adm-to-bytes(var1, number)
+
+
+ * Displays the annotated bytes of a data value. The main goal with this
function is to debug the object serialization
+ process in AsterixDB and to show how data being stored look like. Its mainly
used for debugging purposes.
+ * Arguments:
+ * `var1`: a data value, including record, list, string, and number.
+ * `number`: a number value from 0 to infinty ("INF") specifying the nested
level of a record.
+ `number = 0` prints the raw byte array value, `number = 1` or higher
prints the byte arrays with
+ corresponding annotations. This function is mainly used for debugging.
+ * Return Value:
+ * A record showing the raw byte array or annotaded byte arrays.
+
+
+ * Example:
+
+ let $r1 := {"id":1,
+ "project":"AsterixDB",
+ "address":{"city":"Irvine", "state":"CA"},
+ "related":["Hivestrix", "Preglix", "Apache VXQuery"] }
+ return adm-to-bytes($r1, <Level>);
+
+ * The expected results are:
+
+ Level=0:
+
+ { "RawBytes": "[24, 0, 0, 0, -127, 0, 0, 0, 4, 0, 0, 0, 25, 0, 0, 0,
33, 0, 0, 0, 44,
+ 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 65, 115, 116, 101, 114,
105, 120, 68, 66,
+ 0, 0, 0, 29, 0, 0, 0, 2, 0, 0, 0, 17, 0, 0, 0, 25, 0, 6, 73, 114,
118, 105, 110, 101,
+ 0, 2, 67, 65, 13, 0, 0, 0, 58, 0, 0, 0, 3, 0, 0, 0, 22, 0, 0, 0, 33,
0, 0, 0, 42, 0,
+ 9, 72, 105, 118, 101, 115, 116, 114, 105, 120, 0, 7, 80, 114, 101,
103, 108, 105, 120,
+ 0, 14, 65, 112, 97, 99, 104, 101, 32, 86, 88, 81, 117, 101, 114,
121]" }
+
+ Level=1:
+
+ { "Tag": "[24]", "Length": "[0, 0, 0, -127]", "NumberOfClosedFields":
"[0, 0, 0, 4]",
+ "ClosedFieldOffsets": "[0, 0, 0, 25, 0, 0, 0, 33, 0, 0, 0, 44, 0, 0,
0, 72]",
+ "ClosedFields": "[0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 65, 115, 116, 101,
114, 105, 120,
+ 68, 66, 0, 0, 0, 29, 0, 0, 0, 2, 0, 0, 0, 17, 0, 0, 0, 25, 0, 6, 73,
114, 118, 105,
+ 110, 101, 0, 2, 67, 65, 13, 0, 0, 0, 58, 0, 0, 0, 3, 0, 0, 0, 22, 0,
0, 0, 33, 0, 0,
+ 0, 42, 0, 9, 72, 105, 118, 101, 115, 116, 114, 105, 120, 0, 7, 80,
114, 101, 103,
+ 108, 105, 120, 0, 14, 65, 112, 97, 99, 104, 101, 32, 86, 88, 81, 117,
101, 114, 121]" }
+
+ Level=2:
+
+ { "id": { "Tag": "[4]", "Value": "[0, 0, 0, 0, 0, 0, 0, 1]" },
"project": { "Tag": "[13]",
+ "Value": "[65, 115, 116, 101, 114, 105, 120, 68, 66]", "Length":
"[9]" }, "address":
+ { "Tag": "[24]", "Length": "[0, 0, 0, 29]", "NumberOfClosedFields":
"[0, 0, 0, 2]",
+ "ClosedFieldOffsets": "[0, 0, 0, 17, 0, 0, 0, 25]", "ClosedFields":
"[0, 6, 73, 114, 118,
+ 105, 110, 101, 0, 2, 67, 65]" }, "related": { "Tag": "[22]",
"ItemType": "[13]",
+ "Length": "[0, 0, 0, 58]", "NumberOfItems": "[0, 0, 0, 3]",
"ItemOffsets": "[0, 0, 0, 22,
+ 0, 0, 0, 33, 0, 0, 0, 42]", "Value": "[0, 9, 72, 105, 118, 101, 115,
116, 114, 105, 120,
+ 0, 7, 80, 114, 101, 103, 108, 105, 120, 0, 14, 65, 112, 97, 99, 104,
101, 32, 86, 88, 81,
+ 117, 101, 114, 121]" } }
+
+ Level>=3 (including "INF"):
+
+ { "id": { "Tag": "[4]", "Value": "[0, 0, 0, 0, 0, 0, 0, 1]" },
"project": { "Tag": "[13]",
+ "Length": "[9]", "Value": "[65, 115, 116, 101, 114, 105, 120, 68, 66]"
},
+ "address": { "city": { "Tag": "[13]", "Length": "[0, 6]", "Value":
"[73, 114, 118, 105,
+ 110, 101]" }, "state": { "Tag": "[13]", "Length": "[0, 2]", "Value":
"[67, 65]" } },
+ "related": { "Tag": "[22]", "ItemType": "[13]", "Length": "[0, 0, 0,
58]", "NumberOfItems":
+ "[0, 0, 0, 3]", "ItemOffsets": "[0, 0, 0, 22, 0, 0, 0, 33, 0, 0, 0,
42]", "Value": "[0, 9,
+ 72, 105, 118, 101, 115, 116, 114, 105, 120, 0, 7, 80, 114, 101, 103,
108, 105, 120, 0, 14,
+ 65, 112, 97, 99, 104, 101, 32, 86, 88, 81, 117, 101, 114, 121]" } }
diff --git
a/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
b/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
index ad4831f..fd0a745 100644
---
a/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
+++
b/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
@@ -18,12 +18,6 @@
*/
package org.apache.asterix.om.functions;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.asterix.common.functions.FunctionConstants;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
@@ -40,6 +34,7 @@
import org.apache.asterix.om.typecomputer.impl.AStringTypeComputer;
import org.apache.asterix.om.typecomputer.impl.ATimeTypeComputer;
import org.apache.asterix.om.typecomputer.impl.AUUIDTypeComputer;
+import org.apache.asterix.om.typecomputer.impl.AdmToBytesTypeComputer;
import org.apache.asterix.om.typecomputer.impl.AnyTypeComputer;
import
org.apache.asterix.om.typecomputer.impl.BinaryBooleanOrNullFunctionTypeComputer;
import
org.apache.asterix.om.typecomputer.impl.BinaryStringBoolOrNullTypeComputer;
@@ -117,6 +112,12 @@
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
public class AsterixBuiltinFunctions {
public enum SpatialFilterKind {
@@ -190,6 +191,9 @@
"get-record-fields", 1);
public final static FunctionIdentifier GET_RECORD_FIELD_VALUE = new
FunctionIdentifier(FunctionConstants.ASTERIX_NS,
"get-record-field-value", 2);
+
+ public final static FunctionIdentifier ADM_TO_BYTES = new
FunctionIdentifier(
+ FunctionConstants.ASTERIX_NS, "adm-to-bytes", 2);
// numeric
public final static FunctionIdentifier NUMERIC_UNARY_MINUS = new
FunctionIdentifier(FunctionConstants.ASTERIX_NS,
@@ -942,6 +946,8 @@
addFunction(GET_RECORD_FIELDS, OrderedListOfAnyTypeComputer.INSTANCE,
true);
addFunction(GET_RECORD_FIELD_VALUE,
FieldAccessNestedResultType.INSTANCE, true);
+ addFunction(ADM_TO_BYTES, AdmToBytesTypeComputer.INSTANCE, true);
+
// temporal type accessors
addFunction(ACCESSOR_TEMPORAL_YEAR,
OptionalAInt64TypeComputer.INSTANCE, true);
addFunction(ACCESSOR_TEMPORAL_MONTH,
OptionalAInt64TypeComputer.INSTANCE, true);
diff --git
a/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/AdmToBytesTypeComputer.java
b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/AdmToBytesTypeComputer.java
new file mode 100644
index 0000000..fcaa2d8
--- /dev/null
+++
b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/AdmToBytesTypeComputer.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.typecomputer.impl;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.om.base.AInt64;
+import org.apache.asterix.om.base.AString;
+import org.apache.asterix.om.base.IAObject;
+import org.apache.asterix.om.constants.AsterixConstantValue;
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.AUnorderedListType;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.TypeHelper;
+import org.apache.asterix.om.util.admdebugger.AnnotatedFieldNameComputerUtil;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import
org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import
org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import
org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.HyracksException;
+
+public class AdmToBytesTypeComputer implements IResultTypeComputer {
+ public static final AdmToBytesTypeComputer INSTANCE = new
AdmToBytesTypeComputer();
+ private final long VARIABLE_OUTPULEVEL = -1;
+ private long outputLevel = 0;
+
+ private AdmToBytesTypeComputer() {
+ }
+
+ public static AbstractCollectionType extractListType(IAType type) {
+ if (type.getTypeTag() == ATypeTag.UNORDEREDLIST)
+ return TypeComputerUtils.extractUnorderedListType(type);
+ else
+ return TypeComputerUtils.extractOrderedListType(type);
+ }
+
+ private long getLevel(IAType inputType1, ILogicalExpression expr) throws
AlgebricksException {
+ // Treat null as a level = 0
+ if (inputType1.getTypeTag() == ATypeTag.NULL) {
+ return 0;
+ }
+
+ // If input level is a variable
+ if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ return VARIABLE_OUTPULEVEL;
+ }
+
+ if (expr.getExpressionTag() != LogicalExpressionTag.CONSTANT)
+ throw new AlgebricksException("Expected a variable or either an
integer or a string (INF) constant"
+ + " but got a " + expr.getExpressionTag());
+
+ ConstantExpression constExpr = (ConstantExpression) expr;
+ IAObject value = ((AsterixConstantValue)
constExpr.getValue()).getObject();
+
+ switch (inputType1.getTypeTag()) {
+ case INT64:
+ return ((AInt64) value).getLongValue();
+ case STRING:
+ String valStr = ((AString) value).getStringValue();
+ if (valStr.equals("INF")) { // Check if infinity
+ return Long.MAX_VALUE;
+ } else { // We accept also numbers expressed as string
+ for (int i = 0; i < valStr.length(); i++) {
+ if (!Character.isDigit(valStr.charAt(i))) {
+ throw new AlgebricksException("Invalid number.
Expected argument is a number from 0 "
+ + "to INF, where INF is infinity");
+ }
+ }
+ return Long.parseLong(valStr);
+ }
+ default:
+ throw new AlgebricksException("Invalid argument. Expected a
number from 0 "
+ + "to INF, where INF is infinity, but got " +
inputType1);
+ }
+ }
+
+ @Override
+ public IAType computeType(ILogicalExpression expression,
IVariableTypeEnvironment env,
+ IMetadataProvider<?, ?> metadataProvider) throws
AlgebricksException {
+ AbstractFunctionCallExpression funcExpr =
(AbstractFunctionCallExpression) expression;
+
+ ILogicalExpression expr1 = funcExpr.getArguments().get(1).getValue();
+ IAType type1 = (IAType) env.getType(expr1);
+ outputLevel = getLevel(type1, expr1);
+
+ // If level is unknown at translation time (input is a variable)
+ if (outputLevel == VARIABLE_OUTPULEVEL) {
+ IAType resultType =
DefaultOpenFieldType.getDefaultOpenFieldType(type1.getTypeTag());
+ if (resultType == null) {
+ return DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE;
+ }
+ return resultType;
+ }
+
+ // If for printing of the raw bytes only
+ if (outputLevel == 0) {
+ try {
+ return new ARecordType("RawBytes", new String[] { "RawBytes"
}, new IAType[] { BuiltinType.ASTRING },
+ false);
+ } catch (HyracksDataException | AsterixException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+
+ // For printing annotated byte arrays (outputLevel > 0)
+ return computeResultType((IAType)
env.getType(funcExpr.getArguments().get(0).getValue()), 1);
+ }
+
+ private IAType computeResultType(IAType inType, long nestedLevel) throws
AlgebricksException {
+ IAType inputType = inType;
+ IAType resultType;
+
+ if (TypeHelper.canBeNull(inputType)) {
+ return AUnionType.createNullableType(AnnotatedFieldNameComputerUtil
+ .getNullableAnnotatedFieldsRecordType(true));
+ }
+
+ // If the input type is not a record or level = 1, not a complex
object or reached the lowest nesting level we
+ // print out the annotated byte with no nesting level processing
+ if (outputLevel == 1 || !inputType.getTypeTag().isDerivedType() ||
nestedLevel == outputLevel) {
+ resultType =
AnnotatedFieldNameComputerUtil.getAnnotatedBytesRecordType(inputType.getTypeTag());
+ } else { // Is a list or a record and outputLevel > 1
+ switch (inputType.getTypeTag()) {
+ case RECORD:
+ resultType = getResultRecordType(inputType, nestedLevel);
+ break;
+ case UNION:
+ resultType = computeResultType(inputType, nestedLevel);
+ break;
+ default: //an UNORDEREDLIST or ORDEREDLIST
+ resultType = getResultListType(inputType, nestedLevel);
+ }
+ }
+
+ return resultType;
+ }
+
+ private IAType getResultListType(IAType inputType, long nestedLevel)
throws AlgebricksException {
+ AbstractCollectionType inputListType = extractListType(inputType);
+ if (inputListType.getItemType().getTypeTag() == ATypeTag.ANY ||
inputListType == null) {
+ return creatListType(inputType.getTypeTag(), BuiltinType.ANY,
"annotated-bytes-open-type");
+ }
+ return getNestedList(inputListType, nestedLevel);
+ }
+
+ private IAType getNestedList(AbstractCollectionType inputListType, long
nestedLevel) throws AlgebricksException {
+ IAType inputItemType = inputListType.getItemType();
+ ATypeTag inputItemTypeTypeTag = inputItemType.getTypeTag();
+ IAType reqItemType;
+ if (inputItemTypeTypeTag.isDerivedType() && nestedLevel < outputLevel)
{
+ reqItemType = computeResultType(inputItemType, nestedLevel + 1);
+ } else {
+ reqItemType =
AnnotatedFieldNameComputerUtil.getAnnotatedBytesRecordType(inputItemType.getTypeTag());
+ }
+ return creatListType(inputListType.getTypeTag(), reqItemType,
"annotated-bytes(" + inputListType.getTypeName()
+ + ")");
+ }
+
+ private IAType creatListType(ATypeTag typeTag, IAType itemType, String
typeName) throws AlgebricksException {
+ switch (typeTag) {
+ case ORDEREDLIST:
+ return new AOrderedListType(itemType, typeName);
+ case UNORDEREDLIST:
+ return new AUnorderedListType(itemType, typeName);
+ default:
+ throw new AlgebricksException("Not a valid AsterixDB list");
+ }
+ }
+
+ private IAType getResultRecordType(IAType inputType, long nestedLevel)
throws AlgebricksException {
+ ARecordType inputRecordType =
TypeComputerUtils.extractRecordType(inputType);
+ if (inputRecordType == null) {
+ return DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE;
+ }
+ return getNestedRecordFields(inputRecordType, nestedLevel);
+ }
+
+ private ARecordType getNestedRecordFields(ARecordType inputType, long
nestedLevel) throws AlgebricksException {
+ try {
+ String fieldNames[] = inputType.getFieldNames();
+ IAType fieldTypes[] = inputType.getFieldTypes();
+
+ IAType[] newTypes = new IAType[fieldNames.length];
+ for (int i = 0; i < fieldTypes.length; i++) {
+ ATypeTag fieldTypeTag = fieldTypes[i].getTypeTag();
+ if (fieldTypeTag.isDerivedType() && nestedLevel < outputLevel)
{
+ newTypes[i] = computeResultType(fieldTypes[i], nestedLevel
+ 1);
+ } else {
+ newTypes[i] = AnnotatedFieldNameComputerUtil
+
.getAnnotatedBytesRecordType(fieldTypes[i].getTypeTag());
+ }
+ }
+ String typeName = "annotated-bytes(" + inputType.getTypeName() +
")";
+ return new ARecordType(typeName, fieldNames, newTypes,
inputType.isOpen());
+
+ } catch (AsterixException | HyracksException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+}
diff --git
a/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/TypeComputerUtils.java
b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/TypeComputerUtils.java
new file mode 100644
index 0000000..3834ae4
--- /dev/null
+++
b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/TypeComputerUtils.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.typecomputer.impl;
+
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.AUnorderedListType;
+import org.apache.asterix.om.types.IAType;
+
+public class TypeComputerUtils {
+
+ private TypeComputerUtils() {
+
+ }
+
+ public static ARecordType extractRecordType(IAType t) {
+ if (t.getTypeTag() == ATypeTag.RECORD) {
+ return (ARecordType) t;
+ }
+
+ if (t.getTypeTag() == ATypeTag.UNION) {
+ IAType innerType = ((AUnionType) t).getUnionList().get(1);
+ if (innerType.getTypeTag() == ATypeTag.RECORD) {
+ return (ARecordType) innerType;
+ }
+ }
+
+ return null;
+ }
+
+ public static AOrderedListType extractOrderedListType(IAType t) {
+ if (t.getTypeTag() == ATypeTag.ORDEREDLIST) {
+ return (AOrderedListType) t;
+ }
+
+ if (t.getTypeTag() == ATypeTag.UNION) {
+ IAType innerType = ((AUnionType) t).getUnionList().get(1);
+ if (innerType.getTypeTag() == ATypeTag.ORDEREDLIST) {
+ return (AOrderedListType) innerType;
+ }
+ }
+
+ return null;
+ }
+
+ public static AUnorderedListType extractUnorderedListType(IAType t) {
+ if (t.getTypeTag() == ATypeTag.UNORDEREDLIST) {
+ return (AUnorderedListType) t;
+ }
+
+ if (t.getTypeTag() == ATypeTag.UNION) {
+ IAType innerType = ((AUnionType) t).getUnionList().get(1);
+ if (innerType.getTypeTag() == ATypeTag.UNORDEREDLIST) {
+ return (AUnorderedListType) innerType;
+ }
+ }
+
+ return null;
+ }
+
+
+}
diff --git
a/asterix-om/src/main/java/org/apache/asterix/om/util/admdebugger/AnnotatedFieldNameComputerUtil.java
b/asterix-om/src/main/java/org/apache/asterix/om/util/admdebugger/AnnotatedFieldNameComputerUtil.java
new file mode 100644
index 0000000..a5eae65
--- /dev/null
+++
b/asterix-om/src/main/java/org/apache/asterix/om/util/admdebugger/AnnotatedFieldNameComputerUtil.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.util.admdebugger;
+
+import java.util.EnumSet;
+import java.util.Set;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class AnnotatedFieldNameComputerUtil {
+ private AnnotatedFieldNameComputerUtil() {
+ }
+
+ public static ARecordType getAnnotatedBytesRecordType(ATypeTag typeTag)
throws AlgebricksException {
+ switch (typeTag) {
+ case RECORD:
+ return
createAnnotatedFieldsRecordType(ByteAnnotationField.recordFields, true);
+ case ORDEREDLIST:
+ case UNORDEREDLIST:
+ return
createAnnotatedFieldsRecordType(ByteAnnotationField.listFields, false);
+ case STRING:
+ return
createAnnotatedFieldsRecordType(ByteAnnotationField.stringObjectFields, false);
+ case UNION:
+ return getNullableAnnotatedFieldsRecordType(true);
+ case NULL:
+ return getNullableAnnotatedFieldsRecordType(false);
+ default:
+ return
createAnnotatedFieldsRecordType(ByteAnnotationField.asterixObjectFields, true);
+ }
+ }
+
+ public static ARecordType getNullableAnnotatedFieldsRecordType(boolean
isOpen) throws AlgebricksException {
+ try {
+ return new ARecordType("NullableAnnotationRecord",
+ new String[] { ByteAnnotationField.TAG.fieldName() }, new
IAType[] { BuiltinType.ASTRING }, isOpen);
+ } catch (AsterixException | HyracksDataException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+
+ private static ARecordType
createAnnotatedFieldsRecordType(Set<ByteAnnotationField> fields, boolean isOpen)
+ throws AlgebricksException {
+ String fieldNames[] = new String[fields.size()];
+ IAType fieldTypes[] = new IAType[fields.size()];
+ try {
+ int j = 0;
+ for (ByteAnnotationField baf : fields) {
+ fieldNames[j] = baf.fieldName();
+ fieldTypes[j] = BuiltinType.ASTRING;
+ j++;
+ }
+ return new ARecordType("ByteArrayfields", fieldNames, fieldTypes,
isOpen);
+
+ } catch (HyracksDataException | AsterixException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+
+ public enum ByteAnnotationField {
+ // Common
+ TAG("Tag"),
+
+ // FOR RECORDS
+ // (required)
+ NUMBER_CLOSED_FIELDS("NumberOfClosedFields"),
+ CLOSED_FIELD_OFFSETS("ClosedFieldOffsets"),
+ CLOSED_FIELDS("ClosedFields"),
+ // Optional
+ NULLBITMAP("NullBitMap"),
+ IS_EXPANDED("IsExpanded"),
+ OPENPART_OFFSET("OpenPartOffset"),
+ NUMBER_OF_OPEN_FIELDS("NumberOfOpenFields"),
+ OPEN_FIELDS_HASH_OFFSET("OpenfieldHashCodeAndOffsets"),
+ OPEN_FIELDS("OpenFields"),
+
+ // FOR LISTS
+ ITEM_TYPE("ItemType"),
+ NUMBER_OF_ITEMS("NumberOfItems"),
+ ITEM_OFFSETS("ItemOffsets"),
+
+ // FOR PRIMITIVE TYPES
+ VALUE("Value"),
+
+ // For all but numeric values
+ LENGTH("Length");
+
+ public static EnumSet<ByteAnnotationField> recordFields =
EnumSet.of(TAG, LENGTH,
+ NUMBER_CLOSED_FIELDS, CLOSED_FIELD_OFFSETS, CLOSED_FIELDS);
+ public static EnumSet<ByteAnnotationField> listFields =
EnumSet.of(TAG, ITEM_TYPE, LENGTH, NUMBER_OF_ITEMS,
+ ITEM_OFFSETS, VALUE);
+ public static EnumSet<ByteAnnotationField> stringObjectFields =
EnumSet.of(TAG, LENGTH, VALUE);
+ public static EnumSet<ByteAnnotationField> asterixObjectFields =
EnumSet.of(TAG, VALUE); // All other objects
+
+ private String fieldName;
+
+ ByteAnnotationField(String fieldName) {
+ this.fieldName = fieldName;
+ }
+
+ @Override
+ public String toString() {
+ return "{" + fieldName + "}";
+ }
+
+ public String fieldName() {
+ return this.fieldName;
+ }
+ }
+}
diff --git
a/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AdmToBytesDescriptor.java
b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AdmToBytesDescriptor.java
new file mode 100644
index 0000000..e62e741
--- /dev/null
+++
b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AdmToBytesDescriptor.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.typecomputer.impl.TypeComputerUtils;
+import org.apache.asterix.om.types.IAType;
+import
org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+
+public class AdmToBytesDescriptor extends
AbstractScalarFunctionDynamicDescriptor {
+ private static final long serialVersionUID = 1L;
+
+ private IAType outputType;
+ private IAType inputType;
+
+ public static final IFunctionDescriptorFactory FACTORY = new
IFunctionDescriptorFactory() {
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new AdmToBytesDescriptor();
+ }
+ };
+
+
+ public void reset(IAType outputType, IAType inType) {
+ switch (inType.getTypeTag()) {
+ case RECORD:
+ this.inputType = TypeComputerUtils.extractRecordType(inType);
+ break;
+ case UNORDEREDLIST:
+ this.inputType =
TypeComputerUtils.extractUnorderedListType(inType);
+ break;
+ case ORDEREDLIST:
+ this.inputType =
TypeComputerUtils.extractOrderedListType(inType);
+ break;
+ default:
+ this.inputType = inType;
+ }
+
+ this.outputType = outputType;
+
+ }
+
+
+ private AdmToBytesDescriptor() {
+
+ }
+
+ public ICopyEvaluatorFactory createEvaluatorFactory(final
ICopyEvaluatorFactory[] args) throws AlgebricksException {
+ return new AdmToBytesFactory(args[0], args[1], inputType, outputType);
+ }
+
+ @Override public FunctionIdentifier getIdentifier() {
+ return AsterixBuiltinFunctions.ADM_TO_BYTES;
+ }
+}
diff --git
a/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AdmToBytesFactory.java
b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AdmToBytesFactory.java
new file mode 100644
index 0000000..82e4763
--- /dev/null
+++
b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AdmToBytesFactory.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.runtime.evaluators.functions;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.asterix.builders.IARecordBuilder;
+import org.apache.asterix.builders.RecordBuilder;
+import org.apache.asterix.common.exceptions.AsterixException;
+import
org.apache.asterix.dataflow.data.nontagged.serde.AStringSerializerDeserializer;
+import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
+import org.apache.asterix.om.base.ANull;
+import org.apache.asterix.om.base.AString;
+import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
+import org.apache.asterix.om.pointables.PointableAllocator;
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.EnumDeserializer;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.runtime.RuntimeRecordTypeInfo;
+import org.apache.asterix.om.util.ResettableByteArrayOutputStream;
+import
org.apache.asterix.runtime.evaluators.visitors.admdebugging.AdmToBytesVisitor;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Triple;
+import org.apache.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IDataOutputProvider;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+import org.apache.hyracks.util.string.UTF8StringUtil;
+
+/**
+ * Check each field in a record and prints out the serialized values of the
fields
+ */
+public class AdmToBytesFactory implements ICopyEvaluatorFactory {
+ private static final byte SER_NULL_TYPE_TAG = ATypeTag.NULL.serialize();
+ private ICopyEvaluatorFactory inputEvalFactory;
+ private ICopyEvaluatorFactory levelEvalFactory;
+ private IAType outputType;
+ private IAType inputArgType;
+
+ public AdmToBytesFactory(ICopyEvaluatorFactory inputEvalFactory,
ICopyEvaluatorFactory levelEvalFactory,
+ IAType inputArgType, IAType outputType) {
+ this.levelEvalFactory = levelEvalFactory;
+ this.inputEvalFactory = inputEvalFactory;
+ this.outputType = outputType;
+ this.inputArgType = inputArgType;
+ }
+
+ @Override
+ public ICopyEvaluator createEvaluator(final IDataOutputProvider output)
throws AlgebricksException {
+ @SuppressWarnings("unchecked")
+ final ISerializerDeserializer<ANull> nullSerde =
AqlSerializerDeserializerProvider.INSTANCE
+ .getSerializerDeserializer(BuiltinType.ANULL);
+
+ return new ICopyEvaluator() {
+ private final AString RAW_BYTES_FIELD_NAME = new
AString("RawBytes");
+ private final AString INFINITY_STR = new AString("INF");
+ private final RuntimeRecordTypeInfo runtimeRecordTypeInfo = new
RuntimeRecordTypeInfo();
+ private final AStringSerializerDeserializer aStringSerDer = new
AStringSerializerDeserializer();
+ private final ArrayBackedValueStorage outInput0 = new
ArrayBackedValueStorage();
+ private final ICopyEvaluator eval0 =
inputEvalFactory.createEvaluator(outInput0);
+ private final ArrayBackedValueStorage outInput1 = new
ArrayBackedValueStorage();
+ private final ICopyEvaluator eval1 =
levelEvalFactory.createEvaluator(outInput1);
+ private final ResettableByteArrayOutputStream bos = new
ResettableByteArrayOutputStream();
+ private final DataOutputStream dos = new DataOutputStream(bos);
+ private final IARecordBuilder recordBuilder = new RecordBuilder();
+ private final AdmToBytesHelper admToBytesHelper = new
AdmToBytesHelper(new PointableUtils());
+ private final PointableAllocator allocator = new
PointableAllocator();
+ private final IVisitablePointable levelPointable =
allocator.allocateEmpty();
+ private final IVisitablePointable inputPointable = PointableUtils
+ .allocatePointable(allocator, inputArgType);
+ private final IVisitablePointable tempReference =
allocator.allocateEmpty();
+ private AdmToBytesVisitor visitor;
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple) throws
AlgebricksException {
+ outInput0.reset();
+ eval0.evaluate(tuple);
+ outInput1.reset();
+ eval1.evaluate(tuple);
+ bos.reset();
+
+ if (outInput0.getByteArray()[0] == SER_NULL_TYPE_TAG) {
+ try {
+ nullSerde.serialize(ANull.NULL,
output.getDataOutput());
+ } catch (HyracksDataException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+ // Get the serialized values from an input
+ inputPointable.set(outInput0);
+ if (outputType.getTypeTag() == ATypeTag.RECORD) {
+ runtimeRecordTypeInfo.reset((ARecordType) outputType);
+ }
+ try {
+ // Get the level
+ long outputLevel = getOutputLevel();
+ visitor = new AdmToBytesVisitor(admToBytesHelper,
outputLevel);
+ // Return only an array of raw byte
+ if (outputLevel < 1) {
+ printRawBytes(inputPointable);
+ } else {
+ Triple<IAType, RuntimeRecordTypeInfo, Long> arg = new
Triple<>(outputType,
+ runtimeRecordTypeInfo, 1L);
+ IVisitablePointable resultPointable =
inputPointable.accept(visitor, arg);
+
output.getDataOutput().write(resultPointable.getByteArray(),
resultPointable.getStartOffset(),
+ resultPointable.getLength());
+ }
+ } catch (AsterixException | HyracksDataException e) {
+ throw new AlgebricksException("Unable to display
serialized value of the input " + inputArgType);
+ } catch (IOException e) {
+ new AlgebricksException("Error parsing input argument
#2.");
+ }
+
+ }
+
+ private long getOutputLevel() throws AlgebricksException {
+ if (outInput1.getByteArray()[0] == SER_NULL_TYPE_TAG) {
+ return 0;
+ }
+ // Level input arg type check
+ levelPointable.set(outInput1);
+ ATypeTag tag = PointableUtils.getTypeTag(levelPointable);
+ if (tag != ATypeTag.INT64 && tag != ATypeTag.STRING) {
+ throw new
AlgebricksException(AsterixBuiltinFunctions.ADM_TO_BYTES.getName()
+ + ": expects input type (ADM object, INT32|STRING)
but got ("
+ +
EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(outInput0.getByteArray()[0])
+ ", "
+ + tag + ")");
+ }
+ try {
+ if (tag == ATypeTag.STRING) {
+ // Serialize "INF"
+ int start = bos.size();
+ dos.writeByte(ATypeTag.STRING.serialize());
+ aStringSerDer.serialize(INFINITY_STR, dos);
+ int end = bos.size();
+ tempReference.set(bos.getByteArray(), start, end -
start);
+ if (PointableUtils.isEqual(levelPointable,
tempReference)) {
+ return Long.MAX_VALUE;
+ } else {
+ StringBuilder sb = new StringBuilder();
+ sb = UTF8StringUtil.toString(sb,
levelPointable.getByteArray(),
+ levelPointable.getStartOffset() + 1);
+ // Will throw an exception if not a number
+ return Long.parseLong(sb.toString());
+ }
+ } else {
+ return PointableUtils.getLongValue(levelPointable,
true);
+ }
+ } catch (IOException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+
+ // Print a raw byte array
+ private void printRawBytes(IValueReference vr) throws
AlgebricksException {
+ try {
+ byte[] b = vr.getByteArray();
+ int offset = vr.getStartOffset();
+ int length = vr.getLength();
+
+ ARecordType reqRecType;
+ if (outputType != null && outputType.getTypeTag() ==
ATypeTag.RECORD) {
+ reqRecType = (ARecordType) outputType;
+ } else {
+ reqRecType =
DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE;
+ }
+ recordBuilder.reset(reqRecType);
+ recordBuilder.init();
+
+ int start = bos.size();
+ dos.writeByte(ATypeTag.STRING.serialize());
+ aStringSerDer.serialize(RAW_BYTES_FIELD_NAME, dos);
+ int end = bos.size();
+ tempReference.set(bos.getByteArray(), start, end - start);
+
+ int pos =
runtimeRecordTypeInfo.getFieldIndex(tempReference.getByteArray(),
+ tempReference.getStartOffset() + 1,
tempReference.getLength() - 1);
+ IVisitablePointable byteArrayBuffer =
allocator.allocateFieldValue(BuiltinType.ASTRING);
+ if (pos >= 0) {
+ admToBytesHelper.setByteArrayStringPointableValue(b,
offset, length, byteArrayBuffer, true);
+ recordBuilder.addField(pos, byteArrayBuffer);
+ } else {
+ admToBytesHelper.setByteArrayStringPointableValue(b,
offset, length, byteArrayBuffer, true);
+ recordBuilder.addField(tempReference, byteArrayBuffer);
+ }
+ recordBuilder.write(output.getDataOutput(), true);
+ } catch (AsterixException | IOException e) {
+ throw new AlgebricksException("Error parsing input
argument #2. " + e);
+ }
+ }
+ };
+ }
+}
diff --git
a/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AdmToBytesHelper.java
b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AdmToBytesHelper.java
new file mode 100644
index 0000000..a2164c8
--- /dev/null
+++
b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AdmToBytesHelper.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.runtime.evaluators.functions;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import org.apache.asterix.builders.IARecordBuilder;
+import org.apache.asterix.builders.RecordBuilder;
+import org.apache.asterix.common.exceptions.AsterixException;
+import
org.apache.asterix.dataflow.data.nontagged.serde.AInt32SerializerDeserializer;
+import org.apache.asterix.om.pointables.ARecordVisitablePointable;
+import org.apache.asterix.om.pointables.PointableAllocator;
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.EnumDeserializer;
+import org.apache.asterix.om.util.NonTaggedFormatUtil;
+import
org.apache.asterix.om.util.admdebugger.AnnotatedFieldNameComputerUtil.ByteAnnotationField;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.data.std.util.ByteArrayAccessibleOutputStream;
+import org.apache.hyracks.util.string.UTF8StringUtil;
+
+public class AdmToBytesHelper {
+ private final int MAX_NUMBER_OF_MARKS = 12; // Max number of marks in a
record + 1 for the end mark
+ private final int[] marks = new int[MAX_NUMBER_OF_MARKS];
+ private final String[] markNames = new String[MAX_NUMBER_OF_MARKS];
+ private final StringBuilder valueHolder = new StringBuilder();
+ private final PointableAllocator allocator = new PointableAllocator();
+ private final IARecordBuilder recordBuilder = new RecordBuilder();
+ private final ArrayBackedValueStorage nameBuffer = new
ArrayBackedValueStorage();
+ private final ArrayBackedValueStorage valueBuffer = new
ArrayBackedValueStorage();
+ private final IVisitablePointable byteArrayStringPointable =
allocator.allocateFieldValue(BuiltinType.ASTRING);
+ private final ByteArrayAccessibleOutputStream outputBos = new
ByteArrayAccessibleOutputStream();
+ private final DataOutputStream outputDos = new DataOutputStream(outputBos);
+ private PointableUtils pointableUtils;
+
+ public AdmToBytesHelper(PointableUtils pointableUtils) {
+ this.pointableUtils = pointableUtils;
+ }
+
+ public ARecordVisitablePointable getAnnotatedByteArray(IVisitablePointable
valuePointable) throws AsterixException {
+ return getAnnotatedByteArray(valuePointable,
DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE);
+ }
+
+ public ARecordVisitablePointable getAnnotatedByteArray(IVisitablePointable
valuePointable,
+ ARecordType requiredRecType) throws AsterixException {
+ allocator.reset();
+
+ IVisitablePointable annotationRecordPointable =
allocator.allocateRecordValue(requiredRecType);
+ getAnnotatedByteArray(valuePointable, requiredRecType,
annotationRecordPointable);
+ return (ARecordVisitablePointable) annotationRecordPointable;
+ }
+
+ public void getAnnotatedByteArray(IVisitablePointable valuePointable,
ARecordType requiredRecType,
+ IVisitablePointable annotatedByteArrayPointable) throws
AsterixException {
+ int markCount = getByteArrayMarkers(valuePointable);
+
+ recordBuilder.reset(requiredRecType);
+ recordBuilder.init();
+ try {
+ for (int i = 0; i < markCount; i++) {
+ int id = recordBuilder.getFieldId(markNames[i]);
+
setByteArrayStringPointableValue(valuePointable.getByteArray(), marks[i],
(marks[i + 1] - marks[i]),
+ byteArrayStringPointable, true);
+ if (id >= 0) {
+ recordBuilder.addField(id, byteArrayStringPointable);
+ } else {
+ nameBuffer.reset();
+ pointableUtils.serializeString(markNames[i], nameBuffer,
true);
+ recordBuilder.addField(nameBuffer,
byteArrayStringPointable);
+ }
+ }
+ outputBos.reset();
+ recordBuilder.write(outputDos, true);
+ annotatedByteArrayPointable.set(outputBos.getByteArray(), 0,
outputBos.size());
+ } catch (IOException e) {
+ throw new AsterixException(e);
+ }
+ }
+
+ /**
+ * Print a byte array as a string
+ *
+ * @param bytes
+ * The array of bytes
+ * @param offset
+ * The start offset for to read from
+ * @param length
+ * The number of the bytes to read
+ * @param resultPointable
+ * Pointer to where the string is to be stored after
serialization
+ * @param writeTag
+ * A flag specifying whether we should add a tag byte at the
start of the string pointable
+ */
+ public void setByteArrayStringPointableValue(byte bytes[], int offset, int
length,
+ IVisitablePointable resultPointable, boolean writeTag) throws
AsterixException {
+ int end = offset + length;
+
+ valueHolder.setLength(0);
+ valueHolder.append("[" + bytes[offset]);
+ for (int i = offset + 1; i < end; i++) {
+ valueHolder.append(", " + bytes[i]);
+ }
+ valueHolder.append("]");
+ valueBuffer.reset();
+ pointableUtils.serializeString(valueHolder.toString(), valueBuffer,
writeTag);
+ resultPointable.set(valueBuffer.getByteArray(),
valueBuffer.getStartOffset(), valueBuffer.getLength());
+ }
+
+ private int getByteArrayMarkers(IVisitablePointable valuePointable) throws
AsterixException {
+ byte[] bytes = valuePointable.getByteArray();
+ int offset = valuePointable.getStartOffset();
+ int length = valuePointable.getLength();
+ ATypeTag typeTag =
EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(bytes[offset]);
+
+ int markCount = 0;
+ switch (typeTag) {
+ case RECORD:
+ markCount = getRecordByteArrayMarkers(bytes, offset,
+ ((ARecordVisitablePointable)
valuePointable).getInputRecordType());
+ break;
+ case UNORDEREDLIST:
+ case ORDEREDLIST:
+ markCount = getListByteArrayMarkers(bytes, offset);
+ break;
+ case STRING:
+ markCount = getStringByteArrayMarkes(bytes, offset, length);
+ break;
+ case NULL:
+ markCount = setMark(0, ByteAnnotationField.TAG, offset);
+ break;
+ default:
+ markCount = setMark(0, ByteAnnotationField.TAG, offset);
+ markCount = setMark(markCount, ByteAnnotationField.VALUE,
offset + 1);
+
+ }
+ // Set an end mark
+ marks[markCount] = offset + length;
+ return markCount;
+ }
+
+ /**
+ * 1 byte - tag, 13
+ * x bytes - variable number length bytes
+ * y bytes - value of x-length byte array of the value of the string
+ *
+ */
+ private int getStringByteArrayMarkes(byte bytes[], int offset, int length)
{
+ int s = offset;
+ int i = setMark(0, ByteAnnotationField.TAG, s);
+ s++;
+ i = setMark(i, ByteAnnotationField.LENGTH, s);
+
+ // Get the string length
+ int utflen = UTF8StringUtil.getUTFLength(bytes, offset + 1);
+ if (utflen > 0) { // If not an empty string
+ s = offset + (length - utflen);
+ i = setMark(i, ByteAnnotationField.VALUE, s);
+ }
+ return i;
+ }
+
+ /*
+ *
+ * 1 byte - Orderedlist (or Unorderedlist) tag, 22 (excluded when type is
known)
+ * 1 byte - Type of items on the list
+ * 4 bytes - Total number of bytes
+ * 4 bytes - Number of items
+ * if (type of list items is string, record, or list (length is not
constant))
+ * 4 bytes per item (offsets)
+ * for each item
+ * Bytes of the item (In the case of a list of ANY, the items will
include type tags)
+ */
+ private int getListByteArrayMarkers(byte bytes[], int offset) throws
AsterixException {
+ int s = offset;
+ int i = setMark(0, ByteAnnotationField.TAG, s);
+ s++;
+ // 1 byte - Type of items on the list
+ i = setMark(i, ByteAnnotationField.ITEM_TYPE, s);
+ s++;
+ //4 bytes - Total number of bytes
+ i = setMark(i, ByteAnnotationField.LENGTH, s);
+ s += 4;
+ //4 bytes - Number of items
+ i = setMark(i, ByteAnnotationField.NUMBER_OF_ITEMS, s);
+ int numberOfItems = AInt32SerializerDeserializer.getInt(bytes, s);
+ s += 4;
+ // 4 bytes for each item
+ i = setMark(i, ByteAnnotationField.ITEM_OFFSETS, s);
+ s += 4 * numberOfItems;
+ // The rest is the values of the items
+ return setMark(i, ByteAnnotationField.VALUE, s);
+ }
+
+ /*
+ *
+ * 1 byte - Record tag, 24 (excluded when type is known)
+ * 4 bytes - Total number of bytes
+ * if (recordType is not closed)
+ * 1 byte - Boolean isExpanded
+ * if (isExpanded)
+ * 4 bytes - Offset to open part
+ * 4 bytes - Number of closed fields
+ * if (recordType hasNullableFields)
+ * ceil (numberOfFields / 8) bytes - Nullbitmap (1 bit per field, "1"
means field is Null for this record)
+ * for each closed field that is not Null for this record
+ * 4 bytes - Closed field offset
+ * for each closed field that is not Null for this record
+ * Bytes of the field (type is known from recordtype, so the bytes will
not have a type tag)
+ * if (isExpanded)
+ * 4 bytes - Number of open fields
+ * for each open field
+ * 4 bytes - Hash code
+ * 4 bytes - Offset
+ * for each open field
+ * Bytes of the field name (String, no type tag)
+ * Bytes of the field (with type tag)
+ *
+ */
+ private int getRecordByteArrayMarkers(byte bytes[], int offset,
ARecordType inputRecType) throws AsterixException {
+ int s = offset;
+ int openPartOffset = 0;
+ boolean isExpanded = false;
+
+ int numberOfSchemaFields = 0;
+
+ // Tag mark
+ // 1 byte - Record tag, 24 (excluded when type is known)
+ int i = setMark(0, ByteAnnotationField.TAG, s);
+
+ // Length
+ // 4 bytes - Total number of bytes
+ i = setMark(i, ByteAnnotationField.LENGTH, s + 1);
+
+ if (inputRecType == null) {
+ openPartOffset = s + AInt32SerializerDeserializer.getInt(bytes, s
+ 6);
+ i = setMark(i, ByteAnnotationField.OPENPART_OFFSET,
openPartOffset);
+ s += 8;
+ isExpanded = true;
+ } else {
+ numberOfSchemaFields = inputRecType.getFieldNames().length;
+ if (inputRecType.isOpen()) {
+ i = setMark(i, ByteAnnotationField.IS_EXPANDED, s + 5);
+ isExpanded = bytes[s + 5] == 1 ? true : false;
+ if (isExpanded) {
+ openPartOffset = s +
AInt32SerializerDeserializer.getInt(bytes, s + 6);
+ i = setMark(i, ByteAnnotationField.OPENPART_OFFSET, s + 6);
+ s += 10;
+ } else {
+ s += 6;
+ }
+ } else {
+ s += 5;
+ }
+ }
+
+ //4 bytes - Number of closed fields
+ i = setMark(i, ByteAnnotationField.NUMBER_CLOSED_FIELDS, s);
+ s += 4;
+
+ if (numberOfSchemaFields > 0) {
+ boolean hasNullableFields =
NonTaggedFormatUtil.hasNullableField(inputRecType);
+ int nullBitMapLength = 0;
+ if (hasNullableFields) {
+ // Nullbitmap bytes
+ i = setMark(i, ByteAnnotationField.NULLBITMAP, s);
+ nullBitMapLength = (numberOfSchemaFields % 8 == 0 ?
numberOfSchemaFields / 8
+ : numberOfSchemaFields / 8 + 1);
+ }
+
+ // Offset of the closed field values
+ s += nullBitMapLength;
+ i = setMark(i, ByteAnnotationField.CLOSED_FIELD_OFFSETS, s);
+
+ // Offset of the closed field values
+ s += 4 * numberOfSchemaFields;
+ i = setMark(i, ByteAnnotationField.CLOSED_FIELDS, s);
+ }
+
+ if (isExpanded) {
+ int numberOfOpenFields =
AInt32SerializerDeserializer.getInt(bytes, openPartOffset);
+ // 4 bytes - Number of open fields
+ s = openPartOffset;
+ i = setMark(i, ByteAnnotationField.NUMBER_OF_OPEN_FIELDS, s);
+ s += 4;
+
+ // 4 bytes openfield offsets and 4 Hash code for each field
+ i = setMark(i, ByteAnnotationField.OPEN_FIELDS_HASH_OFFSET, s);
+ s += 8 * numberOfOpenFields;
+
+ // Open fields
+ i = setMark(i, ByteAnnotationField.OPEN_FIELDS, s);
+ }
+
+ return i;
+ }
+
+ private int setMark(int index, ByteAnnotationField field, int offset) {
+ markNames[index] = field.fieldName();
+ marks[index] = offset;
+ return index + 1;
+ }
+}
diff --git
a/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PointableUtils.java
b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PointableUtils.java
new file mode 100644
index 0000000..e12573f
--- /dev/null
+++
b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PointableUtils.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import
org.apache.asterix.dataflow.data.nontagged.serde.AStringSerializerDeserializer;
+import org.apache.asterix.om.base.AMutableString;
+import org.apache.asterix.om.pointables.PointableAllocator;
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.EnumDeserializer;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import org.apache.hyracks.data.std.api.IMutableValueStorage;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.data.std.primitive.LongPointable;
+import org.apache.hyracks.data.std.primitive.ShortPointable;
+import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
+
+/**
+ * An utility class for some frequently used methods like checking the
equality between two pointables (binary values)
+ * (e.g., field names), string value of a fieldname pointable, getting the
typetag of a pointable, etc.
+ * Note: To get the typetag of a fieldvalue (i) in a record, it is recommended
to use the getFieldTypeTags().get(i)
+ * method rather than getting it from fhe field value itself.
+ */
+
+public class PointableUtils {
+ private static final IBinaryComparator STRING_BINARY_COMPARATOR =
PointableBinaryComparatorFactory.of(
+ UTF8StringPointable.FACTORY).createBinaryComparator();
+ private final ISerializerDeserializer strSerde = new
AStringSerializerDeserializer();
+ private final AMutableString aString = new AMutableString("");
+
+ public PointableUtils() {
+ }
+
+ public static int compareStringBinValues(IValueReference a,
IValueReference b) throws HyracksDataException {
+ // start+1 and len-1 due to type tag ignore (only interested in String
value)
+ return STRING_BINARY_COMPARATOR.compare(a.getByteArray(),
a.getStartOffset() + 1, a.getLength() - 1,
+ b.getByteArray(), b.getStartOffset() + 1, b.getLength() - 1);
+ }
+
+ public static boolean isEqual(IValueReference a, IValueReference b) throws
HyracksDataException {
+ return (compareStringBinValues(a, b) == 0);
+ }
+
+ public static boolean byteArrayEqual(IValueReference valueRef1,
IValueReference valueRef2)
+ throws HyracksDataException {
+ return byteArrayEqual(valueRef1, valueRef2, 3);
+ }
+
+ public static boolean byteArrayEqual(IValueReference valueRef1,
IValueReference valueRef2, int dataOffset)
+ throws HyracksDataException {
+ if (valueRef1 == null || valueRef2 == null) {
+ return false;
+ }
+ if (valueRef1 == valueRef2) {
+ return true;
+ }
+
+ int length1 = valueRef1.getLength();
+ int length2 = valueRef2.getLength();
+
+ if (length1 != length2) {
+ return false;
+ }
+
+ byte[] bytes1 = valueRef1.getByteArray();
+ byte[] bytes2 = valueRef2.getByteArray();
+ int start1 = valueRef1.getStartOffset() + dataOffset;
+ int start2 = valueRef2.getStartOffset() + dataOffset;
+
+ int end = start1 + length1 - dataOffset;
+
+ for (int i = start1, j = start2; i < end; i++, j++) {
+ if (bytes1[i] != bytes2[j]) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ public static IVisitablePointable allocatePointable(PointableAllocator pa,
IAType type) {
+ if (type == null) {
+ return pa.allocateEmpty();
+ }
+
+ switch (type.getTypeTag()) {
+ case RECORD:
+ return pa.allocateRecordValue(type);
+ case ORDEREDLIST:
+ case UNORDEREDLIST:
+ return pa.allocateListValue(type);
+ default:
+ return pa.allocateFieldValue(type);
+ }
+
+ }
+
+ public static long getLongValue(IValueReference valuePointable, boolean
isTagged) throws AlgebricksException {
+ ATypeTag tag = getTypeTag(valuePointable);
+
+ int offset = isTagged ? valuePointable.getStartOffset() + 1 :
valuePointable.getStartOffset();
+
+ switch (tag) {
+ case INT64:
+ return LongPointable.getLong(valuePointable.getByteArray(),
offset);
+ case INT32:
+ return
IntegerPointable.getInteger(valuePointable.getByteArray(), offset);
+ case INT16:
+ return ShortPointable.getShort(valuePointable.getByteArray(),
offset);
+ default:
+ throw new AlgebricksException("Unsupported type " + tag);
+ }
+ }
+
+ public static boolean sameType(ATypeTag typeTag, IVisitablePointable
visitablePointable) {
+ return (getTypeTag(visitablePointable) == typeTag);
+ }
+
+ public static ATypeTag getTypeTag(IValueReference visitablePointable) {
+ byte[] bytes = visitablePointable.getByteArray();
+ int s = visitablePointable.getStartOffset();
+ return EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(bytes[s]);
+ }
+
+ /**
+ * @param str
+ * The input string
+ * @param vs
+ * The storage buffer
+ * @param writeTag
+ * Specifying whether a tag for the string should also be
written
+ * @throws AlgebricksException
+ */
+ public void serializeString(String str, IMutableValueStorage vs, boolean
writeTag) throws AsterixException {
+ vs.reset();
+ try {
+ DataOutput output = vs.getDataOutput();
+ if (writeTag) {
+ output.write(ATypeTag.STRING.serialize());
+ }
+ aString.setValue(str);
+ strSerde.serialize(aString, output);
+ } catch (IOException e) {
+ throw new AsterixException("Could not serialize " + str);
+ }
+ }
+
+ public void serializeString(String str, IMutableValueStorage vs) throws
AsterixException {
+ serializeString(str, vs, false);
+ }
+}
diff --git
a/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/admdebugging/AdmToBytesVisitor.java
b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/admdebugging/AdmToBytesVisitor.java
new file mode 100644
index 0000000..65d5f85
--- /dev/null
+++
b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/admdebugging/AdmToBytesVisitor.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.visitors.admdebugging;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.om.pointables.AFlatValuePointable;
+import org.apache.asterix.om.pointables.AListVisitablePointable;
+import org.apache.asterix.om.pointables.ARecordVisitablePointable;
+import org.apache.asterix.om.pointables.PointableAllocator;
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.pointables.visitor.IVisitablePointableVisitor;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.runtime.RuntimeRecordTypeInfo;
+import org.apache.asterix.runtime.evaluators.functions.AdmToBytesHelper;
+import org.apache.hyracks.algebricks.common.utils.Triple;
+
+public class AdmToBytesVisitor implements
+ IVisitablePointableVisitor<IVisitablePointable, Triple<IAType,
RuntimeRecordTypeInfo, Long>> {
+
+ private final Map<IVisitablePointable, RecordBytesProcessor>
rProcessorToAnnotatedBytes = new HashMap<>();
+ private final Map<IVisitablePointable, ListBytesProcessor>
lProcessorToAnnotatedBytes = new HashMap<>();
+ private final PointableAllocator allocator = new PointableAllocator();
+ private final IVisitablePointable resultPointable =
allocator.allocateEmpty();
+ private RecordBytesProcessor recordBytesProcessor;
+ private ListBytesProcessor listBytesProcessor;
+ private AdmToBytesHelper admToBytesHelper;
+
+ private long outputLevel = 0;
+
+ public AdmToBytesVisitor(AdmToBytesHelper admToBytesHelper, long
outputLevel) {
+ this.admToBytesHelper = admToBytesHelper;
+ this.outputLevel = outputLevel;
+ }
+
+ @Override
+ public IVisitablePointable visit(AListVisitablePointable pointable,
Triple<IAType, RuntimeRecordTypeInfo, Long> arg)
+ throws AsterixException {
+ if (outputLevel == 1
+ || (arg.third == outputLevel && arg.first.getTypeTag() !=
ATypeTag.UNORDEREDLIST && arg.first
+ .getTypeTag() != ATypeTag.ORDEREDLIST)) {
+ computeResultPointable(pointable, arg.first, resultPointable);
+ return resultPointable;
+ }
+
+ listBytesProcessor = lProcessorToAnnotatedBytes.get(pointable);
+ if (listBytesProcessor == null) {
+ listBytesProcessor = new ListBytesProcessor();
+ lProcessorToAnnotatedBytes.put(pointable, listBytesProcessor);
+ }
+ if (arg.first.getTypeTag() == ATypeTag.ANY) {
+ arg.first = DefaultOpenFieldType.NESTED_OPEN_AUNORDERED_LIST_TYPE;
+ if (pointable.ordered()) {
+ arg.first =
DefaultOpenFieldType.NESTED_OPEN_AORDERED_LIST_TYPE;
+ }
+ }
+ listBytesProcessor.accessList(pointable, outputLevel, this, arg.first,
arg.third, resultPointable);
+ return resultPointable;
+ }
+
+ @Override
+ public IVisitablePointable visit(ARecordVisitablePointable pointable,
+ Triple<IAType, RuntimeRecordTypeInfo, Long> arg) throws
AsterixException {
+
+ if (outputLevel == 1 || arg.third == outputLevel) {
+ computeResultPointable(pointable, arg.first, resultPointable);
+ return resultPointable;
+ }
+
+ RuntimeRecordTypeInfo runtimeRecordTypeInfo = arg.second;
+ if (arg.first.getTypeTag() == ATypeTag.ANY) {
+ arg.first = DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE;
+ }
+ ARecordType requiredType = (ARecordType) arg.first;
+
+ recordBytesProcessor = rProcessorToAnnotatedBytes.get(pointable);
+ if (recordBytesProcessor == null) {
+ recordBytesProcessor = new RecordBytesProcessor();
+ rProcessorToAnnotatedBytes.put(pointable, recordBytesProcessor);
+ }
+ recordBytesProcessor.accessRecord(pointable, this, requiredType,
runtimeRecordTypeInfo, arg.third,
+ resultPointable);
+ return resultPointable;
+ }
+
+ @Override
+ public IVisitablePointable visit(AFlatValuePointable pointable,
Triple<IAType, RuntimeRecordTypeInfo, Long> arg)
+ throws AsterixException {
+ computeResultPointable(pointable, arg.first, resultPointable);
+ return resultPointable;
+ }
+
+ public void computeResultPointable(IVisitablePointable inputPointable,
IAType requiredType,
+ IVisitablePointable outputPointable) throws AsterixException {
+ if (requiredType != null && requiredType.getTypeTag() ==
ATypeTag.RECORD) {
+ admToBytesHelper.getAnnotatedByteArray(inputPointable,
(ARecordType) requiredType, outputPointable);
+ } else {
+ admToBytesHelper.getAnnotatedByteArray(inputPointable,
DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE,
+ outputPointable);
+ }
+ }
+}
diff --git
a/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/admdebugging/ListBytesProcessor.java
b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/admdebugging/ListBytesProcessor.java
new file mode 100644
index 0000000..0f5acd1
--- /dev/null
+++
b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/admdebugging/ListBytesProcessor.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.visitors.admdebugging;
+
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.util.List;
+
+import org.apache.asterix.builders.OrderedListBuilder;
+import org.apache.asterix.builders.UnorderedListBuilder;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.om.pointables.AListVisitablePointable;
+import org.apache.asterix.om.pointables.PointableAllocator;
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnorderedListType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.runtime.RuntimeRecordTypeInfo;
+import org.apache.asterix.runtime.evaluators.functions.PointableUtils;
+import org.apache.hyracks.algebricks.common.utils.Triple;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.util.ByteArrayAccessibleOutputStream;
+
+public class ListBytesProcessor {
+ // pointable allocator
+ private final PointableAllocator allocator = new PointableAllocator();
+ private final IVisitablePointable itemTempReference =
allocator.allocateEmpty();
+ private final Triple<IAType, RuntimeRecordTypeInfo, Long> itemVisitorArg =
new Triple<>(null,
+ new RuntimeRecordTypeInfo(), 1L);
+ private final UnorderedListBuilder unOrderedListBuilder = new
UnorderedListBuilder();
+ private final OrderedListBuilder orderedListBuilder = new
OrderedListBuilder();
+ private final ByteArrayAccessibleOutputStream dataBos = new
ByteArrayAccessibleOutputStream();
+ private final DataOutput dataDos = new DataOutputStream(dataBos);
+ private IAType reqItemType;
+
+ public void accessList(AListVisitablePointable pointable, long
outputLevel, AdmToBytesVisitor visitor,
+ IAType requiredType, long nestedLevel, IVisitablePointable
resultPointable) throws AsterixException {
+ // Printing just the highest level of the annotated bytes
+ ATypeTag reqTypeTag = requiredType.getTypeTag();
+
+ if (requiredType != null && reqTypeTag != ATypeTag.ANY) {
+ if (reqTypeTag == ATypeTag.UNORDEREDLIST) {
+ unOrderedListBuilder.reset((AUnorderedListType) requiredType);
+ reqItemType = ((AUnorderedListType)
requiredType).getItemType();
+ }
+ if (reqTypeTag == ATypeTag.ORDEREDLIST) {
+ orderedListBuilder.reset((AOrderedListType) requiredType);
+ reqItemType = ((AOrderedListType) requiredType).getItemType();
+ }
+ } else {
+ if (pointable.ordered()) {
+
orderedListBuilder.reset(DefaultOpenFieldType.NESTED_OPEN_AORDERED_LIST_TYPE);
+ } else {
+
orderedListBuilder.reset(DefaultOpenFieldType.NESTED_OPEN_AUNORDERED_LIST_TYPE);
+ }
+ reqItemType = BuiltinType.ANY;
+ }
+
+ List<IVisitablePointable> items = pointable.getItems();
+ List<IVisitablePointable> itemTypeTags = pointable.getItemTags();
+
+ try {
+ for (int i = 0; i < items.size(); i++) {
+ IVisitablePointable item = items.get(i);
+ IVisitablePointable itemTypeTag = itemTypeTags.get(i);
+ ATypeTag typeTag = PointableUtils.getTypeTag(itemTypeTag);
+
+ // If reached the max nesting level just "print" the annotated
byte record for this item
+ if (nestedLevel == outputLevel) {
+ visitor.computeResultPointable(item, reqItemType,
itemTempReference);
+ addListItem(reqTypeTag, itemTempReference);
+ } else {
+ if (reqItemType == null ||
reqItemType.getTypeTag().equals(ATypeTag.ANY)) {
+ itemVisitorArg.first =
DefaultOpenFieldType.getDefaultOpenFieldType(typeTag);
+ } else {
+ itemVisitorArg.first = reqItemType;
+ if (reqItemType.getTypeTag().equals(ATypeTag.RECORD)) {
+ itemVisitorArg.second.reset((ARecordType)
reqItemType);
+ }
+ }
+ itemVisitorArg.third = nestedLevel + 1;
+ addListItem(reqTypeTag, (item.accept(visitor,
itemVisitorArg)));
+ }
+ }
+ dataBos.reset();
+ if (reqTypeTag == ATypeTag.ORDEREDLIST) {
+ orderedListBuilder.write(dataDos, true);
+ }
+ if (reqTypeTag == ATypeTag.UNORDEREDLIST) {
+ unOrderedListBuilder.write(dataDos, true);
+ }
+ resultPointable.set(dataBos.getByteArray(), 0, dataBos.size());
+ } catch (HyracksDataException e) {
+ throw new AsterixException(e);
+ }
+ }
+
+ private void addListItem(ATypeTag listTypeTag, IValueReference
itemReference) throws HyracksDataException {
+ if (listTypeTag == ATypeTag.ORDEREDLIST) {
+ orderedListBuilder.addItem(itemReference);
+ }
+ if (listTypeTag == ATypeTag.UNORDEREDLIST) {
+ unOrderedListBuilder.addItem(itemReference);
+ }
+ }
+}
diff --git
a/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/admdebugging/RecordBytesProcessor.java
b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/admdebugging/RecordBytesProcessor.java
new file mode 100644
index 0000000..4867071
--- /dev/null
+++
b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/visitors/admdebugging/RecordBytesProcessor.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.runtime.evaluators.visitors.admdebugging;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+import org.apache.asterix.builders.RecordBuilder;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.om.pointables.ARecordVisitablePointable;
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.runtime.RuntimeRecordTypeInfo;
+import org.apache.asterix.om.util.NonTaggedFormatUtil;
+import org.apache.asterix.runtime.evaluators.functions.PointableUtils;
+import org.apache.hyracks.algebricks.common.utils.Triple;
+import org.apache.hyracks.data.std.util.ByteArrayAccessibleOutputStream;
+
+class RecordBytesProcessor {
+ private final ByteArrayAccessibleOutputStream outputBos = new
ByteArrayAccessibleOutputStream();
+ private final DataOutputStream outputDos = new DataOutputStream(outputBos);
+
+ private final Triple<IAType, RuntimeRecordTypeInfo, Long> nestedVisitorArg
= new Triple<>(null,
+ new RuntimeRecordTypeInfo(), 1L);
+
+ private final RecordBuilder recordBuilder = new RecordBuilder();
+
+ public void accessRecord(ARecordVisitablePointable pointable,
AdmToBytesVisitor visitor, ARecordType requiredType,
+ RuntimeRecordTypeInfo runtimeRecordTypeInfo, long nestedLevel,
IVisitablePointable resultPointable)
+ throws AsterixException {
+ try {
+ List<IVisitablePointable> fieldNames = pointable.getFieldNames();
+ List<IVisitablePointable> fieldValues = pointable.getFieldValues();
+ List<IVisitablePointable> fieldTypeTags =
pointable.getFieldTypeTags();
+ outputBos.reset();
+
+ IAType reqFieldTypes[] = requiredType.getFieldTypes();
+ recordBuilder.reset(requiredType);
+ recordBuilder.init();
+
+ for (int i = 0; i < fieldNames.size(); i++) {
+ IVisitablePointable fieldValue = fieldValues.get(i);
+ IVisitablePointable fieldName = fieldNames.get(i);
+ ATypeTag fieldTypeTag =
PointableUtils.getTypeTag(fieldTypeTags.get(i));
+
+ int pos =
runtimeRecordTypeInfo.getFieldIndex(fieldName.getByteArray(),
fieldName.getStartOffset() + 1,
+ fieldName.getLength() - 1);
+
+ if (pos >= 0 && fieldTypeTag != null) {
+ IAType reqfieldType = reqFieldTypes[pos];
+ if (NonTaggedFormatUtil.isOptional(reqFieldTypes[pos])) {
+ if (fieldTypeTags.get(pos) == null
+ || PointableUtils.sameType(ATypeTag.NULL,
fieldTypeTags.get(pos))) {
+ reqfieldType = ((AUnionType)
reqFieldTypes[pos]).getUnionList().get(0);
+ } else {
+ reqfieldType = ((AUnionType)
reqFieldTypes[pos]).getNullableType();
+ }
+ }
+ nestedVisitorArg.first = reqfieldType;
+ if (reqfieldType.getTypeTag() == ATypeTag.RECORD) {
+ nestedVisitorArg.second.reset((ARecordType)
reqfieldType);
+ }
+ } else {
+ nestedVisitorArg.first =
DefaultOpenFieldType.getDefaultOpenFieldType(fieldTypeTag);
+ }
+ nestedVisitorArg.third = nestedLevel + 1;
+ IVisitablePointable tempFieldReference =
fieldValue.accept(visitor, nestedVisitorArg);
+
+ if (pos >= 0) {
+ recordBuilder.addField(pos, tempFieldReference);
+ } else {
+ recordBuilder.addField(fieldNames.get(i),
tempFieldReference);
+ }
+ }
+ recordBuilder.write(outputDos, true);
+ resultPointable.set(outputBos.getByteArray(), 0, outputBos.size());
+ } catch (IOException e) {
+ throw new AsterixException(e);
+ }
+ }
+}
diff --git
a/asterix-runtime/src/main/java/org/apache/asterix/runtime/formats/NonTaggedDataFormat.java
b/asterix-runtime/src/main/java/org/apache/asterix/runtime/formats/NonTaggedDataFormat.java
index ef7a6c8..e3ae85d 100644
---
a/asterix-runtime/src/main/java/org/apache/asterix/runtime/formats/NonTaggedDataFormat.java
+++
b/asterix-runtime/src/main/java/org/apache/asterix/runtime/formats/NonTaggedDataFormat.java
@@ -23,6 +23,7 @@
import org.apache.asterix.common.parse.IParseFileSplitsDecl;
import org.apache.asterix.dataflow.data.nontagged.AqlNullWriterFactory;
import org.apache.asterix.formats.base.IDataFormat;
+import org.apache.asterix.formats.nontagged.AqlADMPrinterFactoryProvider;
import org.apache.asterix.formats.nontagged.AqlBinaryBooleanInspectorImpl;
import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
import
org.apache.asterix.formats.nontagged.AqlBinaryHashFunctionFactoryProvider;
@@ -33,7 +34,6 @@
import
org.apache.asterix.formats.nontagged.AqlLosslessJSONPrinterFactoryProvider;
import
org.apache.asterix.formats.nontagged.AqlNormalizedKeyComputerFactoryProvider;
import
org.apache.asterix.formats.nontagged.AqlPredicateEvaluatorFactoryProvider;
-import org.apache.asterix.formats.nontagged.AqlADMPrinterFactoryProvider;
import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
import org.apache.asterix.formats.nontagged.AqlTypeTraitProvider;
import org.apache.asterix.om.base.ABoolean;
@@ -161,12 +161,12 @@
import
org.apache.asterix.runtime.evaluators.constructors.AYearMonthDurationConstructorDescriptor;
import
org.apache.asterix.runtime.evaluators.constructors.ClosedRecordConstructorDescriptor;
import
org.apache.asterix.runtime.evaluators.constructors.OpenRecordConstructorDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.AdmToBytesDescriptor;
import org.apache.asterix.runtime.evaluators.functions.AndDescriptor;
import
org.apache.asterix.runtime.evaluators.functions.AnyCollectionMemberDescriptor;
import org.apache.asterix.runtime.evaluators.functions.CastListDescriptor;
import org.apache.asterix.runtime.evaluators.functions.CastRecordDescriptor;
import
org.apache.asterix.runtime.evaluators.functions.CodePointToStringDescriptor;
-import
org.apache.asterix.runtime.evaluators.functions.StringContainsDescriptor;
import
org.apache.asterix.runtime.evaluators.functions.CountHashedGramTokensDescriptor;
import
org.apache.asterix.runtime.evaluators.functions.CountHashedWordTokensDescriptor;
import org.apache.asterix.runtime.evaluators.functions.CreateCircleDescriptor;
@@ -182,7 +182,6 @@
import
org.apache.asterix.runtime.evaluators.functions.EditDistanceListIsFilterable;
import
org.apache.asterix.runtime.evaluators.functions.EditDistanceStringIsFilterable;
import org.apache.asterix.runtime.evaluators.functions.EmbedTypeDescriptor;
-import
org.apache.asterix.runtime.evaluators.functions.StringEndsWithDescriptor;
import org.apache.asterix.runtime.evaluators.functions.FlowRecordDescriptor;
import org.apache.asterix.runtime.evaluators.functions.FuzzyEqDescriptor;
import org.apache.asterix.runtime.evaluators.functions.GetItemDescriptor;
@@ -193,7 +192,6 @@
import org.apache.asterix.runtime.evaluators.functions.IsNullDescriptor;
import org.apache.asterix.runtime.evaluators.functions.IsSystemNullDescriptor;
import org.apache.asterix.runtime.evaluators.functions.LenDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.StringLikeDescriptor;
import org.apache.asterix.runtime.evaluators.functions.NotDescriptor;
import org.apache.asterix.runtime.evaluators.functions.NotNullDescriptor;
import org.apache.asterix.runtime.evaluators.functions.NumericAbsDescriptor;
@@ -223,16 +221,19 @@
import org.apache.asterix.runtime.evaluators.functions.SpatialCellDescriptor;
import
org.apache.asterix.runtime.evaluators.functions.SpatialDistanceDescriptor;
import
org.apache.asterix.runtime.evaluators.functions.SpatialIntersectDescriptor;
-import
org.apache.asterix.runtime.evaluators.functions.StringStartsWithDescriptor;
import org.apache.asterix.runtime.evaluators.functions.StringConcatDescriptor;
+import
org.apache.asterix.runtime.evaluators.functions.StringContainsDescriptor;
+import
org.apache.asterix.runtime.evaluators.functions.StringEndsWithDescriptor;
import org.apache.asterix.runtime.evaluators.functions.StringEqualDescriptor;
import org.apache.asterix.runtime.evaluators.functions.StringJoinDescriptor;
import org.apache.asterix.runtime.evaluators.functions.StringLengthDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.StringLikeDescriptor;
import
org.apache.asterix.runtime.evaluators.functions.StringLowerCaseDescriptor;
import org.apache.asterix.runtime.evaluators.functions.StringMatchesDescriptor;
import
org.apache.asterix.runtime.evaluators.functions.StringMatchesWithFlagDescriptor;
import org.apache.asterix.runtime.evaluators.functions.StringReplaceDescriptor;
import
org.apache.asterix.runtime.evaluators.functions.StringReplaceWithFlagsDescriptor;
+import
org.apache.asterix.runtime.evaluators.functions.StringStartsWithDescriptor;
import
org.apache.asterix.runtime.evaluators.functions.StringToCodePointDescriptor;
import
org.apache.asterix.runtime.evaluators.functions.StringUpperCaseDescriptor;
import org.apache.asterix.runtime.evaluators.functions.Substring2Descriptor;
@@ -311,6 +312,8 @@
import org.apache.asterix.runtime.unnestingfunctions.std.RangeDescriptor;
import
org.apache.asterix.runtime.unnestingfunctions.std.ScanCollectionDescriptor;
import
org.apache.asterix.runtime.unnestingfunctions.std.SubsetCollectionDescriptor;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
import org.apache.hyracks.algebricks.common.utils.Triple;
@@ -350,8 +353,6 @@
import org.apache.hyracks.dataflow.common.data.parsers.LongParserFactory;
import org.apache.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
import org.apache.hyracks.dataflow.std.file.ITupleParserFactory;
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
import java.io.DataOutput;
import java.io.IOException;
@@ -688,6 +689,9 @@
temp.add(AIntervalStartFromDateTimeConstructorDescriptor.FACTORY);
temp.add(AIntervalStartFromTimeConstructorDescriptor.FACTORY);
+ // For debugging
+ temp.add(AdmToBytesDescriptor.FACTORY);
+
IFunctionManager mgr = new FunctionManagerImpl();
for (IFunctionDescriptorFactory fdFactory : temp) {
mgr.registerFunction(fdFactory);
@@ -934,6 +938,16 @@
((RecordMergeDescriptor) fd).reset(outType, type0, type1);
}
});
+ functionTypeInferers.put(AsterixBuiltinFunctions.ADM_TO_BYTES, new
FunctionTypeInferer() {
+
+ @Override public void infer(ILogicalExpression expr,
IFunctionDescriptor fd,
+ IVariableTypeEnvironment context) throws
AlgebricksException {
+ AbstractFunctionCallExpression f =
(AbstractFunctionCallExpression) expr;
+ IAType outType = (IAType) context.getType(expr);
+ IAType intype0 = (IAType)
context.getType(f.getArguments().get(0).getValue());
+ ((AdmToBytesDescriptor) fd).reset(outType, intype0);
+ }
+ });
functionTypeInferers.put(AsterixBuiltinFunctions.CAST_RECORD, new
FunctionTypeInferer() {
public void infer(ILogicalExpression expr, IFunctionDescriptor fd,
IVariableTypeEnvironment context) throws AlgebricksException {
AbstractFunctionCallExpression funcExpr =
(AbstractFunctionCallExpression) expr;
--
To view, visit https://asterix-gerrit.ics.uci.edu/523
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I31d0b2ec2d8686531833811937596c3dca660b1e
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Heri Ramampiaro <[email protected]>
