This is an automated email from the ASF dual-hosted git repository.
dlych pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new e381b2c [NO ISSUE][IDX] Refactoring of array index code.
e381b2c is described below
commit e381b2cbdadc3e33a488678e3b2384a00d97f301
Author: ggalvizo <[email protected]>
AuthorDate: Tue Jun 29 15:27:30 2021 -0700
[NO ISSUE][IDX] Refactoring of array index code.
- user model changes: no
- storage format changes: no
- interface changes: no
Refactor: no longer using depth indicators, rather using UNNEST flags with
flattened field names. We can no longer have back-to-back UNNESTs (i.e.
arrays
within arrays without an ASSIGN intermediate), as no such array index
can be created..
Change-Id: Id337d1032796e1d1c2ce68ea2861b4a81dd19aa5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12063
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Glenn Galvizo <[email protected]>
---
.../IntroduceSecondaryIndexInsertDeleteRule.java | 26 +--
.../optimizer/rules/am/AccessMethodUtils.java | 1 -
.../optimizer/rules/am/ArrayBTreeAccessMethod.java | 1 -
.../ArrayBTreeResourceFactoryProvider.java | 9 +-
.../asterix/metadata/utils/ArrayIndexUtil.java | 214 ++++++++++++---------
.../asterix/metadata/utils/KeyFieldTypeUtil.java | 4 +-
.../SecondaryArrayIndexBTreeOperationsHelper.java | 62 +++---
.../apache/asterix/metadata/utils/TypeUtil.java | 71 +++----
8 files changed, 197 insertions(+), 191 deletions(-)
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
index c3859be..81c6c2a 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
@@ -728,10 +728,10 @@ public class IntroduceSecondaryIndexInsertDeleteRule
implements IAlgebraicRewrit
// Walk the array path.
List<String> flatFirstFieldName =
ArrayIndexUtil.getFlattenedKeyFieldNames(
workingElement.getUnnestList(),
workingElement.getProjectList().get(0));
- List<Integer> firstArrayIndicator = ArrayIndexUtil
-
.getArrayDepthIndicator(workingElement.getUnnestList(),
workingElement.getProjectList().get(0));
+ List<Boolean> firstUnnestFlags =
ArrayIndexUtil.getUnnestFlags(workingElement.getUnnestList(),
+ workingElement.getProjectList().get(0));
ArrayIndexUtil.walkArrayPath((isOpenOrNestedField) ? null :
recordType, flatFirstFieldName,
- firstArrayIndicator, branchCreator);
+ firstUnnestFlags, branchCreator);
// For all other elements in the PROJECT list, add an assign.
for (int j = 1; j < workingElement.getProjectList().size();
j++) {
@@ -1054,25 +1054,19 @@ public class IntroduceSecondaryIndexInsertDeleteRule
implements IAlgebraicRewrit
@Override
public void executeActionOnEachArrayStep(ARecordType
startingStepRecordType, IAType workingType,
- List<String> fieldName, boolean isFirstArrayStep, boolean
isFirstUnnestInStep,
- boolean isLastUnnestInIntermediateStep) throws
AlgebricksException {
+ List<String> fieldName, boolean isFirstArrayStep, boolean
isLastUnnestInIntermediateStep)
+ throws AlgebricksException {
if (!isFirstWalk) {
// We have already built the UNNEST path, do not build again.
return;
}
+ // Get the field we want to UNNEST from our record.
ILogicalExpression accessToUnnestVar;
- if (isFirstUnnestInStep) {
- // This is the first UNNEST step. Get the field we want to
UNNEST from our record.
- accessToUnnestVar = (startingStepRecordType != null)
- ? getFieldAccessFunction(new
MutableObject<>(createLastRecordVarRef()),
-
startingStepRecordType.getFieldIndex(fieldName.get(0)), fieldName)
- : getFieldAccessFunction(new
MutableObject<>(createLastRecordVarRef()), -1, fieldName);
- } else {
- // This is the second+ UNNEST step. Refer back to the
previously unnested variable.
- accessToUnnestVar = new
VariableReferenceExpression(this.lastFieldVars.get(0));
- this.lastFieldVars.clear();
- }
+ accessToUnnestVar = (startingStepRecordType != null)
+ ? getFieldAccessFunction(new
MutableObject<>(createLastRecordVarRef()),
+
startingStepRecordType.getFieldIndex(fieldName.get(0)), fieldName)
+ : getFieldAccessFunction(new
MutableObject<>(createLastRecordVarRef()), -1, fieldName);
UnnestingFunctionCallExpression scanCollection = new
UnnestingFunctionCallExpression(
BuiltinFunctions.getBuiltinFunctionInfo(BuiltinFunctions.SCAN_COLLECTION),
Collections.singletonList(new
MutableObject<>(accessToUnnestVar)));
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
index c78e89f..3f4d3f2 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -3060,7 +3060,6 @@ public class AccessMethodUtils {
MutableInt fieldSource = new MutableInt(0);
ARecordType workingRecordType = subTree.getRecordType();
- // TODO: (GLENN) Refactor this to use ArrayIndexUtil.
// Iterate through our array index structure. We must match the depth
and field names for the caller's variable
// to qualify for an array-index optimization.
LogicalVariable varFromParent = assignVar;
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/ArrayBTreeAccessMethod.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/ArrayBTreeAccessMethod.java
index 653d15b..d364b53 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/ArrayBTreeAccessMethod.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/ArrayBTreeAccessMethod.java
@@ -179,7 +179,6 @@ public class ArrayBTreeAccessMethod extends
BTreeAccessMethod {
@Override
protected IAType getIndexedKeyType(Index.IIndexDetails chosenIndexDetails,
int keyPos) throws CompilationException {
- // TODO (GLENN): This assumes a flattened key list. Refactor / clarify
this when removing depth indicators.
Index.ArrayIndexDetails arrayIndexDetails = (Index.ArrayIndexDetails)
chosenIndexDetails;
int elementPos = 0;
for (Index.ArrayIndexElement e : arrayIndexDetails.getElementList()) {
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/ArrayBTreeResourceFactoryProvider.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/ArrayBTreeResourceFactoryProvider.java
index 5654d16..c0bdc75 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/ArrayBTreeResourceFactoryProvider.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/ArrayBTreeResourceFactoryProvider.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.metadata.declared;
-import java.util.List;
import java.util.Map;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
@@ -134,10 +133,8 @@ public class ArrayBTreeResourceFactoryProvider implements
IResourceFactoryProvid
sourceType = metaType;
}
for (int i = 0; i < e.getProjectList().size(); i++) {
- List<String> project = e.getProjectList().get(i);
Pair<IAType, Boolean> keyTypePair =
ArrayIndexUtil.getNonNullableOpenFieldType(e.getTypeList().get(i),
-
ArrayIndexUtil.getFlattenedKeyFieldNames(e.getUnnestList(), project),
sourceType,
-
ArrayIndexUtil.getArrayDepthIndicator(e.getUnnestList(), project));
+ e.getUnnestList(), e.getProjectList().get(i),
sourceType);
IAType keyType = keyTypePair.first;
secondaryTypeTraits[secondaryTypeTraitPos++] =
typeTraitProvider.getTypeTrait(keyType);
}
@@ -175,10 +172,8 @@ public class ArrayBTreeResourceFactoryProvider implements
IResourceFactoryProvid
sourceType = metaType;
}
for (int i = 0; i < e.getProjectList().size(); i++) {
- List<String> project = e.getProjectList().get(i);
Pair<IAType, Boolean> keyTypePair =
ArrayIndexUtil.getNonNullableOpenFieldType(e.getTypeList().get(i),
-
ArrayIndexUtil.getFlattenedKeyFieldNames(e.getUnnestList(), project),
sourceType,
-
ArrayIndexUtil.getArrayDepthIndicator(e.getUnnestList(), project));
+ e.getUnnestList(), e.getProjectList().get(i),
sourceType);
IAType keyType = keyTypePair.first;
secondaryCmpFactories[secondaryCmpFactoriesPos++] =
cmpFactoryProvider.getBinaryComparatorFactory(keyType,
true);
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
index f4dfe56..0778ad9 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
@@ -39,14 +39,18 @@ import org.apache.hyracks.algebricks.common.utils.Pair;
public class ArrayIndexUtil {
/**
- * @deprecated Use the project + unnest scheme instead of array indicators.
+ * Similar function to Index's "getSubFieldType", but accounts for array
fields as well.
*/
- public static IAType getSubFieldInArrayType(ARecordType recordType,
List<String> subFieldName,
- List<Integer> arrayDepthIndicators) throws AlgebricksException {
- IAType subType = recordType.getFieldType(subFieldName.get(0));
- for (int i = 1; i < subFieldName.size(); i++) {
+ public static IAType getSubFieldType(ARecordType recordType,
List<List<String>> unnestList,
+ List<String> projectList) throws AlgebricksException {
+ List<String> flattenedFieldName =
ArrayIndexUtil.getFlattenedKeyFieldNames(unnestList, projectList);
+ List<Boolean> unnestFlags = ArrayIndexUtil.getUnnestFlags(unnestList,
projectList);
+ IAType subType = recordType.getFieldType(flattenedFieldName.get(0));
+
+ for (int i = 1; i < flattenedFieldName.size(); i++) {
if (subType == null) {
return null;
+
} else if (subType.getTypeTag().equals(ATypeTag.UNION)) {
// Support enforced types here.
subType = ((AUnionType) subType).getActualType();
@@ -56,31 +60,30 @@ public class ArrayIndexUtil {
"Field accessor is not defined for values of type
" + subType.getTypeTag());
}
}
- if (subType.getTypeTag().equals(ATypeTag.OBJECT) &&
arrayDepthIndicators.get(i - 1) == 0) {
- subType = ((ARecordType)
subType).getFieldType(subFieldName.get(i));
+
+ if (subType.getTypeTag().equals(ATypeTag.OBJECT) &&
!unnestFlags.get(i - 1)) {
+ subType = ((ARecordType)
subType).getFieldType(flattenedFieldName.get(i));
+
} else if ((subType.getTypeTag().equals(ATypeTag.ARRAY) ||
subType.getTypeTag().equals(ATypeTag.MULTISET))
- && arrayDepthIndicators.get(i - 1) > 0) {
- for (int j = 0; j < arrayDepthIndicators.get(i - 1); j++) {
- subType = TypeComputeUtils.extractListItemType(subType);
- }
- subType = (subType != null) ? ((ARecordType)
subType).getFieldType(subFieldName.get(i)) : null;
+ && unnestFlags.get(i - 1)) {
+ subType = TypeComputeUtils.extractListItemType(subType);
+ subType = (subType != null) ? ((ARecordType)
subType).getFieldType(flattenedFieldName.get(i)) : null;
+
} else {
throw new AsterixException(ErrorCode.COMPILATION_ERROR,
- (arrayDepthIndicators.get(i - 1) > 0)
- ? "Object type given, but array depth
indicator is " + "non-zero."
- : "Array/multiset type given, but array depth
indicator is zero.");
+ unnestFlags.get(i - 1) ? "Object type given, but
unnest flag is also raised."
+ : "Array/multiset type given, but unnest flag
is lowered.");
}
}
- if (subType != null &&
arrayDepthIndicators.get(arrayDepthIndicators.size() - 1) > 0) {
+
+ if (subType != null && unnestFlags.get(unnestFlags.size() - 1)) {
// If the end field is an array, we must extract the list item
here as well.
- for (int j = 0; j <
arrayDepthIndicators.get(arrayDepthIndicators.size() - 1); j++) {
- if (subType instanceof AbstractCollectionType) {
- subType = TypeComputeUtils.extractListItemType(subType);
- } else {
- throw new AsterixException(ErrorCode.COMPILATION_ERROR,
- "Array type expected for last term, but given: "
- + ((subType != null) ?
subType.getTypeTag() : "null"));
- }
+ if (subType instanceof AbstractCollectionType) {
+ subType = TypeComputeUtils.extractListItemType(subType);
+
+ } else {
+ throw new AsterixException(ErrorCode.COMPILATION_ERROR,
+ "Array type expected for last term, but given: " +
subType.getTypeTag());
}
}
return subType;
@@ -88,15 +91,18 @@ public class ArrayIndexUtil {
/**
* Given a path of complex types (i.e. lists + records), determine the
nullability of the field.
- * @deprecated Use the project + unnest scheme instead of array indicators.
*/
- public static boolean isSubFieldNullable(ARecordType recordType,
List<String> subFieldName,
- List<Integer> arrayIndicators) throws AlgebricksException {
- IAType subType = recordType.getFieldType(subFieldName.get(0));
- for (int i = 1; i < subFieldName.size(); i++) {
+ public static boolean isSubFieldNullable(ARecordType recordType,
List<List<String>> unnestList,
+ List<String> projectList) throws AlgebricksException {
+ List<String> flattenedFieldName =
ArrayIndexUtil.getFlattenedKeyFieldNames(unnestList, projectList);
+ List<Boolean> unnestFlags = ArrayIndexUtil.getUnnestFlags(unnestList,
projectList);
+ IAType subType = recordType.getFieldType(flattenedFieldName.get(0));
+
+ for (int i = 1; i < flattenedFieldName.size(); i++) {
if (subType == null) {
return true;
}
+
if (subType.getTypeTag().equals(ATypeTag.UNION)) {
if (NonTaggedFormatUtil.isOptional(subType)) {
return true;
@@ -109,12 +115,12 @@ public class ArrayIndexUtil {
}
if (subType instanceof ARecordType) {
- subType = ((ARecordType)
subType).getFieldType(subFieldName.get(i));
- } else if (subType instanceof AbstractCollectionType &&
arrayIndicators.get(i - 1) > 0) {
- for (int j = 0; j < arrayIndicators.get(i - 1); j++) {
- subType = TypeComputeUtils.extractListItemType(subType);
- }
- subType = (subType != null) ? ((ARecordType)
subType).getFieldType(subFieldName.get(i)) : null;
+ subType = ((ARecordType)
subType).getFieldType(flattenedFieldName.get(i));
+
+ } else if (subType instanceof AbstractCollectionType &&
unnestFlags.get(i - 1)) {
+ subType = TypeComputeUtils.extractListItemType(subType);
+ subType = (subType != null) ? ((ARecordType)
subType).getFieldType(flattenedFieldName.get(i)) : null;
+
} else {
throw
CompilationException.create(ErrorCode.COMPILATION_ILLEGAL_STATE,
"Illegal field type " + subType.getTypeTag() + " when
checking field nullability");
@@ -125,32 +131,37 @@ public class ArrayIndexUtil {
/**
* Similar function to Index's "getNonNullableOpenFieldType", but accounts
for array fields as well.
- * @deprecated Use the project + unnest scheme instead of array indicators.
*/
- public static Pair<IAType, Boolean> getNonNullableOpenFieldType(IAType
fieldType, List<String> fieldName,
- ARecordType recType, List<Integer> arrayIndicators) throws
AlgebricksException {
+ public static Pair<IAType, Boolean> getNonNullableOpenFieldType(IAType
fieldType, List<List<String>> unnestList,
+ List<String> projectList, ARecordType recType) throws
AlgebricksException {
Pair<IAType, Boolean> keyPairType = null;
IAType subType = recType;
boolean nullable = false;
- for (int i = 0; i < fieldName.size(); i++) {
+
+ List<String> flattenedFieldName =
ArrayIndexUtil.getFlattenedKeyFieldNames(unnestList, projectList);
+ List<Boolean> unnestFlags = ArrayIndexUtil.getUnnestFlags(unnestList,
projectList);
+ for (int i = 0; i < flattenedFieldName.size(); i++) {
if (subType instanceof AUnionType) {
nullable = nullable || ((AUnionType)
subType).isUnknownableType();
subType = ((AUnionType) subType).getActualType();
}
if (subType instanceof ARecordType) {
- subType = ((ARecordType)
subType).getFieldType(fieldName.get(i));
+ subType = ((ARecordType)
subType).getFieldType(flattenedFieldName.get(i));
+
} else if ((subType instanceof AOrderedListType || subType
instanceof AUnorderedListType)
- && arrayIndicators.get(i - 1) > 0) {
- for (int j = 0; j < arrayIndicators.get(i - 1); j++) {
- subType = TypeComputeUtils.extractListItemType(subType);
- }
+ && unnestFlags.get(i - 1)) {
+ subType = TypeComputeUtils.extractListItemType(subType);
if (subType instanceof ARecordType) {
- subType = ((ARecordType)
subType).getFieldType(fieldName.get(i));
+ subType = ((ARecordType)
subType).getFieldType(flattenedFieldName.get(i));
+
} else {
- throw
AsterixException.create(ErrorCode.COMPILATION_ILLEGAL_STATE, "Unexpected type "
+ fieldType);
+ throw
AsterixException.create(ErrorCode.COMPILATION_ILLEGAL_STATE,
+ "Unexpected type " + subType + ", expected
record.");
}
+
} else {
- throw
AsterixException.create(ErrorCode.COMPILATION_ILLEGAL_STATE, "Unexpected type "
+ fieldType);
+ throw
AsterixException.create(ErrorCode.COMPILATION_ILLEGAL_STATE,
+ "Unexpected type " + subType + ", expected record,
array, or multi-set.");
}
if (subType == null) {
@@ -158,18 +169,20 @@ public class ArrayIndexUtil {
break;
}
}
+
if (subType != null) {
- IAType keyType = ArrayIndexUtil.getSubFieldInArrayType(recType,
fieldName, arrayIndicators);
+ IAType keyType = ArrayIndexUtil.getSubFieldType(recType,
unnestList, projectList);
Pair<IAType, Boolean> pair = Index.getNonNullableType(keyType);
- pair.second = pair.second ||
ArrayIndexUtil.isSubFieldNullable(recType, fieldName, arrayIndicators);
+ pair.second = pair.second ||
ArrayIndexUtil.isSubFieldNullable(recType, unnestList, projectList);
keyPairType = pair;
}
+
keyPairType.second = keyPairType.second || nullable;
return keyPairType;
}
/**
- * @deprecated Use new unnestList and projectList scheme.
+ * @return The concatenation of the unnest list fields and the project
field (for use in creating a unique name).
*/
public static List<String> getFlattenedKeyFieldNames(List<List<String>>
unnestList, List<String> projectList) {
if (unnestList == null) {
@@ -188,6 +201,41 @@ public class ArrayIndexUtil {
}
/**
+ * @return Mapping to the flattened key field names, determine where the
UNNESTs occur.
+ */
+ public static List<Boolean> getUnnestFlags(List<List<String>> unnestList,
List<String> projectList) {
+ if (unnestList.isEmpty()) {
+ // A simple element has no UNNEST flags raised..
+ List<Boolean> unnestFlags = new ArrayList<>();
+ for (String ignored : projectList) {
+ unnestFlags.add(false);
+ }
+ return unnestFlags;
+
+ } else {
+ List<Boolean> unnestFlagsPrefix = new ArrayList<>();
+ for (List<String> unnestField : unnestList) {
+ for (int i = 0; i < unnestField.size() - 1; i++) {
+ unnestFlagsPrefix.add(false);
+ }
+ unnestFlagsPrefix.add(true);
+ }
+
+ if (projectList == null) {
+ // Stop here. The prefix is the flag vector itself.
+ return unnestFlagsPrefix;
+
+ } else {
+ List<Boolean> unnestFlags = new ArrayList<>(unnestFlagsPrefix);
+ for (int i = 0; i < projectList.size(); i++) {
+ unnestFlags.add(false);
+ }
+ return unnestFlags;
+ }
+ }
+ }
+
+ /**
* @deprecated Use new unnestList and projectList scheme.
*/
public static List<Integer> getArrayDepthIndicator(List<List<String>>
unnestList, List<String> projectList) {
@@ -244,18 +292,14 @@ public class ArrayIndexUtil {
}
/**
- * Given the {@code Index}'s representation of an array path (i.e. a
concatenation of record paths, with array
- * steps specified in depths corresponding to an index in the
aforementioned record path array), traverse each
- * distinct record path and invoke the appropriate commands for each
scenario.
- * <p>
- * Here, we keep track of the record/list type at each step and give this
to each command.
+ * Traverse each distinct record path and invoke the appropriate commands
for each scenario. Here, we keep track
+ * of the record/list type at each step and give this to each command.
*/
public static void walkArrayPath(ARecordType baseRecordType, List<String>
flattenedFieldName,
- List<Integer> flattenedDepthIndicators, TypeTrackerCommandExecutor
commandExecutor)
- throws AlgebricksException {
- ArrayPath arrayPath = new ArrayPath(flattenedFieldName,
flattenedDepthIndicators).invoke();
+ List<Boolean> unnestFlags, TypeTrackerCommandExecutor
commandExecutor) throws AlgebricksException {
+ ArrayPath arrayPath = new ArrayPath(flattenedFieldName,
unnestFlags).invoke();
List<List<String>> fieldNamesPerArray = arrayPath.fieldNamesPerArray;
- List<Integer> depthOfArraySteps = arrayPath.depthOfArraySteps;
+ List<Boolean> unnestFlagsPerArray = arrayPath.unnestFlagsPerArray;
// If we are given no base record type, then we do not need to keep
track of the record type. We are solely
// using this walk for its flags.
@@ -275,7 +319,7 @@ public class ArrayIndexUtil {
startingStepRecordType).first;
}
- for (int j = 0; j < depthOfArraySteps.get(i); j++) {
+ if (unnestFlagsPerArray.get(i)) {
if (isTrackingType) {
workingType =
TypeComputeUtils.extractListItemType(workingType);
if (workingType == null) {
@@ -284,17 +328,14 @@ public class ArrayIndexUtil {
}
}
boolean isFirstArrayStep = i == 0;
- boolean isFirstUnnestInStep = j == 0;
- boolean isLastUnnestInIntermediateStep =
- j == depthOfArraySteps.get(i) - 1 && i <
fieldNamesPerArray.size() - 1;
+ boolean isLastUnnestInIntermediateStep = i <
fieldNamesPerArray.size() - 1;
commandExecutor.executeActionOnEachArrayStep(startingStepRecordType,
workingType,
- fieldNamesPerArray.get(i), isFirstArrayStep,
isFirstUnnestInStep,
- isLastUnnestInIntermediateStep);
+ fieldNamesPerArray.get(i), isFirstArrayStep,
isLastUnnestInIntermediateStep);
}
if (i == fieldNamesPerArray.size() - 1) {
- boolean requiresOnlyOneUnnest =
depthOfArraySteps.stream().reduce(0, Integer::sum).equals(1);
- boolean isNonArrayStep = depthOfArraySteps.get(i) == 0;
+ boolean requiresOnlyOneUnnest = fieldNamesPerArray.size() == 1;
+ boolean isNonArrayStep = !unnestFlagsPerArray.get(i);
commandExecutor.executeActionOnFinalArrayStep(startingStepRecordType,
fieldNamesPerArray.get(i),
isNonArrayStep, requiresOnlyOneUnnest);
}
@@ -302,28 +343,25 @@ public class ArrayIndexUtil {
}
/**
- * Given the {@code Index}'s representation of an array path (i.e. a
concatenation of record paths, with array
- * steps specified in depths corresponding to an index in the
aforementioned record path array), traverse each
- * distinct record path and invoke the appropriate commands for each
scenario.
- * <p>
- * Here, we keep track of the total number of actions performed and give
this to each command.
+ * Traverse each distinct record path and invoke the appropriate commands
for each scenario. Here, we keep track
+ * of the total number of actions performed and give this to each command.
*/
- public static void walkArrayPath(List<String> flattenedFieldName,
List<Integer> flattenedDepthIndicators,
+ public static void walkArrayPath(List<String> flattenedFieldName,
List<Boolean> unnestFlags,
ActionCounterCommandExecutor commandExecutor) throws
AlgebricksException {
- ArrayPath arrayPath = new ArrayPath(flattenedFieldName,
flattenedDepthIndicators).invoke();
+ ArrayPath arrayPath = new ArrayPath(flattenedFieldName,
unnestFlags).invoke();
List<List<String>> fieldNamesPerArray = arrayPath.fieldNamesPerArray;
- List<Integer> depthOfArraySteps = arrayPath.depthOfArraySteps;
+ List<Boolean> unnestFlagsPerArray = arrayPath.unnestFlagsPerArray;
int numberOfActionsPerformed = 0;
for (int i = 0; i < fieldNamesPerArray.size(); i++) {
- int unnestLevel = depthOfArraySteps.get(i);
+ boolean isUnnestFlagRaised = unnestFlagsPerArray.get(i);
if (i == 0) {
commandExecutor.executeActionOnFirstArrayStep();
numberOfActionsPerformed++;
- unnestLevel--;
+ isUnnestFlagRaised = false;
}
- for (int j = 0; j < unnestLevel; j++) {
+ if (isUnnestFlagRaised) {
commandExecutor.executeActionOnIntermediateArrayStep(numberOfActionsPerformed++);
}
@@ -343,8 +381,8 @@ public class ArrayIndexUtil {
public interface TypeTrackerCommandExecutor {
void executeActionOnEachArrayStep(ARecordType startingStepRecordType,
IAType workingType,
- List<String> fieldName, boolean isFirstArrayStep, boolean
isFirstUnnestInStep,
- boolean isLastUnnestInIntermediateStep) throws
AlgebricksException;
+ List<String> fieldName, boolean isFirstArrayStep, boolean
isLastUnnestInIntermediateStep)
+ throws AlgebricksException;
void executeActionOnFinalArrayStep(ARecordType startingStepRecordType,
List<String> fieldName,
boolean isNonArrayStep, boolean requiresOnlyOneUnnest) throws
AlgebricksException;
@@ -352,24 +390,24 @@ public class ArrayIndexUtil {
private static class ArrayPath {
private final List<String> flattenedFieldName;
- private final List<Integer> flattenedDepthIndicators;
+ private final List<Boolean> unnestFlags;
private List<List<String>> fieldNamesPerArray;
- private List<Integer> depthOfArraySteps;
+ private List<Boolean> unnestFlagsPerArray;
- public ArrayPath(List<String> flattenedFieldName, List<Integer>
flattenedDepthIndicators) {
+ public ArrayPath(List<String> flattenedFieldName, List<Boolean>
unnestFlags) {
this.flattenedFieldName = flattenedFieldName;
- this.flattenedDepthIndicators = flattenedDepthIndicators;
+ this.unnestFlags = unnestFlags;
}
public ArrayPath invoke() {
fieldNamesPerArray = new ArrayList<>();
- depthOfArraySteps = new ArrayList<>();
+ unnestFlagsPerArray = new ArrayList<>();
List<String> workingRecordPath = new ArrayList<>();
- for (int i = 0; i < flattenedDepthIndicators.size(); i++) {
+ for (int i = 0; i < unnestFlags.size(); i++) {
workingRecordPath.add(flattenedFieldName.get(i));
- if (i == flattenedDepthIndicators.size() - 1 ||
flattenedDepthIndicators.get(i) > 0) {
- depthOfArraySteps.add(flattenedDepthIndicators.get(i));
+ if (i == unnestFlags.size() - 1 || unnestFlags.get(i)) {
+ unnestFlagsPerArray.add(unnestFlags.get(i));
fieldNamesPerArray.add(workingRecordPath);
workingRecordPath = new ArrayList<>();
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtil.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtil.java
index b2026d7..6a0c44f 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtil.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtil.java
@@ -153,9 +153,7 @@ public class KeyFieldTypeUtil {
ARecordType sourceType =
(e.getSourceIndicator() == Index.RECORD_INDICATOR) ?
recordType : metaRecordType;
Pair<IAType, Boolean> keyPairType =
ArrayIndexUtil.getNonNullableOpenFieldType(e.getTypeList().get(i),
-
ArrayIndexUtil.getFlattenedKeyFieldNames(e.getUnnestList(),
e.getProjectList().get(i)),
- sourceType,
-
ArrayIndexUtil.getArrayDepthIndicator(e.getUnnestList(),
e.getProjectList().get(i)));
+ e.getUnnestList(), e.getProjectList().get(i),
sourceType);
indexKeyTypes.add(keyPairType.first);
}
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
index fdb21a2..c1f8c7b 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
@@ -57,7 +57,6 @@ import
org.apache.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
import org.apache.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
import org.apache.hyracks.algebricks.runtime.base.IUnnestingEvaluatorFactory;
-import
org.apache.hyracks.algebricks.runtime.evaluators.ColumnAccessEvalFactory;
import
org.apache.hyracks.algebricks.runtime.operators.aggreg.SimpleAlgebricksAccumulatingAggregatorFactory;
import org.apache.hyracks.algebricks.runtime.operators.base.SinkRuntimeFactory;
import
org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
@@ -78,14 +77,13 @@ import
org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory;
public class SecondaryArrayIndexBTreeOperationsHelper extends
SecondaryTreeIndexOperationsHelper {
private final int numAtomicSecondaryKeys, numArraySecondaryKeys,
numTotalSecondaryKeys;
- private final Index.ArrayIndexDetails arrayIndexDetails;
private final EvalFactoryAndRecDescStackBuilder
evalFactoryAndRecDescStackBuilder =
new EvalFactoryAndRecDescStackBuilder();
- // TODO (GLENN): Phase these out and use the UNNEST / PROJECT scheme
instead.
+ private final Index.ArrayIndexDetails arrayIndexDetails;
private final List<List<String>> flattenedFieldNames;
private final List<IAType> flattenedKeyTypes;
- private final List<List<Integer>> depthIndicators;
+ private final List<List<Boolean>> unnestFlags;
protected SecondaryArrayIndexBTreeOperationsHelper(Dataset dataset, Index
index, MetadataProvider metadataProvider,
SourceLocation sourceLoc) throws AlgebricksException {
@@ -94,19 +92,19 @@ public class SecondaryArrayIndexBTreeOperationsHelper
extends SecondaryTreeIndex
flattenedFieldNames = new ArrayList<>();
flattenedKeyTypes = new ArrayList<>();
- depthIndicators = new ArrayList<>();
+ unnestFlags = new ArrayList<>();
for (Index.ArrayIndexElement e : arrayIndexDetails.getElementList()) {
if (e.getUnnestList().isEmpty()) {
flattenedFieldNames.add(e.getProjectList().get(0));
flattenedKeyTypes.add(e.getTypeList().get(0));
- depthIndicators
-
.add(ArrayIndexUtil.getArrayDepthIndicator(e.getUnnestList(),
e.getProjectList().get(0)));
+
unnestFlags.add(ArrayIndexUtil.getUnnestFlags(e.getUnnestList(),
e.getProjectList().get(0)));
+
} else {
for (int i = 0; i < e.getProjectList().size(); i++) {
List<String> project = e.getProjectList().get(i);
flattenedFieldNames.add(ArrayIndexUtil.getFlattenedKeyFieldNames(e.getUnnestList(),
project));
-
depthIndicators.add(ArrayIndexUtil.getArrayDepthIndicator(e.getUnnestList(),
project));
flattenedKeyTypes.add(e.getTypeList().get(i));
+
unnestFlags.add(ArrayIndexUtil.getUnnestFlags(e.getUnnestList(), project));
}
}
}
@@ -127,7 +125,7 @@ public class SecondaryArrayIndexBTreeOperationsHelper
extends SecondaryTreeIndex
numArraySecondaryKeys = numTotalSecondaryKeys - numAtomicSecondaryKeys;
}
- private int findPosOfArrayIndex() throws AsterixException {
+ private int findPosOfArrayIndexElement() throws AsterixException {
for (int i = 0; i < arrayIndexDetails.getElementList().size(); i++) {
if
(!arrayIndexDetails.getElementList().get(i).getUnnestList().isEmpty()) {
return i;
@@ -163,9 +161,7 @@ public class SecondaryArrayIndexBTreeOperationsHelper
extends SecondaryTreeIndex
ARecordType sourceType = (e.getSourceIndicator() == 0) ?
itemType : metaType;
addSKEvalFactories(isOverridingKeyFieldTypes ?
enforcedItemType : sourceType, flattenedListPos, false);
Pair<IAType, Boolean> keyTypePair =
ArrayIndexUtil.getNonNullableOpenFieldType(e.getTypeList().get(i),
-
ArrayIndexUtil.getFlattenedKeyFieldNames(e.getUnnestList(),
e.getProjectList().get(i)),
- sourceType,
-
ArrayIndexUtil.getArrayDepthIndicator(e.getUnnestList(),
e.getProjectList().get(i)));
+ e.getUnnestList(), e.getProjectList().get(i),
sourceType);
IAType keyType = keyTypePair.first;
anySecondaryKeyIsNullable = anySecondaryKeyIsNullable ||
keyTypePair.second;
ISerializerDeserializer keySerde =
serdeProvider.getSerializerDeserializer(keyType);
@@ -242,14 +238,15 @@ public class SecondaryArrayIndexBTreeOperationsHelper
extends SecondaryTreeIndex
return;
}
- List<Integer> arrayDepthIndicators = depthIndicators.get(fieldPos);
- List<String> fieldNames = flattenedFieldNames.get(fieldPos);
- if (arrayDepthIndicators.stream().noneMatch(b -> b > 0)) {
+ List<String> flattenedFieldName = flattenedFieldNames.get(fieldPos);
+ List<Boolean> workingUnnestFlags = unnestFlags.get(fieldPos);
+ if (workingUnnestFlags.stream().noneMatch(b -> b)) {
addAtomicFieldToBuilder(recordType, fieldPos);
+
} else {
EvalFactoryAndRecDescInvoker commandExecutor =
new
EvalFactoryAndRecDescInvoker(!evalFactoryAndRecDescStackBuilder.isUnnestEvalPopulated());
- ArrayIndexUtil.walkArrayPath(recordType, fieldNames,
arrayDepthIndicators, commandExecutor);
+ ArrayIndexUtil.walkArrayPath(recordType, flattenedFieldName,
workingUnnestFlags, commandExecutor);
}
}
@@ -278,13 +275,12 @@ public class SecondaryArrayIndexBTreeOperationsHelper
extends SecondaryTreeIndex
sourceOp = targetOp;
}
- // TODO (GLENN): Refactor to use UNNEST + PROJECT scheme.
// Perform the unnest work.
final Mutable<IOperatorDescriptor> sourceOpRef = new
MutableObject<>(sourceOp);
final Mutable<IOperatorDescriptor> targetOpRef = new
MutableObject<>(targetOp);
LoadingJobBuilder jobBuilder = new LoadingJobBuilder(spec,
sourceOpRef, targetOpRef);
- int posOfArrayIndex = findPosOfArrayIndex();
-
ArrayIndexUtil.walkArrayPath(flattenedFieldNames.get(posOfArrayIndex),
depthIndicators.get(posOfArrayIndex),
+ int posOfArrayElement = findPosOfArrayIndexElement();
+
ArrayIndexUtil.walkArrayPath(flattenedFieldNames.get(posOfArrayElement),
unnestFlags.get(posOfArrayElement),
jobBuilder);
sourceOp = sourceOpRef.getValue();
@@ -450,8 +446,7 @@ public class SecondaryArrayIndexBTreeOperationsHelper
extends SecondaryTreeIndex
: inputWidth + numTotalSecondaryKeys +
numFilterFields).toArray();
for (int i = 0; i < numTotalSecondaryKeys; i++) {
int sizeOfFieldNamesForI = flattenedFieldNames.get(i).size();
- if (depthIndicators.get(i).get(sizeOfFieldNamesForI - 1) != 0
- && (depthIndicators.get(i).stream().anyMatch(b -> b >
0))) {
+ if (unnestFlags.get(i).get(sizeOfFieldNamesForI - 1)) {
projectionList[i] = numPrimaryKeys + 1;
} else {
projectionList[i] = outColumns[outColumnsCursor++];
@@ -469,9 +464,9 @@ public class SecondaryArrayIndexBTreeOperationsHelper
extends SecondaryTreeIndex
outColumns = IntStream.range(inputWidth, inputWidth +
numArraySecondaryKeys).toArray();
for (int i = 0; i < numTotalSecondaryKeys; i++) {
int sizeOfFieldNamesForI = flattenedFieldNames.get(i).size();
- if (depthIndicators.get(i).stream().noneMatch(b -> b > 0)) {
+ if (unnestFlags.get(i).stream().noneMatch(b -> b)) {
projectionList[i] = numPrimaryKeys + atomicSKCursor++;
- } else if (depthIndicators.get(i).get(sizeOfFieldNamesForI -
1) == 0) {
+ } else if (!unnestFlags.get(i).get(sizeOfFieldNamesForI - 1)) {
projectionList[i] = outColumns[arraySKCursor++];
} else {
projectionList[i] = numPrimaryKeys +
numAtomicSecondaryKeys + numFilterFields + 1;
@@ -525,25 +520,20 @@ public class SecondaryArrayIndexBTreeOperationsHelper
extends SecondaryTreeIndex
@Override
public void executeActionOnEachArrayStep(ARecordType
startingStepRecordType, IAType workingType,
- List<String> fieldName, boolean isFirstArrayStep, boolean
isFirstUnnestInStep,
- boolean isLastUnnestInIntermediateStep) throws
AlgebricksException {
+ List<String> fieldName, boolean isFirstArrayStep, boolean
isLastUnnestInIntermediateStep)
+ throws AlgebricksException {
if (!this.isFirstWalk) {
// We have already added the appropriate UNNESTs.
return;
}
int sourceColumnForNestedArrays = numPrimaryKeys +
numAtomicSecondaryKeys + numFilterFields;
- if (isFirstUnnestInStep) {
- int sourceColumnForFirstUnnestInAtomicPath =
- isFirstArrayStep ? numPrimaryKeys :
sourceColumnForNestedArrays;
- IScalarEvaluatorFactory sef =
metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
- metadataProvider.getFunctionManager(),
startingStepRecordType, fieldName,
- sourceColumnForFirstUnnestInAtomicPath, sourceLoc);
- evalFactoryAndRecDescStackBuilder.addUnnest(sef, workingType);
- } else {
- IScalarEvaluatorFactory sef = new
ColumnAccessEvalFactory(sourceColumnForNestedArrays);
- evalFactoryAndRecDescStackBuilder.addUnnest(sef, workingType);
- }
+ int sourceColumnForFirstUnnestInAtomicPath =
+ isFirstArrayStep ? numPrimaryKeys :
sourceColumnForNestedArrays;
+ IScalarEvaluatorFactory sef =
metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
+ metadataProvider.getFunctionManager(),
startingStepRecordType, fieldName,
+ sourceColumnForFirstUnnestInAtomicPath, sourceLoc);
+ evalFactoryAndRecDescStackBuilder.addUnnest(sef, workingType);
}
@Override
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java
index dd303fc..b1d610f 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java
@@ -61,8 +61,8 @@ public class TypeUtil {
}
private static class EnforcedTypeBuilder {
- private final Deque<Triple<IAType, String, Integer>> typeStack = new
ArrayDeque<>();
- private List<Integer> keyDepthIndicators;
+ private final Deque<Triple<IAType, String, Boolean>> typeStack = new
ArrayDeque<>();
+ private List<Boolean> keyUnnestFlags;
private List<String> keyFieldNames;
private ARecordType baseRecordType;
private IAType keyFieldType;
@@ -72,11 +72,11 @@ public class TypeUtil {
private IAType endOfOpenTypeBuild;
private int indexOfOpenPart;
- public void reset(ARecordType baseRecordType, List<String>
keyFieldNames, List<Integer> keyDepthIndicators,
+ public void reset(ARecordType baseRecordType, List<String>
keyFieldNames, List<Boolean> keyUnnestFlags,
IAType keyFieldType) {
this.baseRecordType = baseRecordType;
this.keyFieldNames = keyFieldNames;
- this.keyDepthIndicators = keyDepthIndicators;
+ this.keyUnnestFlags = keyUnnestFlags;
this.keyFieldType = keyFieldType;
}
@@ -90,21 +90,19 @@ public class TypeUtil {
IAType typeIntermediate = baseRecordType;
List<String> subFieldName = new ArrayList<>();
for (int i = 0; i < keyFieldNames.size() - 1; i++) {
- typeStack.push(new Triple<>(typeIntermediate,
keyFieldNames.get(i),
- (i == 0) ? 0 : keyDepthIndicators.get(i - 1)));
+ typeStack.push(
+ new Triple<>(typeIntermediate, keyFieldNames.get(i), i
!= 0 && keyUnnestFlags.get(i - 1)));
bridgeNameFoundFromOpenTypeBuild =
typeIntermediate.getTypeName();
- if (i == 0 || keyDepthIndicators.get(i - 1) == 0) {
+ if (i == 0 || !keyUnnestFlags.get(i - 1)) {
subFieldName.add(keyFieldNames.get(i));
} else {
- // We have a multi-valued intermediate. Traverse the array
first, then add our field name.
- for (int j = 0; j < keyDepthIndicators.get(i - 1); j++) {
- typeIntermediate =
TypeComputeUtils.extractListItemType(typeIntermediate);
- if (typeIntermediate == null) {
- String fName = String.join(".", subFieldName);
- throw new
AsterixException(ErrorCode.COMPILATION_ERROR,
- "Wrong level of array nesting for field: "
+ fName);
- }
+ // We have a multi-valued intermediate. Perform our UNNEST
then add our field name.
+ typeIntermediate =
TypeComputeUtils.extractListItemType(typeIntermediate);
+ if (typeIntermediate == null) {
+ String fName = String.join(".", subFieldName);
+ throw new AsterixException(ErrorCode.COMPILATION_ERROR,
+ "No list item type found. Wrong type given
from field " + fName);
}
subFieldName.add(keyFieldNames.get(i));
}
@@ -133,27 +131,27 @@ public class TypeUtil {
}
private IAType buildNewForOpenType() {
- int depthOfOpenType = keyDepthIndicators.subList(indexOfOpenPart +
1, keyDepthIndicators.size()).stream()
- .filter(i -> i != 0).findFirst().orElse(0);
- IAType resultant = nestArrayType(keyFieldType, depthOfOpenType);
+ boolean isTypeWithUnnest = keyUnnestFlags.subList(indexOfOpenPart
+ 1, keyUnnestFlags.size()).stream()
+ .filter(i -> i).findFirst().orElse(false);
+ IAType resultant = nestArrayType(keyFieldType, isTypeWithUnnest);
// Build the type (list or record) that holds the type (list or
record) above.
resultant = nestArrayType(
new ARecordType(keyFieldNames.get(keyFieldNames.size() -
2),
new String[] {
keyFieldNames.get(keyFieldNames.size() - 1) },
new IAType[] {
AUnionType.createUnknownableType(resultant) }, true),
- keyDepthIndicators.get(indexOfOpenPart));
+ keyUnnestFlags.get(indexOfOpenPart));
// Create open part of the nested field.
for (int i = keyFieldNames.size() - 3; i > (indexOfOpenPart - 1);
i--) {
resultant = nestArrayType(
new ARecordType(keyFieldNames.get(i), new String[] {
keyFieldNames.get(i + 1) },
new IAType[] {
AUnionType.createUnknownableType(resultant) }, true),
- keyDepthIndicators.get(i));
+ keyUnnestFlags.get(i));
}
// Now update the parent to include this optional field,
accounting for intermediate arrays.
- Triple<IAType, String, Integer> gapTriple = this.typeStack.pop();
+ Triple<IAType, String, Boolean> gapTriple = this.typeStack.pop();
ARecordType parentRecord =
(ARecordType)
unnestArrayType(TypeComputeUtils.getActualType(gapTriple.first),
gapTriple.third);
IAType[] parentFieldTypes =
ArrayUtils.addAll(parentRecord.getFieldTypes().clone(),
@@ -168,9 +166,9 @@ public class TypeUtil {
private IAType buildNewForFullyClosedType() throws AsterixException {
// The schema is closed all the way to the field itself.
IAType typeIntermediate =
TypeComputeUtils.getActualType(endOfOpenTypeBuild);
- int depthOfOpenType = (indexOfOpenPart == 0) ? 0 :
keyDepthIndicators.get(indexOfOpenPart - 1);
- int depthOfKeyType = keyDepthIndicators.get(indexOfOpenPart);
- ARecordType lastNestedRecord = (ARecordType)
unnestArrayType(typeIntermediate, depthOfOpenType);
+ boolean isOpenTypeWithUnnest = indexOfOpenPart != 0 &&
keyUnnestFlags.get(indexOfOpenPart - 1);
+ boolean isKeyTypeWithUnnest = keyUnnestFlags.get(indexOfOpenPart);
+ ARecordType lastNestedRecord = (ARecordType)
unnestArrayType(typeIntermediate, isOpenTypeWithUnnest);
Map<String, IAType> recordNameTypesMap =
createRecordNameTypeMap(lastNestedRecord);
// If an enforced field already exists, verify that the type is
correct.
@@ -186,21 +184,21 @@ public class TypeUtil {
}
if (enforcedFieldType == null) {
recordNameTypesMap.put(keyFieldNames.get(keyFieldNames.size()
- 1),
-
AUnionType.createUnknownableType(nestArrayType(keyFieldType, depthOfKeyType)));
+
AUnionType.createUnknownableType(nestArrayType(keyFieldType,
isKeyTypeWithUnnest)));
}
// Build the nested record, and account for the wrapping array.
IAType resultant = nestArrayType(
new ARecordType(lastNestedRecord.getTypeName(),
recordNameTypesMap.keySet().toArray(new String[0]),
recordNameTypesMap.values().toArray(new
IAType[0]), lastNestedRecord.isOpen()),
- depthOfOpenType);
+ isOpenTypeWithUnnest);
return keepUnknown(endOfOpenTypeBuild, resultant);
}
private ARecordType buildRestOfRecord(IAType newTypeToAdd) {
IAType resultant = TypeComputeUtils.getActualType(newTypeToAdd);
while (!typeStack.isEmpty()) {
- Triple<IAType, String, Integer> typeFromStack =
typeStack.pop();
+ Triple<IAType, String, Boolean> typeFromStack =
typeStack.pop();
IAType typeIntermediate = unnestArrayType(typeFromStack.first,
typeFromStack.third);
ARecordType recordType = (ARecordType) typeIntermediate;
IAType[] fieldTypes = recordType.getFieldTypes().clone();
@@ -228,18 +226,13 @@ public class TypeUtil {
return updatedRecordType;
}
- private static IAType nestArrayType(IAType originalType, int
depthOfArrays) {
- IAType resultant = originalType;
- for (int i = 0; i < depthOfArrays; i++) {
- resultant =
- new AOrderedListType(resultant, (i == depthOfArrays -
1) ? originalType.getTypeName() : null);
- }
- return resultant;
+ private static IAType nestArrayType(IAType originalType, boolean
isWithinArray) {
+ return (isWithinArray) ? new AOrderedListType(originalType,
originalType.getTypeName()) : originalType;
}
- private static IAType unnestArrayType(IAType originalType, int
depthOfArrays) {
+ private static IAType unnestArrayType(IAType originalType, boolean
isWithinArray) {
IAType resultant = originalType;
- for (int i = 0; i < depthOfArrays; i++) {
+ if (isWithinArray) {
resultant = TypeComputeUtils.extractListItemType(resultant);
if (resultant != null) {
resultant = TypeComputeUtils.getActualType(resultant);
@@ -299,7 +292,7 @@ public class TypeUtil {
"Indexing an open field is only supported on the
record part");
}
enforcedTypeBuilder.reset(enforcedRecordType, keyFieldNames.get(i),
- Collections.nCopies(keyFieldNames.get(i).size(), 0),
keyFieldTypes.get(i));
+ Collections.nCopies(keyFieldNames.get(i).size(), false),
keyFieldTypes.get(i));
validateRecord(enforcedRecordType);
enforcedRecordType = enforcedTypeBuilder.build();
}
@@ -319,7 +312,7 @@ public class TypeUtil {
"Indexing an open field is only supported on the
record part");
}
enforcedTypeBuilder.reset(enforcedRecordType, keyFieldNames.get(i),
- Collections.nCopies(keyFieldNames.get(i).size(), 0),
keyFieldTypes.get(i));
+ Collections.nCopies(keyFieldNames.get(i).size(), false),
keyFieldTypes.get(i));
validateRecord(enforcedRecordType);
enforcedRecordType = enforcedTypeBuilder.build();
}
@@ -342,7 +335,7 @@ public class TypeUtil {
List<String> project = projectList.get(i);
enforcedTypeBuilder.reset(enforcedRecordType,
ArrayIndexUtil.getFlattenedKeyFieldNames(unnestList,
project),
- ArrayIndexUtil.getArrayDepthIndicator(unnestList,
project), typeList.get(i));
+ ArrayIndexUtil.getUnnestFlags(unnestList, project),
typeList.get(i));
validateRecord(enforcedRecordType);
enforcedRecordType = enforcedTypeBuilder.build();
}
