This is an automated email from the ASF dual-hosted git repository.
capistrant pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 9bb5fedaec1 fix issue with nested column processing/storage of empty
fields (#19072)
9bb5fedaec1 is described below
commit 9bb5fedaec1934b322db16850159463ec7dd713e
Author: Clint Wylie <[email protected]>
AuthorDate: Tue Mar 3 17:42:08 2026 -0800
fix issue with nested column processing/storage of empty fields (#19072)
* fix issue with nested column processing/storage of empty fields
changes:
* fix bug in `NestedPathFinder.toNormalizedJsonPath` creating incorrect
paths when given an empty field name
* fix `NestedPathFinder.parseJsonPath` to correctly detect illegal empty
paths consisting of consecutive . characters
* added `NestedPathFinder.parseBadJsonPath` to read nested column fields
dictionaries and detect and fixup illegal path expressions, using a newly added
`FieldsFixupIndexed` to swap the bad values with good values
* `NestedDataColumnSupplier` on column read attempts to detect bad paths
written by the bugged version of `NestedPathFinder.toNormalizedJsonPath`
* added 'pathParserVersion' field to nested column part serde so that newly
written nested columns after the bug fix can skip checking for the bug
* fixes
* better name for parameter
* fix javadoc
* add serde test
* adjust
* add better test with prebuilt segment containing the bugged field names
---
.../segment/nested/NestedDataColumnSupplier.java | 151 ++++++-
.../segment/nested/NestedDataColumnSupplierV4.java | 32 +-
.../druid/segment/nested/NestedDataColumnV3.java | 9 +-
.../druid/segment/nested/NestedDataColumnV4.java | 9 +-
.../druid/segment/nested/NestedPathFinder.java | 96 ++++-
.../serde/NestedCommonFormatColumnPartSerde.java | 60 ++-
...NestedDataColumnSupplierEmptyFieldsBugTest.java | 106 +++++
...DataColumnSupplierFieldDictionaryFixupTest.java | 143 +++++++
.../nested/NestedDataColumnSupplierTest.java | 7 +-
.../nested/NestedDataColumnSupplierV4Test.java | 229 +---------
.../druid/segment/nested/NestedPathFinderTest.java | 459 ++++++++++++---------
.../NestedCommonFormatColumnPartSerdeTest.java | 61 +++
.../druid.segment | Bin 0 -> 3136 bytes
13 files changed, 880 insertions(+), 482 deletions(-)
diff --git
a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSupplier.java
b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSupplier.java
index 5c8067c131c..14af54d86f5 100644
---
a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSupplier.java
+++
b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSupplier.java
@@ -19,10 +19,15 @@
package org.apache.druid.segment.nested;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Supplier;
+import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
+import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
+import org.apache.druid.error.DruidException;
import org.apache.druid.java.util.common.RE;
import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import org.apache.druid.segment.column.ColumnBuilder;
import org.apache.druid.segment.column.ColumnConfig;
import org.apache.druid.segment.column.ColumnIndexSupplier;
@@ -39,16 +44,20 @@ import
org.apache.druid.segment.index.semantic.NullValueIndex;
import org.apache.druid.segment.serde.ColumnSerializerUtils;
import org.apache.druid.segment.serde.NestedCommonFormatColumnPartSerde;
+import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
+import java.util.Iterator;
+import java.util.List;
public class NestedDataColumnSupplier implements
Supplier<NestedCommonFormatColumn>, ColumnIndexSupplier
{
public static NestedDataColumnSupplier read(
ColumnType logicalType,
boolean hasNulls,
+ byte pathParserVersion,
ByteBuffer bb,
ColumnBuilder columnBuilder,
ColumnConfig columnConfig,
@@ -82,7 +91,11 @@ public class NestedDataColumnSupplier implements
Supplier<NestedCommonFormatColu
doubleDictionarySupplier = parent.doubleDictionarySupplier;
arrayDictionarySupplier = parent.arrayDictionarySupplier;
} else {
- fieldsSupplier =
StringEncodingStrategies.getStringDictionarySupplier(mapper, bb, byteOrder);
+ if (pathParserVersion == 0x00) {
+ fieldsSupplier = getAndFixFieldsSupplier(bb, byteOrder, mapper);
+ } else {
+ fieldsSupplier =
StringEncodingStrategies.getStringDictionarySupplier(mapper, bb, byteOrder);
+ }
fieldInfo = FieldTypeInfo.read(bb, fieldsSupplier.get().size());
final ByteBuffer stringDictionaryBuffer =
NestedCommonFormatColumnPartSerde.loadInternalFile(
mapper,
@@ -182,6 +195,52 @@ public class NestedDataColumnSupplier implements
Supplier<NestedCommonFormatColu
}
}
+
+ /**
+ * Detects if field dictionary contains any invalid entries from a bug which
previously existed in
+ * {@link NestedPathFinder#toNormalizedJsonPath(List)} to generate invalid
path expressions when faced with empty
+ * field names - for example {"":{"a":1}} would incorrectly store the path
as $..a instead of $[''].a.
+ * <p>
+ * If this method detects any illegal paths, the field dictionary is wrapped
using {@link FieldsFixupIndexed}, which
+ * replaces the invalid values with corrected values, using {@link
NestedPathFinder#parseBadJsonPath(String)} and
+ * feeding that back into the now fixed {@link
NestedPathFinder#toNormalizedJsonPath(List)}.
+ * <p>
+ * Columns written after the bug was fixed will store {@link
NestedCommonFormatColumnPartSerde#pathParserVersion} as
+ * 0x01 or greater, to indicate that we do not need to call this method to
check for fixing up paths.
+ * <p>
+ * see https://github.com/apache/druid/pull/19072 for additional details.
+ */
+ @VisibleForTesting
+ static Supplier<? extends Indexed<ByteBuffer>> getAndFixFieldsSupplier(
+ ByteBuffer bb,
+ ByteOrder byteOrder,
+ SegmentFileMapper mapper
+ )
+ {
+ final Supplier<? extends Indexed<ByteBuffer>> fieldsSupplier;
+ Supplier<? extends Indexed<ByteBuffer>> _fieldsSupplier =
+ StringEncodingStrategies.getStringDictionarySupplier(mapper, bb,
byteOrder);
+ // check for existence of bug to detect if we need a fixup adapter or not
+ Indexed<ByteBuffer> fields = _fieldsSupplier.get();
+ Int2ObjectMap<ByteBuffer> fixupMap = new Int2ObjectOpenHashMap<>();
+ for (int i = 0; i < fields.size(); i++) {
+ String path = StringUtils.fromUtf8Nullable(fields.get(i));
+ try {
+ NestedPathFinder.parseJsonPath(path);
+ }
+ catch (DruidException d) {
+ String fixed =
NestedPathFinder.toNormalizedJsonPath(NestedPathFinder.parseBadJsonPath(path));
+ fixupMap.put(i, StringUtils.toUtf8ByteBuffer(fixed));
+ }
+ }
+ if (fixupMap.isEmpty()) {
+ fieldsSupplier = _fieldsSupplier;
+ } else {
+ fieldsSupplier = () -> new FieldsFixupIndexed(_fieldsSupplier.get(),
fixupMap);
+ }
+ return fieldsSupplier;
+ }
+
private final String columnName;
private final Supplier<? extends Indexed<ByteBuffer>> fieldSupplier;
private final FieldTypeInfo fieldInfo;
@@ -268,4 +327,94 @@ public class NestedDataColumnSupplier implements
Supplier<NestedCommonFormatColu
}
return null;
}
+
+ /**
+ * {@link Indexed} implementation which contains a map of positions to
replace with corrected values by
+ * {@link #getAndFixFieldsSupplier(ByteBuffer, ByteOrder,
SegmentFileMapper)}.
+ * <p>
+ * This implementation is no longer {@link #isSorted()}, despite the
underlying {@link Indexed} being so, as the
+ * replaced values might violate the old sort order, so this cannot provide
that guarantee. Despite this,
+ * {@link #indexOf(ByteBuffer)} will still function for finding if items
exist in the dictionary, the replaced values
+ * are searched prior to calling {@link #indexOf(ByteBuffer)} on the
underlying dictionary, so for the values we are
+ * trying to find will always report their correct positions.
+ * It is important that the fixed dictionary retains its original order
because the positions are used as the internal
+ * field file names (instead of the paths themselves) in the segment file.
Callers however should not expect
+ * iterating the dictionary to provide values in sorted order.
+ */
+ @VisibleForTesting
+ public static class FieldsFixupIndexed implements Indexed<ByteBuffer>
+ {
+ private final Indexed<ByteBuffer> delegate;
+ private final Int2ObjectMap<ByteBuffer> fixup;
+
+ private FieldsFixupIndexed(Indexed<ByteBuffer> delegate,
Int2ObjectMap<ByteBuffer> fixup)
+ {
+ this.delegate = delegate;
+ this.fixup = fixup;
+ }
+
+ @Override
+ public int size()
+ {
+ return delegate.size();
+ }
+
+ @Nullable
+ @Override
+ public ByteBuffer get(int index)
+ {
+ if (fixup.containsKey(index)) {
+ return fixup.get(index).asReadOnlyBuffer();
+ }
+ return delegate.get(index);
+ }
+
+ @Override
+ public int indexOf(@Nullable ByteBuffer value)
+ {
+ for (Int2ObjectMap.Entry<ByteBuffer> entry : fixup.int2ObjectEntrySet())
{
+ if (entry.getValue().equals(value)) {
+ return entry.getIntKey();
+ }
+ }
+ return delegate.indexOf(value);
+ }
+
+ @Nonnull
+ @Override
+ public Iterator<ByteBuffer> iterator()
+ {
+ return new Iterator<>()
+ {
+ int pos = 0;
+ final int size = delegate.size();
+ final Iterator<ByteBuffer> delegateIterator = delegate.iterator();
+
+ @Override
+ public boolean hasNext()
+ {
+ return pos < size;
+ }
+
+ @Override
+ public ByteBuffer next()
+ {
+ if (fixup.containsKey(pos)) {
+ // move delegate iterator forward, but we're going to return our
own value
+ delegateIterator.next();
+ // this is sad, but downstream stuff wants ByteBuffer, and is less
sad than the original bug
+ return fixup.get(pos++).asReadOnlyBuffer();
+ }
+ pos++;
+ return delegateIterator.next();
+ }
+ };
+ }
+
+ @Override
+ public void inspectRuntimeShape(RuntimeShapeInspector inspector)
+ {
+
+ }
+ }
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4.java
b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4.java
index 4363fd7eb33..5ce4848459e 100644
---
a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4.java
+++
b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4.java
@@ -80,7 +80,7 @@ public class NestedDataColumnSupplierV4 implements
Supplier<ComplexColumn>
try {
final SegmentFileMapper mapper = columnBuilder.getFileMapper();
final ComplexColumnMetadata metadata;
- final GenericIndexed<ByteBuffer> fields;
+ final Supplier<? extends Indexed<ByteBuffer>> fields;
final FieldTypeInfo fieldInfo;
final CompressedVariableSizedBlobColumnSupplier
compressedRawColumnSupplier;
final ImmutableBitmap nullValues;
@@ -95,17 +95,23 @@ public class NestedDataColumnSupplierV4 implements
Supplier<ComplexColumn>
IndexMerger.SERIALIZER_UTILS.readString(bb),
ComplexColumnMetadata.class
);
- fields = GenericIndexed.read(bb, GenericIndexed.UTF8_STRATEGY, mapper);
- fieldInfo = FieldTypeInfo.read(bb, fields.size());
+ if (version < 0x04) {
+ // older than v4 uses jq paths
+ fields = GenericIndexed.read(bb, GenericIndexed.UTF8_STRATEGY,
mapper)::singleThreaded;
+ } else {
+ fields = NestedDataColumnSupplier.getAndFixFieldsSupplier(bb,
metadata.getByteOrder(), mapper);
+ }
+ Indexed<ByteBuffer> fieldsDict = fields.get();
+ fieldInfo = FieldTypeInfo.read(bb, fieldsDict.size());
- if (fields.size() == 0) {
+ if (fieldsDict.size() == 0) {
// all nulls, in the future we'll deal with this better... but for
now lets just call it a string because
// it is the most permissive (besides json)
simpleType = ColumnType.STRING;
- } else if (fields.size() == 1 &&
- ((version == 0x03 &&
NestedPathFinder.JQ_PATH_ROOT.equals(StringUtils.fromUtf8(fields.get(0)))) ||
+ } else if (fieldsDict.size() == 1 &&
+ ((version == 0x03 &&
NestedPathFinder.JQ_PATH_ROOT.equals(StringUtils.fromUtf8(fieldsDict.get(0))))
||
((version == 0x04 || version == 0x05)
- &&
NestedPathFinder.JSON_PATH_ROOT.equals(StringUtils.fromUtf8(fields.get(0)))))
+ &&
NestedPathFinder.JSON_PATH_ROOT.equals(StringUtils.fromUtf8(fieldsDict.get(0)))))
) {
simpleType = fieldInfo.getTypes(0).getSingleType();
} else {
@@ -213,7 +219,7 @@ public class NestedDataColumnSupplierV4 implements
Supplier<ComplexColumn>
private final byte version;
private final String columnName;
private final ColumnConfig columnConfig;
- private final GenericIndexed<ByteBuffer> fields;
+ private final Supplier<? extends Indexed<ByteBuffer>> fieldsSupplier;
private final FieldTypeInfo fieldInfo;
private final CompressedVariableSizedBlobColumnSupplier
compressedRawColumnSupplier;
private final ImmutableBitmap nullValues;
@@ -233,7 +239,7 @@ public class NestedDataColumnSupplierV4 implements
Supplier<ComplexColumn>
byte version,
String columnName,
ColumnConfig columnConfig,
- GenericIndexed<ByteBuffer> fields,
+ Supplier<? extends Indexed<ByteBuffer>> fieldsSupplier,
FieldTypeInfo fieldInfo,
CompressedVariableSizedBlobColumnSupplier compressedRawColumnSupplier,
ImmutableBitmap nullValues,
@@ -250,7 +256,7 @@ public class NestedDataColumnSupplierV4 implements
Supplier<ComplexColumn>
this.version = version;
this.columnName = columnName;
this.columnConfig = columnConfig;
- this.fields = fields;
+ this.fieldsSupplier = fieldsSupplier;
this.fieldInfo = fieldInfo;
this.compressedRawColumnSupplier = compressedRawColumnSupplier;
this.nullValues = nullValues;
@@ -291,7 +297,7 @@ public class NestedDataColumnSupplierV4 implements
Supplier<ComplexColumn>
columnConfig,
compressedRawColumnSupplier,
nullValues,
- fields,
+ fieldsSupplier,
fieldInfo,
stringDictionarySupplier,
longDictionarySupplier,
@@ -310,7 +316,7 @@ public class NestedDataColumnSupplierV4 implements
Supplier<ComplexColumn>
columnConfig,
compressedRawColumnSupplier,
nullValues,
- fields,
+ fieldsSupplier,
fieldInfo,
stringDictionarySupplier,
longDictionarySupplier,
@@ -329,7 +335,7 @@ public class NestedDataColumnSupplierV4 implements
Supplier<ComplexColumn>
columnConfig,
compressedRawColumnSupplier,
nullValues,
- fields::singleThreaded,
+ fieldsSupplier,
fieldInfo,
stringDictionarySupplier,
longDictionarySupplier,
diff --git
a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnV3.java
b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnV3.java
index d19981fb929..c2a5df93174 100644
---
a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnV3.java
+++
b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnV3.java
@@ -26,7 +26,6 @@ import org.apache.druid.segment.column.ColumnConfig;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.data.CompressedVariableSizedBlobColumnSupplier;
import org.apache.druid.segment.data.FixedIndexed;
-import org.apache.druid.segment.data.GenericIndexed;
import org.apache.druid.segment.data.Indexed;
import org.apache.druid.segment.file.SegmentFileMapper;
@@ -34,8 +33,8 @@ import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.List;
-public final class NestedDataColumnV3<TStringDictionary extends
Indexed<ByteBuffer>>
- extends CompressedNestedDataComplexColumn<Indexed<ByteBuffer>,
TStringDictionary>
+public final class NestedDataColumnV3<TKeyDictionary extends
Indexed<ByteBuffer>, TStringDictionary extends Indexed<ByteBuffer>>
+ extends CompressedNestedDataComplexColumn<TKeyDictionary,
TStringDictionary>
{
public NestedDataColumnV3(
String columnName,
@@ -43,7 +42,7 @@ public final class NestedDataColumnV3<TStringDictionary
extends Indexed<ByteBuff
ColumnConfig columnConfig,
CompressedVariableSizedBlobColumnSupplier compressedRawColumnSupplier,
ImmutableBitmap nullValues,
- GenericIndexed<ByteBuffer> fields,
+ Supplier<TKeyDictionary> fieldsSupplier,
FieldTypeInfo fieldInfo,
Supplier<TStringDictionary> stringDictionary,
Supplier<FixedIndexed<Long>> longDictionarySupplier,
@@ -59,7 +58,7 @@ public final class NestedDataColumnV3<TStringDictionary
extends Indexed<ByteBuff
columnConfig,
compressedRawColumnSupplier,
nullValues,
- fields::singleThreaded,
+ fieldsSupplier,
fieldInfo,
stringDictionary,
longDictionarySupplier,
diff --git
a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnV4.java
b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnV4.java
index be07c61d78f..4614150b62c 100644
---
a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnV4.java
+++
b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnV4.java
@@ -25,7 +25,6 @@ import org.apache.druid.segment.column.ColumnConfig;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.data.CompressedVariableSizedBlobColumnSupplier;
import org.apache.druid.segment.data.FixedIndexed;
-import org.apache.druid.segment.data.GenericIndexed;
import org.apache.druid.segment.data.Indexed;
import org.apache.druid.segment.file.SegmentFileMapper;
import org.apache.druid.segment.serde.ColumnSerializerUtils;
@@ -34,8 +33,8 @@ import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.List;
-public final class NestedDataColumnV4<TStringDictionary extends
Indexed<ByteBuffer>>
- extends CompressedNestedDataComplexColumn<Indexed<ByteBuffer>,
TStringDictionary>
+public final class NestedDataColumnV4<TKeyDictionary extends
Indexed<ByteBuffer>, TStringDictionary extends Indexed<ByteBuffer>>
+ extends CompressedNestedDataComplexColumn<TKeyDictionary,
TStringDictionary>
{
public NestedDataColumnV4(
String columnName,
@@ -43,7 +42,7 @@ public final class NestedDataColumnV4<TStringDictionary
extends Indexed<ByteBuff
ColumnConfig columnConfig,
CompressedVariableSizedBlobColumnSupplier compressedRawColumnSupplier,
ImmutableBitmap nullValues,
- GenericIndexed<ByteBuffer> fields,
+ Supplier<TKeyDictionary> fieldsSupplier,
FieldTypeInfo fieldInfo,
Supplier<TStringDictionary> stringDictionary,
Supplier<FixedIndexed<Long>> longDictionarySupplier,
@@ -59,7 +58,7 @@ public final class NestedDataColumnV4<TStringDictionary
extends Indexed<ByteBuff
columnConfig,
compressedRawColumnSupplier,
nullValues,
- fields::singleThreaded,
+ fieldsSupplier,
fieldInfo,
stringDictionary,
longDictionarySupplier,
diff --git
a/processing/src/main/java/org/apache/druid/segment/nested/NestedPathFinder.java
b/processing/src/main/java/org/apache/druid/segment/nested/NestedPathFinder.java
index 2fc52f88df6..597d229375e 100644
---
a/processing/src/main/java/org/apache/druid/segment/nested/NestedPathFinder.java
+++
b/processing/src/main/java/org/apache/druid/segment/nested/NestedPathFinder.java
@@ -31,6 +31,7 @@ import java.util.stream.IntStream;
public class NestedPathFinder
{
+ public static final byte VERSION = 0x01;
public static final String JSON_PATH_ROOT = "$";
public static final String JQ_PATH_ROOT = ".";
@@ -86,7 +87,7 @@ public class NestedPathFinder
bob.append(JSON_PATH_ROOT);
}
final String id = partFinder.getPartIdentifier();
- if (id.contains(".") || id.contains("'") || id.contains("\"") ||
id.contains("[") || id.contains("]")) {
+ if (id.isEmpty() || id.contains(".") || id.contains("'") ||
id.contains("\"") || id.contains("[") || id.contains("]")) {
bob.append("['").append(id).append("']");
} else {
bob.append(".");
@@ -104,9 +105,38 @@ public class NestedPathFinder
}
/**
- * split a JSONPath path into a series of extractors to find things in stuff
+ * Split a JSONPath expression into a series of {@link NestedPathPart}
representing the structure of the path
+ * expression. This allows using with {@link NestedPathFinder#find(Object,
List)} in order to extract values from
+ * json objects represented as java {@link Map} and {@link List}.
*/
public static List<NestedPathPart> parseJsonPath(@Nullable String path)
+ {
+ return parseJsonPathInternal(path, false);
+ }
+
+ /**
+ * Split a JSONPath expression into a series of {@link NestedPathPart}
representing the structure of the path
+ * expression. This method is a variant of {@link #parseJsonPath(String)}
which allows fixing up any illegal
+ * expressions encountered from a previously bugged version of {@link
#toNormalizedJsonPath(List)}, which incorrectly
+ * created path expressions like '$..a' or '$.[0].a' when encountering empty
fields instead of the correct '$[''].a'
+ * and '$[''][0].a'. This method should only be used for converting field
dictionaries stored in columns to fix the
+ * incorrectly stored paths, and never for parsing user input expressions.
+ */
+ public static List<NestedPathPart> parseBadJsonPath(@Nullable String path)
+ {
+ return parseJsonPathInternal(path, true);
+ }
+
+ /**
+ * Split a JSONPath expression into a series of {@link NestedPathPart}
representing the structure of the path
+ * expression. This allows using with {@link NestedPathFinder#find(Object,
List)} in order to extract values from
+ * json objects represented as java {@link Map} and {@link List}.
+ * <p>
+ * If 'allowFixEmptyFieldsBadPaths is true, this method allows fixing up any
illegal expressions encountered from a
+ * previously bugged version of {@link #toNormalizedJsonPath(List)}, which
incorrectly created path expressions like
+ * '$..a' or '$.[0].a' when encountering empty fields instead of the correct
'$[''].a' and '$[''][0].a'.
+ */
+ private static List<NestedPathPart> parseJsonPathInternal(@Nullable String
path, boolean allowFixEmptyFieldsBadPaths)
{
if (path == null || path.isEmpty()) {
return Collections.emptyList();
@@ -130,23 +160,42 @@ public class NestedPathFinder
for (int i = 1; i < path.length(); i++) {
final char current = path.charAt(i);
if (current == '.' && arrayMark < 0 && quoteMark < 0) {
+ if (dotMark == (i - 1)) {
+ // in mode for fixing up empty fields bug which could have added 2
consecutive dots, we allow converting the
+ // empty space into ['']
+ if (!allowFixEmptyFieldsBadPaths) {
+ badFormatJsonPath(
+ path,
+ "found '.' at invalid position [%s], must not follow '.' or
must be contained with '",
+ i
+ );
+ }
+ }
if (dotMark >= 0) {
- parts.add(new NestedPathField(getPathSubstring(path, partMark, i)));
+ parts.add(new NestedPathField(path.substring(partMark, i)));
}
dotMark = i;
partMark = i + 1;
} else if (current == '[' && arrayMark < 0 && quoteMark < 0) {
- if (dotMark == (i - 1) && dotMark != 0) {
- badFormatJsonPath(path, "found '[' at invalid position [%s], must
not follow '.' or must be contained with '", i);
+ if (dotMark == (i - 1)) {
+ // in mode for fixing up empty fields bug which could have added a
dot immediately before an array, we allow
+ // converting the empty space into ['']
+ if (!allowFixEmptyFieldsBadPaths) {
+ badFormatJsonPath(
+ path,
+ "found '[' at invalid position [%s], must not follow '.' or
must be contained with '",
+ i
+ );
+ }
}
if (dotMark >= 0 && i > 1) {
- parts.add(new NestedPathField(getPathSubstring(path, partMark, i)));
+ parts.add(new NestedPathField(path.substring(partMark, i)));
dotMark = -1;
}
arrayMark = i;
partMark = i + 1;
} else if (current == ']' && arrayMark >= 0 && quoteMark < 0) {
- String maybeNumber = getPathSubstring(path, partMark, i);
+ String maybeNumber = path.substring(partMark, i);
try {
int index = Integer.parseInt(maybeNumber);
parts.add(new NestedPathArrayElement(index));
@@ -155,7 +204,11 @@ public class NestedPathFinder
partMark = i + 1;
}
catch (NumberFormatException ignored) {
- badFormatJsonPath(path, "array specifier [%s] should be a number, it
was not. Use ' if this value was meant to be a field name", maybeNumber);
+ badFormatJsonPath(
+ path,
+ "array specifier [%s] should be a number, it was not. Use ' if
this value was meant to be a field name",
+ maybeNumber
+ );
}
} else if (dotMark == -1 && arrayMark == -1) {
badFormatJsonPath(path, "path parts must be separated with '.'");
@@ -173,7 +226,7 @@ public class NestedPathFinder
badFormatJsonPath(path, "closing single-quote (') must immediately
precede ']'");
}
- parts.add(new NestedPathField(getPathSubstring(path, partMark, i)));
+ parts.add(new NestedPathField(path.substring(partMark, i)));
dotMark = -1;
quoteMark = -1;
// chomp to next char to eat closing array
@@ -198,6 +251,7 @@ public class NestedPathFinder
return parts;
}
+
/**
* Given a list of part finders, convert it to a "normalized" 'jq' path
format that is consistent with how
* {@link StructuredDataProcessor} constructs field path names
@@ -235,7 +289,7 @@ public class NestedPathFinder
List<NestedPathPart> parts = new ArrayList<>();
if (path.charAt(0) != '.') {
- badFormat(path, "it must start with '.'");
+ badFormatJq(path, "it must start with '.'");
}
int partMark = -1; // position to start the next substring to build the
path part
@@ -257,13 +311,13 @@ public class NestedPathFinder
parts.add(new NestedPathField(getPathSubstring(path, partMark,
i)));
dotMark = -1;
} else {
- badFormat(path, "found '?' at invalid position [%s]", i);
+ badFormatJq(path, "found '?' at invalid position [%s]", i);
}
}
partMark = i + 1;
} else if (current == '[' && arrayMark < 0 && quoteMark < 0) {
if (dotMark == (i - 1) && dotMark != 0) {
- badFormat(path, "found '[' at invalid position [%s], must not follow
'.' or must be contained with '\"'", i);
+ badFormatJq(path, "found '[' at invalid position [%s], must not
follow '.' or must be contained with '\"'", i);
}
if (dotMark >= 0 && i > 1) {
parts.add(new NestedPathField(getPathSubstring(path, partMark, i)));
@@ -281,16 +335,16 @@ public class NestedPathFinder
partMark = i + 1;
}
catch (NumberFormatException ignored) {
- badFormat(path, "array specifier [%s] should be a number, it was
not. Use \"\" if this value was meant to be a field name", maybeNumber);
+ badFormatJq(path, "array specifier [%s] should be a number, it was
not. Use \"\" if this value was meant to be a field name", maybeNumber);
}
} else if (dotMark == -1 && arrayMark == -1) {
- badFormat(path, "path parts must be separated with '.'");
+ badFormatJq(path, "path parts must be separated with '.'");
} else if (current == '"' && quoteMark < 0) {
if (partMark != i) {
- badFormat(path, "found '\"' at invalid position [%s], it must
immediately follow '.' or '['", i);
+ badFormatJq(path, "found '\"' at invalid position [%s], it must
immediately follow '.' or '['", i);
}
if (arrayMark > 0 && arrayMark != i - 1) {
- badFormat(path, "'\"' within '[', must be immediately after");
+ badFormatJq(path, "'\"' within '[', must be immediately after");
}
quoteMark = i;
partMark = i + 1;
@@ -304,7 +358,7 @@ public class NestedPathFinder
break;
}
if (path.charAt(i) != ']') {
- badFormat(path, "closing '\"' must immediately precede ']'");
+ badFormatJq(path, "closing '\"' must immediately precede ']'");
}
partMark = i + 1;
arrayMark = -1;
@@ -316,10 +370,10 @@ public class NestedPathFinder
// add the last element, this should never be an array because they close
themselves
if (partMark < path.length()) {
if (quoteMark != -1) {
- badFormat(path, "unterminated '\"'");
+ badFormatJq(path, "unterminated '\"'");
}
if (arrayMark != -1) {
- badFormat(path, "unterminated '['");
+ badFormatJq(path, "unterminated '['");
}
parts.add(new NestedPathField(path.substring(partMark)));
}
@@ -330,12 +384,12 @@ public class NestedPathFinder
private static String getPathSubstring(String path, int start, int end)
{
if (end - start < 1) {
- badFormat(path, "path parts separated by '.' must not be empty");
+ badFormatJq(path, "path parts separated by '.' must not be empty");
}
return path.substring(start, end);
}
- private static void badFormat(String path, String message, Object... args)
+ private static void badFormatJq(String path, String message, Object... args)
{
throw InvalidInput.exception("jq path [%s] is invalid, %s", path,
StringUtils.format(message, args));
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/serde/NestedCommonFormatColumnPartSerde.java
b/processing/src/main/java/org/apache/druid/segment/serde/NestedCommonFormatColumnPartSerde.java
index f65a126b4c5..5bf50769b55 100644
---
a/processing/src/main/java/org/apache/druid/segment/serde/NestedCommonFormatColumnPartSerde.java
+++
b/processing/src/main/java/org/apache/druid/segment/serde/NestedCommonFormatColumnPartSerde.java
@@ -38,6 +38,7 @@ import org.apache.druid.segment.file.SegmentFileMapper;
import org.apache.druid.segment.nested.NestedCommonFormatColumn;
import org.apache.druid.segment.nested.NestedCommonFormatColumnFormatSpec;
import org.apache.druid.segment.nested.NestedDataColumnSupplier;
+import org.apache.druid.segment.nested.NestedPathFinder;
import org.apache.druid.segment.nested.ObjectStorageEncoding;
import org.apache.druid.segment.nested.ScalarDoubleColumnAndIndexSupplier;
import org.apache.druid.segment.nested.ScalarLongColumnAndIndexSupplier;
@@ -48,6 +49,7 @@ import javax.annotation.Nullable;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
+import java.util.Objects;
/**
* {@link ColumnPartSerde} shared by all {@link NestedCommonFormatColumn}. The
{@link #logicalType} defines the native
@@ -87,7 +89,8 @@ public class NestedCommonFormatColumnPartSerde implements
ColumnPartSerde
@JsonProperty("enforceLogicalType") boolean enforceLogicalType,
@JsonProperty("byteOrder") ByteOrder byteOrder,
@JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory
bitmapSerdeFactory,
- @JsonProperty("columnFormatSpec") @Nullable FormatSpec columnFormatSpec
+ @JsonProperty("columnFormatSpec") @Nullable FormatSpec columnFormatSpec,
+ @JsonProperty("pathParserVersion") @Nullable Byte pathParserVersion
)
{
return new NestedCommonFormatColumnPartSerde(
@@ -98,6 +101,7 @@ public class NestedCommonFormatColumnPartSerde implements
ColumnPartSerde
byteOrder,
bitmapSerdeFactory,
columnFormatSpec,
+ pathParserVersion,
null
);
}
@@ -107,6 +111,7 @@ public class NestedCommonFormatColumnPartSerde implements
ColumnPartSerde
private final boolean isVariantType;
private final boolean enforceLogicalType;
private final ByteOrder byteOrder;
+ private final byte pathParserVersion;
private final BitmapSerdeFactory bitmapSerdeFactory;
@Nullable
private final FormatSpec columnFormatSpec;
@@ -114,7 +119,6 @@ public class NestedCommonFormatColumnPartSerde implements
ColumnPartSerde
@Nullable
private final Serializer serializer;
-
private NestedCommonFormatColumnPartSerde(
ColumnType logicalType,
boolean hasNulls,
@@ -123,6 +127,7 @@ public class NestedCommonFormatColumnPartSerde implements
ColumnPartSerde
ByteOrder byteOrder,
BitmapSerdeFactory bitmapSerdeFactory,
@Nullable FormatSpec columnFormatSpec,
+ @Nullable Byte pathParserVersion,
@Nullable Serializer serializer
)
{
@@ -131,6 +136,7 @@ public class NestedCommonFormatColumnPartSerde implements
ColumnPartSerde
this.isVariantType = isVariant;
this.enforceLogicalType = enforceLogicalType;
this.byteOrder = byteOrder;
+ this.pathParserVersion = pathParserVersion == null ? 0x00 :
pathParserVersion;
this.bitmapSerdeFactory = bitmapSerdeFactory;
this.serializer = serializer;
this.columnFormatSpec = columnFormatSpec;
@@ -162,13 +168,13 @@ public class NestedCommonFormatColumnPartSerde implements
ColumnPartSerde
return new NestedColumnDeserializer();
}
- @JsonProperty
+ @JsonProperty("logicalType")
public ColumnType getLogicalType()
{
return logicalType;
}
- @JsonProperty
+ @JsonProperty("hasNulls")
public boolean isHasNulls()
{
return hasNulls;
@@ -186,25 +192,63 @@ public class NestedCommonFormatColumnPartSerde implements
ColumnPartSerde
return enforceLogicalType;
}
- @JsonProperty
+ @JsonProperty("byteOrder")
public ByteOrder getByteOrder()
{
return byteOrder;
}
- @JsonProperty
+ @JsonProperty("bitmapSerdeFactory")
public BitmapSerdeFactory getBitmapSerdeFactory()
{
return bitmapSerdeFactory;
}
@Nullable
- @JsonProperty
+ @JsonProperty("columnFormatSpec")
public FormatSpec getColumnFormatSpec()
{
return columnFormatSpec;
}
+ @JsonProperty("pathParserVersion")
+ public byte getPathParserVersion()
+ {
+ return pathParserVersion;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ NestedCommonFormatColumnPartSerde that =
(NestedCommonFormatColumnPartSerde) o;
+ return hasNulls == that.hasNulls
+ && isVariantType == that.isVariantType
+ && enforceLogicalType == that.enforceLogicalType
+ && pathParserVersion == that.pathParserVersion
+ && Objects.equals(logicalType, that.logicalType)
+ && Objects.equals(byteOrder, that.byteOrder)
+ && Objects.equals(bitmapSerdeFactory, that.bitmapSerdeFactory)
+ && Objects.equals(columnFormatSpec, that.columnFormatSpec);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(
+ logicalType,
+ hasNulls,
+ isVariantType,
+ enforceLogicalType,
+ byteOrder,
+ pathParserVersion,
+ bitmapSerdeFactory,
+ columnFormatSpec
+ );
+ }
+
private class StringColumnDeserializer implements Deserializer
{
@Override
@@ -344,6 +388,7 @@ public class NestedCommonFormatColumnPartSerde implements
ColumnPartSerde
NestedDataColumnSupplier supplier = NestedDataColumnSupplier.read(
logicalType,
hasNulls,
+ pathParserVersion,
buffer,
builder,
columnConfig,
@@ -437,6 +482,7 @@ public class NestedCommonFormatColumnPartSerde implements
ColumnPartSerde
byteOrder,
bitmapSerdeFactory,
FormatSpec.forSerde(columnFormatSpec),
+ NestedPathFinder.VERSION,
serializer
);
}
diff --git
a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierEmptyFieldsBugTest.java
b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierEmptyFieldsBugTest.java
new file mode 100644
index 00000000000..25ad0281862
--- /dev/null
+++
b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierEmptyFieldsBugTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.nested;
+
+import org.apache.druid.guice.BuiltInTypesModule;
+import org.apache.druid.java.util.common.io.Closer;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
+import org.apache.druid.segment.QueryableIndex;
+import org.apache.druid.segment.QueryableIndexCursorFactory;
+import org.apache.druid.segment.TestHelper;
+import org.apache.druid.segment.VirtualColumns;
+import org.apache.druid.segment.column.ColumnHolder;
+import org.apache.druid.segment.vector.VectorCursor;
+import org.apache.druid.segment.vector.VectorObjectSelector;
+import org.apache.druid.segment.virtual.NestedFieldVirtualColumn;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+
+public class NestedDataColumnSupplierEmptyFieldsBugTest
+{
+ @BeforeAll
+ public static void staticSetup()
+ {
+ BuiltInTypesModule.registerHandlersAndSerde();
+ }
+
+ @TempDir
+ File tempDir;
+
+ @Test
+ public void testReadSegmentWithBuggyPaths() throws IOException
+ {
+ // segment data:
+ // {"timestamp": "2026-01-01T00:00:00", "str":"b", "obj":{"": {"a":
123}}}
+ // {"timestamp": "2026-01-01T00:00:00", "str":"a", "obj":{"":
[{"a":456}]}}
+ // prior to https://github.com/apache/druid/pull/19072 this would fail
with an error like
+ // org.apache.druid.error.DruidException: jq path [$[''].a] is invalid,
path parts separated by '.' must not be empty
+ // (which was also incorrect since it is a JSONPath not jq path)
+ File tmpLocation = new File(tempDir, "druid.segment");
+ Files.copy(
+ NestedDataColumnSupplierEmptyFieldsBugTest.class.getClassLoader()
+
.getResourceAsStream("nested_segment_empty_fieldname_bug/druid.segment"),
+ tmpLocation.toPath(),
+ StandardCopyOption.REPLACE_EXISTING
+ );
+ try (Closer closer = Closer.create()) {
+ QueryableIndex theIndex =
closer.register(TestHelper.getTestIndexIO().loadIndex(tempDir));
+ ColumnHolder columnHolder = theIndex.getColumnHolder("obj");
+ Assert.assertNotNull(columnHolder);
+ NestedDataColumnV5<?, ?> v5 = closer.register((NestedDataColumnV5<?, ?>)
columnHolder.getColumn());
+ Assert.assertNotNull(v5);
+ NestedFieldVirtualColumn vc1 = new NestedFieldVirtualColumn(
+ "obj",
+ "$[''].a",
+ "v0"
+ );
+ NestedFieldVirtualColumn vc2 = new NestedFieldVirtualColumn(
+ "obj",
+ "$[''][0].a",
+ "v1"
+ );
+ VirtualColumns vc = VirtualColumns.create(vc1, vc2);
+ QueryableIndexCursorFactory cursorFactory = new
QueryableIndexCursorFactory(theIndex);
+ CursorHolder cursorHolder =
closer.register(cursorFactory.makeCursorHolder(CursorBuildSpec.builder().setVirtualColumns(vc).build()));
+
+ VectorCursor cursor = cursorHolder.asVectorCursor();
+ VectorObjectSelector v0Selector =
cursor.getColumnSelectorFactory().makeObjectSelector("v0");
+ VectorObjectSelector v1Selector =
cursor.getColumnSelectorFactory().makeObjectSelector("v1");
+
+ Object[] v0vals = v0Selector.getObjectVector();
+ Object[] v1vals = v1Selector.getObjectVector();
+
+ Assertions.assertEquals("123", v0vals[0]);
+ Assertions.assertNull(v0vals[1]);
+
+ Assertions.assertNull(v1vals[0]);
+ Assertions.assertEquals("456", v1vals[1]);
+ }
+ }
+}
diff --git
a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierFieldDictionaryFixupTest.java
b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierFieldDictionaryFixupTest.java
new file mode 100644
index 00000000000..818b2f5336d
--- /dev/null
+++
b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierFieldDictionaryFixupTest.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.nested;
+
+import com.google.common.base.Supplier;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.segment.column.StringEncodingStrategies;
+import org.apache.druid.segment.column.StringEncodingStrategy;
+import org.apache.druid.segment.data.DictionaryWriter;
+import org.apache.druid.segment.data.FrontCodedIndexed;
+import org.apache.druid.segment.data.Indexed;
+import org.apache.druid.segment.writeout.OnHeapMemorySegmentWriteOutMedium;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.WritableByteChannel;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+class NestedDataColumnSupplierFieldDictionaryFixupTest
+{
+ @Test
+ void testFieldDictionaryEmptyPathsFixup() throws IOException
+ {
+ List<String> values = List.of(
+ "$..a", // bad path, needs fixup
+ "$.a",
+ "$.b",
+ "$.b.[0].a", // bad path, needs fixup
+ "$.b[0].a"
+ );
+ ByteBuffer buffer = ByteBuffer.allocate(1 << 12);
+ buffer.order(ByteOrder.nativeOrder());
+ persistToBuffer(buffer, values);
+
+ buffer.position(0);
+ Supplier<? extends Indexed<ByteBuffer>> fieldSupplier =
+ NestedDataColumnSupplier.getAndFixFieldsSupplier(buffer,
ByteOrder.nativeOrder(), null);
+ Indexed<ByteBuffer> fieldsDictionary = fieldSupplier.get();
+
Assertions.assertInstanceOf(NestedDataColumnSupplier.FieldsFixupIndexed.class,
fieldsDictionary);
+ // expect get to spit out corrected path values
+ Assertions.assertEquals("$[''].a",
StringUtils.fromUtf8Nullable(fieldsDictionary.get(0)));
+ Assertions.assertEquals("$.a",
StringUtils.fromUtf8Nullable(fieldsDictionary.get(1)));
+ Assertions.assertEquals("$.b",
StringUtils.fromUtf8Nullable(fieldsDictionary.get(2)));
+ Assertions.assertEquals("$.b[''][0].a",
StringUtils.fromUtf8Nullable(fieldsDictionary.get(3)));
+ Assertions.assertEquals("$.b[0].a",
StringUtils.fromUtf8Nullable(fieldsDictionary.get(4)));
+
+ // expect to be able to find index of expected correct path instead of bad
path
+ Assertions.assertEquals(0,
fieldsDictionary.indexOf(StringUtils.toUtf8ByteBuffer("$[''].a")));
+ Assertions.assertEquals(0,
fieldsDictionary.indexOf(StringUtils.toUtf8ByteBuffer("$..a")));
+ Assertions.assertEquals(1,
fieldsDictionary.indexOf(StringUtils.toUtf8ByteBuffer("$.a")));
+ Assertions.assertEquals(2,
fieldsDictionary.indexOf(StringUtils.toUtf8ByteBuffer("$.b")));
+ Assertions.assertEquals(3,
fieldsDictionary.indexOf(StringUtils.toUtf8ByteBuffer("$.b[''][0].a")));
+ Assertions.assertEquals(3,
fieldsDictionary.indexOf(StringUtils.toUtf8ByteBuffer("$.b.[0].a")));
+ Assertions.assertEquals(4,
fieldsDictionary.indexOf(StringUtils.toUtf8ByteBuffer("$.b[0].a")));
+
+ List<String> fromIterator = new ArrayList<>();
+ for (ByteBuffer byteBuffer : fieldsDictionary) {
+ fromIterator.add(StringUtils.fromUtf8Nullable(byteBuffer));
+ }
+ List<String> expected = List.of(
+ "$[''].a",
+ "$.a",
+ "$.b",
+ "$.b[''][0].a",
+ "$.b[0].a"
+ );
+ Assertions.assertEquals(expected, fromIterator);
+ }
+
+ private static long persistToBuffer(
+ ByteBuffer buffer,
+ Iterable<String> sortedIterable
+ ) throws IOException
+ {
+ Iterator<String> sortedStrings = sortedIterable.iterator();
+ buffer.position(0);
+ OnHeapMemorySegmentWriteOutMedium medium = new
OnHeapMemorySegmentWriteOutMedium();
+ DictionaryWriter<String> writer =
StringEncodingStrategies.getStringDictionaryWriter(
+ new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1),
+ medium,
+ "test"
+ );
+ writer.open();
+ int index = 0;
+ while (sortedStrings.hasNext()) {
+ Assertions.assertEquals(index, writer.write(sortedStrings.next()));
+ index++;
+ }
+ Assertions.assertEquals(index, writer.getCardinality());
+ Assertions.assertTrue(writer.isSorted());
+
+
+ WritableByteChannel channel = new WritableByteChannel()
+ {
+ @Override
+ public int write(ByteBuffer src)
+ {
+ int size = src.remaining();
+ buffer.put(src);
+ return size;
+ }
+
+ @Override
+ public boolean isOpen()
+ {
+ return true;
+ }
+
+ @Override
+ public void close()
+ {
+ }
+ };
+ long size = writer.getSerializedSize();
+ buffer.position(0);
+ writer.writeTo(channel, null);
+ Assertions.assertEquals(size, buffer.position());
+ buffer.position(0);
+ return size;
+ }
+}
diff --git
a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java
b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java
index 34d096f9c10..de3e0c17ad7 100644
---
a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java
@@ -302,7 +302,8 @@ public class NestedDataColumnSupplierTest extends
InitializedNullHandlingTest
false,
ByteOrder.nativeOrder(),
RoaringBitmapSerdeFactory.getInstance(),
- NestedCommonFormatColumnPartSerde.FormatSpec.forSerde(columnFormatSpec)
+
NestedCommonFormatColumnPartSerde.FormatSpec.forSerde(columnFormatSpec),
+ NestedPathFinder.VERSION
);
bob.setFileMapper(fileMapper);
ColumnPartSerde.Deserializer deserializer = partSerde.getDeserializer();
@@ -327,7 +328,8 @@ public class NestedDataColumnSupplierTest extends
InitializedNullHandlingTest
false,
ByteOrder.nativeOrder(),
RoaringBitmapSerdeFactory.getInstance(),
- NestedCommonFormatColumnPartSerde.FormatSpec.forSerde(columnFormatSpec)
+
NestedCommonFormatColumnPartSerde.FormatSpec.forSerde(columnFormatSpec),
+ NestedPathFinder.VERSION
);
bob.setFileMapper(arrayFileMapper);
ColumnPartSerde.Deserializer deserializer = partSerde.getDeserializer();
@@ -350,6 +352,7 @@ public class NestedDataColumnSupplierTest extends
InitializedNullHandlingTest
NestedDataColumnSupplier supplier = NestedDataColumnSupplier.read(
ColumnType.NESTED_DATA,
false,
+ NestedPathFinder.VERSION,
baseBuffer,
bob,
ColumnConfig.SELECTION_SIZE,
diff --git
a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java
b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java
index 93b3b024d63..87d05925b81 100644
---
a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java
+++
b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java
@@ -21,18 +21,11 @@ package org.apache.druid.segment.nested;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
import org.apache.druid.collections.bitmap.RoaringBitmapFactory;
import org.apache.druid.guice.BuiltInTypesModule;
import org.apache.druid.java.util.common.io.Closer;
-import org.apache.druid.java.util.common.io.smoosh.SmooshedFileMapper;
import org.apache.druid.query.DefaultBitmapResultFactory;
-import org.apache.druid.query.filter.SelectorPredicateFactory;
-import org.apache.druid.query.filter.StringPredicateDruidPredicateFactory;
-import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import org.apache.druid.segment.ColumnValueSelector;
-import org.apache.druid.segment.DimensionSelector;
-import org.apache.druid.segment.ObjectColumnSelector;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.SimpleAscendingOffset;
import org.apache.druid.segment.TestHelper;
@@ -40,8 +33,6 @@ import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.index.BitmapColumnIndex;
-import org.apache.druid.segment.index.semantic.DruidPredicateIndexes;
-import org.apache.druid.segment.index.semantic.NullValueIndex;
import org.apache.druid.segment.index.semantic.StringValueSetIndexes;
import org.apache.druid.testing.InitializedNullHandlingTest;
import org.apache.druid.utils.CompressionUtils;
@@ -52,13 +43,10 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-import javax.annotation.Nullable;
import java.io.File;
import java.io.IOException;
-import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
-import java.util.TreeSet;
public class NestedDataColumnSupplierV4Test extends InitializedNullHandlingTest
{
@@ -76,39 +64,8 @@ public class NestedDataColumnSupplierV4Test extends
InitializedNullHandlingTest
TestHelper.makeMap("x", 4L, "y", 2.0, "z", "e", "v", 11111L, "nullish",
null)
);
- List<Map<String, Object>> arrayTestData = ImmutableList.of(
- TestHelper.makeMap("s", new Object[]{"a", "b", "c"}, "l", new
Object[]{1L, 2L, 3L}, "d", new Object[]{1.1, 2.2}),
- TestHelper.makeMap(
- "s",
- new Object[]{null, "b", "c"},
- "l",
- new Object[]{1L, null, 3L},
- "d",
- new Object[]{2.2, 2.2}
- ),
- TestHelper.makeMap(
- "s",
- new Object[]{"b", "c"},
- "l",
- new Object[]{null, null},
- "d",
- new Object[]{1.1, null, 2.2}
- ),
- TestHelper.makeMap("s", new Object[]{"a", "b", "c", "d"}, "l", new
Object[]{4L, 2L, 3L}),
- TestHelper.makeMap("s", new Object[]{"d", "b", "c", "a"}, "d", new
Object[]{1.1, 2.2}),
- TestHelper.makeMap("l", new Object[]{1L, 2L, 3L}, "d", new Object[]{3.1,
2.2, 1.9})
- );
-
Closer closer = Closer.create();
- SmooshedFileMapper fileMapper;
-
- ByteBuffer baseBuffer;
-
- SmooshedFileMapper arrayFileMapper;
-
- ByteBuffer arrayBaseBuffer;
-
@BeforeClass
public static void staticSetup()
{
@@ -136,7 +93,7 @@ public class NestedDataColumnSupplierV4Test extends
InitializedNullHandlingTest
ColumnHolder holder = theIndex.getColumnHolder(columnName);
Assert.assertNotNull(holder);
Assert.assertEquals(ColumnType.NESTED_DATA,
holder.getCapabilities().toColumnType());
- NestedDataColumnV3<?> v3 = closer.register((NestedDataColumnV3<?>)
holder.getColumn());
+ NestedDataColumnV3<?, ?> v3 = closer.register((NestedDataColumnV3<?, ?>)
holder.getColumn());
Assert.assertNotNull(v3);
List<NestedPathPart> path = ImmutableList.of(new
NestedPathField("lastName"));
ColumnHolder nestedColumnHolder = v3.getColumnHolder(path);
@@ -157,6 +114,7 @@ public class NestedDataColumnSupplierV4Test extends
InitializedNullHandlingTest
Assert.assertTrue(indexForValue.computeBitmapResult(resultFactory,
false).get(0));
}
}
+
@Test
public void testLegacyV4ReaderFormat() throws IOException
{
@@ -173,7 +131,7 @@ public class NestedDataColumnSupplierV4Test extends
InitializedNullHandlingTest
ColumnHolder holder = theIndex.getColumnHolder(columnName);
Assert.assertNotNull(holder);
Assert.assertEquals(ColumnType.NESTED_DATA,
holder.getCapabilities().toColumnType());
- NestedDataColumnV4<?> v4 = closer.register((NestedDataColumnV4<?>)
holder.getColumn());
+ NestedDataColumnV4<?, ?> v4 = closer.register((NestedDataColumnV4<?, ?>)
holder.getColumn());
Assert.assertNotNull(v4);
List<NestedPathPart> path = ImmutableList.of(new
NestedPathField("lastName"));
ColumnHolder nestedColumnHolder = v4.getColumnHolder(path);
@@ -194,185 +152,4 @@ public class NestedDataColumnSupplierV4Test extends
InitializedNullHandlingTest
Assert.assertTrue(indexForValue.computeBitmapResult(resultFactory,
false).get(0));
}
}
-
- private void smokeTest(NestedDataComplexColumn column) throws IOException
- {
- SimpleAscendingOffset offset = new SimpleAscendingOffset(data.size());
- ColumnValueSelector<?> rawSelector =
column.makeColumnValueSelector(offset);
- final List<NestedPathPart> xPath = NestedPathFinder.parseJsonPath("$.x");
- Assert.assertEquals(ImmutableSet.of(ColumnType.LONG),
column.getFieldTypes(xPath));
- Assert.assertEquals(ColumnType.LONG,
column.getColumnHolder(xPath).getCapabilities().toColumnType());
- ColumnValueSelector<?> xSelector = column.makeColumnValueSelector(xPath,
null, offset);
- DimensionSelector xDimSelector = column.makeDimensionSelector(xPath, null,
null, offset);
- ColumnIndexSupplier xIndexSupplier = column.getColumnIndexSupplier(xPath);
- Assert.assertNotNull(xIndexSupplier);
- StringValueSetIndexes xValueIndex =
xIndexSupplier.as(StringValueSetIndexes.class);
- DruidPredicateIndexes xPredicateIndex =
xIndexSupplier.as(DruidPredicateIndexes.class);
- NullValueIndex xNulls = xIndexSupplier.as(NullValueIndex.class);
- final List<NestedPathPart> yPath = NestedPathFinder.parseJsonPath("$.y");
- Assert.assertEquals(ImmutableSet.of(ColumnType.DOUBLE),
column.getFieldTypes(yPath));
- Assert.assertEquals(ColumnType.DOUBLE,
column.getColumnHolder(yPath).getCapabilities().toColumnType());
- ColumnValueSelector<?> ySelector = column.makeColumnValueSelector(yPath,
null, offset);
- DimensionSelector yDimSelector = column.makeDimensionSelector(yPath, null,
null, offset);
- ColumnIndexSupplier yIndexSupplier = column.getColumnIndexSupplier(yPath);
- Assert.assertNotNull(yIndexSupplier);
- StringValueSetIndexes yValueIndex =
yIndexSupplier.as(StringValueSetIndexes.class);
- DruidPredicateIndexes yPredicateIndex =
yIndexSupplier.as(DruidPredicateIndexes.class);
- NullValueIndex yNulls = yIndexSupplier.as(NullValueIndex.class);
- final List<NestedPathPart> zPath = NestedPathFinder.parseJsonPath("$.z");
- Assert.assertEquals(ImmutableSet.of(ColumnType.STRING),
column.getFieldTypes(zPath));
- Assert.assertEquals(ColumnType.STRING,
column.getColumnHolder(zPath).getCapabilities().toColumnType());
- ColumnValueSelector<?> zSelector = column.makeColumnValueSelector(zPath,
null, offset);
- DimensionSelector zDimSelector = column.makeDimensionSelector(zPath, null,
null, offset);
- ColumnIndexSupplier zIndexSupplier = column.getColumnIndexSupplier(zPath);
- Assert.assertNotNull(zIndexSupplier);
- StringValueSetIndexes zValueIndex =
zIndexSupplier.as(StringValueSetIndexes.class);
- DruidPredicateIndexes zPredicateIndex =
zIndexSupplier.as(DruidPredicateIndexes.class);
- NullValueIndex zNulls = zIndexSupplier.as(NullValueIndex.class);
- final List<NestedPathPart> vPath = NestedPathFinder.parseJsonPath("$.v");
- Assert.assertEquals(
- ImmutableSet.of(ColumnType.STRING, ColumnType.LONG, ColumnType.DOUBLE),
- column.getFieldTypes(vPath)
- );
- Assert.assertEquals(ColumnType.STRING,
column.getColumnHolder(vPath).getCapabilities().toColumnType());
- ColumnValueSelector<?> vSelector = column.makeColumnValueSelector(vPath,
null, offset);
- DimensionSelector vDimSelector = column.makeDimensionSelector(vPath, null,
null, offset);
- ColumnIndexSupplier vIndexSupplier = column.getColumnIndexSupplier(vPath);
- Assert.assertNotNull(vIndexSupplier);
- StringValueSetIndexes vValueIndex =
vIndexSupplier.as(StringValueSetIndexes.class);
- DruidPredicateIndexes vPredicateIndex =
vIndexSupplier.as(DruidPredicateIndexes.class);
- NullValueIndex vNulls = vIndexSupplier.as(NullValueIndex.class);
- final List<NestedPathPart> nullishPath =
NestedPathFinder.parseJsonPath("$.nullish");
- Assert.assertEquals(ImmutableSet.of(ColumnType.STRING),
column.getFieldTypes(nullishPath));
- Assert.assertEquals(ColumnType.STRING,
column.getColumnHolder(nullishPath).getCapabilities().toColumnType());
- ColumnValueSelector<?> nullishSelector =
column.makeColumnValueSelector(nullishPath, null, offset);
- DimensionSelector nullishDimSelector =
column.makeDimensionSelector(nullishPath, null, null, offset);
- ColumnIndexSupplier nullishIndexSupplier =
column.getColumnIndexSupplier(nullishPath);
- Assert.assertNotNull(nullishIndexSupplier);
- StringValueSetIndexes nullishValueIndex =
nullishIndexSupplier.as(StringValueSetIndexes.class);
- DruidPredicateIndexes nullishPredicateIndex =
nullishIndexSupplier.as(DruidPredicateIndexes.class);
- NullValueIndex nullishNulls =
nullishIndexSupplier.as(NullValueIndex.class);
- Assert.assertEquals(ImmutableList.of(nullishPath, vPath, xPath, yPath,
zPath), column.getNestedFields());
- for (int i = 0; i < data.size(); i++) {
- Map row = data.get(i);
- Assert.assertEquals(
- JSON_MAPPER.writeValueAsString(row),
-
JSON_MAPPER.writeValueAsString(StructuredData.unwrap(rawSelector.getObject()))
- );
- testPath(row, i, "v", vSelector, vDimSelector, vValueIndex,
vPredicateIndex, vNulls, null);
- testPath(row, i, "x", xSelector, xDimSelector, xValueIndex,
xPredicateIndex, xNulls, ColumnType.LONG);
- testPath(row, i, "y", ySelector, yDimSelector, yValueIndex,
yPredicateIndex, yNulls, ColumnType.DOUBLE);
- testPath(row, i, "z", zSelector, zDimSelector, zValueIndex,
zPredicateIndex, zNulls, ColumnType.STRING);
- testPath(
- row,
- i,
- "nullish",
- nullishSelector,
- nullishDimSelector,
- nullishValueIndex,
- nullishPredicateIndex,
- nullishNulls,
- ColumnType.STRING
- );
- offset.increment();
- }
- }
-
- private void testPath(
- Map row,
- int rowNumber,
- String path,
- ColumnValueSelector<?> valueSelector,
- DimensionSelector dimSelector,
- StringValueSetIndexes valueSetIndex,
- DruidPredicateIndexes predicateIndex,
- NullValueIndex nullValueIndex,
- @Nullable ColumnType singleType
- )
- {
- final Object inputValue = row.get(path);
- if (row.containsKey(path) && inputValue != null) {
- Assert.assertEquals(inputValue, valueSelector.getObject());
- if (ColumnType.LONG.equals(singleType)) {
- Assert.assertEquals(inputValue, valueSelector.getLong());
- Assert.assertFalse(path + " is not null", valueSelector.isNull());
- } else if (ColumnType.DOUBLE.equals(singleType)) {
- Assert.assertEquals((double) inputValue, valueSelector.getDouble(),
0.0);
- Assert.assertFalse(path + " is not null", valueSelector.isNull());
- }
- final String theString = String.valueOf(inputValue);
- Assert.assertEquals(theString, dimSelector.getObject());
- String dimSelectorLookupVal =
dimSelector.lookupName(dimSelector.getRow().get(0));
- Assert.assertEquals(theString, dimSelectorLookupVal);
-
Assert.assertEquals(dimSelector.idLookup().lookupId(dimSelectorLookupVal),
dimSelector.getRow().get(0));
-
Assert.assertTrue(valueSetIndex.forValue(theString).computeBitmapResult(resultFactory,
false).get(rowNumber));
- Assert.assertTrue(valueSetIndex.forSortedValues(new
TreeSet<>(ImmutableSet.of(theString)))
- .computeBitmapResult(resultFactory, false)
- .get(rowNumber));
- Assert.assertTrue(predicateIndex.forPredicate(new
SelectorPredicateFactory(theString))
- .computeBitmapResult(resultFactory,
false)
- .get(rowNumber));
-
Assert.assertFalse(valueSetIndex.forValue(NO_MATCH).computeBitmapResult(resultFactory,
false).get(rowNumber));
- Assert.assertFalse(valueSetIndex.forSortedValues(new
TreeSet<>(ImmutableSet.of(NO_MATCH)))
- .computeBitmapResult(resultFactory,
false)
- .get(rowNumber));
- Assert.assertFalse(predicateIndex.forPredicate(new
SelectorPredicateFactory(NO_MATCH))
- .computeBitmapResult(resultFactory,
false)
- .get(rowNumber));
-
Assert.assertFalse(nullValueIndex.get().computeBitmapResult(resultFactory,
false).get(rowNumber));
-
Assert.assertTrue(dimSelector.makeValueMatcher(theString).matches(false));
-
Assert.assertFalse(dimSelector.makeValueMatcher(NO_MATCH).matches(false));
-
Assert.assertTrue(dimSelector.makeValueMatcher(StringPredicateDruidPredicateFactory.equalTo(theString)).matches(false));
-
Assert.assertFalse(dimSelector.makeValueMatcher(StringPredicateDruidPredicateFactory.equalTo(NO_MATCH)).matches(false));
- } else {
- Assert.assertNull(valueSelector.getObject());
- Assert.assertTrue(path, valueSelector.isNull());
- Assert.assertEquals(0, dimSelector.getRow().get(0));
- Assert.assertNull(dimSelector.getObject());
- Assert.assertNull(dimSelector.lookupName(dimSelector.getRow().get(0)));
-
Assert.assertTrue(valueSetIndex.forValue(null).computeBitmapResult(resultFactory,
false).get(rowNumber));
-
Assert.assertTrue(nullValueIndex.get().computeBitmapResult(resultFactory,
false).get(rowNumber));
- Assert.assertTrue(predicateIndex.forPredicate(new
SelectorPredicateFactory(null))
- .computeBitmapResult(resultFactory,
false)
- .get(rowNumber));
-
Assert.assertFalse(valueSetIndex.forValue(NO_MATCH).computeBitmapResult(resultFactory,
false).get(rowNumber));
-
Assert.assertFalse(valueSetIndex.forValue(NO_MATCH).computeBitmapResult(resultFactory,
false).get(rowNumber));
- Assert.assertFalse(predicateIndex.forPredicate(new
SelectorPredicateFactory(NO_MATCH))
- .computeBitmapResult(resultFactory,
false)
- .get(rowNumber));
- Assert.assertTrue(dimSelector.makeValueMatcher((String)
null).matches(false));
-
Assert.assertFalse(dimSelector.makeValueMatcher(NO_MATCH).matches(false));
-
Assert.assertTrue(dimSelector.makeValueMatcher(StringPredicateDruidPredicateFactory.equalTo(null)).matches(false));
-
Assert.assertFalse(dimSelector.makeValueMatcher(StringPredicateDruidPredicateFactory.equalTo(NO_MATCH)).matches(false));
- }
- }
-
- private static class SettableSelector extends
ObjectColumnSelector<StructuredData>
- {
- private StructuredData data;
-
- public void setObject(StructuredData o)
- {
- this.data = o;
- }
-
- @Nullable
- @Override
- public StructuredData getObject()
- {
- return data;
- }
-
- @Override
- public Class classOfObject()
- {
- return StructuredData.class;
- }
-
- @Override
- public void inspectRuntimeShape(RuntimeShapeInspector inspector)
- {
-
- }
- }
}
diff --git
a/processing/src/test/java/org/apache/druid/segment/nested/NestedPathFinderTest.java
b/processing/src/test/java/org/apache/druid/segment/nested/NestedPathFinderTest.java
index 40e6f494be7..8765ba17a9c 100644
---
a/processing/src/test/java/org/apache/druid/segment/nested/NestedPathFinderTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/nested/NestedPathFinderTest.java
@@ -21,9 +21,10 @@ package org.apache.druid.segment.nested;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import org.apache.druid.error.DruidException;
import org.apache.druid.error.DruidExceptionMatcher;
-import org.junit.Assert;
-import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Map;
@@ -46,145 +47,145 @@ public class NestedPathFinderTest
List<NestedPathPart> pathParts;
pathParts = NestedPathFinder.parseJqPath(".");
- Assert.assertEquals(0, pathParts.size());
+ Assertions.assertEquals(0, pathParts.size());
// { "z" : "hello" }
pathParts = NestedPathFinder.parseJqPath(".z");
- Assert.assertEquals(1, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("z", pathParts.get(0).getPartIdentifier());
- Assert.assertEquals(".\"z\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
+ Assertions.assertEquals(1, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("z", pathParts.get(0).getPartIdentifier());
+ Assertions.assertEquals(".\"z\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
// { "z" : "hello" }
pathParts = NestedPathFinder.parseJqPath(".\"z\"");
- Assert.assertEquals(1, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("z", pathParts.get(0).getPartIdentifier());
- Assert.assertEquals(".\"z\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
+ Assertions.assertEquals(1, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("z", pathParts.get(0).getPartIdentifier());
+ Assertions.assertEquals(".\"z\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
// { "z" : "hello" }
pathParts = NestedPathFinder.parseJqPath(".[\"z\"]");
- Assert.assertEquals(1, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("z", pathParts.get(0).getPartIdentifier());
- Assert.assertEquals(".\"z\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
+ Assertions.assertEquals(1, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("z", pathParts.get(0).getPartIdentifier());
+ Assertions.assertEquals(".\"z\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
// { "x" : ["a", "b"]}
pathParts = NestedPathFinder.parseJqPath(".x[1]");
- Assert.assertEquals(2, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("x", pathParts.get(0).getPartIdentifier());
- Assert.assertTrue(pathParts.get(1) instanceof NestedPathArrayElement);
- Assert.assertEquals("1", pathParts.get(1).getPartIdentifier());
- Assert.assertEquals(".\"x\"[1]",
NestedPathFinder.toNormalizedJqPath(pathParts));
+ Assertions.assertEquals(2, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("x", pathParts.get(0).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathArrayElement.class,
pathParts.get(1));
+ Assertions.assertEquals("1", pathParts.get(1).getPartIdentifier());
+ Assertions.assertEquals(".\"x\"[1]",
NestedPathFinder.toNormalizedJqPath(pathParts));
// { "x" : ["a", "b"]}
pathParts = NestedPathFinder.parseJqPath(".\"x\"[1]");
- Assert.assertEquals(2, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("x", pathParts.get(0).getPartIdentifier());
- Assert.assertTrue(pathParts.get(1) instanceof NestedPathArrayElement);
- Assert.assertEquals("1", pathParts.get(1).getPartIdentifier());
- Assert.assertEquals(".\"x\"[1]",
NestedPathFinder.toNormalizedJqPath(pathParts));
+ Assertions.assertEquals(2, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("x", pathParts.get(0).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathArrayElement.class,
pathParts.get(1));
+ Assertions.assertEquals("1", pathParts.get(1).getPartIdentifier());
+ Assertions.assertEquals(".\"x\"[1]",
NestedPathFinder.toNormalizedJqPath(pathParts));
// { "x" : ["a", "b"]}
pathParts = NestedPathFinder.parseJqPath(".[\"x\"][1]");
- Assert.assertEquals(2, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("x", pathParts.get(0).getPartIdentifier());
- Assert.assertTrue(pathParts.get(1) instanceof NestedPathArrayElement);
- Assert.assertEquals("1", pathParts.get(1).getPartIdentifier());
- Assert.assertEquals(".\"x\"[1]",
NestedPathFinder.toNormalizedJqPath(pathParts));
+ Assertions.assertEquals(2, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("x", pathParts.get(0).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathArrayElement.class,
pathParts.get(1));
+ Assertions.assertEquals("1", pathParts.get(1).getPartIdentifier());
+ Assertions.assertEquals(".\"x\"[1]",
NestedPathFinder.toNormalizedJqPath(pathParts));
// { "x" : { "1" : "hello" }}
pathParts = NestedPathFinder.parseJqPath(".[\"x\"][\"1\"]");
- Assert.assertEquals(2, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("x", pathParts.get(0).getPartIdentifier());
- Assert.assertTrue(pathParts.get(1) instanceof NestedPathField);
- Assert.assertEquals("1", pathParts.get(1).getPartIdentifier());
- Assert.assertEquals(".\"x\".\"1\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
+ Assertions.assertEquals(2, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("x", pathParts.get(0).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(1));
+ Assertions.assertEquals("1", pathParts.get(1).getPartIdentifier());
+ Assertions.assertEquals(".\"x\".\"1\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
// { "x" : [ { "foo" : { "bar" : "hello" }}, { "foo" : { "bar" : "world"
}}]}
pathParts = NestedPathFinder.parseJqPath(".x[1].foo.bar");
- Assert.assertEquals(4, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("x", pathParts.get(0).getPartIdentifier());
- Assert.assertTrue(pathParts.get(1) instanceof NestedPathArrayElement);
- Assert.assertEquals("1", pathParts.get(1).getPartIdentifier());
- Assert.assertTrue(pathParts.get(2) instanceof NestedPathField);
- Assert.assertEquals("foo", pathParts.get(2).getPartIdentifier());
- Assert.assertTrue(pathParts.get(3) instanceof NestedPathField);
- Assert.assertEquals("bar", pathParts.get(3).getPartIdentifier());
- Assert.assertEquals(".\"x\"[1].\"foo\".\"bar\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
+ Assertions.assertEquals(4, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("x", pathParts.get(0).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathArrayElement.class,
pathParts.get(1));
+ Assertions.assertEquals("1", pathParts.get(1).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(2));
+ Assertions.assertEquals("foo", pathParts.get(2).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(3));
+ Assertions.assertEquals("bar", pathParts.get(3).getPartIdentifier());
+ Assertions.assertEquals(".\"x\"[1].\"foo\".\"bar\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
// { "x" : [ { "foo" : { "bar" : "hello" }}, { "foo" : { "bar" : "world"
}}]}
pathParts = NestedPathFinder.parseJqPath(".x[1].\"foo\".bar");
- Assert.assertEquals(4, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("x", pathParts.get(0).getPartIdentifier());
- Assert.assertTrue(pathParts.get(1) instanceof NestedPathArrayElement);
- Assert.assertEquals("1", pathParts.get(1).getPartIdentifier());
- Assert.assertTrue(pathParts.get(2) instanceof NestedPathField);
- Assert.assertEquals("foo", pathParts.get(2).getPartIdentifier());
- Assert.assertTrue(pathParts.get(3) instanceof NestedPathField);
- Assert.assertEquals("bar", pathParts.get(3).getPartIdentifier());
- Assert.assertEquals(".\"x\"[1].\"foo\".\"bar\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
+ Assertions.assertEquals(4, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("x", pathParts.get(0).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathArrayElement.class,
pathParts.get(1));
+ Assertions.assertEquals("1", pathParts.get(1).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(2));
+ Assertions.assertEquals("foo", pathParts.get(2).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(3));
+ Assertions.assertEquals("bar", pathParts.get(3).getPartIdentifier());
+ Assertions.assertEquals(".\"x\"[1].\"foo\".\"bar\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
// { "x" : [ { "foo" : { "bar" : "hello" }}, { "foo" : { "bar" : "world"
}}]}
pathParts = NestedPathFinder.parseJqPath(".[\"x\"][1].\"foo\"[\"bar\"]");
- Assert.assertEquals(4, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("x", pathParts.get(0).getPartIdentifier());
- Assert.assertTrue(pathParts.get(1) instanceof NestedPathArrayElement);
- Assert.assertEquals("1", pathParts.get(1).getPartIdentifier());
- Assert.assertTrue(pathParts.get(2) instanceof NestedPathField);
- Assert.assertEquals("foo", pathParts.get(2).getPartIdentifier());
- Assert.assertTrue(pathParts.get(3) instanceof NestedPathField);
- Assert.assertEquals("bar", pathParts.get(3).getPartIdentifier());
- Assert.assertEquals(".\"x\"[1].\"foo\".\"bar\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
+ Assertions.assertEquals(4, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("x", pathParts.get(0).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathArrayElement.class,
pathParts.get(1));
+ Assertions.assertEquals("1", pathParts.get(1).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(2));
+ Assertions.assertEquals("foo", pathParts.get(2).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(3));
+ Assertions.assertEquals("bar", pathParts.get(3).getPartIdentifier());
+ Assertions.assertEquals(".\"x\"[1].\"foo\".\"bar\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
// make sure we chomp question marks
// { "x" : [ { "foo" : { "bar" : "hello" }}, { "foo" : { "bar" : "world"
}}]}
pathParts = NestedPathFinder.parseJqPath(".[\"x\"]?[1]?.foo?.\"bar\"?");
- Assert.assertEquals(4, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("x", pathParts.get(0).getPartIdentifier());
- Assert.assertTrue(pathParts.get(1) instanceof NestedPathArrayElement);
- Assert.assertEquals("1", pathParts.get(1).getPartIdentifier());
- Assert.assertTrue(pathParts.get(2) instanceof NestedPathField);
- Assert.assertEquals("foo", pathParts.get(2).getPartIdentifier());
- Assert.assertTrue(pathParts.get(3) instanceof NestedPathField);
- Assert.assertEquals("bar", pathParts.get(3).getPartIdentifier());
- Assert.assertEquals(".\"x\"[1].\"foo\".\"bar\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
+ Assertions.assertEquals(4, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("x", pathParts.get(0).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathArrayElement.class,
pathParts.get(1));
+ Assertions.assertEquals("1", pathParts.get(1).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(2));
+ Assertions.assertEquals("foo", pathParts.get(2).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(3));
+ Assertions.assertEquals("bar", pathParts.get(3).getPartIdentifier());
+ Assertions.assertEquals(".\"x\"[1].\"foo\".\"bar\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
// { "x" : { "1" : { "foo" : { "bar" : "hello" }}}}
pathParts = NestedPathFinder.parseJqPath(".\"x\"[\"1\"].\"foo\".\"bar\"");
- Assert.assertEquals(4, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("x", pathParts.get(0).getPartIdentifier());
- Assert.assertTrue(pathParts.get(1) instanceof NestedPathField);
- Assert.assertEquals("1", pathParts.get(1).getPartIdentifier());
- Assert.assertTrue(pathParts.get(2) instanceof NestedPathField);
- Assert.assertEquals("foo", pathParts.get(2).getPartIdentifier());
- Assert.assertTrue(pathParts.get(3) instanceof NestedPathField);
- Assert.assertEquals("bar", pathParts.get(3).getPartIdentifier());
- Assert.assertEquals(".\"x\".\"1\".\"foo\".\"bar\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
+ Assertions.assertEquals(4, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("x", pathParts.get(0).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(1));
+ Assertions.assertEquals("1", pathParts.get(1).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(2));
+ Assertions.assertEquals("foo", pathParts.get(2).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(3));
+ Assertions.assertEquals("bar", pathParts.get(3).getPartIdentifier());
+ Assertions.assertEquals(".\"x\".\"1\".\"foo\".\"bar\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
// stress out the parser
// { "x.y.z]?[\\\"]][]\" : { "13234.12[]][23" : { "f?o.o" : { ".b?.a.r.":
"hello" }}}}
pathParts =
NestedPathFinder.parseJqPath(".[\"x.y.z]?[\\\"]][]\"]?[\"13234.12[]][23\"].\"f?o.o\"?[\".b?.a.r.\"]");
- Assert.assertEquals(4, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("x.y.z]?[\\\"]][]",
pathParts.get(0).getPartIdentifier());
- Assert.assertTrue(pathParts.get(1) instanceof NestedPathField);
- Assert.assertEquals("13234.12[]][23",
pathParts.get(1).getPartIdentifier());
- Assert.assertTrue(pathParts.get(2) instanceof NestedPathField);
- Assert.assertEquals("f?o.o", pathParts.get(2).getPartIdentifier());
- Assert.assertTrue(pathParts.get(3) instanceof NestedPathField);
- Assert.assertEquals(".b?.a.r.", pathParts.get(3).getPartIdentifier());
- Assert.assertEquals(
+ Assertions.assertEquals(4, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("x.y.z]?[\\\"]][]",
pathParts.get(0).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(1));
+ Assertions.assertEquals("13234.12[]][23",
pathParts.get(1).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(2));
+ Assertions.assertEquals("f?o.o", pathParts.get(2).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(3));
+ Assertions.assertEquals(".b?.a.r.", pathParts.get(3).getPartIdentifier());
+ Assertions.assertEquals(
".\"x.y.z]?[\\\"]][]\".\"13234.12[]][23\".\"f?o.o\".\".b?.a.r.\"",
NestedPathFinder.toNormalizedJqPath(pathParts)
);
@@ -196,138 +197,138 @@ public class NestedPathFinderTest
List<NestedPathPart> pathParts;
pathParts = NestedPathFinder.parseJsonPath("$.");
- Assert.assertEquals(0, pathParts.size());
+ Assertions.assertEquals(0, pathParts.size());
pathParts = NestedPathFinder.parseJsonPath("$");
- Assert.assertEquals(0, pathParts.size());
+ Assertions.assertEquals(0, pathParts.size());
// { "z" : "hello" }
pathParts = NestedPathFinder.parseJsonPath("$.z");
- Assert.assertEquals(1, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("z", pathParts.get(0).getPartIdentifier());
- Assert.assertEquals(".\"z\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
- Assert.assertEquals("$.z",
NestedPathFinder.toNormalizedJsonPath(pathParts));
+ Assertions.assertEquals(1, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("z", pathParts.get(0).getPartIdentifier());
+ Assertions.assertEquals(".\"z\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
+ Assertions.assertEquals("$.z",
NestedPathFinder.toNormalizedJsonPath(pathParts));
// { "z" : "hello" }
pathParts = NestedPathFinder.parseJsonPath("$['z']");
- Assert.assertEquals(1, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("z", pathParts.get(0).getPartIdentifier());
- Assert.assertEquals(".\"z\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
- Assert.assertEquals("$.z",
NestedPathFinder.toNormalizedJsonPath(pathParts));
+ Assertions.assertEquals(1, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("z", pathParts.get(0).getPartIdentifier());
+ Assertions.assertEquals(".\"z\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
+ Assertions.assertEquals("$.z",
NestedPathFinder.toNormalizedJsonPath(pathParts));
// { "x" : ["a", "b"]}
pathParts = NestedPathFinder.parseJsonPath("$.x[1]");
- Assert.assertEquals(2, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("x", pathParts.get(0).getPartIdentifier());
- Assert.assertTrue(pathParts.get(1) instanceof NestedPathArrayElement);
- Assert.assertEquals("1", pathParts.get(1).getPartIdentifier());
- Assert.assertEquals(".\"x\"[1]",
NestedPathFinder.toNormalizedJqPath(pathParts));
- Assert.assertEquals("$.x[1]",
NestedPathFinder.toNormalizedJsonPath(pathParts));
+ Assertions.assertEquals(2, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("x", pathParts.get(0).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathArrayElement.class,
pathParts.get(1));
+ Assertions.assertEquals("1", pathParts.get(1).getPartIdentifier());
+ Assertions.assertEquals(".\"x\"[1]",
NestedPathFinder.toNormalizedJqPath(pathParts));
+ Assertions.assertEquals("$.x[1]",
NestedPathFinder.toNormalizedJsonPath(pathParts));
// { "x" : ["a", "b"]}
pathParts = NestedPathFinder.parseJsonPath("$.x[-1]");
- Assert.assertEquals(2, pathParts.size());
+ Assertions.assertEquals(2, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("x", pathParts.get(0).getPartIdentifier());
- Assert.assertTrue(pathParts.get(1) instanceof NestedPathArrayElement);
- Assert.assertEquals("-1", pathParts.get(1).getPartIdentifier());
- Assert.assertEquals(".\"x\"[-1]",
NestedPathFinder.toNormalizedJqPath(pathParts));
- Assert.assertEquals("$.x[-1]",
NestedPathFinder.toNormalizedJsonPath(pathParts));
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("x", pathParts.get(0).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathArrayElement.class,
pathParts.get(1));
+ Assertions.assertEquals("-1", pathParts.get(1).getPartIdentifier());
+ Assertions.assertEquals(".\"x\"[-1]",
NestedPathFinder.toNormalizedJqPath(pathParts));
+ Assertions.assertEquals("$.x[-1]",
NestedPathFinder.toNormalizedJsonPath(pathParts));
// { "x" : ["a", "b"]}
pathParts = NestedPathFinder.parseJsonPath("$['x'][1]");
- Assert.assertEquals(2, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("x", pathParts.get(0).getPartIdentifier());
- Assert.assertTrue(pathParts.get(1) instanceof NestedPathArrayElement);
- Assert.assertEquals("1", pathParts.get(1).getPartIdentifier());
- Assert.assertEquals(".\"x\"[1]",
NestedPathFinder.toNormalizedJqPath(pathParts));
- Assert.assertEquals("$.x[1]",
NestedPathFinder.toNormalizedJsonPath(pathParts));
+ Assertions.assertEquals(2, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("x", pathParts.get(0).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathArrayElement.class,
pathParts.get(1));
+ Assertions.assertEquals("1", pathParts.get(1).getPartIdentifier());
+ Assertions.assertEquals(".\"x\"[1]",
NestedPathFinder.toNormalizedJqPath(pathParts));
+ Assertions.assertEquals("$.x[1]",
NestedPathFinder.toNormalizedJsonPath(pathParts));
// { "x" : { "1" : "hello" }}
pathParts = NestedPathFinder.parseJsonPath("$['x']['1']");
- Assert.assertEquals(2, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("x", pathParts.get(0).getPartIdentifier());
- Assert.assertTrue(pathParts.get(1) instanceof NestedPathField);
- Assert.assertEquals("1", pathParts.get(1).getPartIdentifier());
- Assert.assertEquals(".\"x\".\"1\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
- Assert.assertEquals("$.x.1",
NestedPathFinder.toNormalizedJsonPath(pathParts));
+ Assertions.assertEquals(2, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("x", pathParts.get(0).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(1));
+ Assertions.assertEquals("1", pathParts.get(1).getPartIdentifier());
+ Assertions.assertEquals(".\"x\".\"1\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
+ Assertions.assertEquals("$.x.1",
NestedPathFinder.toNormalizedJsonPath(pathParts));
// { "x" : [ { "foo" : { "bar" : "hello" }}, { "foo" : { "bar" : "world"
}}]}
pathParts = NestedPathFinder.parseJsonPath("$.x[1].foo.bar");
- Assert.assertEquals(4, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("x", pathParts.get(0).getPartIdentifier());
- Assert.assertTrue(pathParts.get(1) instanceof NestedPathArrayElement);
- Assert.assertEquals("1", pathParts.get(1).getPartIdentifier());
- Assert.assertTrue(pathParts.get(2) instanceof NestedPathField);
- Assert.assertEquals("foo", pathParts.get(2).getPartIdentifier());
- Assert.assertTrue(pathParts.get(3) instanceof NestedPathField);
- Assert.assertEquals("bar", pathParts.get(3).getPartIdentifier());
- Assert.assertEquals(".\"x\"[1].\"foo\".\"bar\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
- Assert.assertEquals("$.x[1].foo.bar",
NestedPathFinder.toNormalizedJsonPath(pathParts));
+ Assertions.assertEquals(4, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("x", pathParts.get(0).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathArrayElement.class,
pathParts.get(1));
+ Assertions.assertEquals("1", pathParts.get(1).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(2));
+ Assertions.assertEquals("foo", pathParts.get(2).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(3));
+ Assertions.assertEquals("bar", pathParts.get(3).getPartIdentifier());
+ Assertions.assertEquals(".\"x\"[1].\"foo\".\"bar\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
+ Assertions.assertEquals("$.x[1].foo.bar",
NestedPathFinder.toNormalizedJsonPath(pathParts));
// { "x" : [ { "foo" : { "bar" : "hello" }}, { "foo" : { "bar" : "world"
}}]}
pathParts = NestedPathFinder.parseJsonPath("$.x[1]['foo'].bar");
- Assert.assertEquals(4, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("x", pathParts.get(0).getPartIdentifier());
- Assert.assertTrue(pathParts.get(1) instanceof NestedPathArrayElement);
- Assert.assertEquals("1", pathParts.get(1).getPartIdentifier());
- Assert.assertTrue(pathParts.get(2) instanceof NestedPathField);
- Assert.assertEquals("foo", pathParts.get(2).getPartIdentifier());
- Assert.assertTrue(pathParts.get(3) instanceof NestedPathField);
- Assert.assertEquals("bar", pathParts.get(3).getPartIdentifier());
- Assert.assertEquals(".\"x\"[1].\"foo\".\"bar\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
- Assert.assertEquals("$.x[1].foo.bar",
NestedPathFinder.toNormalizedJsonPath(pathParts));
+ Assertions.assertEquals(4, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("x", pathParts.get(0).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathArrayElement.class,
pathParts.get(1));
+ Assertions.assertEquals("1", pathParts.get(1).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(2));
+ Assertions.assertEquals("foo", pathParts.get(2).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(3));
+ Assertions.assertEquals("bar", pathParts.get(3).getPartIdentifier());
+ Assertions.assertEquals(".\"x\"[1].\"foo\".\"bar\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
+ Assertions.assertEquals("$.x[1].foo.bar",
NestedPathFinder.toNormalizedJsonPath(pathParts));
// { "x" : [ { "foo" : { "bar" : "hello" }}, { "foo" : { "bar" : "world"
}}]}
pathParts = NestedPathFinder.parseJsonPath("$['x'][1].foo['bar']");
- Assert.assertEquals(4, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("x", pathParts.get(0).getPartIdentifier());
- Assert.assertTrue(pathParts.get(1) instanceof NestedPathArrayElement);
- Assert.assertEquals("1", pathParts.get(1).getPartIdentifier());
- Assert.assertTrue(pathParts.get(2) instanceof NestedPathField);
- Assert.assertEquals("foo", pathParts.get(2).getPartIdentifier());
- Assert.assertTrue(pathParts.get(3) instanceof NestedPathField);
- Assert.assertEquals("bar", pathParts.get(3).getPartIdentifier());
- Assert.assertEquals(".\"x\"[1].\"foo\".\"bar\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
- Assert.assertEquals("$.x[1].foo.bar",
NestedPathFinder.toNormalizedJsonPath(pathParts));
+ Assertions.assertEquals(4, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("x", pathParts.get(0).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathArrayElement.class,
pathParts.get(1));
+ Assertions.assertEquals("1", pathParts.get(1).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(2));
+ Assertions.assertEquals("foo", pathParts.get(2).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(3));
+ Assertions.assertEquals("bar", pathParts.get(3).getPartIdentifier());
+ Assertions.assertEquals(".\"x\"[1].\"foo\".\"bar\"",
NestedPathFinder.toNormalizedJqPath(pathParts));
+ Assertions.assertEquals("$.x[1].foo.bar",
NestedPathFinder.toNormalizedJsonPath(pathParts));
// stress out the parser
// { "x.y.z]?[\\\"]][]\" : { "13234.12[]][23" : { "f?o.o" : { ".b?.a.r.":
"hello" }}}}
pathParts =
NestedPathFinder.parseJsonPath("$['x.y.z][\\']][]']['13234.12[]][23']['fo.o']['.b.a.r.']");
- Assert.assertEquals(4, pathParts.size());
- Assert.assertTrue(pathParts.get(0) instanceof NestedPathField);
- Assert.assertEquals("x.y.z][\\']][]",
pathParts.get(0).getPartIdentifier());
- Assert.assertTrue(pathParts.get(1) instanceof NestedPathField);
- Assert.assertEquals("13234.12[]][23",
pathParts.get(1).getPartIdentifier());
- Assert.assertTrue(pathParts.get(2) instanceof NestedPathField);
- Assert.assertEquals("fo.o", pathParts.get(2).getPartIdentifier());
- Assert.assertTrue(pathParts.get(3) instanceof NestedPathField);
- Assert.assertEquals(".b.a.r.", pathParts.get(3).getPartIdentifier());
- Assert.assertEquals(
+ Assertions.assertEquals(4, pathParts.size());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(0));
+ Assertions.assertEquals("x.y.z][\\']][]",
pathParts.get(0).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(1));
+ Assertions.assertEquals("13234.12[]][23",
pathParts.get(1).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(2));
+ Assertions.assertEquals("fo.o", pathParts.get(2).getPartIdentifier());
+ Assertions.assertInstanceOf(NestedPathField.class, pathParts.get(3));
+ Assertions.assertEquals(".b.a.r.", pathParts.get(3).getPartIdentifier());
+ Assertions.assertEquals(
".\"x.y.z][\\']][]\".\"13234.12[]][23\".\"fo.o\".\".b.a.r.\"",
NestedPathFinder.toNormalizedJqPath(pathParts)
);
- Assert.assertEquals(
+ Assertions.assertEquals(
"$['x.y.z][\\']][]']['13234.12[]][23']['fo.o']['.b.a.r.']",
NestedPathFinder.toNormalizedJsonPath(pathParts)
);
pathParts = NestedPathFinder.parseJsonPath("$['hell'o']");
- Assert.assertEquals(1, pathParts.size());
- Assert.assertEquals("hell'o", pathParts.get(0).getPartIdentifier());
- Assert.assertEquals("$['hell'o']",
NestedPathFinder.toNormalizedJsonPath(pathParts));
+ Assertions.assertEquals(1, pathParts.size());
+ Assertions.assertEquals("hell'o", pathParts.get(0).getPartIdentifier());
+ Assertions.assertEquals("$['hell'o']",
NestedPathFinder.toNormalizedJsonPath(pathParts));
}
@Test
@@ -410,6 +411,60 @@ public class NestedPathFinderTest
).assertThrowsAndMatches(() -> NestedPathFinder.parseJqPath(".x[\"1]"));
}
+ @Test
+ public void testEmptyFields()
+ {
+ List<NestedPathPart> pathParts = List.of(
+ new NestedPathField(""),
+ new NestedPathArrayElement(0),
+ new NestedPathField("a")
+ );
+ Assertions.assertEquals("$[''][0].a",
NestedPathFinder.toNormalizedJsonPath(pathParts));
+ String abc = "abc";
+ Map<String, Object> findit = Map.of(
+ "",
+ List.of(
+ Map.of("a", abc)
+ )
+ );
+ Assertions.assertEquals(abc, NestedPathFinder.find(findit,
NestedPathFinder.parseJsonPath("$[''][0].a")));
+
+ pathParts = List.of(
+ new NestedPathField(""),
+ new NestedPathField(""),
+ new NestedPathField("a")
+ );
+ Assertions.assertEquals("$[''][''].a",
NestedPathFinder.toNormalizedJsonPath(pathParts));
+ }
+
+ @Test
+ public void testFixup()
+ {
+ // test bug fixes for bug https://github.com/apache/druid/pull/19072
+ Throwable t = Assertions.assertThrows(DruidException.class, () ->
NestedPathFinder.parseJsonPath("$.[0].a"));
+ Assertions.assertEquals(
+ "JSONPath [$.[0].a] is invalid, found '[' at invalid position [2],
must not follow '.' or must be contained with '",
+ t.getMessage()
+ );
+ List<NestedPathPart> expectedPathParts = List.of(
+ new NestedPathField(""),
+ new NestedPathArrayElement(0),
+ new NestedPathField("a")
+ );
+ Assertions.assertEquals(expectedPathParts,
NestedPathFinder.parseBadJsonPath("$.[0].a"));
+
+ t = Assertions.assertThrows(DruidException.class, () ->
NestedPathFinder.parseJsonPath("$...a"));
+ Assertions.assertEquals(
+ "JSONPath [$...a] is invalid, found '.' at invalid position [2], must
not follow '.' or must be contained with '",
+ t.getMessage()
+ );
+ expectedPathParts = List.of(
+ new NestedPathField(""),
+ new NestedPathField(""),
+ new NestedPathField("a")
+ );
+ Assertions.assertEquals(expectedPathParts,
NestedPathFinder.parseBadJsonPath("$...a"));
+ }
@Test
public void testPathSplitter()
@@ -417,53 +472,53 @@ public class NestedPathFinderTest
List<NestedPathPart> pathParts;
pathParts = NestedPathFinder.parseJqPath(".");
- Assert.assertEquals(NESTER, NestedPathFinder.find(NESTER, pathParts));
+ Assertions.assertEquals(NESTER, NestedPathFinder.find(NESTER, pathParts));
pathParts = NestedPathFinder.parseJqPath(".z");
- Assert.assertEquals("foo", NestedPathFinder.find(NESTER, pathParts));
+ Assertions.assertEquals("foo", NestedPathFinder.find(NESTER, pathParts));
pathParts = NestedPathFinder.parseJqPath(".x");
- Assert.assertEquals(NESTER.get("x"), NestedPathFinder.find(NESTER,
pathParts));
+ Assertions.assertEquals(NESTER.get("x"), NestedPathFinder.find(NESTER,
pathParts));
pathParts = NestedPathFinder.parseJqPath(".x[1]");
- Assert.assertEquals("b", NestedPathFinder.find(NESTER, pathParts));
+ Assertions.assertEquals("b", NestedPathFinder.find(NESTER, pathParts));
pathParts = NestedPathFinder.parseJqPath(".x[-1]");
- Assert.assertEquals("c", NestedPathFinder.find(NESTER, pathParts));
+ Assertions.assertEquals("c", NestedPathFinder.find(NESTER, pathParts));
pathParts = NestedPathFinder.parseJqPath(".x[-2]");
- Assert.assertEquals("b", NestedPathFinder.find(NESTER, pathParts));
+ Assertions.assertEquals("b", NestedPathFinder.find(NESTER, pathParts));
pathParts = NestedPathFinder.parseJqPath(".x[-4]");
- Assert.assertNull(NestedPathFinder.find(NESTER, pathParts));
+ Assertions.assertNull(NestedPathFinder.find(NESTER, pathParts));
// object array
pathParts = NestedPathFinder.parseJqPath(".objarray[1]");
- Assert.assertEquals("b", NestedPathFinder.find(NESTER, pathParts));
+ Assertions.assertEquals("b", NestedPathFinder.find(NESTER, pathParts));
pathParts = NestedPathFinder.parseJqPath(".objarray[-1]");
- Assert.assertEquals("c", NestedPathFinder.find(NESTER, pathParts));
+ Assertions.assertEquals("c", NestedPathFinder.find(NESTER, pathParts));
pathParts = NestedPathFinder.parseJqPath(".objarray[-2]");
- Assert.assertEquals("b", NestedPathFinder.find(NESTER, pathParts));
+ Assertions.assertEquals("b", NestedPathFinder.find(NESTER, pathParts));
pathParts = NestedPathFinder.parseJqPath(".objarray[-4]");
- Assert.assertNull(NestedPathFinder.find(NESTER, pathParts));
+ Assertions.assertNull(NestedPathFinder.find(NESTER, pathParts));
// nonexistent
pathParts = NestedPathFinder.parseJqPath(".x[1].y.z");
- Assert.assertNull(NestedPathFinder.find(NESTER, pathParts));
+ Assertions.assertNull(NestedPathFinder.find(NESTER, pathParts));
pathParts = NestedPathFinder.parseJqPath(".y.a");
- Assert.assertEquals("hello", NestedPathFinder.find(NESTER, pathParts));
+ Assertions.assertEquals("hello", NestedPathFinder.find(NESTER, pathParts));
pathParts = NestedPathFinder.parseJqPath(".y[1]");
- Assert.assertNull(NestedPathFinder.find(NESTER, pathParts));
+ Assertions.assertNull(NestedPathFinder.find(NESTER, pathParts));
pathParts = NestedPathFinder.parseJqPath(".\"[sneaky]\"");
- Assert.assertEquals("bar", NestedPathFinder.find(NESTER, pathParts));
+ Assertions.assertEquals("bar", NestedPathFinder.find(NESTER, pathParts));
pathParts = NestedPathFinder.parseJqPath(".\"[also_sneaky]\"[1].c");
- Assert.assertEquals("z", NestedPathFinder.find(NESTER, pathParts));
+ Assertions.assertEquals("z", NestedPathFinder.find(NESTER, pathParts));
}
}
diff --git
a/processing/src/test/java/org/apache/druid/segment/serde/NestedCommonFormatColumnPartSerdeTest.java
b/processing/src/test/java/org/apache/druid/segment/serde/NestedCommonFormatColumnPartSerdeTest.java
new file mode 100644
index 00000000000..3e99c0f1891
--- /dev/null
+++
b/processing/src/test/java/org/apache/druid/segment/serde/NestedCommonFormatColumnPartSerdeTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.serde;
+
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import nl.jqno.equalsverifier.EqualsVerifier;
+import org.apache.druid.segment.TestHelper;
+import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.nested.NestedCommonFormatColumnFormatSpec;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.nio.ByteOrder;
+
+class NestedCommonFormatColumnPartSerdeTest
+{
+ @Test
+ void testSerde() throws JsonProcessingException
+ {
+ NestedCommonFormatColumnPartSerde partSerde =
+ NestedCommonFormatColumnPartSerde.serializerBuilder()
+
.withLogicalType(ColumnType.NESTED_DATA)
+ .withHasNulls(true)
+ .withEnforceLogicalType(false)
+ .withColumnFormatSpec(
+
NestedCommonFormatColumnFormatSpec.builder().build()
+ )
+
.withByteOrder(ByteOrder.nativeOrder())
+ .build();
+ ObjectMapper mapper = TestHelper.makeJsonMapper();
+ Assertions.assertEquals(partSerde,
mapper.readValue(mapper.writeValueAsString(partSerde), ColumnPartSerde.class));
+ }
+
+ @Test
+ public void testEqualsAndHashcode()
+ {
+ EqualsVerifier.forClass(NestedCommonFormatColumnPartSerde.class)
+ .withIgnoredFields("serializer")
+ .usingGetClass()
+ .verify();
+ }
+}
diff --git
a/processing/src/test/resources/nested_segment_empty_fieldname_bug/druid.segment
b/processing/src/test/resources/nested_segment_empty_fieldname_bug/druid.segment
new file mode 100644
index 00000000000..a5e57128999
Binary files /dev/null and
b/processing/src/test/resources/nested_segment_empty_fieldname_bug/druid.segment
differ
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]