This is an automated email from the ASF dual-hosted git repository. jhyde pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/calcite.git
commit 3326475c766267d521330006cc80730c4e456191 Author: Oliver Lee <[email protected]> AuthorDate: Tue Mar 14 18:04:47 2023 -0700 [CALCITE-5614] Serialize Sarg values to and from JSON Close apache/calcite#3140 Co-authored-by: Oliver Lee <[email protected]> Co-authored-by: Julian Hyde <[email protected]> --- .../calcite/rel/externalize/RelEnumTypes.java | 2 + .../apache/calcite/rel/externalize/RelJson.java | 237 ++++++++++++++++++++- .../calcite/rel/externalize/RelJsonReader.java | 11 + .../java/org/apache/calcite/rex/RexBuilder.java | 9 +- .../java/org/apache/calcite/sql/SqlCollation.java | 11 +- .../java/org/apache/calcite/util/DateString.java | 8 +- .../java/org/apache/calcite/util/JsonBuilder.java | 2 +- .../java/org/apache/calcite/util/NlsString.java | 18 +- .../java/org/apache/calcite/util/RangeSets.java | 110 ++++++++-- .../java/org/apache/calcite/util/TimeString.java | 6 +- .../org/apache/calcite/plan/RelWriterTest.java | 95 ++++++++- .../java/org/apache/calcite/util/RangeSetTest.java | 57 +++++ 12 files changed, 526 insertions(+), 40 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelEnumTypes.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelEnumTypes.java index 97181d035d..b4ad2d1b90 100644 --- a/core/src/main/java/org/apache/calcite/rel/externalize/RelEnumTypes.java +++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelEnumTypes.java @@ -18,6 +18,7 @@ package org.apache.calcite.rel.externalize; import org.apache.calcite.avatica.util.TimeUnitRange; import org.apache.calcite.rel.core.TableModify; +import org.apache.calcite.rex.RexUnknownAs; import org.apache.calcite.sql.JoinConditionType; import org.apache.calcite.sql.JoinType; import org.apache.calcite.sql.SqlExplain; @@ -66,6 +67,7 @@ public abstract class RelEnumTypes { ImmutableMap.builder(); register(enumByName, JoinConditionType.class); register(enumByName, JoinType.class); + register(enumByName, RexUnknownAs.class); register(enumByName, SqlExplain.Depth.class); register(enumByName, SqlExplainFormat.class); register(enumByName, SqlExplainLevel.class); diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java index c3f8a7c2ca..0987f1f288 100644 --- a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java +++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java @@ -17,6 +17,7 @@ package org.apache.calcite.rel.externalize; import org.apache.calcite.avatica.AvaticaUtils; +import org.apache.calcite.avatica.util.ByteString; import org.apache.calcite.avatica.util.TimeUnit; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptTable; @@ -62,13 +63,25 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.validate.SqlNameMatchers; +import org.apache.calcite.util.DateString; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.JsonBuilder; +import org.apache.calcite.util.NlsString; +import org.apache.calcite.util.RangeSets; +import org.apache.calcite.util.Sarg; +import org.apache.calcite.util.TimeString; import org.apache.calcite.util.Util; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableRangeSet; +import com.google.common.collect.Range; +import com.google.common.collect.RangeSet; +import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; @@ -93,6 +106,14 @@ import static java.util.Objects.requireNonNull; * into JSON format. */ public class RelJson { + private static final ObjectMapper OBJECT_MAPPER = + new ObjectMapper() + .configure(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS, true); + + private static final List<Class> VALUE_CLASSES = + ImmutableList.of(NlsString.class, BigDecimal.class, ByteString.class, + Boolean.class, DateString.class, TimeString.class); + private final Map<String, Constructor> constructorMap = new HashMap<>(); private final @Nullable JsonBuilder jsonBuilder; private final InputTranslator inputTranslator; @@ -422,6 +443,7 @@ public class RelJson { return map; } + @SuppressWarnings({"BetaApi", "UnstableApiUsage"}) // RangeSet GA in Guava 32 public @Nullable Object toJson(@Nullable Object value) { if (value == null || value instanceof Number @@ -460,12 +482,47 @@ public class RelJson { return toJson((RelDataTypeField) value); } else if (value instanceof RelDistribution) { return toJson((RelDistribution) value); + } else if (value instanceof Sarg) { + //noinspection unchecked,rawtypes + return toJson((Sarg) value); + } else if (value instanceof RangeSet) { + //noinspection unchecked,rawtypes + return toJson((RangeSet) value); + } else if (value instanceof Range) { + //noinspection rawtypes,unchecked + return toJson((Range) value); } else { throw new UnsupportedOperationException("type not serializable: " + value + " (type " + value.getClass().getCanonicalName() + ")"); } } + public <C extends Comparable<C>> Object toJson(Sarg<C> node) { + final Map<String, @Nullable Object> map = jsonBuilder().map(); + map.put("rangeSet", toJson(node.rangeSet)); + map.put("nullAs", RelEnumTypes.fromEnum(node.nullAs)); + return map; + } + + @SuppressWarnings({"BetaApi", "UnstableApiUsage"}) // RangeSet GA in Guava 32 + public <C extends Comparable<C>> List<List<String>> toJson( + RangeSet<C> rangeSet) { + final List<List<String>> list = new ArrayList<>(); + try { + RangeSets.forEach(rangeSet, + RangeToJsonConverter.<C>instance().andThen(list::add)); + } catch (Exception e) { + throw new RuntimeException("Failed to serialize RangeSet: ", e); + } + return list; + } + + /** Serializes a {@link Range} that can be deserialized using + * {@link RelJson#rangeFromJson(List)}. */ + public <C extends Comparable<C>> List<String> toJson(Range<C> range) { + return RangeSets.map(range, RangeToJsonConverter.instance()); + } + private Object toJson(RelDataType node) { final Map<String, @Nullable Object> map = jsonBuilder().map(); if (node.isStruct()) { @@ -517,7 +574,7 @@ public class RelJson { return node.getId(); } - private Object toJson(RexNode node) { + public Object toJson(RexNode node) { final Map<String, @Nullable Object> map; switch (node.getKind()) { case FIELD_ACCESS: @@ -530,7 +587,11 @@ public class RelJson { final RexLiteral literal = (RexLiteral) node; final Object value = literal.getValue3(); map = jsonBuilder().map(); - map.put("literal", RelEnumTypes.fromEnum(value)); + //noinspection rawtypes + map.put("literal", + value instanceof Enum + ? RelEnumTypes.fromEnum((Enum) value) + : toJson(value)); map.put("type", toJson(node.getType())); return map; case INPUT_REF: @@ -657,7 +718,8 @@ public class RelJson { final RexBuilder rexBuilder = cluster.getRexBuilder(); if (o == null) { return null; - } else if (o instanceof Map) { + // Support JSON deserializing of non-default Map classes such as gson LinkedHashMap + } else if (Map.class.isAssignableFrom(o.getClass())) { final Map<String, @Nullable Object> map = (Map) o; final RelDataTypeFactory typeFactory = cluster.getTypeFactory(); if (map.containsKey("op")) { @@ -746,11 +808,26 @@ public class RelJson { return toRex(relInput, literal); } final RelDataType type = toType(typeFactory, get(map, "type")); + if (literal instanceof Map + && ((Map<?, ?>) literal).containsKey("rangeSet")) { + Sarg sarg = sargFromJson((Map) literal); + return rexBuilder.makeSearchArgumentLiteral(sarg, type); + } if (type.getSqlTypeName() == SqlTypeName.SYMBOL) { literal = RelEnumTypes.toEnum((String) literal); } return rexBuilder.makeLiteral(literal, type); } + if (map.containsKey("sargLiteral")) { + Object sargObject = map.get("sargLiteral"); + if (sargObject == null) { + final RelDataType type = toType(typeFactory, get(map, "type")); + return rexBuilder.makeNullLiteral(type); + } + final RelDataType type = toType(typeFactory, get(map, "type")); + Sarg sarg = sargFromJson((Map) sargObject); + return rexBuilder.makeSearchArgumentLiteral(sarg, type); + } throw new UnsupportedOperationException("cannot convert to rex " + o); } else if (o instanceof Boolean) { return rexBuilder.makeLiteral((Boolean) o); @@ -770,8 +847,91 @@ public class RelJson { } } - private void addRexFieldCollationList( - List<RexFieldCollation> list, + /** Converts a JSON object to a {@code Sarg}. + * + * <p>For example, + * {@code {rangeSet: [["[", 0, 5, "]"], ["[", 10, "-", ")"]], + * nullAs: "UNKNOWN"}} represents the range x ≥ 0 and x ≤ 5 or + * x > 10. + */ + // BetaApi is no longer a concern; the Beta tag was removed in Guava 32.0 + @SuppressWarnings({"BetaApi", "unchecked"}) + public static <C extends Comparable<C>> Sarg<C> sargFromJson( + Map<String, Object> map) { + final String nullAs = requireNonNull((String) map.get("nullAs"), "nullAs"); + final List<List<String>> rangeSet = + requireNonNull((List<List<String>>) map.get("rangeSet"), "rangeSet"); + return Sarg.of(RelEnumTypes.toEnum(nullAs), + RelJson.<C>rangeSetFromJson(rangeSet)); + } + + /** Converts a JSON list to a {@link RangeSet}. */ + @SuppressWarnings({"BetaApi", "UnstableApiUsage"}) // RangeSet GA in Guava 32 + public static <C extends Comparable<C>> RangeSet<C> rangeSetFromJson( + List<List<String>> rangeSetsJson) { + final ImmutableRangeSet.Builder<C> builder = ImmutableRangeSet.builder(); + try { + rangeSetsJson.forEach(list -> builder.add(rangeFromJson(list))); + } catch (Exception e) { + throw new RuntimeException("Error creating RangeSet from JSON: ", e); + } + return builder.build(); + } + + /** Creates a {@link Range} from a JSON object. + * + * <p>The JSON object is as serialized using {@link RelJson#toJson(Range)}, + * e.g. {@code ["[", ")", 10, "-"]}. + * + * @see RangeToJsonConverter */ + public static <C extends Comparable<C>> Range<C> rangeFromJson( + List<String> list) { + switch (list.get(0)) { + case "all": + return Range.all(); + case "atLeast": + return Range.atLeast(rangeEndPointFromJson(list.get(1))); + case "atMost": + return Range.atMost(rangeEndPointFromJson(list.get(1))); + case "greaterThan": + return Range.greaterThan(rangeEndPointFromJson(list.get(1))); + case "lessThan": + return Range.lessThan(rangeEndPointFromJson(list.get(1))); + case "singleton": + return Range.singleton(rangeEndPointFromJson(list.get(1))); + case "closed": + return Range.closed(rangeEndPointFromJson(list.get(1)), + rangeEndPointFromJson(list.get(2))); + case "closedOpen": + return Range.closedOpen(rangeEndPointFromJson(list.get(1)), + rangeEndPointFromJson(list.get(2))); + case "openClosed": + return Range.openClosed(rangeEndPointFromJson(list.get(1)), + rangeEndPointFromJson(list.get(2))); + case "open": + return Range.open(rangeEndPointFromJson(list.get(1)), + rangeEndPointFromJson(list.get(2))); + default: + throw new AssertionError("unknown range type " + list.get(0)); + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static <C extends Comparable<C>> C rangeEndPointFromJson(Object o) { + Exception e = null; + for (Class clsType : VALUE_CLASSES) { + try { + return (C) OBJECT_MAPPER.readValue((String) o, clsType); + } catch (JsonProcessingException ex) { + e = ex; + } + } + throw new RuntimeException( + "Error deserializing range endpoint (did not find compatible type): ", + e); + } + + private void addRexFieldCollationList(List<RexFieldCollation> list, RelInput relInput, @Nullable List<Map<String, Object>> order) { if (order == null) { return; @@ -1005,4 +1165,71 @@ public class RelJson { RexNode translateInput(RelJson relJson, int input, Map<String, @Nullable Object> map, RelInput relInput); } + + /** Implementation of {@link RangeSets.Handler} that converts a {@link Range} + * event to a list of strings. + * + * @param <V> Range value type + */ + private static class RangeToJsonConverter<V> + implements RangeSets.Handler<@NonNull V, List<String>> { + @SuppressWarnings("rawtypes") + private static final RangeToJsonConverter INSTANCE = + new RangeToJsonConverter<>(); + + private static <C extends Comparable<C>> RangeToJsonConverter<C> instance() { + //noinspection unchecked + return INSTANCE; + } + + @Override public List<String> all() { + return ImmutableList.of("all"); + } + + @Override public List<String> atLeast(@NonNull V lower) { + return ImmutableList.of("atLeast", toJson(lower)); + } + + @Override public List<String> atMost(@NonNull V upper) { + return ImmutableList.of("atMost", toJson(upper)); + } + + @Override public List<String> greaterThan(@NonNull V lower) { + return ImmutableList.of("greaterThan", toJson(lower)); + } + + @Override public List<String> lessThan(@NonNull V upper) { + return ImmutableList.of("lessThan", toJson(upper)); + } + + @Override public List<String> singleton(@NonNull V value) { + return ImmutableList.of("singleton", toJson(value)); + } + + @Override public List<String> closed(@NonNull V lower, @NonNull V upper) { + return ImmutableList.of("closed", toJson(lower), toJson(upper)); + } + + @Override public List<String> closedOpen(@NonNull V lower, + @NonNull V upper) { + return ImmutableList.of("closedOpen", toJson(lower), toJson(upper)); + } + + @Override public List<String> openClosed(@NonNull V lower, + @NonNull V upper) { + return ImmutableList.of("openClosed", toJson(lower), toJson(upper)); + } + + @Override public List<String> open(@NonNull V lower, @NonNull V upper) { + return ImmutableList.of("open", toJson(lower), toJson(upper)); + } + + private static String toJson(Object o) { + try { + return OBJECT_MAPPER.writeValueAsString(o); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize Range endpoint: ", e); + } + } + } } diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonReader.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonReader.java index a98d0e8848..8b6d1e0f03 100644 --- a/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonReader.java +++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonReader.java @@ -109,6 +109,17 @@ public class RelJsonReader { return RelJson.create().toType(typeFactory, o); } + /** Converts a JSON string (such as that produced by + * {@link RelJson#toJson(RexNode)}) into a Calcite expression. */ + public static RexNode readRex(RelOptCluster typeFactory, String s) + throws IOException { + final ObjectMapper mapper = new ObjectMapper(); + Map<String, Object> o = mapper + .configure(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS, true) + .readValue(s, TYPE_REF); + return RelJson.create().toRex(typeFactory, o); + } + private void readRels(List<Map<String, Object>> jsonRels) { for (Map<String, Object> jsonRel : jsonRels) { readRel(jsonRel); diff --git a/core/src/main/java/org/apache/calcite/rex/RexBuilder.java b/core/src/main/java/org/apache/calcite/rex/RexBuilder.java index c19a1001c6..fda55faa0c 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexBuilder.java +++ b/core/src/main/java/org/apache/calcite/rex/RexBuilder.java @@ -1638,6 +1638,10 @@ public class RexBuilder { case INTEGER: case BIGINT: case DECIMAL: + if (value instanceof RexLiteral + && ((RexLiteral) value).getTypeName() == SqlTypeName.SARG) { + return (RexNode) value; + } return makeExactLiteral((BigDecimal) value, type); case FLOAT: case REAL: @@ -1736,10 +1740,13 @@ public class RexBuilder { * {@link org.apache.calcite.rex.RexLiteral#valueMatchesType}. * * <p>Returns null if and only if {@code o} is null. */ - private static @PolyNull Object clean(@PolyNull Object o, RelDataType type) { + private @PolyNull Object clean(@PolyNull Object o, RelDataType type) { if (o == null) { return o; } + if (o instanceof Sarg) { + return makeSearchArgumentLiteral((Sarg) o, type); + } switch (type.getSqlTypeName()) { case TINYINT: case SMALLINT: diff --git a/core/src/main/java/org/apache/calcite/sql/SqlCollation.java b/core/src/main/java/org/apache/calcite/sql/SqlCollation.java index b6576d4edc..b02035f791 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlCollation.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlCollation.java @@ -22,6 +22,10 @@ import org.apache.calcite.util.Glossary; import org.apache.calcite.util.SerializableCharset; import org.apache.calcite.util.Util; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; + import org.checkerframework.checker.initialization.qual.UnderInitialization; import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; @@ -95,9 +99,10 @@ public class SqlCollation implements Serializable { * @param collation Collation specification * @param coercibility Coercibility */ + @JsonCreator public SqlCollation( - String collation, - Coercibility coercibility) { + @JsonProperty("collationName") String collation, + @JsonProperty("coercibility") Coercibility coercibility) { this.coercibility = coercibility; SqlParserUtil.ParsedCollation parseValues = SqlParserUtil.parseCollation(collation); @@ -290,6 +295,7 @@ public class SqlCollation implements Serializable { writer.identifier(collationName, false); } + @JsonIgnore public Charset getCharset() { return wrappedCharset.getCharset(); } @@ -312,6 +318,7 @@ public class SqlCollation implements Serializable { * which case {@link String#compareTo} will be used. */ @Pure + @JsonIgnore public @Nullable Collator getCollator() { return null; } diff --git a/core/src/main/java/org/apache/calcite/util/DateString.java b/core/src/main/java/org/apache/calcite/util/DateString.java index 33e3675353..2dc19b5def 100644 --- a/core/src/main/java/org/apache/calcite/util/DateString.java +++ b/core/src/main/java/org/apache/calcite/util/DateString.java @@ -18,6 +18,9 @@ package org.apache.calcite.util; import org.apache.calcite.avatica.util.DateTimeUtils; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import org.checkerframework.checker.nullness.qual.Nullable; @@ -120,12 +123,15 @@ public class DateString implements Comparable<DateString> { } /** Creates a DateString that is a given number of days since the epoch. */ - public static DateString fromDaysSinceEpoch(int days) { + @JsonCreator + public static DateString fromDaysSinceEpoch( + @JsonProperty("daysSinceEpoch") int days) { return new DateString(DateTimeUtils.unixDateToString(days)); } /** Returns the number of milliseconds since the epoch. Always a multiple of * 86,400,000 (the number of milliseconds in a day). */ + @JsonIgnore public long getMillisSinceEpoch() { return getDaysSinceEpoch() * DateTimeUtils.MILLIS_PER_DAY; } diff --git a/core/src/main/java/org/apache/calcite/util/JsonBuilder.java b/core/src/main/java/org/apache/calcite/util/JsonBuilder.java index 68496fb161..eeb59699cf 100644 --- a/core/src/main/java/org/apache/calcite/util/JsonBuilder.java +++ b/core/src/main/java/org/apache/calcite/util/JsonBuilder.java @@ -130,7 +130,7 @@ public class JsonBuilder { } else if (o instanceof String) { appendString(buf, (String) o); } else { - assert o instanceof Number || o instanceof Boolean; + assert o instanceof Number || o instanceof Boolean : o; buf.append(o); } } diff --git a/core/src/main/java/org/apache/calcite/util/NlsString.java b/core/src/main/java/org/apache/calcite/util/NlsString.java index 327b699fc0..cbc87b6755 100644 --- a/core/src/main/java/org/apache/calcite/util/NlsString.java +++ b/core/src/main/java/org/apache/calcite/util/NlsString.java @@ -23,6 +23,8 @@ import org.apache.calcite.sql.SqlDialect; import org.apache.calcite.sql.SqlUtil; import org.apache.calcite.sql.dialect.AnsiSqlDialect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; @@ -42,6 +44,8 @@ import java.util.Objects; import static org.apache.calcite.util.Static.RESOURCE; +import static java.util.Objects.requireNonNull; + /** * A string, optionally with {@link Charset character set} and * {@link SqlCollation}. It is immutable. @@ -72,6 +76,7 @@ public class NlsString implements Comparable<NlsString>, Cloneable { }); private final @Nullable String stringValue; + @JsonProperty("valueBytes") private final @Nullable ByteString bytesValue; private final @Nullable String charsetName; private final @Nullable Charset charset; @@ -94,8 +99,8 @@ public class NlsString implements Comparable<NlsString>, Cloneable { */ public NlsString(ByteString bytesValue, String charsetName, @Nullable SqlCollation collation) { - this(null, Objects.requireNonNull(bytesValue, "bytesValue"), - Objects.requireNonNull(charsetName, "charsetName"), collation); + this(null, requireNonNull(bytesValue, "bytesValue"), + requireNonNull(charsetName, "charsetName"), collation); } /** @@ -111,9 +116,12 @@ public class NlsString implements Comparable<NlsString>, Cloneable { * @throws RuntimeException If the given value cannot be represented in the * given charset */ - public NlsString(String stringValue, @Nullable String charsetName, - @Nullable SqlCollation collation) { - this(Objects.requireNonNull(stringValue, "stringValue"), null, charsetName, collation); + @JsonCreator + public NlsString(@JsonProperty("value") String stringValue, + @JsonProperty("charsetName") @Nullable String charsetName, + @JsonProperty("collation") @Nullable SqlCollation collation) { + this(requireNonNull(stringValue, "stringValue"), null, charsetName, + collation); } /** Internal constructor; other constructors must call it. */ diff --git a/core/src/main/java/org/apache/calcite/util/RangeSets.java b/core/src/main/java/org/apache/calcite/util/RangeSets.java index 70b81686d4..a0967e2383 100644 --- a/core/src/main/java/org/apache/calcite/util/RangeSets.java +++ b/core/src/main/java/org/apache/calcite/util/RangeSets.java @@ -22,11 +22,15 @@ import com.google.common.collect.Range; import com.google.common.collect.RangeSet; import com.google.common.collect.TreeRangeSet; +import org.checkerframework.checker.nullness.qual.NonNull; + import java.util.Iterator; import java.util.Set; import java.util.function.BiConsumer; import java.util.function.Function; +import static java.util.Objects.requireNonNull; + /** Utilities for Guava {@link com.google.common.collect.RangeSet}. */ @SuppressWarnings({"BetaApi", "UnstableApiUsage"}) public class RangeSets { @@ -279,39 +283,101 @@ public class RangeSets { /** Deconstructor for {@link Range} values. * - * @param <C> Value type + * @param <V> Value type * @param <R> Return type * * @see Consumer */ - public interface Handler<C extends Comparable<C>, R> { + public interface Handler<V, R> { R all(); - R atLeast(C lower); - R atMost(C upper); - R greaterThan(C lower); - R lessThan(C upper); - R singleton(C value); - R closed(C lower, C upper); - R closedOpen(C lower, C upper); - R openClosed(C lower, C upper); - R open(C lower, C upper); + R atLeast(V lower); + R atMost(V upper); + R greaterThan(V lower); + R lessThan(V upper); + R singleton(V value); + R closed(V lower, V upper); + R closedOpen(V lower, V upper); + R openClosed(V lower, V upper); + R open(V lower, V upper); + + /** Creates a Consumer that sends output to a given sink. */ + default Consumer<V> andThen(java.util.function.Consumer<R> consumer) { + return new SinkConsumer<>(this, consumer); + } + } + + /** Consumer that deconstructs a range to a handler then sends the resulting + * range to a {@link java.util.function.Consumer}. + * + * @param <V> Value type + * @param <R> Output element type + */ + private static class SinkConsumer<V, R> implements Consumer<V> { + final Handler<V, R> handler; + final java.util.function.Consumer<R> consumer; + + SinkConsumer(Handler<V, R> handler, + java.util.function.Consumer<R> consumer) { + this.handler = requireNonNull(handler, "handler"); + this.consumer = requireNonNull(consumer, "consumer"); + } + + @Override public void all() { + consumer.accept(handler.all()); + } + + @Override public void atLeast(V lower) { + consumer.accept(handler.atLeast(lower)); + } + + @Override public void atMost(V upper) { + consumer.accept(handler.atMost(upper)); + } + + @Override public void greaterThan(V lower) { + consumer.accept(handler.greaterThan(lower)); + } + + @Override public void lessThan(V upper) { + consumer.accept(handler.lessThan(upper)); + } + + @Override public void singleton(V value) { + consumer.accept(handler.singleton(value)); + } + + @Override public void closed(V lower, V upper) { + consumer.accept(handler.closed(lower, upper)); + } + + @Override public void closedOpen(V lower, V upper) { + consumer.accept(handler.closedOpen(lower, upper)); + } + + @Override public void openClosed(V lower, V upper) { + consumer.accept(handler.openClosed(lower, upper)); + } + + @Override public void open(V lower, V upper) { + consumer.accept(handler.open(lower, upper)); + } } /** Consumer of {@link Range} values. * - * @param <C> Value type + * @param <V> Value type * * @see Handler */ - public interface Consumer<C extends Comparable<C>> { + public interface Consumer<@NonNull V> { void all(); - void atLeast(C lower); - void atMost(C upper); - void greaterThan(C lower); - void lessThan(C upper); - void singleton(C value); - void closed(C lower, C upper); - void closedOpen(C lower, C upper); - void openClosed(C lower, C upper); - void open(C lower, C upper); + void atLeast(V lower); + void atMost(V upper); + void greaterThan(V lower); + void lessThan(V upper); + void singleton(V value); + void closed(V lower, V upper); + void closedOpen(V lower, V upper); + void openClosed(V lower, V upper); + void open(V lower, V upper); } /** Handler that converts a Range into another Range of the same type, diff --git a/core/src/main/java/org/apache/calcite/util/TimeString.java b/core/src/main/java/org/apache/calcite/util/TimeString.java index e0d46b837a..249c955030 100644 --- a/core/src/main/java/org/apache/calcite/util/TimeString.java +++ b/core/src/main/java/org/apache/calcite/util/TimeString.java @@ -18,6 +18,8 @@ package org.apache.calcite.util; import org.apache.calcite.avatica.util.DateTimeUtils; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -146,7 +148,8 @@ public class TimeString implements Comparable<TimeString> { .withMillis(calendar.get(Calendar.MILLISECOND)); } - public static TimeString fromMillisOfDay(int i) { + @JsonCreator + public static TimeString fromMillisOfDay(@JsonProperty("millisOfDay") int i) { return new TimeString(DateTimeUtils.unixTimeToString(i)) .withMillis((int) floorMod(i, 1000L)); } @@ -163,7 +166,6 @@ public class TimeString implements Comparable<TimeString> { } return new TimeString(v); } - public int getMillisOfDay() { int h = Integer.valueOf(v.substring(0, 2)); int m = Integer.valueOf(v.substring(3, 5)); diff --git a/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java b/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java index 6c5275ec3f..c493d51ed0 100644 --- a/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java +++ b/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java @@ -47,6 +47,7 @@ import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCorrelVariable; import org.apache.calcite.rex.RexFieldCollation; import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexProgramBuilder; import org.apache.calcite.rex.RexWindowBounds; @@ -65,10 +66,13 @@ import org.apache.calcite.test.schemata.hr.HrSchema; import org.apache.calcite.tools.FrameworkConfig; import org.apache.calcite.tools.Frameworks; import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.DateString; import org.apache.calcite.util.Holder; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.JsonBuilder; +import org.apache.calcite.util.NlsString; import org.apache.calcite.util.TestUtil; +import org.apache.calcite.util.TimeString; import org.apache.calcite.util.TimestampString; import com.fasterxml.jackson.core.JsonProcessingException; @@ -93,6 +97,7 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Stream; @@ -826,6 +831,94 @@ class RelWriterTest { .assertThatPlan(isLinux(expected)); } + @Test void testSearchOperator() { + final FrameworkConfig config = RelBuilderTest.config().build(); + final RelBuilder b = RelBuilder.create(config); + final RexBuilder rexBuilder = b.getRexBuilder(); + + // Test toJson -> toRex -> toJson is the same. + final JsonBuilder jsonBuilder = new JsonBuilder(); + final RelJson relJson = RelJson.create().withJsonBuilder(jsonBuilder); + final Consumer<RexNode> consumer = node -> { + Object jsonRepresentation = relJson.toJson(node); + assertThat(jsonRepresentation, notNullValue()); + + RexNode deserialized = relJson.toRex(b.getCluster(), jsonRepresentation); + assertThat(node, is(deserialized)); + assertThat(jsonRepresentation, is(relJson.toJson(deserialized))); + + // Test that toRex is the same as toJsonString -> readRex + final String s = jsonBuilder.toJsonString(jsonRepresentation); + RexNode deserialized2; + try { + deserialized2 = RelJsonReader.readRex(b.getCluster(), s); + } catch (IOException e) { + throw new RuntimeException(e); + } + assertThat(deserialized2, is(deserialized)); + }; + + // Commented out but we should also get this passing! SEARCH in a RelNode + // using the JSON writer also leads to failures. + if (false) { + final RelNode rel = b + .scan("EMP") + .project(b.between(b.field("DEPTNO"), b.literal(20), b.literal(30))) + .build(); + final RelJsonWriter jsonWriter = + new RelJsonWriter(new JsonBuilder(), RelJson::withLibraryOperatorTable); + rel.explain(jsonWriter); + String relJsonString = jsonWriter.asString(); + String result = deserializeAndDumpToTextFormat(getSchema(rel), relJsonString); + final String expected = "<TODO>"; + assertThat(result, isLinux(expected)); + } + + RexNode between = + rexBuilder.makeBetween(b.literal(45), + b.literal(20), + b.literal(30)); + consumer.accept(between); + + RexNode inNode = + rexBuilder.makeIn(b.literal(12), + ImmutableList.of( + b.literal(20), + b.literal(14))); + consumer.accept(inNode); + + // Test Calcite DateString class works in a Range + final DateString d1 = + DateString.fromCalendarFields( + new TimestampString(1970, 2, 1, 1, 1, 0).toCalendar()); + final DateString d2 = DateString.fromDaysSinceEpoch(100); + final DateString d3 = DateString.fromDaysSinceEpoch(1000); + RexNode dateNode = + rexBuilder.makeBetween(rexBuilder.makeDateLiteral(d2), + rexBuilder.makeDateLiteral(d1), + rexBuilder.makeDateLiteral(d3)); + consumer.accept(dateNode); + + // Test Calcite TimeString + final RexLiteral t1 = rexBuilder.makeTimeLiteral(new TimeString(1, 0, 0), 0); + final RexLiteral t2 = rexBuilder.makeTimeLiteral(new TimeString(2, 2, 2), 6); + final RexLiteral t3 = rexBuilder.makeTimeLiteral(new TimeString(3, 3, 3), 9); + + RexNode timeNode = rexBuilder.makeBetween(t2, t1, t3); + consumer.accept(timeNode); + + // Test Calcite NlsString + final NlsString nls1 = new NlsString("one", null, null); + final NlsString nls2 = new NlsString("ten", null, null); + final NlsString nls3 = new NlsString("sixteen", null, null); + RexNode nlsNode = + rexBuilder.makeIn( + rexBuilder.makeCharLiteral(nls2), + ImmutableList.of(rexBuilder.makeCharLiteral(nls1), + rexBuilder.makeCharLiteral(nls3))); + consumer.accept(nlsNode); + } + @ParameterizedTest @MethodSource("explainFormats") void testAggregateWithAlias(SqlExplainFormat format) { @@ -872,7 +965,7 @@ class RelWriterTest { /** Test case for * <a href="https://issues.apache.org/jira/browse/CALCITE-4804";>[CALCITE-4804] - * Support Snapshot operator serialization and deserizalization</a>. */ + * Support Snapshot operator serialization and deserialization</a>. */ @Test void testSnapshot() { // Equivalent SQL: // SELECT * diff --git a/core/src/test/java/org/apache/calcite/util/RangeSetTest.java b/core/src/test/java/org/apache/calcite/util/RangeSetTest.java index 904979ae38..5de76de31e 100644 --- a/core/src/test/java/org/apache/calcite/util/RangeSetTest.java +++ b/core/src/test/java/org/apache/calcite/util/RangeSetTest.java @@ -17,6 +17,7 @@ package org.apache.calcite.util; import org.apache.calcite.linq4j.Ord; +import org.apache.calcite.rel.externalize.RelJson; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableRangeSet; @@ -27,6 +28,7 @@ import com.google.common.collect.TreeRangeSet; import org.junit.jupiter.api.Test; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -44,6 +46,61 @@ import static org.hamcrest.MatcherAssert.assertThat; */ @SuppressWarnings("UnstableApiUsage") class RangeSetTest { + + /** Tests {@link org.apache.calcite.rel.externalize.RelJson#toJson(Range)} + * and {@link RelJson#rangeFromJson(List)}. */ + @Test void testRangeSetSerializeDeserialize() { + RelJson relJson = RelJson.create(); + final Range<BigDecimal> point = Range.singleton(BigDecimal.valueOf(0)); + final Range<BigDecimal> closedRange1 = + Range.closed(BigDecimal.valueOf(0), BigDecimal.valueOf(5)); + final Range<BigDecimal> closedRange2 = + Range.closed(BigDecimal.valueOf(6), BigDecimal.valueOf(10)); + + final Range<BigDecimal> gt1 = Range.greaterThan(BigDecimal.valueOf(7)); + final Range<BigDecimal> al1 = Range.atLeast(BigDecimal.valueOf(8)); + final Range<BigDecimal> lt1 = Range.lessThan(BigDecimal.valueOf(4)); + final Range<BigDecimal> am1 = Range.atMost(BigDecimal.valueOf(3)); + + // Test serialize/deserialize Range + // Point + assertThat(RelJson.rangeFromJson(relJson.toJson(point)), is(point)); + // Closed Range + assertThat(RelJson.rangeFromJson(relJson.toJson(closedRange1)), + is(closedRange1)); + // Open Range + assertThat(RelJson.rangeFromJson(relJson.toJson(gt1)), is(gt1)); + assertThat(RelJson.rangeFromJson(relJson.toJson(al1)), is(al1)); + assertThat(RelJson.rangeFromJson(relJson.toJson(lt1)), is(lt1)); + assertThat(RelJson.rangeFromJson(relJson.toJson(am1)), is(am1)); + // Test closed single RangeSet + final RangeSet<BigDecimal> closedRangeSet = ImmutableRangeSet.of(closedRange1); + assertThat(RelJson.rangeSetFromJson(relJson.toJson(closedRangeSet)), + is(closedRangeSet)); + // Test complex RangeSets + final RangeSet<BigDecimal> complexClosedRangeSet1 = + ImmutableRangeSet.<BigDecimal>builder() + .add(closedRange1) + .add(closedRange2) + .build(); + assertThat( + RelJson.rangeSetFromJson(relJson.toJson(complexClosedRangeSet1)), + is(complexClosedRangeSet1)); + final RangeSet<BigDecimal> complexClosedRangeSet2 = + ImmutableRangeSet.<BigDecimal>builder() + .add(gt1) + .add(am1) + .build(); + assertThat(RelJson.rangeSetFromJson(relJson.toJson(complexClosedRangeSet2)), + is(complexClosedRangeSet2)); + + // Test None and All + final RangeSet<BigDecimal> setNone = ImmutableRangeSet.of(); + final RangeSet<BigDecimal> setAll = setNone.complement(); + assertThat(RelJson.rangeSetFromJson(relJson.toJson(setNone)), is(setNone)); + assertThat(RelJson.rangeSetFromJson(relJson.toJson(setAll)), is(setAll)); + } + /** Tests {@link RangeSets#minus(RangeSet, Range)}. */ @SuppressWarnings("UnstableApiUsage") @Test void testRangeSetMinus() {
