This is an automated email from the ASF dual-hosted git repository.
siddteotia pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new b3263337a8 Support JSON column type in recommender. (#8965)
b3263337a8 is described below
commit b3263337a8763d3d3415790270d7b2dbf8cc3194
Author: Amrish Lal <[email protected]>
AuthorDate: Tue Jun 28 19:55:55 2022 -0700
Support JSON column type in recommender. (#8965)
* Support JSON column type in recommender.
* Fix code format.
* codereview changes.
* Add JSON to TestConfigEngine test case.
* Code format.
* Rebuild.
* codereview changes + cleanup.
* Rebuild.
* Rule for recommending NoDictionary and JsonIndex on JSON columns.
* Add licence header.
---
.../data/generator/GeneratorFactory.java | 3 +
.../recommender/data/generator/JsonGenerator.java | 62 ++++++++++++++++++++
.../controller/recommender/io/InputManager.java | 3 +
.../realtime/provisioning/MemoryEstimator.java | 6 ++
.../recommender/rules/RulesToExecute.java | 14 +++++
.../recommender/rules/impl/JsonIndexRule.java | 55 +++++++++++++++++
.../recommender/rules/io/configs/IndexConfig.java | 14 +++++
.../rules/io/params/RecommenderConstants.java | 1 +
.../controller/recommender/TestConfigEngine.java | 16 ++++-
.../data/generator/JsonGeneratorTest.java | 68 ++++++++++++++++++++++
.../recommenderInput/SegmentSizeRuleInput.json | 11 +++-
11 files changed, 250 insertions(+), 3 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/GeneratorFactory.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/GeneratorFactory.java
index 3496935fb4..4186543eec 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/GeneratorFactory.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/GeneratorFactory.java
@@ -36,6 +36,9 @@ public class GeneratorFactory {
if (type == DataType.STRING) {
return new StringGenerator(cardinality, numberOfValuesPerEntry,
entryLength);
}
+ if (type == DataType.JSON) {
+ return new JsonGenerator(entryLength);
+ }
if (type == DataType.BYTES) {
return new BytesGenerator(cardinality, entryLength);
}
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/JsonGenerator.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/JsonGenerator.java
new file mode 100644
index 0000000000..cb7d4ec0c5
--- /dev/null
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/JsonGenerator.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.controller.recommender.data.generator;
+
+import java.util.Random;
+
+
+public class JsonGenerator implements Generator {
+ // Length of each key-value pair int the JSON string.
+ static final int DEFAULT_JSON_ELEMENT_LENGTH = 5;
+
+ private final int _jsonStringLength;
+ private final Random _random;
+
+ public JsonGenerator(Integer jsonSize) {
+ _jsonStringLength = jsonSize == null || jsonSize <= 0 ? 0 : jsonSize;
+ _random = new Random(System.currentTimeMillis());
+ }
+
+ @Override
+ public void init() {
+ }
+
+ @Override
+ public Object next() {
+ // Return empty string if size of json string is zero or if number of
elements is zero.
+ if (_jsonStringLength == 0 || _jsonStringLength /
DEFAULT_JSON_ELEMENT_LENGTH == 0) {
+ return "{}";
+ }
+
+ // Create JSON string { "<character>":<integer>, "<character>":<integer>,
...} as per length specified. Escape
+ // comma's in JSON since comma is used as delimiter in CSV data file that
will be used to generate segment.
+ StringBuffer jsonBuffer = new StringBuffer();
+ int elementCount = _jsonStringLength / DEFAULT_JSON_ELEMENT_LENGTH;
+ jsonBuffer.append("{");
+ for (int i = 0; i < elementCount; i++) {
+ if (jsonBuffer.length() > 1) {
+ jsonBuffer.append("\\,");
+ }
+ String item = "\"" + (char) ('a' + _random.nextInt(26)) + "\":" +
_random.nextInt(10);
+ jsonBuffer.append(item);
+ }
+
+ return jsonBuffer.append("}").toString();
+ }
+}
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/io/InputManager.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/io/InputManager.java
index fb74b85a48..02aca45911 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/io/InputManager.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/io/InputManager.java
@@ -135,6 +135,7 @@ public class InputManager {
put(FieldSpec.DataType.DOUBLE, Double.BYTES);
put(FieldSpec.DataType.BYTES, Byte.BYTES);
put(FieldSpec.DataType.STRING, Character.BYTES);
+ put(FieldSpec.DataType.JSON, Character.BYTES);
put(FieldSpec.DataType.BOOLEAN, Integer.BYTES); // Stored internally as an
INTEGER
put(null, DEFAULT_NULL_SIZE);
}};
@@ -220,6 +221,7 @@ public class InputManager {
String sortedColumn =
_overWrittenConfigs.getIndexConfig().getSortedColumn();
Set<String> invertedIndexColumns =
_overWrittenConfigs.getIndexConfig().getInvertedIndexColumns();
Set<String> rangeIndexColumns =
_overWrittenConfigs.getIndexConfig().getRangeIndexColumns();
+ Set<String> jsonIndexColumns =
_overWrittenConfigs.getIndexConfig().getJsonIndexColumns();
Set<String> noDictionaryColumns =
_overWrittenConfigs.getIndexConfig().getNoDictionaryColumns();
/*Validate if there's conflict between NoDictionaryColumns and
dimNamesWithAnyIndex*/
@@ -227,6 +229,7 @@ public class InputManager {
dimNamesWithAnyIndex.add(sortedColumn);
dimNamesWithAnyIndex.addAll(invertedIndexColumns);
dimNamesWithAnyIndex.addAll(rangeIndexColumns);
+ dimNamesWithAnyIndex.addAll(jsonIndexColumns);
for (String colName : noDictionaryColumns) {
if (dimNamesWithAnyIndex.contains(colName)) {
throw new InvalidInputException(
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/realtime/provisioning/MemoryEstimator.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/realtime/provisioning/MemoryEstimator.java
index fd915e1609..baa2ea4b52 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/realtime/provisioning/MemoryEstimator.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/realtime/provisioning/MemoryEstimator.java
@@ -40,6 +40,7 @@ import
org.apache.pinot.controller.recommender.io.metadata.FieldMetadata;
import org.apache.pinot.controller.recommender.io.metadata.SchemaWithMetaData;
import
org.apache.pinot.controller.recommender.io.metadata.TimeFieldSpecMetadata;
import
org.apache.pinot.controller.recommender.io.metadata.TimeGranularitySpecMetadata;
+import org.apache.pinot.plugin.inputformat.csv.CSVRecordReaderConfig;
import
org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
import org.apache.pinot.segment.local.indexsegment.mutable.MutableSegmentImpl;
import
org.apache.pinot.segment.local.io.readerwriter.RealtimeIndexOffHeapMemoryManager;
@@ -583,6 +584,11 @@ public class MemoryEstimator {
segmentGeneratorConfig.setOutDir(outDir);
segmentGeneratorConfig.setTableName(_tableConfig.getTableName());
segmentGeneratorConfig.setSequenceId(0);
+
+ CSVRecordReaderConfig recordReaderConfig = new CSVRecordReaderConfig();
+ recordReaderConfig.setEscapeCharacter('\\');
+ segmentGeneratorConfig.setReaderConfig(recordReaderConfig);
+
return segmentGeneratorConfig;
}
}
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/RulesToExecute.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/RulesToExecute.java
index 01d744461f..54e9e17ede 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/RulesToExecute.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/RulesToExecute.java
@@ -26,6 +26,7 @@ import
org.apache.pinot.controller.recommender.rules.impl.AggregateMetricsRule;
import org.apache.pinot.controller.recommender.rules.impl.BloomFilterRule;
import org.apache.pinot.controller.recommender.rules.impl.FlagQueryRule;
import
org.apache.pinot.controller.recommender.rules.impl.InvertedSortedIndexJointRule;
+import org.apache.pinot.controller.recommender.rules.impl.JsonIndexRule;
import org.apache.pinot.controller.recommender.rules.impl.KafkaPartitionRule;
import
org.apache.pinot.controller.recommender.rules.impl.NoDictionaryOnHeapDictionaryJointRule;
import
org.apache.pinot.controller.recommender.rules.impl.PinotTablePartitionRule;
@@ -61,6 +62,8 @@ public class RulesToExecute {
return new BloomFilterRule(inputManager, outputManager);
case RangeIndexRule:
return new RangeIndexRule(inputManager, outputManager);
+ case JsonIndexRule:
+ return new JsonIndexRule(inputManager, outputManager);
case NoDictionaryOnHeapDictionaryJointRule:
return new NoDictionaryOnHeapDictionaryJointRule(inputManager,
outputManager);
case VariedLengthDictionaryRule:
@@ -82,6 +85,7 @@ public class RulesToExecute {
boolean _recommendInvertedSortedIndexJoint =
DEFAULT_RECOMMEND_INVERTED_SORTED_INDEX_JOINT;
boolean _recommendBloomFilter = DEFAULT_RECOMMEND_BLOOM_FILTER;
boolean _recommendRangeIndex = DEFAULT_RECOMMEND_RANGE_INDEX;
+ boolean _recommendJsonIndex = DEFAULT_RECOMMEND_JSON_INDEX;
boolean _recommendNoDictionaryOnHeapDictionaryJoint =
DEFAULT_RECOMMEND_NO_DICTIONARY_ONHEAP_DICTIONARY_JOINT;
boolean _recommendVariedLengthDictionary =
DEFAULT_RECOMMEND_VARIED_LENGTH_DICTIONARY;
boolean _recommendFlagQuery = DEFAULT_RECOMMEND_FLAG_QUERY;
@@ -128,6 +132,11 @@ public class RulesToExecute {
_recommendRangeIndex = recommendRangeIndex;
}
+ @JsonSetter(nulls = Nulls.SKIP)
+ public void setRecommendJsonIndex(boolean recommendJsonIndex) {
+ _recommendJsonIndex = recommendJsonIndex;
+ }
+
@JsonSetter(nulls = Nulls.SKIP)
public void setRecommendAggregateMetrics(boolean aggregateMetrics) {
_recommendAggregateMetrics = aggregateMetrics;
@@ -175,6 +184,10 @@ public class RulesToExecute {
return _recommendRangeIndex;
}
+ public boolean isRecommendJsonIndex() {
+ return _recommendJsonIndex;
+ }
+
public boolean isRecommendAggregateMetrics() {
return _recommendAggregateMetrics;
}
@@ -203,6 +216,7 @@ public class RulesToExecute {
// partitions, after NoDictionaryOnHeapDictionaryJointRule to correctly
calculate record size
BloomFilterRule,
RangeIndexRule,
+ JsonIndexRule,
AggregateMetricsRule,
RealtimeProvisioningRule // this rule must be the last one because it
needs the output of other rules as its input
}
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/impl/JsonIndexRule.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/impl/JsonIndexRule.java
new file mode 100644
index 0000000000..df36f742d7
--- /dev/null
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/impl/JsonIndexRule.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.controller.recommender.rules.impl;
+
+import
org.apache.pinot.controller.recommender.exceptions.InvalidInputException;
+import org.apache.pinot.controller.recommender.io.ConfigManager;
+import org.apache.pinot.controller.recommender.io.InputManager;
+import org.apache.pinot.controller.recommender.rules.AbstractRule;
+import org.apache.pinot.controller.recommender.rules.io.configs.IndexConfig;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/** JSON columns must be NoDictionary columns with JsonIndex. */
+public class JsonIndexRule extends AbstractRule {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(RangeIndexRule.class);
+
+ public JsonIndexRule(InputManager input, ConfigManager output) {
+ super(input, output);
+ }
+
+ @Override
+ public void run()
+ throws InvalidInputException {
+ int numColumns = _input.getNumCols();
+ IndexConfig indexConfig = _output.getIndexConfig();
+ for (int i = 0; i < numColumns; i++) {
+ String columnName = _input.intToColName(i);
+ FieldSpec.DataType columnType = _input.getFieldType(columnName);
+ if (columnType == FieldSpec.DataType.JSON) {
+ // JSON columns must be NoDictionary columns and have a JsonIndex.
+ LOGGER.info("Recommending NoDictionary and JsonIndex on JSON column
{}.", columnName);
+ indexConfig.getJsonIndexColumns().add(columnName);
+ indexConfig.getNoDictionaryColumns().add(columnName);
+ }
+ }
+ }
+}
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/io/configs/IndexConfig.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/io/configs/IndexConfig.java
index 160491a6b5..d54d84424c 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/io/configs/IndexConfig.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/io/configs/IndexConfig.java
@@ -33,6 +33,7 @@ public class IndexConfig {
Set<String> _rangeIndexColumns = new HashSet<>();
String _sortedColumn = "";
Set<String> _bloomFilterColumns = new HashSet<>();
+ Set<String> _jsonIndexColumns = new HashSet<>();
Set<String> _noDictionaryColumns = new HashSet<>();
Set<String> _onHeapDictionaryColumns = new HashSet<>();
@@ -84,6 +85,11 @@ public class IndexConfig {
_isSortedColumnOverwritten = sortedColumnOverwritten;
}
+ @JsonSetter(nulls = Nulls.SKIP)
+ public void setJsonIndexColumns(Set<String> jsonIndexColumns) {
+ _jsonIndexColumns = jsonIndexColumns;
+ }
+
public Set<String> getVarLengthDictionaryColumns() {
return _varLengthDictionaryColumns;
}
@@ -112,6 +118,10 @@ public class IndexConfig {
return _rangeIndexColumns;
}
+ public Set<String> getJsonIndexColumns() {
+ return _jsonIndexColumns;
+ }
+
public boolean hasInvertedIndex(String colname) {
return _invertedIndexColumns.contains(colname);
}
@@ -124,6 +134,10 @@ public class IndexConfig {
return _rangeIndexColumns.contains(colName);
}
+ public boolean hasJsonIndex(String colName) {
+ return _jsonIndexColumns.contains(colName);
+ }
+
public boolean hasAnyIndex() {
return !_sortedColumn.isEmpty() || !_rangeIndexColumns.isEmpty() ||
!_invertedIndexColumns.isEmpty();
}
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/io/params/RecommenderConstants.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/io/params/RecommenderConstants.java
index 2cda521399..1a707fb4f4 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/io/params/RecommenderConstants.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/io/params/RecommenderConstants.java
@@ -48,6 +48,7 @@ public class RecommenderConstants {
public static final boolean DEFAULT_RECOMMEND_INVERTED_SORTED_INDEX_JOINT
= true;
public static final boolean DEFAULT_RECOMMEND_BLOOM_FILTER = true;
public static final boolean DEFAULT_RECOMMEND_RANGE_INDEX = true;
+ public static final boolean DEFAULT_RECOMMEND_JSON_INDEX = true;
public static final boolean
DEFAULT_RECOMMEND_NO_DICTIONARY_ONHEAP_DICTIONARY_JOINT = true;
public static final boolean DEFAULT_RECOMMEND_AGGREGATE_METRICS = true;
public static final boolean DEFAULT_RECOMMEND_REALTIME_PROVISIONING = true;
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/recommender/TestConfigEngine.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/recommender/TestConfigEngine.java
index 26d33570b0..58362560d6 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/recommender/TestConfigEngine.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/recommender/TestConfigEngine.java
@@ -281,6 +281,18 @@ public class TestConfigEngine {
assertEquals(output.getIndexConfig().getRangeIndexColumns().toString(),
"[t, j]");
}
+ /** Verifiy rule that recommends JsonIndex and NoDictionary on JSON columns.
*/
+ @Test
+ void testJsonIndexRule()
+ throws InvalidInputException, IOException {
+ loadInput("recommenderInput/SegmentSizeRuleInput.json");
+ ConfigManager output = new ConfigManager();
+ AbstractRule abstractRule =
RulesToExecute.RuleFactory.getRule(RulesToExecute.Rule.JsonIndexRule, _input,
output);
+ abstractRule.run();
+ assertEquals(output.getIndexConfig().getJsonIndexColumns().toString(),
"[q]");
+ assertEquals(output.getIndexConfig().getNoDictionaryColumns().toString(),
"[q]");
+ }
+
@Test
void testNoDictionaryOnHeapDictionaryJointRule()
throws InvalidInputException, IOException {
@@ -464,8 +476,8 @@ public class TestConfigEngine {
throws Exception {
ConfigManager output =
runRecommenderDriver("recommenderInput/SegmentSizeRuleInput.json");
SegmentSizeRecommendations segmentSizeRecommendations =
output.getSegmentSizeRecommendations();
- assertEquals(segmentSizeRecommendations.getNumSegments(), 2);
- assertEquals(segmentSizeRecommendations.getNumRowsPerSegment(), 50_000);
+ assertEquals(segmentSizeRecommendations.getNumSegments(), 3);
+ assertEquals(segmentSizeRecommendations.getNumRowsPerSegment(), 33_333);
}
@Test
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/recommender/data/generator/JsonGeneratorTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/recommender/data/generator/JsonGeneratorTest.java
new file mode 100644
index 0000000000..c4bcc1ccc3
--- /dev/null
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/recommender/data/generator/JsonGeneratorTest.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.controller.recommender.data.generator;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.io.IOException;
+import org.apache.commons.lang.StringUtils;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
+public class JsonGeneratorTest {
+
+ @Test
+ public void testNext()
+ throws IOException {
+ // JsonGenerator generates empty json when size is less than
JsonGenerator.DEFAULT_JSON_ELEMENT_LENGTH
+ JsonGenerator jsonGenerator1 = new JsonGenerator(0);
+ Assert.assertEquals(jsonGenerator1.next(), "{}");
+
+ JsonGenerator jsonGenerator2 = new JsonGenerator(4);
+ Assert.assertEquals(jsonGenerator2.next(), "{}");
+
+ JsonGenerator jsonGenerator3 = new JsonGenerator(8);
+ checkJson((String) jsonGenerator3.next(), 8);
+
+ JsonGenerator jsonGenerator4 = new JsonGenerator(20);
+ checkJson((String) jsonGenerator4.next(), 20);
+ }
+
+ public static void checkJson(String jsonString, int desiredLength)
+ throws IOException {
+
+ // Number of expected elements in the generated JSON string
+ int elementCount = desiredLength /
JsonGenerator.DEFAULT_JSON_ELEMENT_LENGTH;
+
+ // Remove escape characters from jsonString for verification purposes.
Escape character were added before comma
+ // since json string is written to a CSV file where comma is used as
delimiter.
+ jsonString = StringUtils.remove(jsonString, "\\");
+
+ // Make sure we are generating JSON string that is close to the desired
length. Length of JSON string should be 2
+ // (length of opening and closing parentheses) + number of commas + size
of all the elements.
+ Assert.assertEquals(jsonString.length(),
+ "{}".length() + (elementCount - 1) + elementCount *
JsonGenerator.DEFAULT_JSON_ELEMENT_LENGTH);
+
+ // Check if json string is valid json (i.e does not throw parse
exceptions) and doesn't result in a
+ // JsonNode "missing node".
+ JsonNode jsonNode = JsonUtils.stringToJsonNode(jsonString);
+ Assert.assertFalse(jsonNode.isMissingNode());
+ }
+}
diff --git
a/pinot-controller/src/test/resources/recommenderInput/SegmentSizeRuleInput.json
b/pinot-controller/src/test/resources/recommenderInput/SegmentSizeRuleInput.json
index fc1170f1ca..4c11943acd 100644
---
a/pinot-controller/src/test/resources/recommenderInput/SegmentSizeRuleInput.json
+++
b/pinot-controller/src/test/resources/recommenderInput/SegmentSizeRuleInput.json
@@ -72,6 +72,14 @@
"cardinality":4,
"numValuesPerEntry":1.00000001,
"singleValueField": false
+ },
+ {
+ "name": "q",
+ "dataType": "JSON",
+ "cardinality":4,
+ "numValuesPerEntry":1,
+ "averageLength" : 25,
+ "singleValueField": true
}
],
"metricFieldSpecs": [
@@ -131,7 +139,8 @@
"select j from tableName where (a=3 and (h = 5 or f >34) and
REGEXP_LIKE(i, 'as*')) or ((f = 3 or j in ('#VALUES', 4)) and REGEXP_LIKE(d,
'fl*'))": 2,
"select f from tableName where (a=0 or (b=1 and (e in ('#VALUES',2) or
c=7))) and TEXT_MATCH(d, 'dasd') and MAX(MAX(h,i),j)=4 and t<3": 4,
"select f from tableName where (a=0 and b=1) or c=7 or (d = 7 and e =1)":
2,
- "select f from tableName where t between 1 and 1000": 2
+ "select f from tableName where t between 1 and 1000": 2,
+ "select q from tableName where JSON_MATCH(q, '\"$.a\" IS NOT NULL')": 1
},
"qps": 400,
"numMessagesPerSecInKafkaTopic":1000,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
