yashmayya commented on code in PR #16626:
URL: https://github.com/apache/pinot/pull/16626#discussion_r2284141207
##########
pinot-common/src/main/java/org/apache/pinot/common/config/provider/TableCache.java:
##########
@@ -61,11 +61,11 @@
/**
- * An implementation of {@link PinotConfigProvider}
+ * An implementation of {@link TableCacheProvider}
* The {@code TableCache} caches all the table configs and schemas within the
cluster, and listens on ZK changes to keep
* them in sync. It also maintains the table name map and the column name map
for case-insensitive queries.
*/
-public class TableCache implements PinotConfigProvider {
+public class TableCache implements TableCacheProvider, PinotConfigProvider {
Review Comment:
Can you add `@Override` annotations to all the interface method
implementations here?
##########
pinot-common/src/main/java/org/apache/pinot/common/config/provider/TableCache.java:
##########
@@ -646,7 +646,7 @@ public synchronized void handleDataDeleted(String path) {
}
}
- private static Map<Expression, Expression>
createExpressionOverrideMap(String physicalOrLogicalTableName,
+ public static Map<Expression, Expression> createExpressionOverrideMap(String
physicalOrLogicalTableName,
Review Comment:
This can be moved into the parent interface.
##########
pinot-common/src/main/java/org/apache/pinot/common/config/provider/TableCacheQueryValidator.java:
##########
@@ -0,0 +1,251 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.config.provider;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import javax.annotation.Nullable;
+import org.apache.pinot.common.request.Expression;
+import org.apache.pinot.spi.config.provider.LogicalTableConfigChangeListener;
+import org.apache.pinot.spi.config.provider.SchemaChangeListener;
+import org.apache.pinot.spi.config.provider.TableConfigChangeListener;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.LogicalTableConfig;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.CommonConstants.Segment.BuiltInVirtualColumn;
+import org.apache.pinot.spi.utils.TimestampIndexUtils;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A static implementation that works with pre-loaded table configs and
schemas jsons.
+ * This is useful for validation scenarios where you want to test query
compilation against a specific
+ * set of table configs and schemas without needing a live cluster.
+ */
+public class TableCacheQueryValidator implements TableCacheProvider {
Review Comment:
nit: what do you think about calling it `StaticTableCache` instead? Keeps it
more generic as compared to the current name which is tied to a specific use
case.
##########
pinot-common/src/main/java/org/apache/pinot/common/config/provider/TableCacheQueryValidator.java:
##########
@@ -0,0 +1,251 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.config.provider;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import javax.annotation.Nullable;
+import org.apache.pinot.common.request.Expression;
+import org.apache.pinot.spi.config.provider.LogicalTableConfigChangeListener;
+import org.apache.pinot.spi.config.provider.SchemaChangeListener;
+import org.apache.pinot.spi.config.provider.TableConfigChangeListener;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.LogicalTableConfig;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.CommonConstants.Segment.BuiltInVirtualColumn;
+import org.apache.pinot.spi.utils.TimestampIndexUtils;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A static implementation that works with pre-loaded table configs and
schemas jsons.
+ * This is useful for validation scenarios where you want to test query
compilation against a specific
+ * set of table configs and schemas without needing a live cluster.
+ */
+public class TableCacheQueryValidator implements TableCacheProvider {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(TableCacheQueryValidator.class);
+
+
+ private final boolean _ignoreCase;
+ private final Map<String, TableConfig> _tableConfigMap = new HashMap<>();
+ private final Map<String, Schema> _schemaMap = new HashMap<>();
+ private final Map<String, LogicalTableConfig> _logicalTableConfigMap = new
HashMap<>();
+ private final Map<String, String> _tableNameMap = new HashMap<>();
+ private final Map<String, String> _logicalTableNameMap = new HashMap<>();
+ private final Map<String, Map<String, String>> _columnNameMaps = new
HashMap<>();
+
+ public TableCacheQueryValidator(List<TableConfig> tableConfigs, List<Schema>
schemas, boolean ignoreCase) {
+ this(tableConfigs, schemas, Collections.emptyList(), ignoreCase);
+ }
Review Comment:
Let's avoid this constructor since it's only being used in a test and makes
it seem like logical table are an optional part of the table cache.
##########
pinot-common/src/main/java/org/apache/pinot/common/config/provider/TableCacheQueryValidator.java:
##########
@@ -0,0 +1,251 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.config.provider;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import javax.annotation.Nullable;
+import org.apache.pinot.common.request.Expression;
+import org.apache.pinot.spi.config.provider.LogicalTableConfigChangeListener;
+import org.apache.pinot.spi.config.provider.SchemaChangeListener;
+import org.apache.pinot.spi.config.provider.TableConfigChangeListener;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.LogicalTableConfig;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.CommonConstants.Segment.BuiltInVirtualColumn;
+import org.apache.pinot.spi.utils.TimestampIndexUtils;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A static implementation that works with pre-loaded table configs and
schemas jsons.
Review Comment:
nit: the `json` aspect isn't really relevant here.
##########
pinot-common/src/main/java/org/apache/pinot/common/config/provider/TableCacheQueryValidator.java:
##########
@@ -0,0 +1,251 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.config.provider;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import javax.annotation.Nullable;
+import org.apache.pinot.common.request.Expression;
+import org.apache.pinot.spi.config.provider.LogicalTableConfigChangeListener;
+import org.apache.pinot.spi.config.provider.SchemaChangeListener;
+import org.apache.pinot.spi.config.provider.TableConfigChangeListener;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.LogicalTableConfig;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.CommonConstants.Segment.BuiltInVirtualColumn;
+import org.apache.pinot.spi.utils.TimestampIndexUtils;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A static implementation that works with pre-loaded table configs and
schemas jsons.
+ * This is useful for validation scenarios where you want to test query
compilation against a specific
+ * set of table configs and schemas without needing a live cluster.
+ */
+public class TableCacheQueryValidator implements TableCacheProvider {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(TableCacheQueryValidator.class);
+
+
+ private final boolean _ignoreCase;
+ private final Map<String, TableConfig> _tableConfigMap = new HashMap<>();
+ private final Map<String, Schema> _schemaMap = new HashMap<>();
+ private final Map<String, LogicalTableConfig> _logicalTableConfigMap = new
HashMap<>();
+ private final Map<String, String> _tableNameMap = new HashMap<>();
+ private final Map<String, String> _logicalTableNameMap = new HashMap<>();
+ private final Map<String, Map<String, String>> _columnNameMaps = new
HashMap<>();
+
+ public TableCacheQueryValidator(List<TableConfig> tableConfigs, List<Schema>
schemas, boolean ignoreCase) {
+ this(tableConfigs, schemas, Collections.emptyList(), ignoreCase);
+ }
+
+ public TableCacheQueryValidator(List<TableConfig> tableConfigs, List<Schema>
schemas,
+ List<LogicalTableConfig> logicalTableConfigs, boolean ignoreCase) {
+ _ignoreCase = ignoreCase;
+
+ for (TableConfig tableConfig : tableConfigs) {
+ String tableNameWithType = tableConfig.getTableName();
+ String rawTableName =
TableNameBuilder.extractRawTableName(tableNameWithType);
+
+ _tableConfigMap.put(tableNameWithType, tableConfig);
+
+ if (_ignoreCase) {
+ _tableNameMap.put(tableNameWithType.toLowerCase(), tableNameWithType);
+ _tableNameMap.put(rawTableName.toLowerCase(), rawTableName);
+ } else {
+ _tableNameMap.put(tableNameWithType, tableNameWithType);
+ _tableNameMap.put(rawTableName, rawTableName);
+ }
+ }
+
+ // Initialize schemas
+ for (Schema schema : schemas) {
+ String schemaName = schema.getSchemaName();
+ _schemaMap.put(schemaName, schema);
+ Map<String, String> columnNameMap = new
TreeMap<>(String.CASE_INSENSITIVE_ORDER);
Review Comment:
Why do we need an ordered tree map here?
##########
pinot-common/src/main/java/org/apache/pinot/common/config/provider/TableCacheProvider.java:
##########
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.config.provider;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.annotation.Nullable;
+import org.apache.pinot.common.request.Expression;
+import org.apache.pinot.spi.config.provider.PinotConfigProvider;
+import org.apache.pinot.spi.data.LogicalTableConfig;
+
+
+public interface TableCacheProvider extends PinotConfigProvider {
+
+ /**
+ * Returns {@code true} if the TableCache is case-insensitive, {@code false}
otherwise.
+ */
Review Comment:
nit: you can remove these Javadocs from the existing `TableCache` class now
- no point in duplicating it across the interface and implementation.
##########
pinot-common/src/main/java/org/apache/pinot/common/config/provider/TableCacheQueryValidator.java:
##########
@@ -0,0 +1,251 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.config.provider;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import javax.annotation.Nullable;
+import org.apache.pinot.common.request.Expression;
+import org.apache.pinot.spi.config.provider.LogicalTableConfigChangeListener;
+import org.apache.pinot.spi.config.provider.SchemaChangeListener;
+import org.apache.pinot.spi.config.provider.TableConfigChangeListener;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.LogicalTableConfig;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.CommonConstants.Segment.BuiltInVirtualColumn;
+import org.apache.pinot.spi.utils.TimestampIndexUtils;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A static implementation that works with pre-loaded table configs and
schemas jsons.
+ * This is useful for validation scenarios where you want to test query
compilation against a specific
+ * set of table configs and schemas without needing a live cluster.
+ */
+public class TableCacheQueryValidator implements TableCacheProvider {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(TableCacheQueryValidator.class);
+
+
+ private final boolean _ignoreCase;
+ private final Map<String, TableConfig> _tableConfigMap = new HashMap<>();
+ private final Map<String, Schema> _schemaMap = new HashMap<>();
+ private final Map<String, LogicalTableConfig> _logicalTableConfigMap = new
HashMap<>();
+ private final Map<String, String> _tableNameMap = new HashMap<>();
+ private final Map<String, String> _logicalTableNameMap = new HashMap<>();
+ private final Map<String, Map<String, String>> _columnNameMaps = new
HashMap<>();
+
+ public TableCacheQueryValidator(List<TableConfig> tableConfigs, List<Schema>
schemas, boolean ignoreCase) {
+ this(tableConfigs, schemas, Collections.emptyList(), ignoreCase);
+ }
+
+ public TableCacheQueryValidator(List<TableConfig> tableConfigs, List<Schema>
schemas,
+ List<LogicalTableConfig> logicalTableConfigs, boolean ignoreCase) {
+ _ignoreCase = ignoreCase;
+
+ for (TableConfig tableConfig : tableConfigs) {
+ String tableNameWithType = tableConfig.getTableName();
+ String rawTableName =
TableNameBuilder.extractRawTableName(tableNameWithType);
+
+ _tableConfigMap.put(tableNameWithType, tableConfig);
+
+ if (_ignoreCase) {
+ _tableNameMap.put(tableNameWithType.toLowerCase(), tableNameWithType);
+ _tableNameMap.put(rawTableName.toLowerCase(), rawTableName);
+ } else {
+ _tableNameMap.put(tableNameWithType, tableNameWithType);
+ _tableNameMap.put(rawTableName, rawTableName);
+ }
+ }
+
+ // Initialize schemas
+ for (Schema schema : schemas) {
+ String schemaName = schema.getSchemaName();
+ _schemaMap.put(schemaName, schema);
+ Map<String, String> columnNameMap = new
TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+ for (FieldSpec fieldSpec : schema.getAllFieldSpecs()) {
+ String columnName = fieldSpec.getName();
+ if (_ignoreCase) {
+ columnNameMap.put(columnName.toLowerCase(), columnName);
+ } else {
+ columnNameMap.put(columnName, columnName);
+ }
+ }
+ addBuiltInVirtualColumns(columnNameMap);
+ _columnNameMaps.put(schemaName,
Collections.unmodifiableMap(columnNameMap));
+ }
+
+ for (LogicalTableConfig logicalTableConfig : logicalTableConfigs) {
+ String logicalTableName = logicalTableConfig.getTableName();
+ _logicalTableConfigMap.put(logicalTableName, logicalTableConfig);
+
+ if (_ignoreCase) {
+ _logicalTableNameMap.put(logicalTableName.toLowerCase(),
logicalTableName);
+ } else {
+ _logicalTableNameMap.put(logicalTableName, logicalTableName);
+ }
+ }
+
+ LOGGER.info(
+ "Initialized QueryValidator with {} table configs, {} schemas, {}
logical table configs (ignoreCase: {})",
+ tableConfigs.size(), schemas.size(), logicalTableConfigs.size(),
ignoreCase);
+ }
+
+ @Override
+ public boolean isIgnoreCase() {
+ return _ignoreCase;
+ }
+
+ @Override
+ @Nullable
+ public String getActualTableName(String tableName) {
+ if (_ignoreCase) {
+ return _tableNameMap.get(tableName.toLowerCase());
+ } else {
+ return _tableNameMap.get(tableName);
+ }
+ }
+
+ @Override
+ @Nullable
+ public String getActualLogicalTableName(String logicalTableName) {
+ return _ignoreCase ?
_logicalTableNameMap.get(logicalTableName.toLowerCase())
+ : _logicalTableNameMap.get(logicalTableName);
+ }
+
+ @Override
+ public Map<String, String> getTableNameMap() {
+ return Collections.unmodifiableMap(_tableNameMap);
+ }
+
+ @Override
+ public Map<String, String> getLogicalTableNameMap() {
+ return Collections.unmodifiableMap(_logicalTableNameMap);
+ }
+
+ @Override
+ public List<String> getAllDimensionTables() {
+ List<String> dimensionTables = new ArrayList<>();
+ for (TableConfig tableConfig : _tableConfigMap.values()) {
+ if (tableConfig.getTableType() == TableType.OFFLINE &&
tableConfig.isDimTable()) {
Review Comment:
Why is the additional `OFFLINE` check needed?
##########
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java:
##########
@@ -1721,6 +1721,151 @@ public void testValidateQueryApiError()
assertFalse(result.get("errorMessage").isNull());
}
+ @Test
+ public void testValidateQueryApiWithStaticConfigs()
+ throws Exception {
+ JsonNode tableConfigsNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/tables/mytable"));
+ JsonNode schemaNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/schemas/mytable"));
+
+ JsonNode offlineConfig = tableConfigsNode.get("OFFLINE");
+ JsonNode realtimeConfig = tableConfigsNode.get("REALTIME");
+
+ String offlineJson = offlineConfig != null ? offlineConfig.toString() :
"{}";
+ String realtimeJson = realtimeConfig != null ? realtimeConfig.toString() :
"{}";
+
+ String requestJson = String.format(
+ "{\"sql\": \"SELECT * FROM mytable\", \"tableConfigs\": [%s, %s],
\"schemas\": [%s]}",
Review Comment:
Let's use a separate class for this request structure and use that for JSON
serde here.
##########
pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/PinotQueryResourceStaticValidationTest.java:
##########
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.controller.api.resources;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.ws.rs.core.HttpHeaders;
+import org.apache.pinot.common.config.provider.TableCacheQueryValidator;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/**
+ * Unit test for the static table cache functionality in PinotQueryResource.
+ */
+public class PinotQueryResourceStaticValidationTest {
+
+ @Mock
+ private HttpHeaders _httpHeaders;
Review Comment:
Unused?
##########
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java:
##########
@@ -1721,6 +1721,151 @@ public void testValidateQueryApiError()
assertFalse(result.get("errorMessage").isNull());
}
+ @Test
+ public void testValidateQueryApiWithStaticConfigs()
+ throws Exception {
+ JsonNode tableConfigsNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/tables/mytable"));
+ JsonNode schemaNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/schemas/mytable"));
+
+ JsonNode offlineConfig = tableConfigsNode.get("OFFLINE");
+ JsonNode realtimeConfig = tableConfigsNode.get("REALTIME");
+
+ String offlineJson = offlineConfig != null ? offlineConfig.toString() :
"{}";
+ String realtimeJson = realtimeConfig != null ? realtimeConfig.toString() :
"{}";
+
+ String requestJson = String.format(
+ "{\"sql\": \"SELECT * FROM mytable\", \"tableConfigs\": [%s, %s],
\"schemas\": [%s]}",
+ offlineJson,
+ realtimeJson,
+ schemaNode.toString());
+
+ JsonNode result = JsonUtils.stringToJsonNode(
+ sendPostRequest(getControllerBaseApiUrl() +
"/validateMultiStageQuery", requestJson, null));
+ assertTrue(result.get("compiledSuccessfully").asBoolean());
+ assertTrue(result.get("errorCode").isNull());
+ assertTrue(result.get("errorMessage").isNull());
+
+ // Test with invalid query
+ String invalidRequestJson = String.format(
+ "{\"sql\": \"SELECT nonExistentColumn FROM mytable\",
\"tableConfigs\": [%s, %s], \"schemas\": [%s]}",
+ offlineConfig,
+ realtimeConfig,
+ schemaNode.toString());
+
+ result = JsonUtils.stringToJsonNode(
+ sendPostRequest(getControllerBaseApiUrl() +
"/validateMultiStageQuery", invalidRequestJson, null));
+ assertFalse(result.get("compiledSuccessfully").asBoolean());
+ assertFalse(result.get("errorMessage").isNull());
+ }
+
+ @Test
+ public void testValidateQueryApiSuccessfulQueries()
+ throws Exception {
+ JsonNode tableConfigsNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/tables/mytable"));
+ JsonNode schemaNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/schemas/mytable"));
+
+ String[] successfulQueries = {
+ "SELECT COUNT(*) FROM mytable",
+ "SELECT DivAirportSeqIDs, COUNT(*) FROM mytable GROUP BY
DivAirportSeqIDs",
+ "SELECT DivAirportSeqIDs FROM mytable WHERE
arrayToMV(DivAirportSeqIDs) > 0 LIMIT 10",
+ "SELECT DivAirportSeqIDs, AirlineID FROM mytable ORDER BY
DivAirportSeqIDs LIMIT 5",
+ "SELECT SUM(arrayToMV(DivAirportSeqIDs)) AS total FROM mytable;",
+ "SELECT AVG(arrayToMV(DivAirportSeqIDs)) FROM mytable WHERE AirlineID
IS NOT NULL"
+ };
+
+ for (String query : successfulQueries) {
+ JsonNode offlineConfig = tableConfigsNode.get("OFFLINE");
+ JsonNode realtimeConfig = tableConfigsNode.get("REALTIME");
+
+ String offlineJson = offlineConfig != null ? offlineConfig.toString() :
"{}";
+ String realtimeJson = realtimeConfig != null ? realtimeConfig.toString()
: "{}";
+ String requestJson = String.format(
+ "{\"sql\": \"%s\", \"tableConfigs\": [%s, %s], \"schemas\": [%s],
\"ignoreCase\": false}",
+ query, offlineJson, realtimeJson, schemaNode.toString());
+
+ JsonNode result = JsonUtils.stringToJsonNode(
+ sendPostRequest(getControllerBaseApiUrl() +
"/validateMultiStageQuery", requestJson, null));
+
+ assertTrue(result.get("compiledSuccessfully").asBoolean(), "Query should
compile successfully: " + query);
+ assertTrue(result.get("errorCode").isNull(), "Error code should be null
for: " + query);
+ assertTrue(result.get("errorMessage").isNull(), "Error message should be
null for: " + query);
+ }
+ }
+
+ @Test
+ public void testValidateQueryApiUnSuccessfulQuery()
Review Comment:
Why isn't this combined with `testValidateQueryApiWithStaticConfigs`?
##########
pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/PinotQueryResourceStaticValidationTest.java:
##########
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.controller.api.resources;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.ws.rs.core.HttpHeaders;
+import org.apache.pinot.common.config.provider.TableCacheQueryValidator;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/**
+ * Unit test for the static table cache functionality in PinotQueryResource.
+ */
+public class PinotQueryResourceStaticValidationTest {
+
+ @Mock
+ private HttpHeaders _httpHeaders;
+
+ private ObjectMapper _objectMapper;
+
+ @BeforeClass
+ public void setUp() {
+ MockitoAnnotations.openMocks(this);
+ _objectMapper = new ObjectMapper();
+ }
+
+ @Test
+ public void testStaticTableCacheProvider() {
+ TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE)
+ .setTableName("testTable")
+ .build();
+
+ Schema schema = new Schema.SchemaBuilder()
+ .setSchemaName("testTable")
+ .addSingleValueDimension("dimensionCol", FieldSpec.DataType.STRING)
+ .addMetric("metricCol", FieldSpec.DataType.LONG)
+ .build();
+
+ List<TableConfig> tableConfigs = Arrays.asList(tableConfig);
+ List<Schema> schemas = Arrays.asList(schema);
+
+ TableCacheQueryValidator provider = new
TableCacheQueryValidator(tableConfigs, schemas, false);
+
+ Assert.assertFalse(provider.isIgnoreCase());
+ Assert.assertEquals(provider.getActualTableName("testTable_OFFLINE"),
"testTable_OFFLINE");
+ Assert.assertEquals(provider.getActualTableName("testTable"), "testTable");
+ Assert.assertNotNull(provider.getTableConfig("testTable_OFFLINE"));
+ Assert.assertNotNull(provider.getSchema("testTable"));
+ Assert.assertNotNull(provider.getColumnNameMap("testTable"));
+ Assert.assertEquals(provider.getColumnNameMap("testTable").size(), 4); //
2 columns + 2 built-in virtual columns
Review Comment:
There's 3 virtual columns right?
##########
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java:
##########
@@ -1721,6 +1721,151 @@ public void testValidateQueryApiError()
assertFalse(result.get("errorMessage").isNull());
}
+ @Test
+ public void testValidateQueryApiWithStaticConfigs()
+ throws Exception {
+ JsonNode tableConfigsNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/tables/mytable"));
+ JsonNode schemaNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/schemas/mytable"));
+
+ JsonNode offlineConfig = tableConfigsNode.get("OFFLINE");
+ JsonNode realtimeConfig = tableConfigsNode.get("REALTIME");
+
+ String offlineJson = offlineConfig != null ? offlineConfig.toString() :
"{}";
+ String realtimeJson = realtimeConfig != null ? realtimeConfig.toString() :
"{}";
+
+ String requestJson = String.format(
+ "{\"sql\": \"SELECT * FROM mytable\", \"tableConfigs\": [%s, %s],
\"schemas\": [%s]}",
+ offlineJson,
+ realtimeJson,
+ schemaNode.toString());
+
+ JsonNode result = JsonUtils.stringToJsonNode(
+ sendPostRequest(getControllerBaseApiUrl() +
"/validateMultiStageQuery", requestJson, null));
+ assertTrue(result.get("compiledSuccessfully").asBoolean());
+ assertTrue(result.get("errorCode").isNull());
+ assertTrue(result.get("errorMessage").isNull());
+
+ // Test with invalid query
+ String invalidRequestJson = String.format(
+ "{\"sql\": \"SELECT nonExistentColumn FROM mytable\",
\"tableConfigs\": [%s, %s], \"schemas\": [%s]}",
+ offlineConfig,
+ realtimeConfig,
+ schemaNode.toString());
+
+ result = JsonUtils.stringToJsonNode(
+ sendPostRequest(getControllerBaseApiUrl() +
"/validateMultiStageQuery", invalidRequestJson, null));
+ assertFalse(result.get("compiledSuccessfully").asBoolean());
+ assertFalse(result.get("errorMessage").isNull());
+ }
+
+ @Test
+ public void testValidateQueryApiSuccessfulQueries()
+ throws Exception {
+ JsonNode tableConfigsNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/tables/mytable"));
+ JsonNode schemaNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/schemas/mytable"));
+
+ String[] successfulQueries = {
+ "SELECT COUNT(*) FROM mytable",
+ "SELECT DivAirportSeqIDs, COUNT(*) FROM mytable GROUP BY
DivAirportSeqIDs",
+ "SELECT DivAirportSeqIDs FROM mytable WHERE
arrayToMV(DivAirportSeqIDs) > 0 LIMIT 10",
+ "SELECT DivAirportSeqIDs, AirlineID FROM mytable ORDER BY
DivAirportSeqIDs LIMIT 5",
+ "SELECT SUM(arrayToMV(DivAirportSeqIDs)) AS total FROM mytable;",
+ "SELECT AVG(arrayToMV(DivAirportSeqIDs)) FROM mytable WHERE AirlineID
IS NOT NULL"
+ };
+
+ for (String query : successfulQueries) {
+ JsonNode offlineConfig = tableConfigsNode.get("OFFLINE");
+ JsonNode realtimeConfig = tableConfigsNode.get("REALTIME");
+
+ String offlineJson = offlineConfig != null ? offlineConfig.toString() :
"{}";
+ String realtimeJson = realtimeConfig != null ? realtimeConfig.toString()
: "{}";
+ String requestJson = String.format(
+ "{\"sql\": \"%s\", \"tableConfigs\": [%s, %s], \"schemas\": [%s],
\"ignoreCase\": false}",
+ query, offlineJson, realtimeJson, schemaNode.toString());
+
+ JsonNode result = JsonUtils.stringToJsonNode(
+ sendPostRequest(getControllerBaseApiUrl() +
"/validateMultiStageQuery", requestJson, null));
+
+ assertTrue(result.get("compiledSuccessfully").asBoolean(), "Query should
compile successfully: " + query);
+ assertTrue(result.get("errorCode").isNull(), "Error code should be null
for: " + query);
+ assertTrue(result.get("errorMessage").isNull(), "Error message should be
null for: " + query);
+ }
+ }
+
+ @Test
+ public void testValidateQueryApiUnSuccessfulQuery()
+ throws Exception {
+ JsonNode tableConfigsNode =
+ JsonUtils.stringToJsonNode(sendGetRequest(getControllerBaseApiUrl() +
"/tables/mytable"));
+ JsonNode schemaNode =
JsonUtils.stringToJsonNode(sendGetRequest(getControllerBaseApiUrl() +
"/schemas/mytable"));
+
+ String query = "SELECT DivAirportSeqIDs FROM mytable WHERE
DivAirportSeqIDs > 0 LIMIT 10";
+
+ JsonNode offlineConfig = tableConfigsNode.get("OFFLINE");
+ JsonNode realtimeConfig = tableConfigsNode.get("REALTIME");
+
+ String offlineJson = offlineConfig != null ? offlineConfig.toString() :
"{}";
+ String realtimeJson = realtimeConfig != null ? realtimeConfig.toString() :
"{}";
+ String requestJson =
+ String.format("{\"sql\": \"%s\", \"tableConfigs\": [%s, %s],
\"schemas\": [%s], \"ignoreCase\": false}", query,
+ offlineJson, realtimeJson, schemaNode.toString());
+
+ JsonNode result = JsonUtils.stringToJsonNode(
+ sendPostRequest(getControllerBaseApiUrl() +
"/validateMultiStageQuery", requestJson, null));
+
+ // Cannot apply '>' to arguments of type '<INTEGER> to <ARRAY>
+ assertFalse(result.get("compiledSuccessfully").asBoolean(), "Query should
not compile successfully: " + query);
+ assertFalse(result.get("errorCode").isNull(), "Error code should not be
null for: " + query);
Review Comment:
Can add assertions for the error code too.
##########
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java:
##########
@@ -1721,6 +1721,151 @@ public void testValidateQueryApiError()
assertFalse(result.get("errorMessage").isNull());
}
+ @Test
+ public void testValidateQueryApiWithStaticConfigs()
+ throws Exception {
+ JsonNode tableConfigsNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/tables/mytable"));
+ JsonNode schemaNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/schemas/mytable"));
+
+ JsonNode offlineConfig = tableConfigsNode.get("OFFLINE");
+ JsonNode realtimeConfig = tableConfigsNode.get("REALTIME");
+
+ String offlineJson = offlineConfig != null ? offlineConfig.toString() :
"{}";
+ String realtimeJson = realtimeConfig != null ? realtimeConfig.toString() :
"{}";
+
+ String requestJson = String.format(
+ "{\"sql\": \"SELECT * FROM mytable\", \"tableConfigs\": [%s, %s],
\"schemas\": [%s]}",
+ offlineJson,
+ realtimeJson,
+ schemaNode.toString());
+
+ JsonNode result = JsonUtils.stringToJsonNode(
+ sendPostRequest(getControllerBaseApiUrl() +
"/validateMultiStageQuery", requestJson, null));
+ assertTrue(result.get("compiledSuccessfully").asBoolean());
+ assertTrue(result.get("errorCode").isNull());
+ assertTrue(result.get("errorMessage").isNull());
+
+ // Test with invalid query
+ String invalidRequestJson = String.format(
+ "{\"sql\": \"SELECT nonExistentColumn FROM mytable\",
\"tableConfigs\": [%s, %s], \"schemas\": [%s]}",
+ offlineConfig,
+ realtimeConfig,
+ schemaNode.toString());
+
+ result = JsonUtils.stringToJsonNode(
+ sendPostRequest(getControllerBaseApiUrl() +
"/validateMultiStageQuery", invalidRequestJson, null));
+ assertFalse(result.get("compiledSuccessfully").asBoolean());
+ assertFalse(result.get("errorMessage").isNull());
Review Comment:
Let's add an assertion on the error code.
##########
pinot-common/src/main/java/org/apache/pinot/common/config/provider/TableCacheProvider.java:
##########
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.config.provider;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.annotation.Nullable;
+import org.apache.pinot.common.request.Expression;
+import org.apache.pinot.spi.config.provider.PinotConfigProvider;
+import org.apache.pinot.spi.data.LogicalTableConfig;
+
+
+public interface TableCacheProvider extends PinotConfigProvider {
Review Comment:
Does `PinotConfigProvider` even need to be a separate interface?
##########
pinot-common/src/main/java/org/apache/pinot/common/config/provider/TableCacheQueryValidator.java:
##########
@@ -0,0 +1,251 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.config.provider;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import javax.annotation.Nullable;
+import org.apache.pinot.common.request.Expression;
+import org.apache.pinot.spi.config.provider.LogicalTableConfigChangeListener;
+import org.apache.pinot.spi.config.provider.SchemaChangeListener;
+import org.apache.pinot.spi.config.provider.TableConfigChangeListener;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.LogicalTableConfig;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.CommonConstants.Segment.BuiltInVirtualColumn;
+import org.apache.pinot.spi.utils.TimestampIndexUtils;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A static implementation that works with pre-loaded table configs and
schemas jsons.
+ * This is useful for validation scenarios where you want to test query
compilation against a specific
+ * set of table configs and schemas without needing a live cluster.
+ */
+public class TableCacheQueryValidator implements TableCacheProvider {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(TableCacheQueryValidator.class);
+
+
+ private final boolean _ignoreCase;
+ private final Map<String, TableConfig> _tableConfigMap = new HashMap<>();
+ private final Map<String, Schema> _schemaMap = new HashMap<>();
+ private final Map<String, LogicalTableConfig> _logicalTableConfigMap = new
HashMap<>();
+ private final Map<String, String> _tableNameMap = new HashMap<>();
+ private final Map<String, String> _logicalTableNameMap = new HashMap<>();
+ private final Map<String, Map<String, String>> _columnNameMaps = new
HashMap<>();
+
+ public TableCacheQueryValidator(List<TableConfig> tableConfigs, List<Schema>
schemas, boolean ignoreCase) {
+ this(tableConfigs, schemas, Collections.emptyList(), ignoreCase);
+ }
+
+ public TableCacheQueryValidator(List<TableConfig> tableConfigs, List<Schema>
schemas,
+ List<LogicalTableConfig> logicalTableConfigs, boolean ignoreCase) {
+ _ignoreCase = ignoreCase;
+
+ for (TableConfig tableConfig : tableConfigs) {
+ String tableNameWithType = tableConfig.getTableName();
+ String rawTableName =
TableNameBuilder.extractRawTableName(tableNameWithType);
+
+ _tableConfigMap.put(tableNameWithType, tableConfig);
+
+ if (_ignoreCase) {
+ _tableNameMap.put(tableNameWithType.toLowerCase(), tableNameWithType);
+ _tableNameMap.put(rawTableName.toLowerCase(), rawTableName);
+ } else {
+ _tableNameMap.put(tableNameWithType, tableNameWithType);
+ _tableNameMap.put(rawTableName, rawTableName);
+ }
+ }
+
+ // Initialize schemas
+ for (Schema schema : schemas) {
+ String schemaName = schema.getSchemaName();
+ _schemaMap.put(schemaName, schema);
+ Map<String, String> columnNameMap = new
TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+ for (FieldSpec fieldSpec : schema.getAllFieldSpecs()) {
+ String columnName = fieldSpec.getName();
+ if (_ignoreCase) {
+ columnNameMap.put(columnName.toLowerCase(), columnName);
+ } else {
+ columnNameMap.put(columnName, columnName);
+ }
+ }
+ addBuiltInVirtualColumns(columnNameMap);
+ _columnNameMaps.put(schemaName,
Collections.unmodifiableMap(columnNameMap));
+ }
+
+ for (LogicalTableConfig logicalTableConfig : logicalTableConfigs) {
+ String logicalTableName = logicalTableConfig.getTableName();
+ _logicalTableConfigMap.put(logicalTableName, logicalTableConfig);
+
+ if (_ignoreCase) {
+ _logicalTableNameMap.put(logicalTableName.toLowerCase(),
logicalTableName);
+ } else {
+ _logicalTableNameMap.put(logicalTableName, logicalTableName);
+ }
+ }
+
+ LOGGER.info(
+ "Initialized QueryValidator with {} table configs, {} schemas, {}
logical table configs (ignoreCase: {})",
+ tableConfigs.size(), schemas.size(), logicalTableConfigs.size(),
ignoreCase);
+ }
+
+ @Override
+ public boolean isIgnoreCase() {
+ return _ignoreCase;
+ }
+
+ @Override
+ @Nullable
+ public String getActualTableName(String tableName) {
+ if (_ignoreCase) {
+ return _tableNameMap.get(tableName.toLowerCase());
+ } else {
+ return _tableNameMap.get(tableName);
+ }
+ }
+
+ @Override
+ @Nullable
+ public String getActualLogicalTableName(String logicalTableName) {
+ return _ignoreCase ?
_logicalTableNameMap.get(logicalTableName.toLowerCase())
+ : _logicalTableNameMap.get(logicalTableName);
+ }
+
+ @Override
+ public Map<String, String> getTableNameMap() {
+ return Collections.unmodifiableMap(_tableNameMap);
+ }
+
+ @Override
+ public Map<String, String> getLogicalTableNameMap() {
+ return Collections.unmodifiableMap(_logicalTableNameMap);
+ }
+
+ @Override
+ public List<String> getAllDimensionTables() {
+ List<String> dimensionTables = new ArrayList<>();
+ for (TableConfig tableConfig : _tableConfigMap.values()) {
+ if (tableConfig.getTableType() == TableType.OFFLINE &&
tableConfig.isDimTable()) {
+ dimensionTables.add(tableConfig.getTableName());
+ }
+ }
+ return dimensionTables;
+ }
+
+ @Override
+ public Map<String, String> getColumnNameMap(String rawTableName) {
+ Map<String, String> columnNameMap = _columnNameMaps.get(rawTableName);
+ return columnNameMap != null ? columnNameMap : Collections.emptyMap();
+ }
+
+ @Override
+ public Map<Expression, Expression> getExpressionOverrideMap(String
physicalOrLogicalTableName) {
+ if (isLogicalTable(physicalOrLogicalTableName)) {
+ LogicalTableConfig logicalTableConfig =
getLogicalTableConfig(physicalOrLogicalTableName);
+ if (logicalTableConfig != null) {
+ return
TableCache.createExpressionOverrideMap(physicalOrLogicalTableName,
logicalTableConfig.getQueryConfig());
+ }
+ } else {
+ TableConfig tableConfig = getTableConfig(physicalOrLogicalTableName);
+ if (tableConfig != null) {
+ return
TableCache.createExpressionOverrideMap(physicalOrLogicalTableName,
tableConfig.getQueryConfig());
+ }
+ }
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public Set<String> getTimestampIndexColumns(String tableNameWithType) {
+ TableConfig tableConfig = getTableConfig(tableNameWithType);
+ if (tableConfig != null) {
+ return TimestampIndexUtils.extractColumnsWithGranularity(tableConfig);
+ }
+ return Collections.emptySet();
+ }
Review Comment:
Can we use the same `TableConfigInfo` abstraction from the other
implementation to avoid recomputing this stuff every time?
##########
pinot-common/src/main/java/org/apache/pinot/common/config/provider/TableCacheQueryValidator.java:
##########
@@ -0,0 +1,251 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.config.provider;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import javax.annotation.Nullable;
+import org.apache.pinot.common.request.Expression;
+import org.apache.pinot.spi.config.provider.LogicalTableConfigChangeListener;
+import org.apache.pinot.spi.config.provider.SchemaChangeListener;
+import org.apache.pinot.spi.config.provider.TableConfigChangeListener;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.LogicalTableConfig;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.CommonConstants.Segment.BuiltInVirtualColumn;
+import org.apache.pinot.spi.utils.TimestampIndexUtils;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A static implementation that works with pre-loaded table configs and
schemas jsons.
+ * This is useful for validation scenarios where you want to test query
compilation against a specific
+ * set of table configs and schemas without needing a live cluster.
+ */
+public class TableCacheQueryValidator implements TableCacheProvider {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(TableCacheQueryValidator.class);
+
+
+ private final boolean _ignoreCase;
+ private final Map<String, TableConfig> _tableConfigMap = new HashMap<>();
+ private final Map<String, Schema> _schemaMap = new HashMap<>();
+ private final Map<String, LogicalTableConfig> _logicalTableConfigMap = new
HashMap<>();
+ private final Map<String, String> _tableNameMap = new HashMap<>();
+ private final Map<String, String> _logicalTableNameMap = new HashMap<>();
+ private final Map<String, Map<String, String>> _columnNameMaps = new
HashMap<>();
+
+ public TableCacheQueryValidator(List<TableConfig> tableConfigs, List<Schema>
schemas, boolean ignoreCase) {
+ this(tableConfigs, schemas, Collections.emptyList(), ignoreCase);
+ }
+
+ public TableCacheQueryValidator(List<TableConfig> tableConfigs, List<Schema>
schemas,
+ List<LogicalTableConfig> logicalTableConfigs, boolean ignoreCase) {
+ _ignoreCase = ignoreCase;
+
+ for (TableConfig tableConfig : tableConfigs) {
+ String tableNameWithType = tableConfig.getTableName();
+ String rawTableName =
TableNameBuilder.extractRawTableName(tableNameWithType);
+
+ _tableConfigMap.put(tableNameWithType, tableConfig);
+
+ if (_ignoreCase) {
+ _tableNameMap.put(tableNameWithType.toLowerCase(), tableNameWithType);
+ _tableNameMap.put(rawTableName.toLowerCase(), rawTableName);
+ } else {
+ _tableNameMap.put(tableNameWithType, tableNameWithType);
+ _tableNameMap.put(rawTableName, rawTableName);
+ }
+ }
+
+ // Initialize schemas
+ for (Schema schema : schemas) {
+ String schemaName = schema.getSchemaName();
+ _schemaMap.put(schemaName, schema);
+ Map<String, String> columnNameMap = new
TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+ for (FieldSpec fieldSpec : schema.getAllFieldSpecs()) {
+ String columnName = fieldSpec.getName();
+ if (_ignoreCase) {
+ columnNameMap.put(columnName.toLowerCase(), columnName);
+ } else {
+ columnNameMap.put(columnName, columnName);
+ }
+ }
+ addBuiltInVirtualColumns(columnNameMap);
+ _columnNameMaps.put(schemaName,
Collections.unmodifiableMap(columnNameMap));
+ }
+
+ for (LogicalTableConfig logicalTableConfig : logicalTableConfigs) {
+ String logicalTableName = logicalTableConfig.getTableName();
+ _logicalTableConfigMap.put(logicalTableName, logicalTableConfig);
+
+ if (_ignoreCase) {
+ _logicalTableNameMap.put(logicalTableName.toLowerCase(),
logicalTableName);
+ } else {
+ _logicalTableNameMap.put(logicalTableName, logicalTableName);
+ }
+ }
+
+ LOGGER.info(
+ "Initialized QueryValidator with {} table configs, {} schemas, {}
logical table configs (ignoreCase: {})",
+ tableConfigs.size(), schemas.size(), logicalTableConfigs.size(),
ignoreCase);
+ }
+
+ @Override
+ public boolean isIgnoreCase() {
+ return _ignoreCase;
+ }
+
+ @Override
+ @Nullable
+ public String getActualTableName(String tableName) {
+ if (_ignoreCase) {
+ return _tableNameMap.get(tableName.toLowerCase());
+ } else {
+ return _tableNameMap.get(tableName);
+ }
+ }
+
+ @Override
+ @Nullable
+ public String getActualLogicalTableName(String logicalTableName) {
+ return _ignoreCase ?
_logicalTableNameMap.get(logicalTableName.toLowerCase())
+ : _logicalTableNameMap.get(logicalTableName);
+ }
+
+ @Override
+ public Map<String, String> getTableNameMap() {
+ return Collections.unmodifiableMap(_tableNameMap);
+ }
+
+ @Override
+ public Map<String, String> getLogicalTableNameMap() {
+ return Collections.unmodifiableMap(_logicalTableNameMap);
+ }
+
+ @Override
+ public List<String> getAllDimensionTables() {
+ List<String> dimensionTables = new ArrayList<>();
+ for (TableConfig tableConfig : _tableConfigMap.values()) {
+ if (tableConfig.getTableType() == TableType.OFFLINE &&
tableConfig.isDimTable()) {
+ dimensionTables.add(tableConfig.getTableName());
+ }
+ }
+ return dimensionTables;
+ }
+
+ @Override
+ public Map<String, String> getColumnNameMap(String rawTableName) {
+ Map<String, String> columnNameMap = _columnNameMaps.get(rawTableName);
+ return columnNameMap != null ? columnNameMap : Collections.emptyMap();
+ }
+
+ @Override
+ public Map<Expression, Expression> getExpressionOverrideMap(String
physicalOrLogicalTableName) {
+ if (isLogicalTable(physicalOrLogicalTableName)) {
+ LogicalTableConfig logicalTableConfig =
getLogicalTableConfig(physicalOrLogicalTableName);
+ if (logicalTableConfig != null) {
+ return
TableCache.createExpressionOverrideMap(physicalOrLogicalTableName,
logicalTableConfig.getQueryConfig());
+ }
+ } else {
+ TableConfig tableConfig = getTableConfig(physicalOrLogicalTableName);
+ if (tableConfig != null) {
+ return
TableCache.createExpressionOverrideMap(physicalOrLogicalTableName,
tableConfig.getQueryConfig());
+ }
+ }
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public Set<String> getTimestampIndexColumns(String tableNameWithType) {
+ TableConfig tableConfig = getTableConfig(tableNameWithType);
+ if (tableConfig != null) {
+ return TimestampIndexUtils.extractColumnsWithGranularity(tableConfig);
+ }
+ return Collections.emptySet();
+ }
+
+ @Override
+ @Nullable
+ public TableConfig getTableConfig(String tableNameWithType) {
+ return _tableConfigMap.get(tableNameWithType);
+ }
+
+ @Override
+ @Nullable
+ public LogicalTableConfig getLogicalTableConfig(String logicalTableName) {
+ return _logicalTableConfigMap.get(logicalTableName);
+ }
+
+ @Override
+ public boolean registerTableConfigChangeListener(TableConfigChangeListener
tableConfigChangeListener) {
+ // Static implementation doesn't support change listeners
+ return false;
+ }
+
+ @Override
+ @Nullable
+ public Schema getSchema(String rawTableName) {
+ return _schemaMap.get(rawTableName);
+ }
+
+ @Override
+ public boolean registerSchemaChangeListener(SchemaChangeListener
schemaChangeListener) {
+ return false;
+ }
+
+ @Override
+ public boolean registerLogicalTableConfigChangeListener(
+ LogicalTableConfigChangeListener logicalTableConfigChangeListener) {
+ return false;
+ }
+
+ @Override
+ public List<LogicalTableConfig> getLogicalTableConfigs() {
+ return new ArrayList<>(_logicalTableConfigMap.values());
+ }
+
+ @Override
+ public boolean isLogicalTable(String logicalTableName) {
+ return _logicalTableConfigMap.containsKey(logicalTableName);
+ }
+
+ /**
+ * Add built-in virtual columns to the column name map.
+ */
+ private void addBuiltInVirtualColumns(Map<String, String> columnNameMap) {
+ // Add known built-in virtual columns
+ String[] builtInColumns =
+ {BuiltInVirtualColumn.DOCID, BuiltInVirtualColumn.HOSTNAME,
BuiltInVirtualColumn.SEGMENTNAME};
Review Comment:
Can't we use `BuiltInVirtualColumn.BUILT_IN_VIRTUAL_COLUMNS` here?
##########
pinot-common/src/main/java/org/apache/pinot/common/config/provider/TableCacheQueryValidator.java:
##########
@@ -0,0 +1,251 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.config.provider;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import javax.annotation.Nullable;
+import org.apache.pinot.common.request.Expression;
+import org.apache.pinot.spi.config.provider.LogicalTableConfigChangeListener;
+import org.apache.pinot.spi.config.provider.SchemaChangeListener;
+import org.apache.pinot.spi.config.provider.TableConfigChangeListener;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.LogicalTableConfig;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.CommonConstants.Segment.BuiltInVirtualColumn;
+import org.apache.pinot.spi.utils.TimestampIndexUtils;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A static implementation that works with pre-loaded table configs and
schemas jsons.
+ * This is useful for validation scenarios where you want to test query
compilation against a specific
+ * set of table configs and schemas without needing a live cluster.
+ */
+public class TableCacheQueryValidator implements TableCacheProvider {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(TableCacheQueryValidator.class);
+
+
Review Comment:
```suggestion
```
nit: extra whitespace
##########
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java:
##########
@@ -1721,6 +1721,151 @@ public void testValidateQueryApiError()
assertFalse(result.get("errorMessage").isNull());
}
+ @Test
+ public void testValidateQueryApiWithStaticConfigs()
+ throws Exception {
+ JsonNode tableConfigsNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/tables/mytable"));
+ JsonNode schemaNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/schemas/mytable"));
+
+ JsonNode offlineConfig = tableConfigsNode.get("OFFLINE");
+ JsonNode realtimeConfig = tableConfigsNode.get("REALTIME");
+
+ String offlineJson = offlineConfig != null ? offlineConfig.toString() :
"{}";
+ String realtimeJson = realtimeConfig != null ? realtimeConfig.toString() :
"{}";
+
+ String requestJson = String.format(
+ "{\"sql\": \"SELECT * FROM mytable\", \"tableConfigs\": [%s, %s],
\"schemas\": [%s]}",
+ offlineJson,
+ realtimeJson,
+ schemaNode.toString());
+
+ JsonNode result = JsonUtils.stringToJsonNode(
+ sendPostRequest(getControllerBaseApiUrl() +
"/validateMultiStageQuery", requestJson, null));
+ assertTrue(result.get("compiledSuccessfully").asBoolean());
+ assertTrue(result.get("errorCode").isNull());
+ assertTrue(result.get("errorMessage").isNull());
+
+ // Test with invalid query
+ String invalidRequestJson = String.format(
+ "{\"sql\": \"SELECT nonExistentColumn FROM mytable\",
\"tableConfigs\": [%s, %s], \"schemas\": [%s]}",
+ offlineConfig,
+ realtimeConfig,
+ schemaNode.toString());
+
+ result = JsonUtils.stringToJsonNode(
+ sendPostRequest(getControllerBaseApiUrl() +
"/validateMultiStageQuery", invalidRequestJson, null));
+ assertFalse(result.get("compiledSuccessfully").asBoolean());
+ assertFalse(result.get("errorMessage").isNull());
+ }
+
+ @Test
+ public void testValidateQueryApiSuccessfulQueries()
+ throws Exception {
+ JsonNode tableConfigsNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/tables/mytable"));
+ JsonNode schemaNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/schemas/mytable"));
+
+ String[] successfulQueries = {
+ "SELECT COUNT(*) FROM mytable",
+ "SELECT DivAirportSeqIDs, COUNT(*) FROM mytable GROUP BY
DivAirportSeqIDs",
+ "SELECT DivAirportSeqIDs FROM mytable WHERE
arrayToMV(DivAirportSeqIDs) > 0 LIMIT 10",
+ "SELECT DivAirportSeqIDs, AirlineID FROM mytable ORDER BY
DivAirportSeqIDs LIMIT 5",
+ "SELECT SUM(arrayToMV(DivAirportSeqIDs)) AS total FROM mytable;",
+ "SELECT AVG(arrayToMV(DivAirportSeqIDs)) FROM mytable WHERE AirlineID
IS NOT NULL"
+ };
+
+ for (String query : successfulQueries) {
+ JsonNode offlineConfig = tableConfigsNode.get("OFFLINE");
+ JsonNode realtimeConfig = tableConfigsNode.get("REALTIME");
+
+ String offlineJson = offlineConfig != null ? offlineConfig.toString() :
"{}";
+ String realtimeJson = realtimeConfig != null ? realtimeConfig.toString()
: "{}";
+ String requestJson = String.format(
+ "{\"sql\": \"%s\", \"tableConfigs\": [%s, %s], \"schemas\": [%s],
\"ignoreCase\": false}",
+ query, offlineJson, realtimeJson, schemaNode.toString());
+
+ JsonNode result = JsonUtils.stringToJsonNode(
+ sendPostRequest(getControllerBaseApiUrl() +
"/validateMultiStageQuery", requestJson, null));
+
+ assertTrue(result.get("compiledSuccessfully").asBoolean(), "Query should
compile successfully: " + query);
+ assertTrue(result.get("errorCode").isNull(), "Error code should be null
for: " + query);
+ assertTrue(result.get("errorMessage").isNull(), "Error message should be
null for: " + query);
+ }
+ }
+
+ @Test
+ public void testValidateQueryApiUnSuccessfulQuery()
Review Comment:
```suggestion
public void testValidateQueryApiUnsuccessfulQuery()
```
nit
##########
pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotQueryResource.java:
##########
@@ -162,18 +166,64 @@ public MultiStageQueryValidationResponse
validateMultiStageQuery(String requestJ
LOGGER.warn("Caught exception while parsing request {}", e.getMessage());
return new MultiStageQueryValidationResponse(false, "Failed to parse
request JSON: " + e.getMessage(), null);
}
- if (!requestJson.has("sql")) {
- return new MultiStageQueryValidationResponse(false, "JSON Payload is
missing the query string field 'sql'", null);
+
+ if (!requestJson.has("sql") ||
requestJson.get("sql").asText().trim().isEmpty()) {
+ return new MultiStageQueryValidationResponse(false, "Request is missing
the query string field 'sql'", null);
}
+
String sqlQuery = requestJson.get("sql").asText();
Map<String, String> queryOptionsMap =
RequestUtils.parseQuery(sqlQuery).getOptions();
String database =
DatabaseUtils.extractDatabaseFromQueryRequest(queryOptionsMap, httpHeaders);
- try (QueryEnvironment.CompiledQuery compiledQuery = new
QueryEnvironment(database,
- _pinotHelixResourceManager.getTableCache(), null).compile(sqlQuery)) {
- return new MultiStageQueryValidationResponse(true, null, null);
+
+ try {
+ TableCacheProvider tableCache;
+
+ if (requestJson.has("tableConfigs") && requestJson.has("schemas")) {
Review Comment:
At this point, it's probably better to extract the request structure out
into a separate class like `MultiStageQueryValidationRequest` and use that for
JSON serde.
##########
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java:
##########
@@ -1721,6 +1721,151 @@ public void testValidateQueryApiError()
assertFalse(result.get("errorMessage").isNull());
}
+ @Test
+ public void testValidateQueryApiWithStaticConfigs()
+ throws Exception {
+ JsonNode tableConfigsNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/tables/mytable"));
+ JsonNode schemaNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/schemas/mytable"));
+
+ JsonNode offlineConfig = tableConfigsNode.get("OFFLINE");
+ JsonNode realtimeConfig = tableConfigsNode.get("REALTIME");
+
+ String offlineJson = offlineConfig != null ? offlineConfig.toString() :
"{}";
+ String realtimeJson = realtimeConfig != null ? realtimeConfig.toString() :
"{}";
+
+ String requestJson = String.format(
+ "{\"sql\": \"SELECT * FROM mytable\", \"tableConfigs\": [%s, %s],
\"schemas\": [%s]}",
+ offlineJson,
+ realtimeJson,
+ schemaNode.toString());
+
+ JsonNode result = JsonUtils.stringToJsonNode(
+ sendPostRequest(getControllerBaseApiUrl() +
"/validateMultiStageQuery", requestJson, null));
+ assertTrue(result.get("compiledSuccessfully").asBoolean());
+ assertTrue(result.get("errorCode").isNull());
+ assertTrue(result.get("errorMessage").isNull());
+
+ // Test with invalid query
+ String invalidRequestJson = String.format(
+ "{\"sql\": \"SELECT nonExistentColumn FROM mytable\",
\"tableConfigs\": [%s, %s], \"schemas\": [%s]}",
+ offlineConfig,
+ realtimeConfig,
+ schemaNode.toString());
+
+ result = JsonUtils.stringToJsonNode(
+ sendPostRequest(getControllerBaseApiUrl() +
"/validateMultiStageQuery", invalidRequestJson, null));
+ assertFalse(result.get("compiledSuccessfully").asBoolean());
+ assertFalse(result.get("errorMessage").isNull());
+ }
+
+ @Test
+ public void testValidateQueryApiSuccessfulQueries()
+ throws Exception {
+ JsonNode tableConfigsNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/tables/mytable"));
+ JsonNode schemaNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/schemas/mytable"));
+
+ String[] successfulQueries = {
+ "SELECT COUNT(*) FROM mytable",
+ "SELECT DivAirportSeqIDs, COUNT(*) FROM mytable GROUP BY
DivAirportSeqIDs",
+ "SELECT DivAirportSeqIDs FROM mytable WHERE
arrayToMV(DivAirportSeqIDs) > 0 LIMIT 10",
+ "SELECT DivAirportSeqIDs, AirlineID FROM mytable ORDER BY
DivAirportSeqIDs LIMIT 5",
+ "SELECT SUM(arrayToMV(DivAirportSeqIDs)) AS total FROM mytable;",
+ "SELECT AVG(arrayToMV(DivAirportSeqIDs)) FROM mytable WHERE AirlineID
IS NOT NULL"
+ };
+
+ for (String query : successfulQueries) {
+ JsonNode offlineConfig = tableConfigsNode.get("OFFLINE");
+ JsonNode realtimeConfig = tableConfigsNode.get("REALTIME");
+
+ String offlineJson = offlineConfig != null ? offlineConfig.toString() :
"{}";
+ String realtimeJson = realtimeConfig != null ? realtimeConfig.toString()
: "{}";
+ String requestJson = String.format(
+ "{\"sql\": \"%s\", \"tableConfigs\": [%s, %s], \"schemas\": [%s],
\"ignoreCase\": false}",
+ query, offlineJson, realtimeJson, schemaNode.toString());
+
+ JsonNode result = JsonUtils.stringToJsonNode(
+ sendPostRequest(getControllerBaseApiUrl() +
"/validateMultiStageQuery", requestJson, null));
+
+ assertTrue(result.get("compiledSuccessfully").asBoolean(), "Query should
compile successfully: " + query);
+ assertTrue(result.get("errorCode").isNull(), "Error code should be null
for: " + query);
+ assertTrue(result.get("errorMessage").isNull(), "Error message should be
null for: " + query);
+ }
+ }
+
+ @Test
+ public void testValidateQueryApiUnSuccessfulQuery()
+ throws Exception {
+ JsonNode tableConfigsNode =
+ JsonUtils.stringToJsonNode(sendGetRequest(getControllerBaseApiUrl() +
"/tables/mytable"));
+ JsonNode schemaNode =
JsonUtils.stringToJsonNode(sendGetRequest(getControllerBaseApiUrl() +
"/schemas/mytable"));
+
+ String query = "SELECT DivAirportSeqIDs FROM mytable WHERE
DivAirportSeqIDs > 0 LIMIT 10";
+
+ JsonNode offlineConfig = tableConfigsNode.get("OFFLINE");
+ JsonNode realtimeConfig = tableConfigsNode.get("REALTIME");
+
+ String offlineJson = offlineConfig != null ? offlineConfig.toString() :
"{}";
+ String realtimeJson = realtimeConfig != null ? realtimeConfig.toString() :
"{}";
+ String requestJson =
+ String.format("{\"sql\": \"%s\", \"tableConfigs\": [%s, %s],
\"schemas\": [%s], \"ignoreCase\": false}", query,
+ offlineJson, realtimeJson, schemaNode.toString());
+
+ JsonNode result = JsonUtils.stringToJsonNode(
+ sendPostRequest(getControllerBaseApiUrl() +
"/validateMultiStageQuery", requestJson, null));
+
+ // Cannot apply '>' to arguments of type '<INTEGER> to <ARRAY>
+ assertFalse(result.get("compiledSuccessfully").asBoolean(), "Query should
not compile successfully: " + query);
+ assertFalse(result.get("errorCode").isNull(), "Error code should not be
null for: " + query);
+ assertFalse(result.get("errorMessage").isNull(), "Error message should not
be null for: " + query);
+ }
+
+ @Test
+ public void testValidateQueryApiWithIgnoreCaseOption()
+ throws Exception {
+ JsonNode tableConfigsNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/tables/mytable"));
+ JsonNode schemaNode = JsonUtils.stringToJsonNode(
+ sendGetRequest(getControllerBaseApiUrl() + "/schemas/mytable"));
+
+ JsonNode offlineConfig = tableConfigsNode.get("OFFLINE");
+ JsonNode realtimeConfig = tableConfigsNode.get("REALTIME"); // might be
null
+
+ String offlineJson = offlineConfig != null ? offlineConfig.toString() :
"{}";
+ String realtimeJson = realtimeConfig != null ? realtimeConfig.toString() :
"{}";
+
+ // Test case-sensitive mode (default)
+ String query = String.format(
+ "{\"sql\": \"SELECT divairportseqids FROM mytable\", \"tableConfigs\":
[%s, %s], \"schemas\": [%s], "
+ + "\"ignoreCase\": false}",
+ offlineJson, realtimeJson, schemaNode.toString());
+
+ JsonNode result = JsonUtils.stringToJsonNode(
+ sendPostRequest(getControllerBaseApiUrl() +
"/validateMultiStageQuery", query, null));
+ assertTrue(result.get("compiledSuccessfully").asBoolean(), "Query should
compile successfully: " + query);
+ assertTrue(result.get("errorCode").isNull(), "Error code should be null
for: " + query);
+ assertTrue(result.get("errorMessage").isNull(), "Error message should be
null for: " + query);
+
+ // Test case-insensitive mode
+ query = String.format(
+ "{\"sql\": \"SELECT divairportseqids FROM mytable\", \"tableConfigs\":
[%s, %s], \"schemas\": [%s], "
+ + "\"ignoreCase\": true}",
+ offlineConfig,
+ realtimeConfig,
+ schemaNode.toString());
+
+ result = JsonUtils.stringToJsonNode(
+ sendPostRequest(getControllerBaseApiUrl() +
"/validateMultiStageQuery", query, null));
+
Review Comment:
```suggestion
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
