shaofengshi closed pull request #255: KYLIN-3496 Make calcite extras props
available in JDBC Driver
URL: https://github.com/apache/kylin/pull/255
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index efc5030fa9..0496470f30 100644
---
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -18,8 +18,13 @@
package org.apache.kylin.common.debug;
+import java.io.IOException;
+import java.io.StringReader;
import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import com.google.common.collect.Sets;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.Pair;
@@ -156,6 +161,35 @@ public static void cleanToggles() {
_backdoorToggles.remove();
}
+ /**
+ * get extra calcite props from jdbc client
+ */
+ public static Properties getJdbcDriverClientCalciteProps() {
+ Properties props = new Properties();
+ String propsStr = getString(JDBC_CLIENT_CALCITE_PROPS);
+ if (propsStr == null) {
+ return props;
+ }
+ try {
+ props.load(new StringReader(propsStr));
+ } catch (IOException ignored) {
+ // ignored
+ }
+ final Set<String> allowedPropsNames = Sets.newHashSet(
+ "caseSensitive",
+ "unquotedCasing",
+ "quoting",
+ "conformance"
+ );
+ // remove un-allowed props
+ for (String key : props.stringPropertyNames()) {
+ if (!allowedPropsNames.contains(key)) {
+ props.remove(key);
+ }
+ }
+ return props;
+ }
+
/**
* set DEBUG_TOGGLE_DISABLE_FUZZY_KEY=true to disable fuzzy key for
debug/profile usage
*
@@ -313,4 +347,9 @@ public static void cleanToggles() {
}
*/
public final static String DEBUG_TOGGLE_HTRACE_ENABLED =
"DEBUG_TOGGLE_HTRACE_ENABLED";
+
+ /**
+ * extra calcite props from jdbc client
+ */
+ public static final String JDBC_CLIENT_CALCITE_PROPS =
"JDBC_CLIENT_CALCITE_PROPS";
}
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java
b/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java
index 33d82f8099..5200b8c764 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java
@@ -20,7 +20,9 @@
import java.sql.DriverManager;
import java.sql.SQLException;
+import java.util.Set;
+import com.google.common.collect.Sets;
import org.apache.calcite.avatica.AvaticaConnection;
import org.apache.calcite.avatica.DriverVersion;
import org.apache.calcite.avatica.Meta;
@@ -40,8 +42,8 @@
* Supported Statements:
* </p>
* <ul>
- * <li>{@link KylinStatementImpl}</li>
- * <li>{@link KylinPrepareStatementImpl}</li>
+ * <li>{@link KylinStatement}</li>
+ * <li>{@link KylinPreparedStatement}</li>
* </ul>
*
* <p>
@@ -50,6 +52,7 @@
* <li>user: username</li>
* <li>password: password</li>
* <li>ssl: true/false</li>
+ * <li>{@link #CLIENT_CALCITE_PROP_NAMES extras calcite props} like:
caseSensitive, unquotedCasing, quoting, conformance</li>
* </ul>
* </p>
*
@@ -59,9 +62,10 @@
* <pre>
* Driver driver = (Driver)
Class.forName("org.apache.kylin.kylin.jdbc.Driver").newInstance();
* Properties info = new Properties();
- * info.put("user", "user");
- * info.put("password", "password");
- * info.put("ssl", true);
+ * info.setProperty("user", "user");
+ * info.setProperty("password", "password");
+ * info.setProperty("ssl", "true");
+ * info.setProperty("caseSensitive", "true");
* Connection conn =
driver.connect("jdbc:kylin://{domain}/{project}", info);
* </pre>
*
@@ -70,6 +74,17 @@
public class Driver extends UnregisteredDriver {
public static final String CONNECT_STRING_PREFIX = "jdbc:kylin:";
+
+ /**
+ * These calcite props can be configured by jdbc connection
+ */
+ public static final Set<String> CLIENT_CALCITE_PROP_NAMES =
Sets.newHashSet(
+ "caseSensitive",
+ "unquotedCasing",
+ "quoting",
+ "conformance"
+ );
+
static {
try {
DriverManager.registerDriver(new Driver());
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
index 6814f2d8bc..da856d7c25 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
@@ -379,7 +379,7 @@ private SQLResponseStub executeKylinQuery(String sql,
List<StatementParameter> p
addHttpHeaders(post);
String postBody = jsonMapper.writeValueAsString(request);
- logger.debug("Post body:\n " + postBody);
+ logger.debug("Post body:\n {}", postBody);
StringEntity requestEntity = new StringEntity(postBody,
ContentType.create("application/json", "UTF-8"));
post.setEntity(requestEntity);
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
index 731b75a9f4..5465bd055f 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
@@ -19,11 +19,13 @@
package org.apache.kylin.jdbc;
import java.io.IOException;
+import java.io.StringWriter;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.TimeZone;
import org.apache.calcite.avatica.AvaticaParameter;
@@ -58,11 +60,13 @@ protected AvaticaResultSet execute() throws SQLException {
paramValues = ((KylinPreparedStatement)
statement).getParameterJDBCValues();
}
- IRemoteClient client = ((KylinConnection)
statement.connection).getRemoteClient();
+ KylinConnection connection = (KylinConnection) statement.connection;
+ IRemoteClient client = connection.getRemoteClient();
Map<String, String> queryToggles = new HashMap<>();
int maxRows = statement.getMaxRows();
queryToggles.put("ATTR_STATEMENT_MAX_ROWS", String.valueOf(maxRows));
+ addServerProps(queryToggles, connection);
QueryResult result;
try {
@@ -78,4 +82,30 @@ protected AvaticaResultSet execute() throws SQLException {
return super.execute2(cursor, columnMetaDataList);
}
+ /**
+ * add calcite props into queryToggles
+ */
+ private void addServerProps(Map<String, String> queryToggles,
KylinConnection connection) {
+ Properties connProps = connection.getConnectionProperties();
+ Properties props = new Properties();
+ for (String key : connProps.stringPropertyNames()) {
+ if (Driver.CLIENT_CALCITE_PROP_NAMES.contains(key)) {
+ props.put(key, connProps.getProperty(key));
+ }
+ }
+
+ if (props.isEmpty()) {
+ return;
+ }
+
+ StringWriter writer = new StringWriter();
+ try {
+ props.store(writer, "");
+ } catch (IOException ignored) {
+ // ignored
+ return;
+ }
+ queryToggles.put("JDBC_CLIENT_CALCITE_PROPS", writer.toString());
+ }
+
}
diff --git a/jdbc/src/test/java/org/apache/kylin/jdbc/KylinConnectionTest.java
b/jdbc/src/test/java/org/apache/kylin/jdbc/KylinConnectionTest.java
index ae15472328..65c7b7ba8a 100644
--- a/jdbc/src/test/java/org/apache/kylin/jdbc/KylinConnectionTest.java
+++ b/jdbc/src/test/java/org/apache/kylin/jdbc/KylinConnectionTest.java
@@ -19,12 +19,14 @@
package org.apache.kylin.jdbc;
import java.io.IOException;
+import java.io.StringReader;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Types;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -36,13 +38,18 @@
import org.apache.http.message.BasicStatusLine;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.ArgumentMatcher;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+import javax.annotation.Nonnull;
+
import static org.apache.http.HttpVersion.HTTP_1_1;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyString;
import static org.mockito.Mockito.doAnswer;
@@ -55,10 +62,10 @@
public class KylinConnectionTest {
- private Driver driver = new Driver();
- private KylinJdbcFactory factory = spy(new KylinJdbcFactory.Version41());
- private IRemoteClient client = mock(IRemoteClient.class);
- private HttpClient httpClient = mock(HttpClient.class);
+ private final Driver driver = new Driver();
+ private final KylinJdbcFactory factory = spy(new
KylinJdbcFactory.Version41());
+ private final IRemoteClient client = mock(IRemoteClient.class);
+ private final HttpClient httpClient = mock(HttpClient.class);
@Before
public void setUp() throws Exception {
@@ -68,54 +75,50 @@ public void setUp() throws Exception {
@Test
public void testPrepareStatementWithMockKylinClient() throws SQLException,
IOException {
String sql = "select 1 as val";
- ArrayList<ColumnMetaData> columnMeta = new ArrayList<>();
- columnMeta.add(new ColumnMetaData(0, false, true, false,
- false, 1, true, 1,
- "VAL", "VAL", null,
- 10, 0, null, null,
- ColumnMetaData.scalar(Types.INTEGER, "INTEGER",
ColumnMetaData.Rep.INTEGER),
- true, false, false, "java.lang.Integer"));
- ArrayList<Object> list = new ArrayList<>();
- list.add(new Object[]{1});
- IRemoteClient.QueryResult result = new
IRemoteClient.QueryResult(columnMeta, list);
// mock client
- when(client.executeQuery(anyString(), Mockito.<List<Object>>any(),
Mockito.<Map<String, String>>any())).thenReturn(result);
+ when(client.executeQuery(anyString(), Mockito.<List<Object>>any(),
Mockito.<Map<String, String>>any())).thenReturn(getMockResult());
- PreparedStatement preparedStatement =
getConnectionWithMockClient().prepareStatement(sql);
- ResultSet resultSet = preparedStatement.executeQuery();
+ try (KylinConnection conn = getConnectionWithMockClient()) {
+ PreparedStatement preparedStatement = conn.prepareStatement(sql);
+ try (ResultSet resultSet = preparedStatement.executeQuery()) {
+ verify(client).executeQuery(eq(sql),
Mockito.<List<Object>>any(), Mockito.<Map<String, String>>any());
- verify(client).executeQuery(eq(sql), Mockito.<List<Object>>any(),
Mockito.<Map<String, String>>any());
-
- assertTrue(resultSet.next());
- ResultSetMetaData metaData = resultSet.getMetaData();
- assertEquals("VAL", metaData.getColumnName(1));
- assertEquals(1, resultSet.getInt("VAL"));
+ assertTrue(resultSet.next());
+ ResultSetMetaData metaData = resultSet.getMetaData();
+ assertEquals("VAL", metaData.getColumnName(1));
+ assertEquals(1, resultSet.getInt("VAL"));
+ }
+ }
}
@Test
public void testPrepareStatementWithMockHttp() throws IOException,
SQLException {
String sql = "select 1 as val";
- KylinConnection connection = getConnectionWithMockHttp();
+ try (KylinConnection connection = getConnectionWithMockHttp()) {
- // mock http
- HttpResponse response = TestUtil.mockHttpResponseWithFile(200, "OK",
"query.json");
-
when(httpClient.execute(any(HttpUriRequest.class))).thenReturn(response);
+ // mock http
+ HttpResponse response = TestUtil.mockHttpResponseWithFile(200,
"OK", "query.json");
+
when(httpClient.execute(any(HttpUriRequest.class))).thenReturn(response);
- ResultSet resultSet = connection.prepareStatement(sql).executeQuery();
-
- assertTrue(resultSet.next());
- ResultSetMetaData metaData = resultSet.getMetaData();
- assertEquals("VAL", metaData.getColumnName(1));
- assertEquals(1, resultSet.getInt("VAL"));
+ try (ResultSet resultSet =
connection.prepareStatement(sql).executeQuery()) {
+ assertTrue(resultSet.next());
+ ResultSetMetaData metaData = resultSet.getMetaData();
+ assertEquals("VAL", metaData.getColumnName(1));
+ assertEquals(1, resultSet.getInt("VAL"));
+ }
+ }
}
private KylinConnection getConnectionWithMockClient() throws SQLException {
- Properties info = new Properties();
+ return getConnectionWithMockClient("jdbc:kylin:test_url/test_db", new
Properties());
+ }
+
+ private KylinConnection getConnectionWithMockClient(String url, @Nonnull
Properties info) throws SQLException {
info.setProperty("user", "ADMIN");
info.setProperty("password", "KYLIN");
doReturn(client).when(factory).newRemoteClient(any(KylinConnectionInfo.class));
- return new KylinConnection(driver, factory,
"jdbc:kylin:test_url/test_db", info);
+ return new KylinConnection(driver, factory, url, info);
}
private KylinConnection getConnectionWithMockHttp() throws SQLException,
IOException {
@@ -141,4 +144,55 @@ public Object answer(InvocationOnMock invo) throws
Throwable {
return new KylinConnection(driver, factory,
"jdbc:kylin:test_url/test_db", info);
}
+
+ @Test
+ public void testJdbcClientCalcitePropsInUrl() throws Exception {
+ String sql = "select 1 as val";
+
+ // mock client
+ when(client.executeQuery(anyString(), Mockito.<List<Object>>any(),
Mockito.<Map<String, String>>any())).thenReturn(getMockResult());
+ Map<String, String> toggles = new HashMap<>();
+ Properties info = new Properties();
+ info.setProperty("caseSensitive", "false");
+ info.setProperty("unquotedCasing", "UNCHANGED");
+ try (KylinConnection conn =
getConnectionWithMockClient("jdbc:kylin:test_url/test_db", info)) {
+ PreparedStatement preparedStatement = conn.prepareStatement(sql);
+ try (ResultSet resultSet = preparedStatement.executeQuery()) {
+ verify(client).executeQuery(eq(sql),
Mockito.<List<Object>>any(), argThat(new ArgumentMatcher<Map<String, String>>()
{
+ @Override
+ public boolean matches(Map<String, String> argument) {
+ String propsStr =
argument.get("JDBC_CLIENT_CALCITE_PROPS");
+ assertNotNull(propsStr);
+ Properties props = new Properties();
+ try {
+ props.load(new StringReader(propsStr));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ assertEquals("false",
props.getProperty("caseSensitive"));
+ assertEquals("UNCHANGED",
props.getProperty("unquotedCasing"));
+ return true;
+ }
+ }));
+
+ assertTrue(resultSet.next());
+ ResultSetMetaData metaData = resultSet.getMetaData();
+ assertEquals("VAL", metaData.getColumnName(1));
+ assertEquals(1, resultSet.getInt("VAL"));
+ }
+ }
+ }
+
+ private IRemoteClient.QueryResult getMockResult() {
+ ArrayList<ColumnMetaData> columnMeta = new ArrayList<>();
+ columnMeta.add(new ColumnMetaData(0, false, true, false,
+ false, 1, true, 1,
+ "VAL", "VAL", null,
+ 10, 0, null, null,
+ ColumnMetaData.scalar(Types.INTEGER, "INTEGER",
ColumnMetaData.Rep.INTEGER),
+ true, false, false, "java.lang.Integer"));
+ ArrayList<Object> list = new ArrayList<>();
+ list.add(new Object[]{1});
+ return new IRemoteClient.QueryResult(columnMeta, list);
+ }
}
diff --git a/query/src/main/java/org/apache/kylin/query/QueryConnection.java
b/query/src/main/java/org/apache/kylin/query/QueryConnection.java
index b2db178c55..1ea715fefb 100644
--- a/query/src/main/java/org/apache/kylin/query/QueryConnection.java
+++ b/query/src/main/java/org/apache/kylin/query/QueryConnection.java
@@ -26,6 +26,7 @@
import org.apache.calcite.jdbc.Driver;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
public class QueryConnection {
@@ -47,6 +48,8 @@ public static Connection getConnection(String project) throws
SQLException {
File olapTmp = OLAPSchemaFactory.createTempOLAPJson(project,
KylinConfig.getInstanceFromEnv());
Properties info = new Properties();
info.putAll(KylinConfig.getInstanceFromEnv().getCalciteExtrasProperties());
+ // Import calcite props from jdbc client(override the kylin.properties)
+ info.putAll(BackdoorToggles.getJdbcDriverClientCalciteProps());
info.put("model", olapTmp.getAbsolutePath());
info.put("typeSystem",
"org.apache.kylin.query.calcite.KylinRelDataTypeSystem");
return DriverManager.getConnection("jdbc:calcite:", info);
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
