This is an automated email from the ASF dual-hosted git repository.
csy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new e498bdba0 [KYUUBI #5582] JDBC Engine supports configurable default
fetchSize
e498bdba0 is described below
commit e498bdba0089ef2e6003544462f56273165420d3
Author: senmiaoliu <[email protected]>
AuthorDate: Wed Nov 15 12:15:05 2023 +0800
[KYUUBI #5582] JDBC Engine supports configurable default fetchSize
### _Why are the changes needed?_
close #5582
JDBC Engine supports configurable default fetchSize
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run
test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests)
locally before make a pull request
### _Was this patch authored or co-authored using generative AI tooling?_
NO
Closes #5614 from lsm1/branch-kyuubi-5582.
Closes #5582
dd58a6d51 [senmiaoliu] fix style
d0eb7c5dc [senmiaoliu] set jdbc fetch size in operation manager
4eee26b7b [senmiaoliu] rename conf key
4d8b74796 [senmiaoliu] add conf for JDBC engine default fetch size
Authored-by: senmiaoliu <[email protected]>
Signed-off-by: Shaoyun Chen <[email protected]>
---
docs/configuration/settings.md | 1 +
.../engine/jdbc/operation/ExecuteStatement.scala | 5 ++--
.../jdbc/operation/JdbcOperationManager.scala | 28 +++++++++++++++++-----
.../org/apache/kyuubi/config/KyuubiConf.scala | 7 ++++++
4 files changed, 33 insertions(+), 8 deletions(-)
diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md
index 4c4eb8be9..5577da674 100644
--- a/docs/configuration/settings.md
+++ b/docs/configuration/settings.md
@@ -155,6 +155,7 @@ You can configure the Kyuubi properties in
`$KYUUBI_HOME/conf/kyuubi-defaults.co
| kyuubi.engine.jdbc.connection.user | <undefined>
| The user is used for connecting to server
[...]
| kyuubi.engine.jdbc.driver.class | <undefined>
| The driver class for JDBC engine connection
[...]
| kyuubi.engine.jdbc.extra.classpath | <undefined>
| The extra classpath for the JDBC query engine, for configuring the
location of the JDBC driver and etc.
[...]
+| kyuubi.engine.jdbc.fetch.size | 1000
| The fetch size of JDBC engine
[...]
| kyuubi.engine.jdbc.initialize.sql | SELECT 1
| SemiColon-separated list of SQL statements to be initialized in the
newly created engine before queries. i.e. use `SELECT 1` to eagerly active
JDBCClient.
[...]
| kyuubi.engine.jdbc.java.options | <undefined>
| The extra Java options for the JDBC query engine
[...]
| kyuubi.engine.jdbc.memory | 1g
| The heap memory for the JDBC query engine
[...]
diff --git
a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/operation/ExecuteStatement.scala
b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/operation/ExecuteStatement.scala
index b7caea014..4292c320b 100644
---
a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/operation/ExecuteStatement.scala
+++
b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/operation/ExecuteStatement.scala
@@ -33,7 +33,8 @@ class ExecuteStatement(
override val statement: String,
override val shouldRunAsync: Boolean,
queryTimeout: Long,
- incrementalCollect: Boolean)
+ incrementalCollect: Boolean,
+ fetchSize: Int)
extends JdbcOperation(session) with Logging {
private val operationLog: OperationLog =
OperationLog.createOperationLog(session, getHandle)
@@ -61,7 +62,7 @@ class ExecuteStatement(
setState(OperationState.RUNNING)
try {
val connection: Connection =
session.asInstanceOf[JdbcSessionImpl].sessionConnection
- jdbcStatement = dialect.createStatement(connection)
+ jdbcStatement = dialect.createStatement(connection, fetchSize)
val hasResult = jdbcStatement.execute(statement)
if (hasResult) {
val resultSetWrapper = new ResultSetWrapper(jdbcStatement)
diff --git
a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/operation/JdbcOperationManager.scala
b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/operation/JdbcOperationManager.scala
index 8b9a7832e..7ced3e6b8 100644
---
a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/operation/JdbcOperationManager.scala
+++
b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/operation/JdbcOperationManager.scala
@@ -20,7 +20,7 @@ import java.util
import org.apache.kyuubi.KyuubiSQLException
import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.OPERATION_INCREMENTAL_COLLECT
+import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_FETCH_SIZE,
OPERATION_INCREMENTAL_COLLECT}
import org.apache.kyuubi.engine.jdbc.dialect.{JdbcDialect, JdbcDialects}
import org.apache.kyuubi.engine.jdbc.session.JdbcSessionImpl
import org.apache.kyuubi.engine.jdbc.util.SupportServiceLoader
@@ -44,13 +44,16 @@ class JdbcOperationManager(conf: KyuubiConf) extends
OperationManager("JdbcOpera
val incrementalCollect =
normalizedConf.get(OPERATION_INCREMENTAL_COLLECT.key).map(
_.toBoolean).getOrElse(
session.sessionManager.getConf.get(OPERATION_INCREMENTAL_COLLECT))
+ val fetchSize = normalizedConf.get(ENGINE_JDBC_FETCH_SIZE.key).map(_.toInt)
+ .getOrElse(session.sessionManager.getConf.get(ENGINE_JDBC_FETCH_SIZE))
val executeStatement =
new ExecuteStatement(
session,
statement,
runAsync,
queryTimeout,
- incrementalCollect)
+ incrementalCollect,
+ fetchSize)
addOperation(executeStatement)
}
@@ -61,8 +64,11 @@ class JdbcOperationManager(conf: KyuubiConf) extends
OperationManager("JdbcOpera
override def newGetCatalogsOperation(session: Session): Operation = {
val query = dialect.getCatalogsOperation()
+ val normalizedConf = session.asInstanceOf[JdbcSessionImpl].normalizedConf
+ val fetchSize = normalizedConf.get(ENGINE_JDBC_FETCH_SIZE.key).map(_.toInt)
+ .getOrElse(session.sessionManager.getConf.get(ENGINE_JDBC_FETCH_SIZE))
val executeStatement =
- new ExecuteStatement(session, query, false, 0L, true)
+ new ExecuteStatement(session, query, false, 0L, true, fetchSize)
addOperation(executeStatement)
}
@@ -71,8 +77,11 @@ class JdbcOperationManager(conf: KyuubiConf) extends
OperationManager("JdbcOpera
catalog: String,
schema: String): Operation = {
val query = dialect.getSchemasOperation(catalog, schema)
+ val normalizedConf = session.asInstanceOf[JdbcSessionImpl].normalizedConf
+ val fetchSize = normalizedConf.get(ENGINE_JDBC_FETCH_SIZE.key).map(_.toInt)
+ .getOrElse(session.sessionManager.getConf.get(ENGINE_JDBC_FETCH_SIZE))
val executeStatement =
- new ExecuteStatement(session, query, false, 0L, true)
+ new ExecuteStatement(session, query, false, 0L, true, fetchSize)
addOperation(executeStatement)
}
@@ -83,8 +92,11 @@ class JdbcOperationManager(conf: KyuubiConf) extends
OperationManager("JdbcOpera
tableName: String,
tableTypes: util.List[String]): Operation = {
val query = dialect.getTablesQuery(catalogName, schemaName, tableName,
tableTypes)
+ val normalizedConf = session.asInstanceOf[JdbcSessionImpl].normalizedConf
+ val fetchSize = normalizedConf.get(ENGINE_JDBC_FETCH_SIZE.key).map(_.toInt)
+ .getOrElse(session.sessionManager.getConf.get(ENGINE_JDBC_FETCH_SIZE))
val executeStatement =
- new ExecuteStatement(session, query, false, 0L, true)
+ new ExecuteStatement(session, query, false, 0L, true, fetchSize)
addOperation(executeStatement)
}
@@ -100,8 +112,12 @@ class JdbcOperationManager(conf: KyuubiConf) extends
OperationManager("JdbcOpera
tableName: String,
columnName: String): Operation = {
val query = dialect.getColumnsQuery(session, catalogName, schemaName,
tableName, columnName)
+ val normalizedConf = session.asInstanceOf[JdbcSessionImpl].normalizedConf
+ val fetchSize = normalizedConf.get(ENGINE_JDBC_FETCH_SIZE.key).map(
+ _.toInt).getOrElse(
+ session.sessionManager.getConf.get(ENGINE_JDBC_FETCH_SIZE))
val executeStatement =
- new ExecuteStatement(session, query, false, 0L, true)
+ new ExecuteStatement(session, query, false, 0L, true, fetchSize)
addOperation(executeStatement)
}
diff --git
a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
index 9ab392c2a..a48f4ba9f 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -2820,6 +2820,13 @@ object KyuubiConf {
.toSequence(";")
.createWithDefault(Nil)
+ val ENGINE_JDBC_FETCH_SIZE: ConfigEntry[Int] =
+ buildConf("kyuubi.engine.jdbc.fetch.size")
+ .doc("The fetch size of JDBC engine")
+ .version("1.9.0")
+ .intConf
+ .createWithDefault(1000)
+
val ENGINE_OPERATION_CONVERT_CATALOG_DATABASE_ENABLED: ConfigEntry[Boolean] =
buildConf("kyuubi.engine.operation.convert.catalog.database.enabled")
.doc("When set to true, The engine converts the JDBC methods of set/get
Catalog " +
