This is an automated email from the ASF dual-hosted git repository.
bowenliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 86b88b7e4 [KYUUBI #5906] [JDBC] Rebase Doris dialect implementation
86b88b7e4 is described below
commit 86b88b7e4952ea5c33783404ba904ad4dbf369a2
Author: Bowen Liang <[email protected]>
AuthorDate: Sat Dec 23 00:12:52 2023 +0800
[KYUUBI #5906] [JDBC] Rebase Doris dialect implementation
# :mag: Description
## Issue References ๐
As decribed.
## Describe Your Solution ๐ง
As Doris is MySQL protocol compatible , use MySQLDialect as base class to
simplify Doris dialect implementation
## Types of changes :bookmark:
- [ ] Bugfix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
## Test Plan ๐งช
#### Behavior Without This Pull Request :coffin:
No behaviour changes.
#### Behavior With This Pull Request :tada:
No behaviour changes.
#### Related Unit Tests
---
# Checklists
## ๐ Author Self Checklist
- [x] My code follows the [style
guidelines](https://kyuubi.readthedocs.io/en/master/contributing/code/style.html)
of this project
- [x] I have performed a self-review
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [ ] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my feature
works
- [ ] New and existing unit tests pass locally with my changes
- [x] This patch was not authored or co-authored using [Generative
Tooling](https://www.apache.org/legal/generative-tooling.html)
## ๐ Committer Pre-Merge Checklist
- [ ] Pull request title is okay.
- [ ] No license issues.
- [ ] Milestone correctly set?
- [ ] Test coverage is ok
- [ ] Assignees are selected.
- [ ] Minimum number of approvals
- [ ] No changes are requested
**Be nice. Be informative.**
Closes #5906 from bowenliang123/jdbc-doris-simp.
Closes #5906
2e1420a88 [liangbowen] skip
e095d28d7 [liangbowen] Revert "simplify MySQLTRowSetGenerator"
7d59e2d28 [liangbowen] simplify MySQLTRowSetGenerator
9214388f7 [Bowen Liang] simplify StarRocksSchemaHelper
5f910426b [Bowen Liang] simplify DorisSchemaHelper
b90f26d04 [Bowen Liang] simplify DorisSchemaHelper
45ccdfeab [Bowen Liang] update
5b4a2c2ba [Bowen Liang] import
f7fdd35ca [Bowen Liang] simplify DorisDialect
Lead-authored-by: Bowen Liang <[email protected]>
Co-authored-by: liangbowen <[email protected]>
Signed-off-by: liangbowen <[email protected]>
---
.../kyuubi/engine/jdbc/dialect/DorisDialect.scala | 109 +--------------------
.../engine/jdbc/doris/DorisSchemaHelper.scala | 10 +-
.../jdbc/schema/DefaultJdbcTRowSetGenerator.scala | 1 -
.../jdbc/starrocks/StarRocksSchemaHelper.scala | 10 +-
.../engine/jdbc/doris/WithDorisContainer.scala | 19 ++--
.../kyuubi/engine/jdbc/doris/WithDorisEngine.scala | 2 +-
6 files changed, 16 insertions(+), 135 deletions(-)
diff --git
a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/DorisDialect.scala
b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/DorisDialect.scala
index 32801c36e..e48a12a89 100644
---
a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/DorisDialect.scala
+++
b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/DorisDialect.scala
@@ -15,118 +15,15 @@
* limitations under the License.
*/
package org.apache.kyuubi.engine.jdbc.dialect
-import java.sql.{Connection, Statement}
-import java.util
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-
-import org.apache.commons.lang3.StringUtils
import org.apache.kyuubi.engine.jdbc.doris.{DorisSchemaHelper,
DorisTRowSetGenerator}
import org.apache.kyuubi.engine.jdbc.schema.{JdbcTRowSetGenerator,
SchemaHelper}
-import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._
-import org.apache.kyuubi.session.Session
-
-class DorisDialect extends JdbcDialect {
-
- override def createStatement(connection: Connection, fetchSize: Int):
Statement = {
- val statement = super.createStatement(connection, fetchSize)
- statement.setFetchSize(Integer.MIN_VALUE)
- statement
- }
-
- override def getTablesQuery(
- catalog: String,
- schema: String,
- tableName: String,
- tableTypes: util.List[String]): String = {
- val tTypes =
- if (tableTypes == null || tableTypes.isEmpty) {
- Set("BASE TABLE", "SYSTEM VIEW", "VIEW")
- } else {
- tableTypes.asScala.toSet
- }
- val query = new StringBuilder(
- s"""
- |SELECT TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, ENGINE,
- |TABLE_ROWS, AVG_ROW_LENGTH, DATA_LENGTH,
- |CREATE_TIME, UPDATE_TIME, TABLE_COLLATION, TABLE_COMMENT
- |FROM INFORMATION_SCHEMA.TABLES
- |""".stripMargin)
-
- val filters = ArrayBuffer[String]()
- if (StringUtils.isNotBlank(catalog)) {
- filters += s"$TABLE_CATALOG = '$catalog'"
- }
-
- if (StringUtils.isNotBlank(schema)) {
- filters += s"$TABLE_SCHEMA LIKE '$schema'"
- }
- if (StringUtils.isNotBlank(tableName)) {
- filters += s"$TABLE_NAME LIKE '$tableName'"
- }
+class DorisDialect extends MySQLDialect {
- if (tTypes.nonEmpty) {
- filters += s"(${tTypes.map { tableType => s"$TABLE_TYPE = '$tableType'" }
- .mkString(" OR ")})"
- }
-
- if (filters.nonEmpty) {
- query.append(" WHERE ")
- query.append(filters.mkString(" AND "))
- }
-
- query.toString()
- }
-
- override def getColumnsQuery(
- session: Session,
- catalogName: String,
- schemaName: String,
- tableName: String,
- columnName: String): String = {
- val query = new StringBuilder(
- """
- |SELECT
- |`TABLE_CATALOG`,`TABLE_SCHEMA`,`TABLE_NAME`,
`COLUMN_NAME`,`ORDINAL_POSITION`,
- |`COLUMN_DEFAULT`,`IS_NULLABLE`,`DATA_TYPE`,`CHARACTER_MAXIMUM_LENGTH`,
-
|`CHARACTER_OCTET_LENGTH`,`NUMERIC_PRECISION`,`NUMERIC_SCALE`,`DATETIME_PRECISION`,
-
|`CHARACTER_SET_NAME`,`COLLATION_NAME`,`COLUMN_TYPE`,`COLUMN_KEY`,`EXTRA`,`PRIVILEGES`,
-
|`COLUMN_COMMENT`,`COLUMN_SIZE`,`DECIMAL_DIGITS`,`GENERATION_EXPRESSION`,`SRS_ID`
- |FROM information_schema.columns
- |""".stripMargin)
-
- val filters = ArrayBuffer[String]()
- if (StringUtils.isNotEmpty(catalogName)) {
- filters += s"$TABLE_CATALOG = '$catalogName'"
- }
- if (StringUtils.isNotEmpty(schemaName)) {
- filters += s"$TABLE_SCHEMA LIKE '$schemaName'"
- }
- if (StringUtils.isNotEmpty(tableName)) {
- filters += s"$TABLE_NAME LIKE '$tableName'"
- }
- if (StringUtils.isNotEmpty(columnName)) {
- filters += s"$COLUMN_NAME LIKE '$columnName'"
- }
-
- if (filters.nonEmpty) {
- query.append(" WHERE ")
- query.append(filters.mkString(" AND "))
- }
-
- query.toString()
- }
+ override def name(): String = "doris"
override def getTRowSetGenerator(): JdbcTRowSetGenerator = new
DorisTRowSetGenerator
- override def getSchemaHelper(): SchemaHelper = {
- new DorisSchemaHelper
- }
-
- override def name(): String = {
- "doris"
- }
+ override def getSchemaHelper(): SchemaHelper = new DorisSchemaHelper
}
diff --git
a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/doris/DorisSchemaHelper.scala
b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/doris/DorisSchemaHelper.scala
index d16c31456..a37ba4a39 100644
---
a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/doris/DorisSchemaHelper.scala
+++
b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/doris/DorisSchemaHelper.scala
@@ -16,12 +16,6 @@
*/
package org.apache.kyuubi.engine.jdbc.doris
-import org.apache.kyuubi.engine.jdbc.schema.SchemaHelper
-import org.apache.kyuubi.shaded.hive.service.rpc.thrift._
+import org.apache.kyuubi.engine.jdbc.mysql.MySQLSchemaHelper
-class DorisSchemaHelper extends SchemaHelper {
-
- override def tinyIntToTTypeId: TTypeId = TTypeId.INT_TYPE
-
- override def smallIntToTTypeId: TTypeId = TTypeId.INT_TYPE
-}
+class DorisSchemaHelper extends MySQLSchemaHelper {}
diff --git
a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/DefaultJdbcTRowSetGenerator.scala
b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/DefaultJdbcTRowSetGenerator.scala
index 2621379c1..2c9ddd6da 100644
---
a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/DefaultJdbcTRowSetGenerator.scala
+++
b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/DefaultJdbcTRowSetGenerator.scala
@@ -124,7 +124,6 @@ class DefaultJdbcTRowSetGenerator extends
JdbcTRowSetGenerator {
case (date: Date, DATE) => formatDate(date)
case (dateTime: LocalDateTime, TIMESTAMP) =>
formatLocalDateTime(dateTime)
case (decimal: java.math.BigDecimal, DECIMAL) => decimal.toPlainString
- case (bigint: java.math.BigInteger, BIGINT) => bigint.toString()
case (other, _) => other.toString
}
}
diff --git
a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/starrocks/StarRocksSchemaHelper.scala
b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/starrocks/StarRocksSchemaHelper.scala
index 18c2f5112..e6b4e1521 100644
---
a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/starrocks/StarRocksSchemaHelper.scala
+++
b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/starrocks/StarRocksSchemaHelper.scala
@@ -16,12 +16,6 @@
*/
package org.apache.kyuubi.engine.jdbc.starrocks
-import org.apache.kyuubi.engine.jdbc.schema.SchemaHelper
-import org.apache.kyuubi.shaded.hive.service.rpc.thrift._
+import org.apache.kyuubi.engine.jdbc.mysql.MySQLSchemaHelper
-class StarRocksSchemaHelper extends SchemaHelper {
-
- override def tinyIntToTTypeId: TTypeId = TTypeId.INT_TYPE
-
- override def smallIntToTTypeId: TTypeId = TTypeId.INT_TYPE
-}
+class StarRocksSchemaHelper extends MySQLSchemaHelper {}
diff --git
a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisContainer.scala
b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisContainer.scala
index 1d63760ff..c37478e99 100644
---
a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisContainer.scala
+++
b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisContainer.scala
@@ -27,12 +27,10 @@ import org.apache.kyuubi.engine.jdbc.WithJdbcServerContainer
trait WithDorisContainer extends WithJdbcServerContainer {
- private val DORIS_FE_PORT = 9030
-
- private val DORIS_BE_PORT = 8040
+ private val DORIS_FE_MYSQL_PORT = 9030
+ private val DORIS_BE_HTTTP_PORT = 8040
private val DORIS_FE_SERVICE_NAME = "doris-fe"
-
private val DORIS_BE_SERVICE_NAME = "doris-be"
override val containerDef: DockerComposeContainer.Def =
@@ -43,19 +41,18 @@ trait WithDorisContainer extends WithJdbcServerContainer {
exposedServices = Seq[ExposedService](
ExposedService(
DORIS_FE_SERVICE_NAME,
- DORIS_FE_PORT,
+ DORIS_FE_MYSQL_PORT,
waitStrategy =
new
DockerHealthcheckWaitStrategy().withStartupTimeout(Duration.ofMinutes(5))),
ExposedService(
DORIS_BE_SERVICE_NAME,
- DORIS_BE_PORT,
+ DORIS_BE_HTTTP_PORT,
waitStrategy =
new
DockerHealthcheckWaitStrategy().withStartupTimeout(Duration.ofMinutes(5)))))
- protected def feUrl: String = withContainers { container =>
- val feHost: String = container.getServiceHost(DORIS_FE_SERVICE_NAME,
DORIS_FE_PORT)
- val fePort: Int = container.getServicePort(DORIS_FE_SERVICE_NAME,
DORIS_FE_PORT)
- val url = s"$feHost:$fePort"
- url
+ protected def feJdbcUrl: String = withContainers { container =>
+ val feHost: String = container.getServiceHost(DORIS_FE_SERVICE_NAME,
DORIS_FE_MYSQL_PORT)
+ val fePort: Int = container.getServicePort(DORIS_FE_SERVICE_NAME,
DORIS_FE_MYSQL_PORT)
+ s"jdbc:mysql://$feHost:$fePort"
}
}
diff --git
a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisEngine.scala
b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisEngine.scala
index 9945fb640..692f37b95 100644
---
a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisEngine.scala
+++
b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisEngine.scala
@@ -23,7 +23,7 @@ trait WithDorisEngine extends WithJdbcEngine with
WithDorisContainer {
override def withKyuubiConf: Map[String, String] = Map(
ENGINE_SHARE_LEVEL.key -> "SERVER",
- ENGINE_JDBC_CONNECTION_URL.key -> s"jdbc:mysql://$feUrl",
+ ENGINE_JDBC_CONNECTION_URL.key -> feJdbcUrl,
ENGINE_JDBC_CONNECTION_USER.key -> "root",
ENGINE_JDBC_CONNECTION_PASSWORD.key -> "",
ENGINE_TYPE.key -> "jdbc",
