This is an automated email from the ASF dual-hosted git repository.
peacewong pushed a commit to branch dev-1.4.0
in repository https://gitbox.apache.org/repos/asf/linkis.git
The following commit(s) were added to refs/heads/dev-1.4.0 by this push:
new 7cab75646 Support Gaussdb dataSources and Fixed pg bug (#4513)
7cab75646 is described below
commit 7cab7564653b24a9adf0093c4ae26fe5555f88f4
Author: ChengJie1053 <[email protected]>
AuthorDate: Thu May 4 15:55:55 2023 +0800
Support Gaussdb dataSources and Fixed pg bug (#4513)
---
linkis-dist/package/db/linkis_dml.sql | 13 +++
.../db/upgrade/1.4.0_schema/mysql/linkis_dml.sql | 14 +++
.../query/common/cache/CacheConfiguration.java | 2 +-
.../metadata/query/service/GaussdbMetaService.java | 103 +++++++++++++++++++++
.../query/service/postgres/SqlConnection.java | 11 ++-
5 files changed, 137 insertions(+), 6 deletions(-)
diff --git a/linkis-dist/package/db/linkis_dml.sql
b/linkis-dist/package/db/linkis_dml.sql
index 2ec20f354..d207beca0 100644
--- a/linkis-dist/package/db/linkis_dml.sql
+++ b/linkis-dist/package/db/linkis_dml.sql
@@ -556,6 +556,7 @@ INSERT INTO `linkis_ps_dm_datasource_type` (`name`,
`description`, `option`, `cl
INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`,
`classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`)
VALUES ('clickhouse', 'clickhouse数据库', 'clickhouse', 'olap', '', 4, 'Clickhouse
Database', 'Clickhouse', 'Olap');
INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`,
`classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`)
VALUES ('tidb', 'tidb数据库', 'tidb', '关系型数据库', '', 3, 'TiDB Database', 'TiDB',
'Relational Database');
INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`,
`classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`)
VALUES ('starrocks', 'starrocks数据库', 'starrocks', 'olap', '', 4, 'StarRocks
Database', 'StarRocks', 'Olap');
+INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`,
`classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`)
VALUES ('gaussdb', 'gaussdb数据库', 'gaussdb', '关系型数据库', '', 3, 'GaussDB
Database', 'GaussDB', 'Relational Database');
select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where
`name` = 'mongodb';
@@ -748,3 +749,15 @@ VALUES (@data_source_type_id, 'address', '地址', 'Address',
NULL, 'TEXT', NULL
(@data_source_type_id, 'username', '用户名(Username)', 'Username', NULL,
'TEXT', NULL, 1, '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL,
NULL, now(), now()),
(@data_source_type_id, 'password', '密码(Password)', 'Password', NULL,
'PASSWORD', NULL, 0, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(),
now()),
(@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL,
'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(),
now());
+
+select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where
`name` = 'gaussdb';
+INSERT INTO `linkis_ps_dm_datasource_type_key`
+(`data_source_type_id`, `key`, `name`, `name_en`, `default_value`,
`value_type`, `scope`, `require`, `description`, `description_en`,
`value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`,
`create_time`)
+VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL,
0, '地址(host1:port1,host2:port2...)', 'Address(host1:port1,host2:port2...)',
NULL, NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'host', '主机名(Host)', 'Host', NULL, 'TEXT', NULL,
1, '主机名(Host)', 'Host', NULL, NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'port', '端口号(Port)', 'Port', NULL, 'TEXT', NULL,
1, '端口号(Port)', 'Port', NULL, NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'driverClassName', '驱动类名(Driver class name)',
'Driver class name', 'org.postgresql.Driver', 'TEXT', NULL, 1, '驱动类名(Driver
class name)', 'Driver class name', NULL, NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'params', '连接参数(Connection params)', 'Connection
params', NULL, 'TEXT', NULL, 0, '输入JSON格式(Input JSON format):
{"param":"value"}', 'Input JSON format: {"param":"value"}', NULL, NULL, NULL,
NULL, now(), now()),
+ (@data_source_type_id, 'username', '用户名(Username)', 'Username', NULL,
'TEXT', NULL, 1, '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL,
NULL, now(), now()),
+ (@data_source_type_id, 'password', '密码(Password)', 'Password', NULL,
'PASSWORD', NULL, 1, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(),
now()),
+ (@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL,
'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(),
now());
diff --git a/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql
b/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql
index 66d9beee1..f19bf7f69 100644
--- a/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql
+++ b/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql
@@ -42,3 +42,17 @@ VALUES (@data_source_type_id, 'address', '地址', 'Address',
NULL, 'TEXT', NULL
(@data_source_type_id, 'username', '用户名(Username)', 'Username', NULL,
'TEXT', NULL, 1, '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL,
NULL, now(), now()),
(@data_source_type_id, 'password', '密码(Password)', 'Password', NULL,
'PASSWORD', NULL, 0, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(),
now()),
(@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL,
'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(),
now());
+
+INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`,
`classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`)
VALUES ('gaussdb', 'gaussdb数据库', 'gaussdb', '关系型数据库', '', 3, 'GaussDB
Database', 'GaussDB', 'Relational Database');
+
+select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where
`name` = 'gaussdb';
+INSERT INTO `linkis_ps_dm_datasource_type_key`
+(`data_source_type_id`, `key`, `name`, `name_en`, `default_value`,
`value_type`, `scope`, `require`, `description`, `description_en`,
`value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`,
`create_time`)
+VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL,
0, '地址(host1:port1,host2:port2...)', 'Address(host1:port1,host2:port2...)',
NULL, NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'host', '主机名(Host)', 'Host', NULL, 'TEXT', NULL,
1, '主机名(Host)', 'Host', NULL, NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'port', '端口号(Port)', 'Port', NULL, 'TEXT', NULL,
1, '端口号(Port)', 'Port', NULL, NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'driverClassName', '驱动类名(Driver class name)',
'Driver class name', 'org.postgresql.Driver', 'TEXT', NULL, 1, '驱动类名(Driver
class name)', 'Driver class name', NULL, NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'params', '连接参数(Connection params)', 'Connection
params', NULL, 'TEXT', NULL, 0, '输入JSON格式(Input JSON format):
{"param":"value"}', 'Input JSON format: {"param":"value"}', NULL, NULL, NULL,
NULL, now(), now()),
+ (@data_source_type_id, 'username', '用户名(Username)', 'Username', NULL,
'TEXT', NULL, 1, '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL,
NULL, now(), now()),
+ (@data_source_type_id, 'password', '密码(Password)', 'Password', NULL,
'PASSWORD', NULL, 1, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(),
now()),
+ (@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL,
'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(),
now());
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/cache/CacheConfiguration.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/cache/CacheConfiguration.java
index 54baec4b7..2aacc7382 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/cache/CacheConfiguration.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/cache/CacheConfiguration.java
@@ -34,5 +34,5 @@ public class CacheConfiguration {
public static final CommonVars<String> MYSQL_RELATIONSHIP_LIST =
CommonVars.apply(
"wds.linkis.server.mdq.mysql.relationship",
-
"mysql,oracle,kingbase,postgresql,sqlserver,db2,greenplum,dm,doris,clickhouse,tidb,starrocks");
+
"mysql,oracle,kingbase,postgresql,sqlserver,db2,greenplum,dm,doris,clickhouse,tidb,starrocks,gaussdb");
}
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/GaussdbMetaService.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/GaussdbMetaService.java
new file mode 100644
index 000000000..956e92065
--- /dev/null
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/GaussdbMetaService.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.metadata.query.service;
+
+import org.apache.linkis.datasourcemanager.common.util.json.Json;
+import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo;
+import org.apache.linkis.metadata.query.common.service.AbstractDbMetaService;
+import org.apache.linkis.metadata.query.common.service.MetadataConnection;
+import org.apache.linkis.metadata.query.service.conf.SqlParamsMapper;
+import org.apache.linkis.metadata.query.service.postgres.SqlConnection;
+
+import org.apache.logging.log4j.util.Strings;
+
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class GaussdbMetaService extends AbstractDbMetaService<SqlConnection> {
+ @Override
+ public MetadataConnection<SqlConnection> getConnection(
+ String operator, Map<String, Object> params) throws Exception {
+ String host =
+
String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_HOST.getValue(),
""));
+ // After deserialize, Integer will be Double, Why?
+ Integer port =
+ (Double.valueOf(
+
String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_PORT.getValue(),
0))))
+ .intValue();
+ String username =
+
String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_USERNAME.getValue(),
""));
+ String password =
+
String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_PASSWORD.getValue(),
""));
+ // In PG, each database under the same instance is completely independent,
and the table is
+ // stored under the catalog with the same library name.
+ // \c (\connect) Behind the dbname command is to close the current
connection and create a
+ // new connection to achieve database switching
+ // Cannot directly switch to another database under the current database
connection, and
+ // cannot show tables from xxxx, select * from database.table like MySQL
+
+ String database =
+
String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_DATABASE.getValue(),
""));
+ Map<String, Object> extraParams = new HashMap<>();
+ Object sqlParamObj =
params.get(SqlParamsMapper.PARAM_SQL_EXTRA_PARAMS.getValue());
+ if (null != sqlParamObj) {
+ if (!(sqlParamObj instanceof Map)) {
+ extraParams =
+ Json.fromJson(String.valueOf(sqlParamObj), Map.class,
String.class, Object.class);
+ } else {
+ extraParams = (Map<String, Object>) sqlParamObj;
+ }
+ }
+ assert extraParams != null;
+ if (Strings.isBlank(database)) {
+ database = "";
+ }
+ return new MetadataConnection<>(
+ new SqlConnection(host, port, username, password, database,
extraParams));
+ }
+
+ @Override
+ public List<String> queryTables(SqlConnection connection, String schemaname)
{
+ try {
+ return connection.getAllTables(schemaname);
+ } catch (SQLException e) {
+ throw new RuntimeException("Fail to get Sql tables(获取表列表失败)", e);
+ }
+ }
+
+ @Override
+ public List<String> queryDatabases(SqlConnection connection) {
+ try {
+ return connection.getAllDatabases();
+ } catch (SQLException e) {
+ throw new RuntimeException("Fail to get Sql databases(获取数据库列表失败)", e);
+ }
+ }
+
+ @Override
+ public List<MetaColumnInfo> queryColumns(
+ SqlConnection connection, String database, String table) {
+ try {
+ return connection.getColumns(database, table);
+ } catch (SQLException | ClassNotFoundException e) {
+ throw new RuntimeException("Fail to get Sql columns(获取字段列表失败)", e);
+ }
+ }
+}
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/postgres/SqlConnection.java
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/postgres/SqlConnection.java
index 3ac2e638b..02acd76a9 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/postgres/SqlConnection.java
+++
b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/postgres/SqlConnection.java
@@ -20,6 +20,7 @@ package org.apache.linkis.metadata.query.service.postgres;
import org.apache.linkis.common.conf.CommonVars;
import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo;
+import org.apache.commons.collections.MapUtils;
import org.apache.logging.log4j.util.Strings;
import java.io.Closeable;
@@ -181,15 +182,15 @@ public class SqlConnection implements Closeable {
*/
private Connection getDBConnection(ConnectMessage connectMessage, String
database)
throws ClassNotFoundException, SQLException {
- String extraParamString =
- connectMessage.extraParams.entrySet().stream()
- .map(e -> String.join("=", e.getKey(),
String.valueOf(e.getValue())))
- .collect(Collectors.joining("&"));
Class.forName(SQL_DRIVER_CLASS.getValue());
String url =
String.format(
SQL_CONNECT_URL.getValue(), connectMessage.host,
connectMessage.port, database);
- if (!connectMessage.extraParams.isEmpty()) {
+ if (MapUtils.isNotEmpty(connectMessage.extraParams)) {
+ String extraParamString =
+ connectMessage.extraParams.entrySet().stream()
+ .map(e -> String.join("=", e.getKey(),
String.valueOf(e.getValue())))
+ .collect(Collectors.joining("&"));
url += "?" + extraParamString;
}
return DriverManager.getConnection(url, connectMessage.username,
connectMessage.password);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
