This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new b1865fffa [KYUUBI #5078] Make `kyuubi_instance` nullable in metadata
table schema
b1865fffa is described below
commit b1865fffa6dadbc9fc509b486a9c70327ae88e21
Author: Cheng Pan <[email protected]>
AuthorDate: Fri Jul 21 16:22:51 2023 +0800
[KYUUBI #5078] Make `kyuubi_instance` nullable in metadata table schema
### _Why are the changes needed?_
This is required by Batch V2, as it allows the batch job queued in
metastore before being picked by Kyuubi Server for scheduling.
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [x] Add screenshots for manual tests if appropriate
```
mysql> CREATE TABLE IF NOT EXISTS metadata(
-> key_id bigint PRIMARY KEY AUTO_INCREMENT COMMENT 'the auto
increment key id',
-> identifier varchar(36) NOT NULL COMMENT 'the identifier id,
which is an UUID',
-> session_type varchar(32) NOT NULL COMMENT 'the session type, SQL
or BATCH',
-> real_user varchar(255) NOT NULL COMMENT 'the real user',
-> user_name varchar(255) NOT NULL COMMENT 'the user name, might be
a proxy user',
-> ip_address varchar(128) COMMENT 'the client ip address',
-> kyuubi_instance varchar(1024) NOT NULL COMMENT 'the kyuubi
instance that creates this',
-> state varchar(128) NOT NULL COMMENT 'the session state',
-> resource varchar(1024) COMMENT 'the main resource',
-> class_name varchar(1024) COMMENT 'the main class name',
-> request_name varchar(1024) COMMENT 'the request name',
-> request_conf mediumtext COMMENT 'the request config map',
-> request_args mediumtext COMMENT 'the request arguments',
-> create_time BIGINT NOT NULL COMMENT 'the metadata create time',
-> engine_type varchar(32) NOT NULL COMMENT 'the engine type',
-> cluster_manager varchar(128) COMMENT 'the engine cluster
manager',
-> engine_open_time bigint COMMENT 'the engine open time',
-> engine_id varchar(128) COMMENT 'the engine application id',
-> engine_name mediumtext COMMENT 'the engine application name',
-> engine_url varchar(1024) COMMENT 'the engine tracking url',
-> engine_state varchar(32) COMMENT 'the engine application state',
-> engine_error mediumtext COMMENT 'the engine application
diagnose',
-> end_time bigint COMMENT 'the metadata end time',
-> peer_instance_closed boolean default '0' COMMENT 'closed by peer
kyuubi instance',
-> UNIQUE INDEX unique_identifier_index(identifier),
-> INDEX user_name_index(user_name),
-> INDEX engine_type_index(engine_type)
-> ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
Query OK, 0 rows affected (0.04 sec)
mysql> ALTER TABLE metadata MODIFY kyuubi_instance varchar(1024) COMMENT
'the kyuubi instance that creates this';
Query OK, 0 rows affected (0.05 sec)
Records: 0 Duplicates: 0 Warnings: 0
mysql> SHOW CREATE TABLE metadata;
mysql> SHOW CREATE TABLE metadata;
+----------+---------------------------------------------------------------------------+
| Table | Create Table
[...]
+----------+---------------------------------------------------------------------------+
| metadata | CREATE TABLE `metadata` (
`key_id` bigint NOT NULL AUTO_INCREMENT COMMENT 'the auto increment key
id',
`identifier` varchar(36) NOT NULL COMMENT 'the identifier id, which is an
UUID',
`session_type` varchar(32) NOT NULL COMMENT 'the session type, SQL or
BATCH',
`real_user` varchar(255) NOT NULL COMMENT 'the real user',
`user_name` varchar(255) NOT NULL COMMENT 'the user name, might be a
proxy user',
`ip_address` varchar(128) DEFAULT NULL COMMENT 'the client ip address',
`kyuubi_instance` varchar(1024) DEFAULT NULL COMMENT 'the kyuubi instance
that creates this',
`state` varchar(128) NOT NULL COMMENT 'the session state',
`resource` varchar(1024) DEFAULT NULL COMMENT 'the main resource',
`class_name` varchar(1024) DEFAULT NULL COMMENT 'the main class name',
`request_name` varchar(1024) DEFAULT NULL COMMENT 'the request name',
`request_conf` mediumtext COMMENT 'the request config map',
`request_args` mediumtext COMMENT 'the request arguments',
`create_time` bigint NOT NULL COMMENT 'the metadata create time',
`engine_type` varchar(32) NOT NULL COMMENT 'the engine type',
`cluster_manager` varchar(128) DEFAULT NULL COMMENT 'the engine cluster
manager',
`engine_open_time` bigint DEFAULT NULL COMMENT 'the engine open time',
`engine_id` varchar(128) DEFAULT NULL COMMENT 'the engine application id',
`engine_name` mediumtext COMMENT 'the engine application name',
`engine_url` varchar(1024) DEFAULT NULL COMMENT 'the engine tracking url',
`engine_state` varchar(32) DEFAULT NULL COMMENT 'the engine application
state',
`engine_error` mediumtext COMMENT 'the engine application diagnose',
`end_time` bigint DEFAULT NULL COMMENT 'the metadata end time',
`peer_instance_closed` tinyint(1) DEFAULT '0' COMMENT 'closed by peer
kyuubi instance',
PRIMARY KEY (`key_id`),
UNIQUE KEY `unique_identifier_index` (`identifier`),
KEY `user_name_index` (`user_name`),
KEY `engine_type_index` (`engine_type`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci |
+----------+---------------------------------------------------------------------------+
1 row in set (0.00 sec)
mysql>
```
The derby SQL also is tested
<img width="1330" alt="image"
src="https://github.com/apache/kyuubi/assets/26535726/4eef0742-05dd-4bd6-a77e-e9de0238375e";>
- [ ] [Run
test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests)
locally before make a pull request
Closes #5078 from pan3793/nullable.
Closes #5078
0c5dec85d [Cheng Pan] Make kyuubi_instance nullable in metadata table schema
Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
.../resources/sql/derby/003-KYUUBI-5078.derby.sql | 1 +
.../metadata-store-schema-1.8.0.derby.sql} | 26 +++++++++---------
.../sql/derby/upgrade-1.7.0-to-1.8.0.derby.sql | 1 +
.../resources/sql/mysql/003-KYUUBI-5078.mysql.sql | 3 +++
.../mysql/metadata-store-schema-1.8.0.mysql.sql | 31 ++++++++++++++++++++++
.../sql/mysql/upgrade-1.7.0-to-1.8.0.mysql.sql | 3 +++
.../sqlite/metadata-store-schema-1.8.0.sqlite.sql | 2 +-
.../server/metadata/MetadataManagerSuite.scala | 2 +-
8 files changed, 55 insertions(+), 14 deletions(-)
diff --git
a/kyuubi-server/src/main/resources/sql/derby/003-KYUUBI-5078.derby.sql
b/kyuubi-server/src/main/resources/sql/derby/003-KYUUBI-5078.derby.sql
new file mode 100644
index 000000000..dfdfe6069
--- /dev/null
+++ b/kyuubi-server/src/main/resources/sql/derby/003-KYUUBI-5078.derby.sql
@@ -0,0 +1 @@
+ALTER TABLE metadata ALTER COLUMN kyuubi_instance DROP NOT NULL;
diff --git
a/kyuubi-server/src/main/resources/sql/sqlite/metadata-store-schema-1.8.0.sqlite.sql
b/kyuubi-server/src/main/resources/sql/derby/metadata-store-schema-1.8.0.derby.sql
similarity index 55%
copy from
kyuubi-server/src/main/resources/sql/sqlite/metadata-store-schema-1.8.0.sqlite.sql
copy to
kyuubi-server/src/main/resources/sql/derby/metadata-store-schema-1.8.0.derby.sql
index 2b05dd5db..b4a66d8d6 100644
---
a/kyuubi-server/src/main/resources/sql/sqlite/metadata-store-schema-1.8.0.sqlite.sql
+++
b/kyuubi-server/src/main/resources/sql/derby/metadata-store-schema-1.8.0.derby.sql
@@ -1,34 +1,36 @@
+-- Derby does not support `CREATE TABLE IF NOT EXISTS`
+
-- the metadata table ddl
-CREATE TABLE IF NOT EXISTS metadata(
- key_id INTEGER PRIMARY KEY AUTOINCREMENT, -- the auto increment key id
+CREATE TABLE metadata(
+ key_id bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, -- the auto
increment key id
identifier varchar(36) NOT NULL, -- the identifier id, which is an UUID
session_type varchar(32) NOT NULL, -- the session type, SQL or BATCH
real_user varchar(255) NOT NULL, -- the real user
user_name varchar(255) NOT NULL, -- the user name, might be a proxy user
ip_address varchar(128), -- the client ip address
- kyuubi_instance varchar(1024) NOT NULL, -- the kyuubi instance that
creates this
+ kyuubi_instance varchar(1024), -- the kyuubi instance that creates this
state varchar(128) NOT NULL, -- the session state
resource varchar(1024), -- the main resource
class_name varchar(1024), -- the main class name
request_name varchar(1024), -- the request name
- request_conf mediumtext, -- the request config map
- request_args mediumtext, -- the request arguments
+ request_conf clob, -- the request config map
+ request_args clob, -- the request arguments
create_time BIGINT NOT NULL, -- the metadata create time
engine_type varchar(32) NOT NULL, -- the engine type
cluster_manager varchar(128), -- the engine cluster manager
engine_open_time bigint, -- the engine open time
engine_id varchar(128), -- the engine application id
- engine_name mediumtext, -- the engine application name
+ engine_name clob, -- the engine application name
engine_url varchar(1024), -- the engine tracking url
engine_state varchar(32), -- the engine application state
- engine_error mediumtext, -- the engine application diagnose
- end_time bigint, -- the metadata end time
- peer_instance_closed boolean default '0' -- closed by peer kyuubi instance
+ engine_error clob, -- the engine application diagnose
+ end_time bigint, -- the metadata end time
+ peer_instance_closed boolean default FALSE -- closed by peer kyuubi
instance
);
-CREATE UNIQUE INDEX IF NOT EXISTS metadata_unique_identifier_index ON
metadata(identifier);
+CREATE UNIQUE INDEX metadata_unique_identifier_index ON metadata(identifier);
-CREATE INDEX IF NOT EXISTS metadata_user_name_index ON metadata(user_name);
+CREATE INDEX metadata_user_name_index ON metadata(user_name);
-CREATE INDEX IF NOT EXISTS metadata_engine_type_index ON metadata(engine_type);
+CREATE INDEX metadata_engine_type_index ON metadata(engine_type);
diff --git
a/kyuubi-server/src/main/resources/sql/derby/upgrade-1.7.0-to-1.8.0.derby.sql
b/kyuubi-server/src/main/resources/sql/derby/upgrade-1.7.0-to-1.8.0.derby.sql
new file mode 100644
index 000000000..b9e6cff91
--- /dev/null
+++
b/kyuubi-server/src/main/resources/sql/derby/upgrade-1.7.0-to-1.8.0.derby.sql
@@ -0,0 +1 @@
+RUN '003-KYUUBI-5078.derby.sql';
diff --git
a/kyuubi-server/src/main/resources/sql/mysql/003-KYUUBI-5078.mysql.sql
b/kyuubi-server/src/main/resources/sql/mysql/003-KYUUBI-5078.mysql.sql
new file mode 100644
index 000000000..1d730cd4c
--- /dev/null
+++ b/kyuubi-server/src/main/resources/sql/mysql/003-KYUUBI-5078.mysql.sql
@@ -0,0 +1,3 @@
+SELECT '< KYUUBI-5078: Make kyuubi_instance nullable in metadata table schema'
AS ' ';
+
+ALTER TABLE metadata MODIFY kyuubi_instance varchar(1024) COMMENT 'the kyuubi
instance that creates this';
diff --git
a/kyuubi-server/src/main/resources/sql/mysql/metadata-store-schema-1.8.0.mysql.sql
b/kyuubi-server/src/main/resources/sql/mysql/metadata-store-schema-1.8.0.mysql.sql
new file mode 100644
index 000000000..4c86fbf76
--- /dev/null
+++
b/kyuubi-server/src/main/resources/sql/mysql/metadata-store-schema-1.8.0.mysql.sql
@@ -0,0 +1,31 @@
+-- the metadata table ddl
+
+CREATE TABLE IF NOT EXISTS metadata(
+ key_id bigint PRIMARY KEY AUTO_INCREMENT COMMENT 'the auto increment key
id',
+ identifier varchar(36) NOT NULL COMMENT 'the identifier id, which is an
UUID',
+ session_type varchar(32) NOT NULL COMMENT 'the session type, SQL or BATCH',
+ real_user varchar(255) NOT NULL COMMENT 'the real user',
+ user_name varchar(255) NOT NULL COMMENT 'the user name, might be a proxy
user',
+ ip_address varchar(128) COMMENT 'the client ip address',
+ kyuubi_instance varchar(1024) COMMENT 'the kyuubi instance that creates
this',
+ state varchar(128) NOT NULL COMMENT 'the session state',
+ resource varchar(1024) COMMENT 'the main resource',
+ class_name varchar(1024) COMMENT 'the main class name',
+ request_name varchar(1024) COMMENT 'the request name',
+ request_conf mediumtext COMMENT 'the request config map',
+ request_args mediumtext COMMENT 'the request arguments',
+ create_time BIGINT NOT NULL COMMENT 'the metadata create time',
+ engine_type varchar(32) NOT NULL COMMENT 'the engine type',
+ cluster_manager varchar(128) COMMENT 'the engine cluster manager',
+ engine_open_time bigint COMMENT 'the engine open time',
+ engine_id varchar(128) COMMENT 'the engine application id',
+ engine_name mediumtext COMMENT 'the engine application name',
+ engine_url varchar(1024) COMMENT 'the engine tracking url',
+ engine_state varchar(32) COMMENT 'the engine application state',
+ engine_error mediumtext COMMENT 'the engine application diagnose',
+ end_time bigint COMMENT 'the metadata end time',
+ peer_instance_closed boolean default '0' COMMENT 'closed by peer kyuubi
instance',
+ UNIQUE INDEX unique_identifier_index(identifier),
+ INDEX user_name_index(user_name),
+ INDEX engine_type_index(engine_type)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
diff --git
a/kyuubi-server/src/main/resources/sql/mysql/upgrade-1.7.0-to-1.8.0.mysql.sql
b/kyuubi-server/src/main/resources/sql/mysql/upgrade-1.7.0-to-1.8.0.mysql.sql
new file mode 100644
index 000000000..a3a4bcdf1
--- /dev/null
+++
b/kyuubi-server/src/main/resources/sql/mysql/upgrade-1.7.0-to-1.8.0.mysql.sql
@@ -0,0 +1,3 @@
+SELECT '< Upgrading MetaStore schema from 1.7.0 to 1.8.0 >' AS ' ';
+SOURCE 003-KYUUBI-5078.mysql.sql;
+SELECT '< Finished upgrading MetaStore schema from 1.7.0 to 1.8.0 >' AS ' ';
diff --git
a/kyuubi-server/src/main/resources/sql/sqlite/metadata-store-schema-1.8.0.sqlite.sql
b/kyuubi-server/src/main/resources/sql/sqlite/metadata-store-schema-1.8.0.sqlite.sql
index 2b05dd5db..6df7405be 100644
---
a/kyuubi-server/src/main/resources/sql/sqlite/metadata-store-schema-1.8.0.sqlite.sql
+++
b/kyuubi-server/src/main/resources/sql/sqlite/metadata-store-schema-1.8.0.sqlite.sql
@@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS metadata(
real_user varchar(255) NOT NULL, -- the real user
user_name varchar(255) NOT NULL, -- the user name, might be a proxy user
ip_address varchar(128), -- the client ip address
- kyuubi_instance varchar(1024) NOT NULL, -- the kyuubi instance that
creates this
+ kyuubi_instance varchar(1024), -- the kyuubi instance that creates this
state varchar(128) NOT NULL, -- the session state
resource varchar(1024), -- the main resource
class_name varchar(1024), -- the main class name
diff --git
a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/metadata/MetadataManagerSuite.scala
b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/metadata/MetadataManagerSuite.scala
index aca416dac..564b5ebe9 100644
---
a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/metadata/MetadataManagerSuite.scala
+++
b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/metadata/MetadataManagerSuite.scala
@@ -116,7 +116,7 @@ class MetadataManagerSuite extends KyuubiFunSuite {
MetricsSystem.meterValue(MetricsConstants.METADATA_REQUEST_RETRYING)
.getOrElse(0L) - retryingRequests === 0)
- val invalidMetadata = metadata.copy(kyuubiInstance = null)
+ val invalidMetadata = metadata.copy(state = null)
intercept[Exception](metadataManager.insertMetadata(invalidMetadata,
false))
assert(
MetricsSystem.meterValue(MetricsConstants.METADATA_REQUEST_TOTAL)
