This is an automated email from the ASF dual-hosted git repository. chengpan pushed a commit to branch release-180 in repository https://gitbox.apache.org/repos/asf/kyuubi-website.git
commit 2982476064c15f51c51278379d00986794e442e5 Author: Cheng Pan <[email protected]> AuthorDate: Tue Nov 7 13:10:17 2023 +0800 Release 1.8.0 --- hugo.toml | 2 +- src/en/news/2023-11-07-release-1.8.0.md | 20 + src/en/release/1.8.0.md | 713 ++++++++++++++++++++++++++++++++ src/zh/news/2023-11-07-release-1.8.0.md | 20 + src/zh/release/1.8.0.md | 713 ++++++++++++++++++++++++++++++++ 5 files changed, 1467 insertions(+), 1 deletion(-) diff --git a/hugo.toml b/hugo.toml index 2c46ceb..10ad595 100644 --- a/hugo.toml +++ b/hugo.toml @@ -337,6 +337,6 @@ defaultContentLanguage = 'en' keepQuotes = true [params] - releases = [ 'v1.7.3', 'v1.6.1-incubating', 'v1.5.2-incubating', 'v1.4.1-incubating', 'v1.3.1-incubating' ] + releases = [ 'v1.8.0', 'v1.7.3', 'v1.6.1-incubating', 'v1.5.2-incubating', 'v1.4.1-incubating', 'v1.3.1-incubating' ] downloadLink = 'https://www.apache.org/dyn/closer.lua/kyuubi/' downloadLinkIncubator = 'https://www.apache.org/dyn/closer.lua/incubator/kyuubi/' diff --git a/src/en/news/2023-11-07-release-1.8.0.md b/src/en/news/2023-11-07-release-1.8.0.md new file mode 100644 index 0000000..af294ec --- /dev/null +++ b/src/en/news/2023-11-07-release-1.8.0.md @@ -0,0 +1,20 @@ +--- +title: Apache Kyuubi 1.8.0 released +date: 2023-11-07 +linked: true +--- +<!--- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +We are happy to announce the availability of [Apache Kyuubi 1.8.0](/release/1.8.0.html) ! Visit the [release notes](/release/1.8.0.html) to read about the new features, or [download](/releases.html) the released today. diff --git a/src/en/release/1.8.0.md b/src/en/release/1.8.0.md new file mode 100644 index 0000000..68eb831 --- /dev/null +++ b/src/en/release/1.8.0.md @@ -0,0 +1,713 @@ +--- +title: Release Notes - 1.8.0 +date: 2023-11-07 +linked: true +meta: +_edit_last: '4' +_wpas_done_all: '1' +--- +<!--- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +### Highlights + +- Fully verified with Java 8, 11, and 17, Scala 2.12, initial support Scala 2.13 +- Fully verified with Spark 3.1, 3.2, 3.3, and 3.4, initial support Spark 3.5 +- Fully verified with Flink 1.16, 1.17, and 1.18 +- New Web UI (experimental) +- New Chat engine with OpenAI ChatGPT backend +- Major improvements of Spark Batch API, optimized for large-scale job submission (experimental) +- Major improvements of Spark AuthZ plugin, including enhanced Iceberg and DSv2 operators support +- Major improvements of Flink engine, including YARN Application Mode support +- Introducing Atlas hook support in Spark Lineage plugin +- Continuously Kyuubi Helm Chart & Spark on K8s improvements + +### Spark Engine, Connectors and Plugins + +[[KYUUBI #1265](https://github.com/apache/kyuubi/pull/1265)] `OPTIMIZE` where clause expression support +[[KYUUBI #3420](https://github.com/apache/kyuubi/pull/3420)] Expose UI url on registering engine service +[[KYUUBI #3444](https://github.com/apache/kyuubi/pull/3444)] Support the planOnly mode of kyuubi spark engine support SQL lineage +[[KYUUBI #3929](https://github.com/apache/kyuubi/pull/3929)] Refactor lineage plugin to add LineageDispatcher +[[KYUUBI #4167](https://github.com/apache/kyuubi/pull/4167)] [AUTHZ] Introduce function support in PrivilegeBuilder with Serde layers +[[KYUUBI #4171](https://github.com/apache/kyuubi/pull/4171)] Support skip retrieving table's properties to speed up GetTables operation +[[KYUUBI #4186](https://github.com/apache/kyuubi/pull/4186)] Spark showProgress with JobInfo +[[KYUUBI #4330](https://github.com/apache/kyuubi/pull/4330)] Non-temporary views do not resolve to a specific real table +[[KYUUBI #4331](https://github.com/apache/kyuubi/pull/4331)] [[KYUUBI #4431](https://github.com/apache/kyuubi/pull/4431)] Lineage supports `Union` and `MergeInto` +[[KYUUBI #4415](https://github.com/apache/kyuubi/pull/4415)] Align the operation handle in server/engine for ExecuteScala, ExecutePython and PlanOnlyStatement +[[KYUUBI #4430](https://github.com/apache/kyuubi/pull/4430)] Lineage supports the `lateral view explode` statement +[[KYUUBI #4433](https://github.com/apache/kyuubi/pull/4433)] Fix the case with `create a view from view` +[[KYUUBI #4443](https://github.com/apache/kyuubi/pull/4443)] Do not set engine session init sql for alive probe session +[[KYUUBI #4450](https://github.com/apache/kyuubi/pull/4450)] Ignore unknown fields `policyPriority` when reading policy json file +[[KYUUBI #4488](https://github.com/apache/kyuubi/pull/4488)] [KSHC] Keep object original name defined in HiveBridgeHelper +[[KYUUBI #4493](https://github.com/apache/kyuubi/pull/4493)] [AUTHZ] Enable Tests for Spark 3.1 with iceberg tables +[[KYUUBI #4497](https://github.com/apache/kyuubi/pull/4497)] Add RowFilteringTestBase to improve the test coverage +[[KYUUBI #4504](https://github.com/apache/kyuubi/pull/4504)] [AUTHZ] Fix source table privilege requirement when querying permanent view in Spark 3.1 and below +[[KYUUBI #4525](https://github.com/apache/kyuubi/pull/4525)] [KSHC] Partitioning predicates should take effect to filter data +[[KYUUBI #4530](https://github.com/apache/kyuubi/pull/4530)] [AUTHZ] Support non-English chars for MASK, MASK_SHOW_FIRST_4, and MASK_SHOW_FIRST_4 +[[KYUUBI #4532](https://github.com/apache/kyuubi/pull/4532)] [AUTHZ] Displays the columns involved in extracting the aggregation operator +[[KYUUBI #4541](https://github.com/apache/kyuubi/pull/4541)] Support to customize the attributes to expose for Spark engine +[[KYUUBI #4560](https://github.com/apache/kyuubi/pull/4560)] [KSHC] Support Kerberized HMS in cluster mode w/o keytab +[[KYUUBI #4585](https://github.com/apache/kyuubi/pull/4585)] [AUTHZ] Authz policy file generation +[[KYUUBI #4587](https://github.com/apache/kyuubi/pull/4587)] Take care of table cache query stage +[[KYUUBI #4592](https://github.com/apache/kyuubi/pull/4592)] Support eagerly kill redundant executors +[[KYUUBI #4615](https://github.com/apache/kyuubi/pull/4615)] Support stage level schedule for final write stage +[[KYUUBI #4617](https://github.com/apache/kyuubi/pull/4617)] [AUTHZ] Collect results for filtered show objects ahead to prevent holding unserializable spark plan +[[KYUUBI #4620](https://github.com/apache/kyuubi/pull/4620)] [KSHC] Cut off transitive dependencies +[[KYUUBI #4636](https://github.com/apache/kyuubi/pull/4636)] Improve eagerly kill redundant executors +[[KYUUBI #4641](https://github.com/apache/kyuubi/pull/4641)] Add MaxFileSizeStrategy to limit max scan file size +[[KYUUBI #4644](https://github.com/apache/kyuubi/pull/4644)] Manually terminate the Py4JServer during engine shutdown +[[KYUUBI #4658](https://github.com/apache/kyuubi/pull/4658)] [AUTHZ] Fix InsertIntoHiveDirCommand classname so that we can extract the query in it when authorization. +[[KYUUBI #4662](https://github.com/apache/kyuubi/pull/4662)] [ARROW] Arrow serialization should not introduce extra shuffle for outermost limit +[[KYUUBI #4664](https://github.com/apache/kyuubi/pull/4664)] Fix empty relation when kill executors +[[KYUUBI #4672](https://github.com/apache/kyuubi/pull/4672)] [AUTHZ] Remove Support for Apache Ranger 0.6.0 +[[KYUUBI #4676](https://github.com/apache/kyuubi/pull/4676)] [AUTHZ] Reuse users and namespaces in both tests and policy file generation +[[KYUUBI #4677](https://github.com/apache/kyuubi/pull/4677)] [AUTHZ] Check generated policy file in test suite +[[KYUUBI #4678](https://github.com/apache/kyuubi/pull/4678)] Improve FinalStageResourceManager kill executors +[[KYUUBI #4681](https://github.com/apache/kyuubi/pull/4681)] Set thread `CreateSparkTimeoutChecker` daemon +[[KYUUBI #4683](https://github.com/apache/kyuubi/pull/4683)] Update `spark.sql.finalWriteStage.resourceIsolation.enabled` version +[[KYUUBI #4702](https://github.com/apache/kyuubi/pull/4702)] [ARROW] CommandResultExec should not trigger job +[[KYUUBI #4710](https://github.com/apache/kyuubi/pull/4710)] [ARROW] LocalTableScanExec should not trigger job +[[KYUUBI #4710](https://github.com/apache/kyuubi/pull/4710)] [ARROW] Post driver-side metrics for LocalTableScanExec/CommandResultExec +[[KYUUBI #4716](https://github.com/apache/kyuubi/pull/4716)] [[KYUUBI #4715](https://github.com/apache/kyuubi/pull/4715)][AUTHZ] Fix the incorrect class name of InsertIntoHiveDirCommand in table spec generator +[[KYUUBI #4717](https://github.com/apache/kyuubi/pull/4717)] [AUTHZ] Check Authz plugin's spec json files in UT +[[KYUUBI #4720](https://github.com/apache/kyuubi/pull/4720)] [ARROW] Fix java.lang.NoSuchFieldError: IpcOption.DEFAULT for Spark-3.1/3.2 +[[KYUUBI #4741](https://github.com/apache/kyuubi/pull/4741)] Kyuubi Spark Engine/TPC connectors support Spark 3.4 +[[KYUUBI #4754](https://github.com/apache/kyuubi/pull/4754)] [ARROW] Use `KyuubiArrowConverters#toBatchIterator` instead of `ArrowConverters#toBatchIterator` +[[KYUUBI #4797](https://github.com/apache/kyuubi/pull/4797)] [ARROW] Reflective calls to the function `ArrowUtils#toArrowSchema` +[[KYUUBI #4812](https://github.com/apache/kyuubi/pull/4812)] Generalize case transformation method for string type config entry +[[KYUUBI #4834](https://github.com/apache/kyuubi/pull/4834)] Reduce the scope of method references in Authz plugin cleanup shutdown hook +[[KYUUBI #4838](https://github.com/apache/kyuubi/pull/4838)] Fix spark operation exception leak in `withLocalProperties` method +[[KYUUBI #4866](https://github.com/apache/kyuubi/pull/4866)] Add annotation for Iceberg tests in Authz plugin +[[KYUUBI #4869](https://github.com/apache/kyuubi/pull/4869)] [AUTHZ] Introduce table extractor for ResolvedIdentifier in Spark 3.4 +[[KYUUBI #4871](https://github.com/apache/kyuubi/pull/4871)] [AUTHZ] Adapt plan changes for CreateNamespace and SetCatalogAndNamespace in Spark 3.4 +[[KYUUBI #4873](https://github.com/apache/kyuubi/pull/4873)] [AUTHZ] Refactor Authz reflection with kyuubi-util's DynMethods +[[KYUUBI #4874](https://github.com/apache/kyuubi/pull/4874)] [AUTHZ] Improve methods in AuthzUtils +[[KYUUBI #4875](https://github.com/apache/kyuubi/pull/4875)] [AUTHZ] Remove checking Spark v2 in tests since Spark v2 not supported +[[KYUUBI #4878](https://github.com/apache/kyuubi/pull/4878)] Delta support for Spark-3.4 +[[KYUUBI #4880](https://github.com/apache/kyuubi/pull/4880)] [BUILD] Skip compiling useless files when making distribution +[[KYUUBI #4888](https://github.com/apache/kyuubi/pull/4888)] [AUTHZ] Remove filtering results for ShowDatabasesCommand in Spark 2.x +[[KYUUBI #4892](https://github.com/apache/kyuubi/pull/4892)] [AUTHZ] Make identifier part name comparison case insensitive in tests of PrivilegeBuilder +[[KYUUBI #4895](https://github.com/apache/kyuubi/pull/4895)] [ARROW] Reflective calls to the function `ArrowConverters#fromBatchIterator` +[[KYUUBI #4899](https://github.com/apache/kyuubi/pull/4899)] [AUTHZ] Extract function from FunctionIdentifier for CreateFunction and DropFunction in Spark 3.4 +[[KYUUBI #4903](https://github.com/apache/kyuubi/pull/4903)] [AUTHZ] Fix NoSuchElementException when listing database in CatalogImpl in Spark 3.4 +[[KYUUBI #4909](https://github.com/apache/kyuubi/pull/4909)] [AUTHZ] Enable authz plugin tests for Spark 3.4 +[[KYUUBI #4910](https://github.com/apache/kyuubi/pull/4910)] Extract table from ResolvedIdentifier for DropTable in Spark 3.4 +[[KYUUBI #4913](https://github.com/apache/kyuubi/pull/4913)] Eliminate unnecessary output in ut "union an unmasked table" +[[KYUUBI #4914](https://github.com/apache/kyuubi/pull/4914)] [AUTHZ] Reuse extractor singleton instance with generalized getter for supported extractor types +[[KYUUBI #4915](https://github.com/apache/kyuubi/pull/4915)] Adapt database not exist error message change for Spark 3.4 +[[KYUUBI #4916](https://github.com/apache/kyuubi/pull/4916)] [AUTHZ] Support ReplaceData and compatible Spark 3.4 and 3.5 +[[KYUUBI #4916](https://github.com/apache/kyuubi/pull/4916)] Fix Iceberg merge into UT +[[KYUUBI #4917](https://github.com/apache/kyuubi/pull/4917)] [AUTHZ] Table owner undefined in Iceberg 1.3.0 on Spark 3.4 +[[KYUUBI #4923](https://github.com/apache/kyuubi/pull/4923)] [ARROW] Update arguments of `ArrowUtils#toArrowSchema` function +[[KYUUBI #4925](https://github.com/apache/kyuubi/pull/4925)] Add default catalog using `spark_catalog` with the lineage result +[[KYUUBI #4935](https://github.com/apache/kyuubi/pull/4935)] More than target num of executors may survive after FinalStageResourceManager did kill +[[KYUUBI #4937](https://github.com/apache/kyuubi/pull/4937)] Cleanup spark catalog shim and renamed to catalog utils +[[KYUUBI #4952](https://github.com/apache/kyuubi/pull/4952)] Enhance AWS Glue default database does not exist error message on opening session +[[KYUUBI #4995](https://github.com/apache/kyuubi/pull/4995)] Use hadoop conf and hive conf from catalog options +[[KYUUBI #4999](https://github.com/apache/kyuubi/pull/4999)] [KSHC] Kyuubi-Spark-Hive-Connector support Apache Spark 3.4 +[[KYUUBI #5002](https://github.com/apache/kyuubi/pull/5002)] Fail the engine fast when no incoming connection in CONNECTION mode +[[KYUUBI #5005](https://github.com/apache/kyuubi/pull/5005)] Remove default settings `spark.sql.execution.topKSortFallbackThreshold` +[[KYUUBI #5007](https://github.com/apache/kyuubi/pull/5007)] Remove blank SparkSimpleStatsReportListener +[[KYUUBI #5009](https://github.com/apache/kyuubi/pull/5009)] Pass Spark Engine Log Path to Spark Conf +[[KYUUBI #5017](https://github.com/apache/kyuubi/pull/5017)] [KSHC] Support Parquet/Orc provider is splittable +[[KYUUBI #5018](https://github.com/apache/kyuubi/pull/5018)] Make kyuubi spark extension compatible with Spark 3.4 +[[KYUUBI #5022](https://github.com/apache/kyuubi/pull/5022)] [KSHC] CreateTable should use the correct provider +[[KYUUBI #5023](https://github.com/apache/kyuubi/pull/5023)] [KSHC] TableIdentify don't attach catalog +[[KYUUBI #5025](https://github.com/apache/kyuubi/pull/5025)] Use formatDuration instead of formatDurationVerbose in Spark UI's engine tab +[[KYUUBI #5028](https://github.com/apache/kyuubi/pull/5028)] Update session hadoop conf to catalog hadoop conf +[[KYUUBI #5030](https://github.com/apache/kyuubi/pull/5030)] Support get query id in Spark engine +[[KYUUBI #5035](https://github.com/apache/kyuubi/pull/5035)] Spark engine session page display session end time and duration +[[KYUUBI #5045](https://github.com/apache/kyuubi/pull/5045)] Initialize EXECUTOR_POD_NAME_PREFIX_MAX_LENGTH before using +[[KYUUBI #5055](https://github.com/apache/kyuubi/pull/5055)] [AUTHZ] Support building function privileges in Spark 3.4 +[[KYUUBI #5073](https://github.com/apache/kyuubi/pull/5073)] Correct the method name in SparkSQLLineageParserHelperSuite +[[KYUUBI #5075](https://github.com/apache/kyuubi/pull/5075)] Refine tests to adapt Spark 4.0 +[[KYUUBI #5076](https://github.com/apache/kyuubi/pull/5076)] Add KDF engine_url +[[KYUUBI #5105](https://github.com/apache/kyuubi/pull/5105)] GetColumns gets duplicate column names when Spark integrates with Iceberg +[[KYUUBI #5136](https://github.com/apache/kyuubi/pull/5136)] Spark App may hang forever if FinalStageResourceManager killed all executors +[[KYUUBI #5148](https://github.com/apache/kyuubi/pull/5148)] Improve spark.driver.host assignment in Spark on K8s client mode +[[KYUUBI #5170](https://github.com/apache/kyuubi/pull/5170)] Identifier compatible with Spark 3.4 +[[KYUUBI #5172](https://github.com/apache/kyuubi/pull/5172)] [AUTHZ] Check USE permissions for DESCRIBE FUNCTION +[[KYUUBI #5180](https://github.com/apache/kyuubi/pull/5180)] Replace deprecated classes by equivalent in scala.tools.nsc.interpreter package +[[KYUUBI #5184](https://github.com/apache/kyuubi/pull/5184)] Rename Kyuubi's StageInfo to SparkStageInfo to fix class mismatch +[[KYUUBI #5188](https://github.com/apache/kyuubi/pull/5188)] Make server module and Spark sql engine module compilable on Scala 2.13 +[[KYUUBI #5189](https://github.com/apache/kyuubi/pull/5189)] [AUTHZ] Make spark authz module compilable on Scala 2.13 +[[KYUUBI #5191](https://github.com/apache/kyuubi/pull/5191)] Make Spark extension plugin compilable on Scala 2.13 +[[KYUUBI #5192](https://github.com/apache/kyuubi/pull/5192)] Make Spark sql lineage plugin compilable on Scala 2.13 +[[KYUUBI #5193](https://github.com/apache/kyuubi/pull/5193)] Make Spark hive connector plugin compilable on Scala 2.13 +[[KYUUBI #5195](https://github.com/apache/kyuubi/pull/5195)] Make Spark TPC-H connector plugin compilable on Scala 2.13 +[[KYUUBI #5196](https://github.com/apache/kyuubi/pull/5196)] Enable CI Test on Scala 2.13 and support custom or spark-core extracted Scala version for Spark's engine +[[KYUUBI #5196](https://github.com/apache/kyuubi/pull/5196)] Extract spark core scala version lazily and respect engine env +[[KYUUBI #5199](https://github.com/apache/kyuubi/pull/5199)] Read all columns of metadata to prevent column missing +[[KYUUBI #5202](https://github.com/apache/kyuubi/pull/5202)] Switch to Spark 3.4 by default +[[KYUUBI #5224](https://github.com/apache/kyuubi/pull/5224)] Modify the README file of authz to make spark 3.4 the default version +[[KYUUBI #5225](https://github.com/apache/kyuubi/pull/5225)] [KSHC] Unify the exception handling of v1 and v2 during dropDatabase +[[KYUUBI #5232](https://github.com/apache/kyuubi/pull/5232)] In SparkOperation#cleanup always calls cancelJobGroup even though it's in the completed state +[[KYUUBI #5257](https://github.com/apache/kyuubi/pull/5257)] [LINEAGE] Adapt `CreateTableAsSelect` plan to Spark 3.5 changes +[[KYUUBI #5268](https://github.com/apache/kyuubi/pull/5268)] [AUTHZ] Extract method for enabling authorization in single call mode +[[KYUUBI #5269](https://github.com/apache/kyuubi/pull/5269)] Duplicated AnalyzesTablesCommand in kyuubi authz +[[KYUUBI #5271](https://github.com/apache/kyuubi/pull/5271)] AnalyzeTableCommand should also add table write privilege +[[KYUUBI #5291](https://github.com/apache/kyuubi/pull/5291)] Verify Spark engine is compatible with Spark 3.5.0 +[[KYUUBI #5317](https://github.com/apache/kyuubi/pull/5317)] [KSHC] Hive Connector throws NotSerializableException on reading Hive Avro partitioned table +[[KYUUBI #5336](https://github.com/apache/kyuubi/pull/5336)] Spark extension supports Spark 3.5 +[[KYUUBI #5339](https://github.com/apache/kyuubi/pull/5339)] Avoid using `kyuubi.session.engine.initialize.timeout` string +[[KYUUBI #5414](https://github.com/apache/kyuubi/pull/5414)] [KSHC] Reader should not pollute the global hiveConf instance +[[KYUUBI #5426](https://github.com/apache/kyuubi/pull/5426)] [KSHC] Avoid use class.newInstance directly +[[KYUUBI #5550](https://github.com/apache/kyuubi/pull/5550)] Optimizing TPC-DS dataset generation for 10x speedup + +### Flink Engine + +[[KYUUBI #1652](https://github.com/apache/kyuubi/pull/1652)] Support Flink YARN application mode +[[KYUUBI #4367](https://github.com/apache/kyuubi/pull/4367)] Support Flink 1.17 +[[KYUUBI #4387](https://github.com/apache/kyuubi/pull/4387)] Remove support for Flink 1.14 +[[KYUUBI #4446](https://github.com/apache/kyuubi/pull/4446)] Fix connections blocked by Flink insert statements +[[KYUUBI #4495](https://github.com/apache/kyuubi/pull/4495)] Support Flink job management statements +[[KYUUBI #4517](https://github.com/apache/kyuubi/pull/4517)] Fix multiple executions lead to abnormal results on Flink 1.14 +[[KYUUBI #4635](https://github.com/apache/kyuubi/pull/4635)] Support flink time type in query operation +[[KYUUBI #4666](https://github.com/apache/kyuubi/pull/4666)] Support flink varbinary type in query operation +[[KYUUBI #4745](https://github.com/apache/kyuubi/pull/4745)] Support Flink's LocalZonedTimestamp DataType +[[KYUUBI #4806](https://github.com/apache/kyuubi/pull/4806)] Support time-out incremental result fetch for Flink engine +[[KYUUBI #4861](https://github.com/apache/kyuubi/pull/4861)] Support Flink session idleness +[[KYUUBI #4938](https://github.com/apache/kyuubi/pull/4938)] Implement Kyuubi UDF in Flink engine +[[KYUUBI #5080](https://github.com/apache/kyuubi/pull/5080)] Fix EmbeddedExecutorFactory not thread-safe during bootstrap +[[KYUUBI #5106](https://github.com/apache/kyuubi/pull/5106)] Improve logs for fatal errors +[[KYUUBI #5108](https://github.com/apache/kyuubi/pull/5108)] Fix FileNotFoundException during Flink engine bootstrap +[[KYUUBI #5160](https://github.com/apache/kyuubi/pull/5160)] Refactor getNextRowSetInternal to support fetch streaming data +[[KYUUBI #5190](https://github.com/apache/kyuubi/pull/5190)] Explicitly name Flink bootstrap SQL in application mode +[[KYUUBI #5200](https://github.com/apache/kyuubi/pull/5200)] Optimize Flink application name generating +[[KYUUBI #5222](https://github.com/apache/kyuubi/pull/5222)] Remove unused provided dependencies in Flink SQL engine +[[KYUUBI #5238](https://github.com/apache/kyuubi/pull/5238)] Fix credentials may break Flink engine launch command +[[KYUUBI #5315](https://github.com/apache/kyuubi/pull/5315)] Propagate HIVE_CONF_DIR on launching Flink engine +[[KYUUBI #5405](https://github.com/apache/kyuubi/pull/5405)] Support Flink 1.18 +[[KYUUBI #5531](https://github.com/apache/kyuubi/pull/5531)] Fix flaky FlinkOperationOnYarnSuite by enlarging the max rows setting + +### Other Engines + +[[KYUUBI #3374](https://github.com/apache/kyuubi/pull/3374)] Support password authentication for Trino engine +[[KYUUBI #4412](https://github.com/apache/kyuubi/pull/4412)] Align the server/engine session handle for flink/hive/trino/jdbc engines +[[KYUUBI #4491](https://github.com/apache/kyuubi/pull/4491)] Fix Trino typo +[[KYUUBI #4522](https://github.com/apache/kyuubi/pull/4522)] `use:catalog` should execute before than `use:database` +[[KYUUBI #4544](https://github.com/apache/kyuubi/pull/4544)] Initial implement Kyuubi Chat Engine +[[KYUUBI #4548](https://github.com/apache/kyuubi/pull/4548)] Kyuubi Chat Engine supports Chinese questions and HTTP proxy +[[KYUUBI #4554](https://github.com/apache/kyuubi/pull/4554)] Code improvement in ChatGPTProvider +[[KYUUBI #4556](https://github.com/apache/kyuubi/pull/4556)] Refactor ChatGPTProvider to use `openai-java` client +[[KYUUBI #4558](https://github.com/apache/kyuubi/pull/4558)] Make ChatGPT model ID configurable +[[KYUUBI #4559](https://github.com/apache/kyuubi/pull/4559)] Pretty chat engine launch command +[[KYUUBI #4711](https://github.com/apache/kyuubi/pull/4711)] JDBC client should catch task failed exception instead of NPE in the incremental mode +[[KYUUBI #4809](https://github.com/apache/kyuubi/pull/4809)] Set session user in ChatGPT request +[[KYUUBI #4810](https://github.com/apache/kyuubi/pull/4810)] Request and use a single choice for chat completion +[[KYUUBI #4881](https://github.com/apache/kyuubi/pull/4881)] JDBCEngine performs initialization sql +[[KYUUBI #4940](https://github.com/apache/kyuubi/pull/4940)] Implement Kyuubi UDF in Hive engine +[[KYUUBI #5120](https://github.com/apache/kyuubi/pull/5120)] Allow using session's user and password for connecting database in JDBC engine +[[KYUUBI #5282](https://github.com/apache/kyuubi/pull/5282)] Support configure Trino session conf in `kyuubi-default.conf` +[[KYUUBI #5333](https://github.com/apache/kyuubi/pull/5333)] Adaptive call fetchLaunchEngineResult +[[KYUUBI #5570](https://github.com/apache/kyuubi/pull/5570)] Fix memory leak when using incremental collect mode in JDBC engine + +### Server + +[[KYUUBI #3653](https://github.com/apache/kyuubi/pull/3653)] AdminResource add list Kyuubi server api +[[KYUUBI #3739](https://github.com/apache/kyuubi/pull/3739)] Remove unused parameters in SessionsResource rest docs +[[KYUUBI #3872](https://github.com/apache/kyuubi/pull/3872)] Skip setting proxy user when keytab is provided on batch session +[[KYUUBI #3876](https://github.com/apache/kyuubi/pull/3876)] Update the rest api docs for open session response +[[KYUUBI #3920](https://github.com/apache/kyuubi/pull/3920)] Fix flaky test - distribute lock +[[KYUUBI #3957](https://github.com/apache/kyuubi/pull/3957)] Rename the config prefix from kyuubi.server.batch.limit to kyuubi.server.limit.batch +[[KYUUBI #4325](https://github.com/apache/kyuubi/pull/4325)] Support replace preparedStatement for Trino-jdbc +[[KYUUBI #4344](https://github.com/apache/kyuubi/pull/4344)] Expose exec pool work queue size metrics +[[KYUUBI #4364](https://github.com/apache/kyuubi/pull/4364)] Add metrics for user opened connections with session type +[[KYUUBI #4376](https://github.com/apache/kyuubi/pull/4376)] Support to config the Kyuubi service administrator with Kyuubi conf +[[KYUUBI #4388](https://github.com/apache/kyuubi/pull/4388)] Limit the max rows for get nextRowSet api +[[KYUUBI #4404](https://github.com/apache/kyuubi/pull/4404)] Support to list/close sessions in AdminResource +[[KYUUBI #4407](https://github.com/apache/kyuubi/pull/4407)] Adapt SLF4J2 +[[KYUUBI #4424](https://github.com/apache/kyuubi/pull/4424)] Catch No Node Exception, when list kyuubi engines +[[KYUUBI #4439](https://github.com/apache/kyuubi/pull/4439)] Add list/close operation method for AdminResource +[[KYUUBI #4462](https://github.com/apache/kyuubi/pull/4462)] Fix variable usage issue in `SessionManager#stop` +[[KYUUBI #4464](https://github.com/apache/kyuubi/pull/4464)] Simplify and improve log for JDBCMetadataStore +[[KYUUBI #4467](https://github.com/apache/kyuubi/pull/4467)] Tolerate Driver Pod ephemerally invisible after submitting +[[KYUUBI #4472](https://github.com/apache/kyuubi/pull/4472)] add session/operation methods in AdminRestApi +[[KYUUBI #4480](https://github.com/apache/kyuubi/pull/4480)] Engine alive probe should close thrift connection on engine lost +[[KYUUBI #4482](https://github.com/apache/kyuubi/pull/4482)] Support to disable batch resource upload function +[[KYUUBI #4483](https://github.com/apache/kyuubi/pull/4483)] Enable REST frontend protocol by default +[[KYUUBI #4496](https://github.com/apache/kyuubi/pull/4496)] Call `super.clusterManager()` when `batchConf` does not have `spark.master` +[[KYUUBI #4498](https://github.com/apache/kyuubi/pull/4498)] Support to list sessions/operations with conditions in AdminRestApi +[[KYUUBI #4501](https://github.com/apache/kyuubi/pull/4501)] Allow administrator to specify the user for engine admin +[[KYUUBI #4506](https://github.com/apache/kyuubi/pull/4506)] Return sessionType in session data +[[KYUUBI #4516](https://github.com/apache/kyuubi/pull/4516)] Return kyuubi instance in SessionData and OperationData +[[KYUUBI #4518](https://github.com/apache/kyuubi/pull/4518)] Return engine id in SessionData and post kyuubi instance in KyuubiOperationEvent +[[KYUUBI #4519](https://github.com/apache/kyuubi/pull/4519)] Update metadata when batch application state changed +[[KYUUBI #4523](https://github.com/apache/kyuubi/pull/4523)] support close engine session gracefully +[[KYUUBI #4526](https://github.com/apache/kyuubi/pull/4526)] Defer evaluation for GroupProvider +[[KYUUBI #4540](https://github.com/apache/kyuubi/pull/4540)] Support to deny some users to make connection +[[KYUUBI #4540](https://github.com/apache/kyuubi/pull/4540)] Create session limiter if user unlimited list or deny list nonempty +[[KYUUBI #4563](https://github.com/apache/kyuubi/pull/4563)] Format error log output in case of metrics json file not found +[[KYUUBI #4575](https://github.com/apache/kyuubi/pull/4575)] Fix the empty last line may causes the session to exit directly +[[KYUUBI #4579](https://github.com/apache/kyuubi/pull/4579)] Fix the unstable situation of test cases for Trino-Fronted-Service +[[KYUUBI #4584](https://github.com/apache/kyuubi/pull/4584)] Post the session exception to EventBus in time +[[KYUUBI #4594](https://github.com/apache/kyuubi/pull/4594)] Support PUT method in REST client +[[KYUUBI #4611](https://github.com/apache/kyuubi/pull/4611)] support all sessions api in SessionRestApi +[[KYUUBI #4618](https://github.com/apache/kyuubi/pull/4618)] Admin Resource list operations with sessionHandle filter +[[KYUUBI #4623](https://github.com/apache/kyuubi/pull/4623)] KubernetesApplicationOperation uses Informer instead of list +[[KYUUBI #4639](https://github.com/apache/kyuubi/pull/4639)] Support to specify confOverlay when executing statement with RESTful API +[[KYUUBI #4652](https://github.com/apache/kyuubi/pull/4652)] Fix JaasConfiguration ClassNotFoundException for Hadoop 3.3.4 and previous +[[KYUUBI #4653](https://github.com/apache/kyuubi/pull/4653)] [KYUUBI 4650] LogDivertAppender supports reading RollingFileAppender pattern +[[KYUUBI #4669](https://github.com/apache/kyuubi/pull/4669)] LDAP authentication allows auth user contains domain when bind.dn/pw enabled +[[KYUUBI #4688](https://github.com/apache/kyuubi/pull/4688)] Fix the failure to read the operation log after executing catalog and database operation +[[KYUUBI #4691](https://github.com/apache/kyuubi/pull/4691)] Configure FAIL_ON_UNKNOWN_PROPERTIES to false for KyuubiScalaObjectMapper +[[KYUUBI #4713](https://github.com/apache/kyuubi/pull/4713)] Fix false positive result in SchedulerPoolSuite +[[KYUUBI #4719](https://github.com/apache/kyuubi/pull/4719)] Support submission timeout for yarn application manager and get the applicationInfo in-memory +[[KYUUBI #4731](https://github.com/apache/kyuubi/pull/4731)] Support batch session conf advisor +[[KYUUBI #4733](https://github.com/apache/kyuubi/pull/4733)] Introduce Kafka event logger for server events +[[KYUUBI #4738](https://github.com/apache/kyuubi/pull/4738)] `AdminResource.getEngineSpace` should use primary group name on GROUP share level +[[KYUUBI #4739](https://github.com/apache/kyuubi/pull/4739)] Add operation lock instead of locking state Enumeration +[[KYUUBI #4746](https://github.com/apache/kyuubi/pull/4746)] Do not recreate async request executor if has been shutdown +[[KYUUBI #4749](https://github.com/apache/kyuubi/pull/4749)] Fix flaky test issues in SchedulerPoolSuite +[[KYUUBI #4753](https://github.com/apache/kyuubi/pull/4753)] KyuubiConf.unset should not log deprecation warning +[[KYUUBI #4767](https://github.com/apache/kyuubi/pull/4767)] Correct the submit time for BatchJobSubmission and check applicationInfo if submitted application +[[KYUUBI #4777](https://github.com/apache/kyuubi/pull/4777)] Deregister event handlers when stopping server with event handler made auto-closeable +[[KYUUBI #4780](https://github.com/apache/kyuubi/pull/4780)] Get engine application info with interval to prevent frequent call to resource manager +[[KYUUBI #4786](https://github.com/apache/kyuubi/pull/4786)] Support yarn-client and yarn-cluster for YarnApplicationOperation +[[KYUUBI #4790](https://github.com/apache/kyuubi/pull/4790)] Initial implement Batch V2 +[[KYUUBI #4791](https://github.com/apache/kyuubi/pull/4791)] Add helper method to simplify REST enabled judgment +[[KYUUBI #4792](https://github.com/apache/kyuubi/pull/4792)] Enhance hardcode session keywords and remove unused code +[[KYUUBI #4796](https://github.com/apache/kyuubi/pull/4796)] Expose JVM attributes to metrics +[[KYUUBI #4798](https://github.com/apache/kyuubi/pull/4798)] Allows BatchJobSubmission to run in sync mode +[[KYUUBI #4801](https://github.com/apache/kyuubi/pull/4801)] Using different engine submit timeout config for kubernetes and yarn +[[KYUUBI #4807](https://github.com/apache/kyuubi/pull/4807)] Fix flaky test "basic batch rest client" +[[KYUUBI #4807](https://github.com/apache/kyuubi/pull/4807)] Retry more times to fix flaky test "spnego batch rest client" +[[KYUUBI #4811](https://github.com/apache/kyuubi/pull/4811)] Do not update app info after batch or application terminated +[[KYUUBI #4813](https://github.com/apache/kyuubi/pull/4813)] Add the fetch orientation parameter to the /v1/operations/:operationId/log interface +[[KYUUBI #4814](https://github.com/apache/kyuubi/pull/4814)] Introduce Apache Atlas hook support in lineage plugin +[[KYUUBI #4816](https://github.com/apache/kyuubi/pull/4816)] Correct the implementation of cleanup K8s terminated appInfo +[[KYUUBI #4829](https://github.com/apache/kyuubi/pull/4829)] Support to expose operation metrics +[[KYUUBI #4830](https://github.com/apache/kyuubi/pull/4830)] Fix flaky test "support to interrupt the thrift request if remote engine is broken" +[[KYUUBI #4835](https://github.com/apache/kyuubi/pull/4835)] Using hive conf to check whether to apply HIVE_DELEGATION_TOKEN +[[KYUUBI #4836](https://github.com/apache/kyuubi/pull/4836)] Set UncaughtExceptionHandler for thread to log exception +[[KYUUBI #4840](https://github.com/apache/kyuubi/pull/4840)] Return cached appInfo iif both op and app are terminated +[[KYUUBI #4841](https://github.com/apache/kyuubi/pull/4841)] Log more info for closing session +[[KYUUBI #4842](https://github.com/apache/kyuubi/pull/4842)] Fix typo in etcd client config +[[KYUUBI #4843](https://github.com/apache/kyuubi/pull/4843)] Support multiple kubernetes contexts and namespaces +[[KYUUBI #4847](https://github.com/apache/kyuubi/pull/4847)] Close the session immediately when engine corrupt +[[KYUUBI #4847](https://github.com/apache/kyuubi/pull/4847)] Close the session immediately when engine connection closed +[[KYUUBI #4850](https://github.com/apache/kyuubi/pull/4850)] Change default EC spec to secp521r1 for internal session variable signature +[[KYUUBI #4855](https://github.com/apache/kyuubi/pull/4855)] Change socket "connection refused" message match pattern +[[KYUUBI #4857](https://github.com/apache/kyuubi/pull/4857)] Fix flaky test TFrontendServiceSuite +[[KYUUBI #4858](https://github.com/apache/kyuubi/pull/4858)] Relax test message assertion in "sync query causes engine crash" +[[KYUUBI #4859](https://github.com/apache/kyuubi/pull/4859)] HttpException message should not be null +[[KYUUBI #4865](https://github.com/apache/kyuubi/pull/4865)] Fix flaky test BackendServiceMetricSuite +[[KYUUBI #4876](https://github.com/apache/kyuubi/pull/4876)] fix Flaky test BackendServiceMetricSuite (#4924) +[[KYUUBI #4889](https://github.com/apache/kyuubi/pull/4889)] Admin command line supports list server command +[[KYUUBI #4890](https://github.com/apache/kyuubi/pull/4890)] Return operation metrics in OperationData and fix typo +[[KYUUBI #4893](https://github.com/apache/kyuubi/pull/4893)] Prevent null collection for rest dto +[[KYUUBI #4898](https://github.com/apache/kyuubi/pull/4898)] Fix logOperation multiple read with missing line +[[KYUUBI #4919](https://github.com/apache/kyuubi/pull/4919)] Fix the flaky tests in the "open batch session" functionality within the "BatchesResourceSuite" +[[KYUUBI #4920](https://github.com/apache/kyuubi/pull/4920)] Add OperationRestApi +[[KYUUBI #4946](https://github.com/apache/kyuubi/pull/4946)] Alter the order of initLoggerEventHandler +[[KYUUBI #4949](https://github.com/apache/kyuubi/pull/4949)] For operation getNextRowSet method, with operation lock required +[[KYUUBI #4950](https://github.com/apache/kyuubi/pull/4950)] Migrate Kyuubi embedded database from Derby to SQLite +[[KYUUBI #4950](https://github.com/apache/kyuubi/pull/4950)] Misc improvement for usage of SQLite +[[KYUUBI #4950](https://github.com/apache/kyuubi/pull/4950)] SQLite schema files should start from 1.8.0 +[[KYUUBI #4951](https://github.com/apache/kyuubi/pull/4951)] Add reload4j support +[[KYUUBI #4977](https://github.com/apache/kyuubi/pull/4977)] Log error message when REST API invocation error occurs +[[KYUUBI #4978](https://github.com/apache/kyuubi/pull/4978)] Fix flaky test: close expired operations +[[KYUUBI #4978](https://github.com/apache/kyuubi/pull/4978)] Fix flaky test: close expired operations +[[KYUUBI #4979](https://github.com/apache/kyuubi/pull/4979)] Fix flaky test: support to interrupt the thrift request if remote engine is broken +[[KYUUBI #4981](https://github.com/apache/kyuubi/pull/4981)] Refactor code of closeBatchSession +[[KYUUBI #4985](https://github.com/apache/kyuubi/pull/4985)] Fix log level restore behavior of `KyuubiFunSuite#withLogAppender` +[[KYUUBI #4986](https://github.com/apache/kyuubi/pull/4986)] Always use Files#deleteIfExists +[[KYUUBI #4994](https://github.com/apache/kyuubi/pull/4994)] Respect the engine type and share level for listing all engines +[[KYUUBI #4994](https://github.com/apache/kyuubi/pull/4994)] Support listing all engines +[[KYUUBI #4996](https://github.com/apache/kyuubi/pull/4996)] Support to refresh kubernetes configs dynamically +[[KYUUBI #5004](https://github.com/apache/kyuubi/pull/5004)] Fix typo for kubernetes allowed context and namespace check +[[KYUUBI #5010](https://github.com/apache/kyuubi/pull/5010)] Make Kyuubi server's connection URL configurable +[[KYUUBI #5029](https://github.com/apache/kyuubi/pull/5029)] Close the alive probe session after engine session closed +[[KYUUBI #5036](https://github.com/apache/kyuubi/pull/5036)] Fix Operation.close not update complete timestamp +[[KYUUBI #5043](https://github.com/apache/kyuubi/pull/5043)] Destroy the build process when waitCompletion is false and the engine is running in cluster mode +[[KYUUBI #5050](https://github.com/apache/kyuubi/pull/5050)] Add admin checking in `getProxyUser` +[[KYUUBI #5056](https://github.com/apache/kyuubi/pull/5056)] Return more helpful error message to http client +[[KYUUBI #5063](https://github.com/apache/kyuubi/pull/5063)] Support to filter batch with batch name +[[KYUUBI #5065](https://github.com/apache/kyuubi/pull/5065)] Call destroy first on killing Spark startup process to allows it release temp files +[[KYUUBI #5065](https://github.com/apache/kyuubi/pull/5065)] Graceful close the process when launch engine timeout +[[KYUUBI #5078](https://github.com/apache/kyuubi/pull/5078)] Make `kyuubi_instance` nullable in metadata table schema +[[KYUUBI #5081](https://github.com/apache/kyuubi/pull/5081)] Minor refactor JDBCMetadataStore +[[KYUUBI #5089](https://github.com/apache/kyuubi/pull/5089)] Limit maximum engine startup concurrency of kyuubi server +[[KYUUBI #5130](https://github.com/apache/kyuubi/pull/5130)] Allow setting Zookeeper SPN using Kyuubi configuration +[[KYUUBI #5131](https://github.com/apache/kyuubi/pull/5131)] Create index on `metastore`.`create_time` +[[KYUUBI #5138](https://github.com/apache/kyuubi/pull/5138)] Fix flaky test BatchesResourceSuite - get batch session list +[[KYUUBI #5145](https://github.com/apache/kyuubi/pull/5145)] Change embedded Zookeeper server to method-local variable +[[KYUUBI #5158](https://github.com/apache/kyuubi/pull/5158)] Allow embedded Zookeeper binding IP address +[[KYUUBI #5165](https://github.com/apache/kyuubi/pull/5165)] Build Spark Driver/Executor Pod Name(Prefix) in process +[[KYUUBI #5185](https://github.com/apache/kyuubi/pull/5185)] Use Set collection for order-insensitive configs +[[KYUUBI #5187](https://github.com/apache/kyuubi/pull/5187)] Fix info message in UserFilter +[[KYUUBI #5196](https://github.com/apache/kyuubi/pull/5196)] Extract spark core scala version lazily and respect engine env +[[KYUUBI #5201](https://github.com/apache/kyuubi/pull/5201)] Allow disabling batch impl v2 on globally +[[KYUUBI #5204](https://github.com/apache/kyuubi/pull/5204)] Thrift HTTP FE checks req conf nullable before evaluating proxy user +[[KYUUBI #5206](https://github.com/apache/kyuubi/pull/5206)] Try to kill pod with label if no ApplicationInfo found to prevent pod leak +[[KYUUBI #5210](https://github.com/apache/kyuubi/pull/5210)] Cancel operation will cause the log file node to leak +[[KYUUBI #5213](https://github.com/apache/kyuubi/pull/5213)] Check config value by enum values +[[KYUUBI #5216](https://github.com/apache/kyuubi/pull/5216)] Workaround for negative counter in SessionLimiter +[[KYUUBI #5218](https://github.com/apache/kyuubi/pull/5218)] Improve logs of KubernetesApplicationOperation +[[KYUUBI #5220](https://github.com/apache/kyuubi/pull/5220)] Batch submitter should only block submitting stage +[[KYUUBI #5220](https://github.com/apache/kyuubi/pull/5220)] Batch submitted considers application state +[[KYUUBI #5243](https://github.com/apache/kyuubi/pull/5243)] Distinguish metadata between batch impl v2 and recovery +[[KYUUBI #5289](https://github.com/apache/kyuubi/pull/5289)] RESTful API should always print audit log +[[KYUUBI #5290](https://github.com/apache/kyuubi/pull/5290)] Batch impl v2 should pick batch jobs in FIFO +[[KYUUBI #5300](https://github.com/apache/kyuubi/pull/5300)] Batch v2 should not update kyuubi_instance when open session +[[KYUUBI #5305](https://github.com/apache/kyuubi/pull/5305)] Return dummy log when batch job before being added to session manager +[[KYUUBI #5306](https://github.com/apache/kyuubi/pull/5306)] YarnApplicationOperation supports proxy user +[[KYUUBI #5310](https://github.com/apache/kyuubi/pull/5310)] Batch session recovery should start after HTTP server getting started +[[KYUUBI #5327](https://github.com/apache/kyuubi/pull/5327)] Add priority field in batch session metadata table definition +[[KYUUBI #5328](https://github.com/apache/kyuubi/pull/5328)] Batch supports priority scheduling +[[KYUUBI #5340](https://github.com/apache/kyuubi/pull/5340)] AdminResourceSuite make engine idle time 3min +[[KYUUBI #5380](https://github.com/apache/kyuubi/pull/5380)] Create PySpark batch jobs tests for RESTful API +[[KYUUBI #5381](https://github.com/apache/kyuubi/pull/5381)] Change the default metrics reporter to Prometheus +[[KYUUBI #5391](https://github.com/apache/kyuubi/pull/5391)] Close batch session after batch operation completion for idle timeout +[[KYUUBI #5392](https://github.com/apache/kyuubi/pull/5392)] Add query timeout monitor on server-side in ExecuteStatement +[[KYUUBI #5408](https://github.com/apache/kyuubi/pull/5408)] MetadataManager tries MySQL 8 driver class first +[[KYUUBI #5412](https://github.com/apache/kyuubi/pull/5412)] Resolve the relative zk configuration dir based on KYUUBI_HOME +[[KYUUBI #5441](https://github.com/apache/kyuubi/pull/5441)] Make the configuration kyuubi.zookeeper.embedded.data.log.dir effective +[[KYUUBI #5451](https://github.com/apache/kyuubi/pull/5451)] Ignore NoSuchFileException during OperationLog.close() +[[KYUUBI #5499](https://github.com/apache/kyuubi/pull/5499)] [[KYUUBI #2503](https://github.com/apache/kyuubi/pull/2503)] Catch any exception when closing idle session +[[KYUUBI #5513](https://github.com/apache/kyuubi/pull/5513)] Always redirect delete batch request to Kyuubi instance that owns batch session +[[KYUUBI #5522](https://github.com/apache/kyuubi/pull/5522)] Ignore main class for PySpark batch job submission +[[KYUUBI #5528](https://github.com/apache/kyuubi/pull/5528)] Remove duplicate properties tag and respect KYUUBI_LOG_DIR +[[KYUUBI #5566](https://github.com/apache/kyuubi/pull/5566)] InternalRestClient respects `kyuubi.engine.security.enabled` to add HTTP auth header +[[KYUUBI #5566](https://github.com/apache/kyuubi/pull/5566)] Check InternalSecurityAccessor is initialized only when `kyuubi.engine.security.enabled` is true +[[KYUUBI #5568](https://github.com/apache/kyuubi/pull/5568)] Check administrator only when security is enabled +[[KYUUBI #5568](https://github.com/apache/kyuubi/pull/5568)] Fix security enabled judgement +[[KYUUBI #5606](https://github.com/apache/kyuubi/pull/5606)] Handle engine listing request properly for users who have not created engine + +### Client + +[[KYUUBI #3887](https://github.com/apache/kyuubi/pull/3887)] Fix kyuubiServerPrincipal logic in KyuubiCommands +[[KYUUBI #4305](https://github.com/apache/kyuubi/pull/4305)] Backport HIVE-15820: comment at the head of beeline -e +[[KYUUBI #4438](https://github.com/apache/kyuubi/pull/4438)] Render the batch info map in kyuubi-ctl +[[KYUUBI #4439](https://github.com/apache/kyuubi/pull/4439)] Add dto class for operation data +[[KYUUBI #4452](https://github.com/apache/kyuubi/pull/4452)] Strip the redundant leading and tailing slash of getZooKeeperNamespace. +[[KYUUBI #4479](https://github.com/apache/kyuubi/pull/4479)] Restore JDBC Kerberos authentication behavior for UGI.doAs +[[KYUUBI #4489](https://github.com/apache/kyuubi/pull/4489)] Fix missing `org.apache.commons.logging` by including `jcl-over-slf4j` dependency in REST client +[[KYUUBI #4492](https://github.com/apache/kyuubi/pull/4492)] Correct engine subdomain calculation in `kyuubi-ctl` +[[KYUUBI #4508](https://github.com/apache/kyuubi/pull/4508)] Beeline should reset stdin after consuming init SQL file +[[KYUUBI #4581](https://github.com/apache/kyuubi/pull/4581)] Add KyuubiCommands parse python snippets unit test +[[KYUUBI #4609](https://github.com/apache/kyuubi/pull/4609)] get engineRefId in KyuubiConnection +[[KYUUBI #4619](https://github.com/apache/kyuubi/pull/4619)] Fix beeline with -e when there are other SQL statements before the source statement, the source statement cannot be executed normally +[[KYUUBI #4657](https://github.com/apache/kyuubi/pull/4657)] Building rest client to kyuubi instance including original host urls +[[KYUUBI #4962](https://github.com/apache/kyuubi/pull/4962)] Backport HIVE-19048: init script errors are ignored +[[KYUUBI #4965](https://github.com/apache/kyuubi/pull/4965)] Beeline supports `--python-mode` option and remove comments for non-python mode +[[KYUUBI #4969](https://github.com/apache/kyuubi/pull/4969)] Run JUnit tests on beeline module +[[KYUUBI #5054](https://github.com/apache/kyuubi/pull/5054)] Refine the kyuubi client version properties file to prevent conflicts +[[KYUUBI #5100](https://github.com/apache/kyuubi/pull/5100)] Fix unpaired quote in message pattern +[[KYUUBI #5101](https://github.com/apache/kyuubi/pull/5101)] Remove unused StringBuilders in ResultSets +[[KYUUBI #5129](https://github.com/apache/kyuubi/pull/5129)] KyuubiBeeline should redirect JDK logging +[[KYUUBI #5179](https://github.com/apache/kyuubi/pull/5179)] Use Iterable instead of Seq in kyuubi-ctl commands +[[KYUUBI #5297](https://github.com/apache/kyuubi/pull/5297)] New RetryableClient get http://null server uri from metadata.kyuubiInstance +[[KYUUBI #5480](https://github.com/apache/kyuubi/pull/5480)] Support setting kyuubi hive jdbc client protocol version +[[KYUUBI #5564](https://github.com/apache/kyuubi/pull/5564)] Support to config ticket cache location in JDBC URL + +### Web UI + +[[KYUUBI #3420](https://github.com/apache/kyuubi/pull/3420)] Kyuubi Server Proxy Engine UI +[[KYUUBI #3646](https://github.com/apache/kyuubi/pull/3646)] Front-end style should bracket same line +[[KYUUBI #3646](https://github.com/apache/kyuubi/pull/3646)] Init Session Statistic Page +[[KYUUBI #3648](https://github.com/apache/kyuubi/pull/3648)] Add Session Detail Page +[[KYUUBI #3650](https://github.com/apache/kyuubi/pull/3650)] Add Operation Statistics Page +[[KYUUBI #3652](https://github.com/apache/kyuubi/pull/3652)] Add Kyuubi Server Management page +[[KYUUBI #3654](https://github.com/apache/kyuubi/pull/3654)] Add Engine Manager Page +[[KYUUBI #4537](https://github.com/apache/kyuubi/pull/4537)] Enable Vite proxy server for web ui development +[[KYUUBI #4565](https://github.com/apache/kyuubi/pull/4565)] Fix out-date `README.md` about `Development Project` +[[KYUUBI #4622](https://github.com/apache/kyuubi/pull/4622)] Session api use admin api for security +[[KYUUBI #4626](https://github.com/apache/kyuubi/pull/4626)] Fix Web UI CI check fail with pnpm the lockfile needs updates +[[KYUUBI #4757](https://github.com/apache/kyuubi/pull/4757)] Move Statistics to Management Menu +[[KYUUBI #4982](https://github.com/apache/kyuubi/pull/4982)] Add query string to forward requests to Engine UI +[[KYUUBI #5259](https://github.com/apache/kyuubi/pull/5259)] Web UI supports Swagger pages +[[KYUUBI #5463](https://github.com/apache/kyuubi/pull/5463)] Adjust the file directory structure +[[KYUUBI #5484](https://github.com/apache/kyuubi/pull/5484)] Remove legacy Web UI +[[KYUUBI #5573](https://github.com/apache/kyuubi/pull/5573)] Delete parts of the Kyuubi Web UI that are not useful +[[KYUUBI #5586](https://github.com/apache/kyuubi/pull/5586)] Kyuubi Web UI show current version +[[KYUUBI #5608](https://github.com/apache/kyuubi/pull/5608)] Kyuubi version should always use the light color +[[KYUUBI #5620](https://github.com/apache/kyuubi/pull/5620)] Remove dummy information card from Overview page + +### Documentation + +[[KYUUBI #4308](https://github.com/apache/kyuubi/pull/4308)] Make README more welcoming +[[KYUUBI #4312](https://github.com/apache/kyuubi/pull/4312)] Fix over-matched README pattern for markdown styling +[[KYUUBI #4367](https://github.com/apache/kyuubi/pull/4367)] Kyuubi requires Flink 1.16/1.17 +[[KYUUBI #4387](https://github.com/apache/kyuubi/pull/4387)] Update Flink version requirements +[[KYUUBI #4568](https://github.com/apache/kyuubi/pull/4568)] Make kyuubi-ctl doc enable variable 'release' automatic substitution +[[KYUUBI #4583](https://github.com/apache/kyuubi/pull/4583)] Rename Flink Table Store to Apache Paimon (Incubating) +[[KYUUBI #4593](https://github.com/apache/kyuubi/pull/4593)] Keep promote the release step update-to-date +[[KYUUBI #4599](https://github.com/apache/kyuubi/pull/4599)] Simplify project version evaluation +[[KYUUBI #4607](https://github.com/apache/kyuubi/pull/4607)] Rename Flink Table Store to Apache Paimon (Incubating) in docs `Connectors for Hive SQL Query Engine` +[[KYUUBI #4608](https://github.com/apache/kyuubi/pull/4608)] Rename Flink Table Store to Apache Paimon (Incubating) in docs `Connectors for Trino SQL Query Engine` +[[KYUUBI #4614](https://github.com/apache/kyuubi/pull/4614)] Rename Flink Table Store to Apache Paimon (Incubating) in docs `Connectors for Flink SQL Query Engine` +[[KYUUBI #4624](https://github.com/apache/kyuubi/pull/4624)] Fix table headers in kyuubi_vs_hive.md +[[KYUUBI #4627](https://github.com/apache/kyuubi/pull/4627)] Fix a typo in rest_api.md. +[[KYUUBI #4655](https://github.com/apache/kyuubi/pull/4655)] Enrich docs for Kyuubi Hive JDBC driver +[[KYUUBI #4655](https://github.com/apache/kyuubi/pull/4655)] Recover the missing JDBC URL +[[KYUUBI #4722](https://github.com/apache/kyuubi/pull/4722)] Kyuubi requires Spark 3.1 and above now +[[KYUUBI #4727](https://github.com/apache/kyuubi/pull/4727)] kyuubi-spark-lineage has no transitive deps +[[KYUUBI #4734](https://github.com/apache/kyuubi/pull/4734)] Fix typo in docs of custom event handler +[[KYUUBI #4742](https://github.com/apache/kyuubi/pull/4742)] Add docs for Flink application mode +[[KYUUBI #4763](https://github.com/apache/kyuubi/pull/4763)] Fix the Kyuubi JDBC kerberos parameters +[[KYUUBI #4800](https://github.com/apache/kyuubi/pull/4800)] Update readthedocs.yaml +[[KYUUBI #4864](https://github.com/apache/kyuubi/pull/4864)] Fix grammar in HA section +[[KYUUBI #4933](https://github.com/apache/kyuubi/pull/4933)] Mark `spark.sql.optimizer.insertRepartitionNum` config for Spark 3.1 only +[[KYUUBI #4954](https://github.com/apache/kyuubi/pull/4954)] Fix variable reference in quick_start_with_jdbc +[[KYUUBI #4960](https://github.com/apache/kyuubi/pull/4960)] Fix doc for pyhive client. +[[KYUUBI #5049](https://github.com/apache/kyuubi/pull/5049)] PyHive Kerberos usage doc +[[KYUUBI #5083](https://github.com/apache/kyuubi/pull/5083)] Add LDAP document +[[KYUUBI #5085](https://github.com/apache/kyuubi/pull/5085)] Update config section of deploy on kubernetes +[[KYUUBI #5094](https://github.com/apache/kyuubi/pull/5094)] Add Python JayDeBeApi doc +[[KYUUBI #5099](https://github.com/apache/kyuubi/pull/5099)] Document link failure +[[KYUUBI #5118](https://github.com/apache/kyuubi/pull/5118)] Add Flink KDF doc +[[KYUUBI #5122](https://github.com/apache/kyuubi/pull/5122)] Hive KDF usage +[[KYUUBI #5127](https://github.com/apache/kyuubi/pull/5127)] Improvements for markdown builder +[[KYUUBI #5146](https://github.com/apache/kyuubi/pull/5146)] Fix link of IntelliJ IDEA Setup Guide +[[KYUUBI #5153](https://github.com/apache/kyuubi/pull/5153)] Minor change w/ to with +[[KYUUBI #5154](https://github.com/apache/kyuubi/pull/5154)] Move configuration docs to the top level +[[KYUUBI #5162](https://github.com/apache/kyuubi/pull/5162)] Add statements about contributions co-authored with AIGC in developer guide page +[[KYUUBI #5178](https://github.com/apache/kyuubi/pull/5178)] Make MarkdownBuilder not extending Growable +[[KYUUBI #5227](https://github.com/apache/kyuubi/pull/5227)] Added deploy mode info +[[KYUUBI #5247](https://github.com/apache/kyuubi/pull/5247)] Correct building command for Spark authz/lineage plugins +[[KYUUBI #5273](https://github.com/apache/kyuubi/pull/5273)] Update release docs +[[KYUUBI #5275](https://github.com/apache/kyuubi/pull/5275)] Improve and fix comparison and regeneration for golden files +[[KYUUBI #5294](https://github.com/apache/kyuubi/pull/5294)] Update supported dialects for JDBC engine +[[KYUUBI #5335](https://github.com/apache/kyuubi/pull/5335)] Set markdown file EOL +[[KYUUBI #5345](https://github.com/apache/kyuubi/pull/5345)] Fix non-kerberized code description in quick_start_with_jdbc +[[KYUUBI #5347](https://github.com/apache/kyuubi/pull/5347)] Fix java code errors in quick_start_with_jdbc +[[KYUUBI #5409](https://github.com/apache/kyuubi/pull/5409)] Update config docs regeneration hints in developer guide +[[KYUUBI #5432](https://github.com/apache/kyuubi/pull/5432)] Fix typo in README.md +[[KYUUBI #5523](https://github.com/apache/kyuubi/pull/5523)] Update the Kyuubi supported components version + +### Building, Infrastructure and Dependency + +[[KYUUBI #4078](https://github.com/apache/kyuubi/pull/4078)] Fix shellcheck violations in scripts of /bin folder +[[KYUUBI #4187](https://github.com/apache/kyuubi/pull/4187)] Windows build-info script supports revision time +[[KYUUBI #4278](https://github.com/apache/kyuubi/pull/4278)] Use new Apache 'closer.lua' syntax to obtain Maven +[[KYUUBI #4295](https://github.com/apache/kyuubi/pull/4295)] Introduce `super-linter` action for linting JSON, XML, ENV files and bash_exec +[[KYUUBI #4299](https://github.com/apache/kyuubi/pull/4299)] Bump JUnit to 4.13.2 +[[KYUUBI #4299](https://github.com/apache/kyuubi/pull/4299)] Bump MySQL JDBC driver to 8.0.32 +[[KYUUBI #4315](https://github.com/apache/kyuubi/pull/4315)] Check modules available before build step in dependency workflow +[[KYUUBI #4347](https://github.com/apache/kyuubi/pull/4347)] Bump maven download plugin from 1.6.6 to 1.6.8 +[[KYUUBI #4348](https://github.com/apache/kyuubi/pull/4348)] Cache engine archives in CI jobs for maven download plugin +[[KYUUBI #4397](https://github.com/apache/kyuubi/pull/4397)] `build/dist` supports `--web-ui` +[[KYUUBI #4413](https://github.com/apache/kyuubi/pull/4413)] Build docker image should include web UI +[[KYUUBI #4447](https://github.com/apache/kyuubi/pull/4447)] Bump log4j from 2.19.0 to 2.20.0 +[[KYUUBI #4448](https://github.com/apache/kyuubi/pull/4448)] Bump Netty from 4.1.87.Final to 4.1.89.Final +[[KYUUBI #4451](https://github.com/apache/kyuubi/pull/4451)] Skip etcd test if no docker env +[[KYUUBI #4453](https://github.com/apache/kyuubi/pull/4453)] Bump Kubernetes Client Version to 6.4.1 +[[KYUUBI #4461](https://github.com/apache/kyuubi/pull/4461)] Add `Kyuubi Kubernetes IT` Module to CI style check +[[KYUUBI #4466](https://github.com/apache/kyuubi/pull/4466)] Use bitnami/minio docker image +[[KYUUBI #4471](https://github.com/apache/kyuubi/pull/4471)] Prompted command for docs generation should skip Java tests +[[KYUUBI #4484](https://github.com/apache/kyuubi/pull/4484)] [K8S][HELM] Set specific Kyuubi version to the chart appVersion +[[KYUUBI #4494](https://github.com/apache/kyuubi/pull/4494)] `bin/kyuubi` should use `exec` to run Kyuubi server +[[KYUUBI #4513](https://github.com/apache/kyuubi/pull/4513)] Bump Kyuubi 1.7.0 in Playground +[[KYUUBI #4546](https://github.com/apache/kyuubi/pull/4546)] Fully exclude `metrics` dir from git +[[KYUUBI #4557](https://github.com/apache/kyuubi/pull/4557)] [K8S][HELM] Kyuubi server should bind Pod IP by default +[[KYUUBI #4572](https://github.com/apache/kyuubi/pull/4572)] Bump Iceberg from 1.1.0 to 1.2.0 +[[KYUUBI #4576](https://github.com/apache/kyuubi/pull/4576)] Multi arch build for ARM support in Docker Hub +[[KYUUBI #4589](https://github.com/apache/kyuubi/pull/4589)] [K8S][HELM] Update template on port-forward usage guide NOTES.txt +[[KYUUBI #4590](https://github.com/apache/kyuubi/pull/4590)] Bump delta from 2.2.0 to 2.3.0 +[[KYUUBI #4598](https://github.com/apache/kyuubi/pull/4598)] Bump Flink-1.15 from 1.15.3 to 1.15.4 +[[KYUUBI #4600](https://github.com/apache/kyuubi/pull/4600)] Bump Iceberg 1.2.0 for playground +[[KYUUBI #4601](https://github.com/apache/kyuubi/pull/4601)] Bump Hadoop 3.3.5 for playground +[[KYUUBI #4605](https://github.com/apache/kyuubi/pull/4605)] [K8S][HELM] Set IfNotPresent pullPolicy by default +[[KYUUBI #4606](https://github.com/apache/kyuubi/pull/4606)] [K8S][HELM] Add command and args configuration support +[[KYUUBI #4615](https://github.com/apache/kyuubi/pull/4615)] Bump Ranger from 2.3.0 to 2.4.0 +[[KYUUBI #4628](https://github.com/apache/kyuubi/pull/4628)] Add Dependency management ISSUE_TEMPLATE +[[KYUUBI #4632](https://github.com/apache/kyuubi/pull/4632)] Add rule for labeler to tag `module:ui` +[[KYUUBI #4638](https://github.com/apache/kyuubi/pull/4638)] Update dependency.yml as label in not permitted in markdown block +[[KYUUBI #4652](https://github.com/apache/kyuubi/pull/4652)] Upgrade Hadoop from 3.3.4 to 3.3.5 +[[KYUUBI #4695](https://github.com/apache/kyuubi/pull/4695)] Bump super-linter action from v4 to v5 +[[KYUUBI #4696](https://github.com/apache/kyuubi/pull/4696)] Upgrade scalafmt from 3.7.1 to 3.7.3 +[[KYUUBI #4697](https://github.com/apache/kyuubi/pull/4697)] [K8S][HELM] Add template rendering info to README +[[KYUUBI #4698](https://github.com/apache/kyuubi/pull/4698)] [K8S][HELM] Centralize Kyuubi labels definition +[[KYUUBI #4706](https://github.com/apache/kyuubi/pull/4706)] [K8S][HELM] Use template comments for the chart license header +[[KYUUBI #4707](https://github.com/apache/kyuubi/pull/4707)] [K8S][HELM] Fix README typo and minor reformatting +[[KYUUBI #4712](https://github.com/apache/kyuubi/pull/4712)] Bump Spark from 3.2.3 to 3.2.4 +[[KYUUBI #4737](https://github.com/apache/kyuubi/pull/4737)] Restore Project & Community Status in README.md +[[KYUUBI #4744](https://github.com/apache/kyuubi/pull/4744)] Remove Hudi integration tests +[[KYUUBI #4747](https://github.com/apache/kyuubi/pull/4747)] Install and cache build/mvn in CI builds +[[KYUUBI #4768](https://github.com/apache/kyuubi/pull/4768)] Guidelines for Document and Code contributions +[[KYUUBI #4772](https://github.com/apache/kyuubi/pull/4772)] Bump Jersey from 2.39 to 2.39.1 +[[KYUUBI #4783](https://github.com/apache/kyuubi/pull/4783)] Use pnpm v8 and update pnpm lockfile to v6 schema +[[KYUUBI #4784](https://github.com/apache/kyuubi/pull/4784)] Use nodejs v18 in CI builds and maven web-ui builds +[[KYUUBI #4788](https://github.com/apache/kyuubi/pull/4788)] [K8S][HELM] Use StatefulSet instead of Deployment +[[KYUUBI #4805](https://github.com/apache/kyuubi/pull/4805)] Bump openai-java from 0.11.1 to 0.12.0 +[[KYUUBI #4808](https://github.com/apache/kyuubi/pull/4808)] Bump Jetty from 9.4.50 to 9.4.51 +[[KYUUBI #4818](https://github.com/apache/kyuubi/pull/4818)] [K8S][HELM] Update default Kyuubi version to 1.7.1 +[[KYUUBI #4819](https://github.com/apache/kyuubi/pull/4819)] [K8S][HELM] Refactor liveliness and readiness probes +[[KYUUBI #4824](https://github.com/apache/kyuubi/pull/4824)] Bump Jackson from 2.14.2 to 2.15.0 +[[KYUUBI #4827](https://github.com/apache/kyuubi/pull/4827)] Bump Apache Arrow from 11.0.0 to 12.0.0 +[[KYUUBI #4828](https://github.com/apache/kyuubi/pull/4828)] Exclude macOS tar extended metadata in build/dist +[[KYUUBI #4849](https://github.com/apache/kyuubi/pull/4849)] Open modules to enable JDK 17 support +[[KYUUBI #4851](https://github.com/apache/kyuubi/pull/4851)] Bump Maven 3.8.8 +[[KYUUBI #4852](https://github.com/apache/kyuubi/pull/4852)] Switch to Kyuubi Shaded Zookeeper +[[KYUUBI #4856](https://github.com/apache/kyuubi/pull/4856)] Add `restore-keys` for `action/cache` +[[KYUUBI #4870](https://github.com/apache/kyuubi/pull/4870)] Add kyuubi-util and kyuubi-util-scala modules +[[KYUUBI #4877](https://github.com/apache/kyuubi/pull/4877)] Bump Flink 1.17 to 1.17.1 +[[KYUUBI #4879](https://github.com/apache/kyuubi/pull/4879)] Refactor and promote reflection utils and cleanup similar reflection methods +[[KYUUBI #4883](https://github.com/apache/kyuubi/pull/4883)] Bump Iceberg 1.3.0 +[[KYUUBI #4887](https://github.com/apache/kyuubi/pull/4887)] Refactor and add ut for ClassUtils +[[KYUUBI #4891](https://github.com/apache/kyuubi/pull/4891)] Bump Flink 1.16.2 +[[KYUUBI #4894](https://github.com/apache/kyuubi/pull/4894)] Bump Mockito from 4.9.0 to 4.11.0 and ScalaTest from 3.2.15 to 3.2.16 +[[KYUUBI #4904](https://github.com/apache/kyuubi/pull/4904)] Move AssertionUtils to kyuubi-util-scala module +[[KYUUBI #4905](https://github.com/apache/kyuubi/pull/4905)] Generalize util method for loading class from service loader +[[KYUUBI #4907](https://github.com/apache/kyuubi/pull/4907)] Add a maven profile 'remote-debug' +[[KYUUBI #4908](https://github.com/apache/kyuubi/pull/4908)] Bump scalafmt from 3.7.3 to 3.7.4 +[[KYUUBI #4911](https://github.com/apache/kyuubi/pull/4911)] Remove duplicate test annotations `org.apache.kyuubi.tags.*` in kyuubi-common +[[KYUUBI #4912](https://github.com/apache/kyuubi/pull/4912)] Replace Scala's assert in tests with scalatest's for prettified error message +[[KYUUBI #4931](https://github.com/apache/kyuubi/pull/4931)] Bump vite from 3.0.0 to 4.2.3 and vitest from 0.22.0 to 0.32.0 +[[KYUUBI #4937](https://github.com/apache/kyuubi/pull/4937)] Remove redundant quoteIfNeeded method +[[KYUUBI #4943](https://github.com/apache/kyuubi/pull/4943)] Bump Guava from 31.1 to 32.0.1 +[[KYUUBI #4944](https://github.com/apache/kyuubi/pull/4944)] Code improvement for Java +[[KYUUBI #4947](https://github.com/apache/kyuubi/pull/4947)] Bump pnpm version in maven's frontend plugin to 8.6.1 for lockfileVersion 6.1 +[[KYUUBI #4948](https://github.com/apache/kyuubi/pull/4948)] Reuse versions of NodeJS and pnpm from maven properties +[[KYUUBI #4959](https://github.com/apache/kyuubi/pull/4959)] Code improvements for Scala +[[KYUUBI #4966](https://github.com/apache/kyuubi/pull/4966)] Bump Apache parent pom from 29 to 30 +[[KYUUBI #4968](https://github.com/apache/kyuubi/pull/4968)] Simplify Option conversion +[[KYUUBI #4970](https://github.com/apache/kyuubi/pull/4970)] Unified reflection methods invokeAs and getField +[[KYUUBI #4974](https://github.com/apache/kyuubi/pull/4974)] Eliminate raw-types compilation warning in KyuubiBeeLine +[[KYUUBI #4975](https://github.com/apache/kyuubi/pull/4975)] Fixed run test link in PR template +[[KYUUBI #4990](https://github.com/apache/kyuubi/pull/4990)] Bump Spark 3.4.1 +[[KYUUBI #4992](https://github.com/apache/kyuubi/pull/4992)] Bump Netty 4.1.93.Final +[[KYUUBI #5001](https://github.com/apache/kyuubi/pull/5001)] Bump Scala from 2.12.17 to 2.12.18 +[[KYUUBI #5003](https://github.com/apache/kyuubi/pull/5003)] [K8S][HELM] Add doc strings to the chart values.yaml +[[KYUUBI #5007](https://github.com/apache/kyuubi/pull/5007)] Bump scalafmt from 3.7.4 to 3.7.5 +[[KYUUBI #5019](https://github.com/apache/kyuubi/pull/5019)] Shows details of compilation info in SparkUI's engine tab +[[KYUUBI #5024](https://github.com/apache/kyuubi/pull/5024)] Bump grpc-protobuf from 1.48.0 to 1.53.0 +[[KYUUBI #5034](https://github.com/apache/kyuubi/pull/5034)] Bump tough-cookie from 4.0.0 to 4.1.3 in /kyuubi-server/web-ui +[[KYUUBI #5039](https://github.com/apache/kyuubi/pull/5039)] Use semantic versions and remove redundant version comparison methods +[[KYUUBI #5058](https://github.com/apache/kyuubi/pull/5058)] [K8S][HELM] Allow the user to set the priorityClass used by Kyuubi server Pod +[[KYUUBI #5059](https://github.com/apache/kyuubi/pull/5059)] Upgrade kubernetes client version to 6.7.2 +[[KYUUBI #5064](https://github.com/apache/kyuubi/pull/5064)] Audit kubernetes application state change in separate log files +[[KYUUBI #5068](https://github.com/apache/kyuubi/pull/5068)] Bump Spark in spark-master profile from 3.5.0-SNAPSHOT to 4.0.0-SNAPSHOT +[[KYUUBI #5070](https://github.com/apache/kyuubi/pull/5070)] Bump word-wrap from 1.2.3 to 1.2.4 in web UI +[[KYUUBI #5072](https://github.com/apache/kyuubi/pull/5072)] Fix KyuubiOperationWithEngineSecuritySuite and related issues +[[KYUUBI #5090](https://github.com/apache/kyuubi/pull/5090)] Fix AllKyuubiConfiguration to generate redundant blank lines in Windows +[[KYUUBI #5113](https://github.com/apache/kyuubi/pull/5113)] Remove dummy Spark Kudu connector module +[[KYUUBI #5116](https://github.com/apache/kyuubi/pull/5116)] Bump Hadoop 3.3.6 +[[KYUUBI #5117](https://github.com/apache/kyuubi/pull/5117)] Code improvements for Java and Scala code +[[KYUUBI #5121](https://github.com/apache/kyuubi/pull/5121)] Extract dev scripts for regenerating the golden files +[[KYUUBI #5132](https://github.com/apache/kyuubi/pull/5132)] [K8S][HELM] Kyuubi add server metrics monitor +[[KYUUBI #5135](https://github.com/apache/kyuubi/pull/5135)] Exclude web-ui coverage reports in rat plugins +[[KYUUBI #5149](https://github.com/apache/kyuubi/pull/5149)] Correct error message of ReflectUtils's invokeAs when method not found +[[KYUUBI #5150](https://github.com/apache/kyuubi/pull/5150)] Bump Spark 3.3.3 +[[KYUUBI #5152](https://github.com/apache/kyuubi/pull/5152)] Check milestone and assignees when merging pull request +[[KYUUBI #5177](https://github.com/apache/kyuubi/pull/5177)] Use Scala binary version placeholder in Maven module's artifactId suffix +[[KYUUBI #5183](https://github.com/apache/kyuubi/pull/5183)] Introducing profile scala-2.13 for development use +[[KYUUBI #5196](https://github.com/apache/kyuubi/pull/5196)] Enable CI Test on Scala 2.13 and support custom or spark-core extracted Scala version for Spark's engine +[[KYUUBI #5202](https://github.com/apache/kyuubi/pull/5202)] Update upload_nexus_staging +[[KYUUBI #5205](https://github.com/apache/kyuubi/pull/5205)] Upgrade `kubernetes-client` to 6.8.1 +[[KYUUBI #5209](https://github.com/apache/kyuubi/pull/5209)] Reformat Scala source code crossing versions +[[KYUUBI #5212](https://github.com/apache/kyuubi/pull/5212)] Fix configuration errors causing by helm charts of prometheus services +[[KYUUBI #5214](https://github.com/apache/kyuubi/pull/5214)] Correct file path for scala tests in util-scala module +[[KYUUBI #5234](https://github.com/apache/kyuubi/pull/5234)] Upgrade Jetty version from 9.4.51.v20230217 to 9.4.52.v20230823 +[[KYUUBI #5235](https://github.com/apache/kyuubi/pull/5235)] Daily deploy snapshot version on master and branch-1.7 +[[KYUUBI #5250](https://github.com/apache/kyuubi/pull/5250)] Bump Iceberg from 1.3.0 to 1.3.1 +[[KYUUBI #5252](https://github.com/apache/kyuubi/pull/5252)] Remove incubator link +[[KYUUBI #5253](https://github.com/apache/kyuubi/pull/5253)] Upgrade testcontainers-scala from 0.40.12 to 0.41.0 +[[KYUUBI #5253](https://github.com/apache/kyuubi/pull/5253)] Supply testcontainers-scala-scalatest deps +[[KYUUBI #5293](https://github.com/apache/kyuubi/pull/5293)] Upgrade snakeyaml from 1.33 to 2.2 +[[KYUUBI #5305](https://github.com/apache/kyuubi/pull/5305)] Fix get local log UT +[[KYUUBI #5308](https://github.com/apache/kyuubi/pull/5308)] Ensure release using Java 8 +[[KYUUBI #5318](https://github.com/apache/kyuubi/pull/5318)] Pin maven-source-plugin 3.2.1 +[[KYUUBI #5358](https://github.com/apache/kyuubi/pull/5358)] Bump iceberg 1.4.0 +[[KYUUBI #5365](https://github.com/apache/kyuubi/pull/5365)] Don't use Log4j2's extended throwable conversion pattern in default logging configurations +[[KYUUBI #5425](https://github.com/apache/kyuubi/pull/5425)] Add thread name and ID to log4j2-defaults.xml +[[KYUUBI #5435](https://github.com/apache/kyuubi/pull/5435)] Improve Kyuubi On Kubernetes IT +[[KYUUBI #5443](https://github.com/apache/kyuubi/pull/5443)] Add kyuubi-kubernetes-it module to reformat script +[[KYUUBI #5444](https://github.com/apache/kyuubi/pull/5444)] Check style with profiles of Spark 3.4 and 3.5 +[[KYUUBI #5483](https://github.com/apache/kyuubi/pull/5483)] Fix deploy command for Scala 2.13 TPC-DS/H connectors +[[KYUUBI #5483](https://github.com/apache/kyuubi/pull/5483)] Release Spark TPC-H/DS Connectors with Scala 2.13 +[[KYUUBI #5486](https://github.com/apache/kyuubi/pull/5486)] Bump Kafka client version from 3.4.0 to 3.5.1 +[[KYUUBI #5508](https://github.com/apache/kyuubi/pull/5508)] Upgrade download-maven-plugin from 1.6.8 to 1.7.1 +[[KYUUBI #5544](https://github.com/apache/kyuubi/pull/5544)] Release contains Spark 3.5 extension +[[KYUUBI #5561](https://github.com/apache/kyuubi/pull/5561)] Binary distribution artifact ships database schema scripts +[[KYUUBI #5561](https://github.com/apache/kyuubi/pull/5561)] Rename folder to db-scripts +[[KYUUBI #5569](https://github.com/apache/kyuubi/pull/5569)] Fix pnpm-lock file version + +### Credits + +Last but not least, this release would not have been possible without the following contributors: + +* Alex Zou +* Anurag Rajawat +* Binjie Yang +* Bowen Liang +* Bruce Wong +* CavemanIV +* Chao Chen +* Cheng Pan +* Chenliang Lu +* David Yuan +* Deng An +* Denis Krivenko +* Derick Cypho +* Drake Youngkun Min +* Fei Wang +* FourSpaces +* Fu Chen +* GitHub Bot +* Guanhua Li +* Hanna Liashchuk +* He Zhao +* Junjie Zu +* Kaifei Yi +* Kang Wang +* Karsonnel +* Kent Yao +* Kyofin +* Miao Wang +* Min Zhao +* Naijie Liu +* Omkar Makhare +* Paul Lin +* Peiyue Liu +* Pengqi Li +* Qiang Zhang +* QianyongY +* Remzi Yang +* Rojin Raju +* Ruguo Yu +* Ruilei Ma +* Senmiao Liu +* Shaoyun Chen +* Sidhant Nagpal +* Tianlin Liao +* Xianxun Ye +* Xiduo You +* Xieming Li +* XorSum +* Xu Guo +* Yang Du +* Yangming Zhao +* Yaodong Zhang +* Yaogai Zhu +* Yi Zhu +* Yifan Zhou +* Zhen Wang +* biangjuang +* camper42 +* fantasticKe +* huangzhir +* huapan123456 +* liaoyt +* mans2singh +* mrtisttt +* phyyou +* rhh777 +* runzhliu +* touchida +* uicosp +* wangyepeng2 +* weixi62961 +* yehere diff --git a/src/zh/news/2023-11-07-release-1.8.0.md b/src/zh/news/2023-11-07-release-1.8.0.md new file mode 100644 index 0000000..dbd6290 --- /dev/null +++ b/src/zh/news/2023-11-07-release-1.8.0.md @@ -0,0 +1,20 @@ +--- +title: Apache Kyuubi 1.8.0 released +date: 2023-11-07 +linked: true +--- +<!--- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +我们很高兴地宣布 [Apache Kyuubi 1.8.0](/zh/release/1.8.0.html) 发布了! 查看 [发布记录](/zh/release/1.8.0.html) 阅读有关新功能的信息，或者马上 [下载](/zh/releases.html) 体验吧。 diff --git a/src/zh/release/1.8.0.md b/src/zh/release/1.8.0.md new file mode 100644 index 0000000..98823de --- /dev/null +++ b/src/zh/release/1.8.0.md @@ -0,0 +1,713 @@ +--- +title: 发布记录 - 1.8.0 +date: 2023-11-07 +linked: true +meta: +_edit_last: '4' +_wpas_done_all: '1' +--- +<!--- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +### 重要变更 + +- 基于 Java 8, 11 和 17，Scala 2.12 做了充分验证，初步支持 Scala 2.13 +- 基于 Spark 3.1, 3.2, 3.3 和 3.4 做了充分验证, 初步支持 Spark 3.5 +- 基于 Flink 1.16, 1.17 和 1.18 做了充分验证 +- 全新 Web UI (实验特性) +- 新增以 OpenAI ChatGPT 作为后端的 Chat 引擎 +- Spark Batch API 重大改进，针对大规模作业提交进行了优化 (实验特性) +- Spark AuthZ 插件重大改进，包括增强了对 Iceberg 和 DSv2 算子的支持 +- Flink 引擎重大改进，包括 YARN Application Mode 的支持 +- Spark 血缘插件引入 Atlas hook 支持 +- 持续改进 Kyuubi Helm Chart 和 Spark on K8s + +### Spark Engine, Connectors and Plugins + +[[KYUUBI #1265](https://github.com/apache/kyuubi/pull/1265)] `OPTIMIZE` where clause expression support +[[KYUUBI #3420](https://github.com/apache/kyuubi/pull/3420)] Expose UI url on registering engine service +[[KYUUBI #3444](https://github.com/apache/kyuubi/pull/3444)] Support the planOnly mode of kyuubi spark engine support SQL lineage +[[KYUUBI #3929](https://github.com/apache/kyuubi/pull/3929)] Refactor lineage plugin to add LineageDispatcher +[[KYUUBI #4167](https://github.com/apache/kyuubi/pull/4167)] [AUTHZ] Introduce function support in PrivilegeBuilder with Serde layers +[[KYUUBI #4171](https://github.com/apache/kyuubi/pull/4171)] Support skip retrieving table's properties to speed up GetTables operation +[[KYUUBI #4186](https://github.com/apache/kyuubi/pull/4186)] Spark showProgress with JobInfo +[[KYUUBI #4330](https://github.com/apache/kyuubi/pull/4330)] Non-temporary views do not resolve to a specific real table +[[KYUUBI #4331](https://github.com/apache/kyuubi/pull/4331)] [[KYUUBI #4431](https://github.com/apache/kyuubi/pull/4431)] Lineage supports `Union` and `MergeInto` +[[KYUUBI #4415](https://github.com/apache/kyuubi/pull/4415)] Align the operation handle in server/engine for ExecuteScala, ExecutePython and PlanOnlyStatement +[[KYUUBI #4430](https://github.com/apache/kyuubi/pull/4430)] Lineage supports the `lateral view explode` statement +[[KYUUBI #4433](https://github.com/apache/kyuubi/pull/4433)] Fix the case with `create a view from view` +[[KYUUBI #4443](https://github.com/apache/kyuubi/pull/4443)] Do not set engine session init sql for alive probe session +[[KYUUBI #4450](https://github.com/apache/kyuubi/pull/4450)] Ignore unknown fields `policyPriority` when reading policy json file +[[KYUUBI #4488](https://github.com/apache/kyuubi/pull/4488)] [KSHC] Keep object original name defined in HiveBridgeHelper +[[KYUUBI #4493](https://github.com/apache/kyuubi/pull/4493)] [AUTHZ] Enable Tests for Spark 3.1 with iceberg tables +[[KYUUBI #4497](https://github.com/apache/kyuubi/pull/4497)] Add RowFilteringTestBase to improve the test coverage +[[KYUUBI #4504](https://github.com/apache/kyuubi/pull/4504)] [AUTHZ] Fix source table privilege requirement when querying permanent view in Spark 3.1 and below +[[KYUUBI #4525](https://github.com/apache/kyuubi/pull/4525)] [KSHC] Partitioning predicates should take effect to filter data +[[KYUUBI #4530](https://github.com/apache/kyuubi/pull/4530)] [AUTHZ] Support non-English chars for MASK, MASK_SHOW_FIRST_4, and MASK_SHOW_FIRST_4 +[[KYUUBI #4532](https://github.com/apache/kyuubi/pull/4532)] [AUTHZ] Displays the columns involved in extracting the aggregation operator +[[KYUUBI #4541](https://github.com/apache/kyuubi/pull/4541)] Support to customize the attributes to expose for Spark engine +[[KYUUBI #4560](https://github.com/apache/kyuubi/pull/4560)] [KSHC] Support Kerberized HMS in cluster mode w/o keytab +[[KYUUBI #4585](https://github.com/apache/kyuubi/pull/4585)] [AUTHZ] Authz policy file generation +[[KYUUBI #4587](https://github.com/apache/kyuubi/pull/4587)] Take care of table cache query stage +[[KYUUBI #4592](https://github.com/apache/kyuubi/pull/4592)] Support eagerly kill redundant executors +[[KYUUBI #4615](https://github.com/apache/kyuubi/pull/4615)] Support stage level schedule for final write stage +[[KYUUBI #4617](https://github.com/apache/kyuubi/pull/4617)] [AUTHZ] Collect results for filtered show objects ahead to prevent holding unserializable spark plan +[[KYUUBI #4620](https://github.com/apache/kyuubi/pull/4620)] [KSHC] Cut off transitive dependencies +[[KYUUBI #4636](https://github.com/apache/kyuubi/pull/4636)] Improve eagerly kill redundant executors +[[KYUUBI #4641](https://github.com/apache/kyuubi/pull/4641)] Add MaxFileSizeStrategy to limit max scan file size +[[KYUUBI #4644](https://github.com/apache/kyuubi/pull/4644)] Manually terminate the Py4JServer during engine shutdown +[[KYUUBI #4658](https://github.com/apache/kyuubi/pull/4658)] [AUTHZ] Fix InsertIntoHiveDirCommand classname so that we can extract the query in it when authorization. +[[KYUUBI #4662](https://github.com/apache/kyuubi/pull/4662)] [ARROW] Arrow serialization should not introduce extra shuffle for outermost limit +[[KYUUBI #4664](https://github.com/apache/kyuubi/pull/4664)] Fix empty relation when kill executors +[[KYUUBI #4672](https://github.com/apache/kyuubi/pull/4672)] [AUTHZ] Remove Support for Apache Ranger 0.6.0 +[[KYUUBI #4676](https://github.com/apache/kyuubi/pull/4676)] [AUTHZ] Reuse users and namespaces in both tests and policy file generation +[[KYUUBI #4677](https://github.com/apache/kyuubi/pull/4677)] [AUTHZ] Check generated policy file in test suite +[[KYUUBI #4678](https://github.com/apache/kyuubi/pull/4678)] Improve FinalStageResourceManager kill executors +[[KYUUBI #4681](https://github.com/apache/kyuubi/pull/4681)] Set thread `CreateSparkTimeoutChecker` daemon +[[KYUUBI #4683](https://github.com/apache/kyuubi/pull/4683)] Update `spark.sql.finalWriteStage.resourceIsolation.enabled` version +[[KYUUBI #4702](https://github.com/apache/kyuubi/pull/4702)] [ARROW] CommandResultExec should not trigger job +[[KYUUBI #4710](https://github.com/apache/kyuubi/pull/4710)] [ARROW] LocalTableScanExec should not trigger job +[[KYUUBI #4710](https://github.com/apache/kyuubi/pull/4710)] [ARROW] Post driver-side metrics for LocalTableScanExec/CommandResultExec +[[KYUUBI #4716](https://github.com/apache/kyuubi/pull/4716)] [[KYUUBI #4715](https://github.com/apache/kyuubi/pull/4715)][AUTHZ] Fix the incorrect class name of InsertIntoHiveDirCommand in table spec generator +[[KYUUBI #4717](https://github.com/apache/kyuubi/pull/4717)] [AUTHZ] Check Authz plugin's spec json files in UT +[[KYUUBI #4720](https://github.com/apache/kyuubi/pull/4720)] [ARROW] Fix java.lang.NoSuchFieldError: IpcOption.DEFAULT for Spark-3.1/3.2 +[[KYUUBI #4741](https://github.com/apache/kyuubi/pull/4741)] Kyuubi Spark Engine/TPC connectors support Spark 3.4 +[[KYUUBI #4754](https://github.com/apache/kyuubi/pull/4754)] [ARROW] Use `KyuubiArrowConverters#toBatchIterator` instead of `ArrowConverters#toBatchIterator` +[[KYUUBI #4797](https://github.com/apache/kyuubi/pull/4797)] [ARROW] Reflective calls to the function `ArrowUtils#toArrowSchema` +[[KYUUBI #4812](https://github.com/apache/kyuubi/pull/4812)] Generalize case transformation method for string type config entry +[[KYUUBI #4834](https://github.com/apache/kyuubi/pull/4834)] Reduce the scope of method references in Authz plugin cleanup shutdown hook +[[KYUUBI #4838](https://github.com/apache/kyuubi/pull/4838)] Fix spark operation exception leak in `withLocalProperties` method +[[KYUUBI #4866](https://github.com/apache/kyuubi/pull/4866)] Add annotation for Iceberg tests in Authz plugin +[[KYUUBI #4869](https://github.com/apache/kyuubi/pull/4869)] [AUTHZ] Introduce table extractor for ResolvedIdentifier in Spark 3.4 +[[KYUUBI #4871](https://github.com/apache/kyuubi/pull/4871)] [AUTHZ] Adapt plan changes for CreateNamespace and SetCatalogAndNamespace in Spark 3.4 +[[KYUUBI #4873](https://github.com/apache/kyuubi/pull/4873)] [AUTHZ] Refactor Authz reflection with kyuubi-util's DynMethods +[[KYUUBI #4874](https://github.com/apache/kyuubi/pull/4874)] [AUTHZ] Improve methods in AuthzUtils +[[KYUUBI #4875](https://github.com/apache/kyuubi/pull/4875)] [AUTHZ] Remove checking Spark v2 in tests since Spark v2 not supported +[[KYUUBI #4878](https://github.com/apache/kyuubi/pull/4878)] Delta support for Spark-3.4 +[[KYUUBI #4880](https://github.com/apache/kyuubi/pull/4880)] [BUILD] Skip compiling useless files when making distribution +[[KYUUBI #4888](https://github.com/apache/kyuubi/pull/4888)] [AUTHZ] Remove filtering results for ShowDatabasesCommand in Spark 2.x +[[KYUUBI #4892](https://github.com/apache/kyuubi/pull/4892)] [AUTHZ] Make identifier part name comparison case insensitive in tests of PrivilegeBuilder +[[KYUUBI #4895](https://github.com/apache/kyuubi/pull/4895)] [ARROW] Reflective calls to the function `ArrowConverters#fromBatchIterator` +[[KYUUBI #4899](https://github.com/apache/kyuubi/pull/4899)] [AUTHZ] Extract function from FunctionIdentifier for CreateFunction and DropFunction in Spark 3.4 +[[KYUUBI #4903](https://github.com/apache/kyuubi/pull/4903)] [AUTHZ] Fix NoSuchElementException when listing database in CatalogImpl in Spark 3.4 +[[KYUUBI #4909](https://github.com/apache/kyuubi/pull/4909)] [AUTHZ] Enable authz plugin tests for Spark 3.4 +[[KYUUBI #4910](https://github.com/apache/kyuubi/pull/4910)] Extract table from ResolvedIdentifier for DropTable in Spark 3.4 +[[KYUUBI #4913](https://github.com/apache/kyuubi/pull/4913)] Eliminate unnecessary output in ut "union an unmasked table" +[[KYUUBI #4914](https://github.com/apache/kyuubi/pull/4914)] [AUTHZ] Reuse extractor singleton instance with generalized getter for supported extractor types +[[KYUUBI #4915](https://github.com/apache/kyuubi/pull/4915)] Adapt database not exist error message change for Spark 3.4 +[[KYUUBI #4916](https://github.com/apache/kyuubi/pull/4916)] [AUTHZ] Support ReplaceData and compatible Spark 3.4 and 3.5 +[[KYUUBI #4916](https://github.com/apache/kyuubi/pull/4916)] Fix Iceberg merge into UT +[[KYUUBI #4917](https://github.com/apache/kyuubi/pull/4917)] [AUTHZ] Table owner undefined in Iceberg 1.3.0 on Spark 3.4 +[[KYUUBI #4923](https://github.com/apache/kyuubi/pull/4923)] [ARROW] Update arguments of `ArrowUtils#toArrowSchema` function +[[KYUUBI #4925](https://github.com/apache/kyuubi/pull/4925)] Add default catalog using `spark_catalog` with the lineage result +[[KYUUBI #4935](https://github.com/apache/kyuubi/pull/4935)] More than target num of executors may survive after FinalStageResourceManager did kill +[[KYUUBI #4937](https://github.com/apache/kyuubi/pull/4937)] Cleanup spark catalog shim and renamed to catalog utils +[[KYUUBI #4952](https://github.com/apache/kyuubi/pull/4952)] Enhance AWS Glue default database does not exist error message on opening session +[[KYUUBI #4995](https://github.com/apache/kyuubi/pull/4995)] Use hadoop conf and hive conf from catalog options +[[KYUUBI #4999](https://github.com/apache/kyuubi/pull/4999)] [KSHC] Kyuubi-Spark-Hive-Connector support Apache Spark 3.4 +[[KYUUBI #5002](https://github.com/apache/kyuubi/pull/5002)] Fail the engine fast when no incoming connection in CONNECTION mode +[[KYUUBI #5005](https://github.com/apache/kyuubi/pull/5005)] Remove default settings `spark.sql.execution.topKSortFallbackThreshold` +[[KYUUBI #5007](https://github.com/apache/kyuubi/pull/5007)] Remove blank SparkSimpleStatsReportListener +[[KYUUBI #5009](https://github.com/apache/kyuubi/pull/5009)] Pass Spark Engine Log Path to Spark Conf +[[KYUUBI #5017](https://github.com/apache/kyuubi/pull/5017)] [KSHC] Support Parquet/Orc provider is splittable +[[KYUUBI #5018](https://github.com/apache/kyuubi/pull/5018)] Make kyuubi spark extension compatible with Spark 3.4 +[[KYUUBI #5022](https://github.com/apache/kyuubi/pull/5022)] [KSHC] CreateTable should use the correct provider +[[KYUUBI #5023](https://github.com/apache/kyuubi/pull/5023)] [KSHC] TableIdentify don't attach catalog +[[KYUUBI #5025](https://github.com/apache/kyuubi/pull/5025)] Use formatDuration instead of formatDurationVerbose in Spark UI's engine tab +[[KYUUBI #5028](https://github.com/apache/kyuubi/pull/5028)] Update session hadoop conf to catalog hadoop conf +[[KYUUBI #5030](https://github.com/apache/kyuubi/pull/5030)] Support get query id in Spark engine +[[KYUUBI #5035](https://github.com/apache/kyuubi/pull/5035)] Spark engine session page display session end time and duration +[[KYUUBI #5045](https://github.com/apache/kyuubi/pull/5045)] Initialize EXECUTOR_POD_NAME_PREFIX_MAX_LENGTH before using +[[KYUUBI #5055](https://github.com/apache/kyuubi/pull/5055)] [AUTHZ] Support building function privileges in Spark 3.4 +[[KYUUBI #5073](https://github.com/apache/kyuubi/pull/5073)] Correct the method name in SparkSQLLineageParserHelperSuite +[[KYUUBI #5075](https://github.com/apache/kyuubi/pull/5075)] Refine tests to adapt Spark 4.0 +[[KYUUBI #5076](https://github.com/apache/kyuubi/pull/5076)] Add KDF engine_url +[[KYUUBI #5105](https://github.com/apache/kyuubi/pull/5105)] GetColumns gets duplicate column names when Spark integrates with Iceberg +[[KYUUBI #5136](https://github.com/apache/kyuubi/pull/5136)] Spark App may hang forever if FinalStageResourceManager killed all executors +[[KYUUBI #5148](https://github.com/apache/kyuubi/pull/5148)] Improve spark.driver.host assignment in Spark on K8s client mode +[[KYUUBI #5170](https://github.com/apache/kyuubi/pull/5170)] Identifier compatible with Spark 3.4 +[[KYUUBI #5172](https://github.com/apache/kyuubi/pull/5172)] [AUTHZ] Check USE permissions for DESCRIBE FUNCTION +[[KYUUBI #5180](https://github.com/apache/kyuubi/pull/5180)] Replace deprecated classes by equivalent in scala.tools.nsc.interpreter package +[[KYUUBI #5184](https://github.com/apache/kyuubi/pull/5184)] Rename Kyuubi's StageInfo to SparkStageInfo to fix class mismatch +[[KYUUBI #5188](https://github.com/apache/kyuubi/pull/5188)] Make server module and Spark sql engine module compilable on Scala 2.13 +[[KYUUBI #5189](https://github.com/apache/kyuubi/pull/5189)] [AUTHZ] Make spark authz module compilable on Scala 2.13 +[[KYUUBI #5191](https://github.com/apache/kyuubi/pull/5191)] Make Spark extension plugin compilable on Scala 2.13 +[[KYUUBI #5192](https://github.com/apache/kyuubi/pull/5192)] Make Spark sql lineage plugin compilable on Scala 2.13 +[[KYUUBI #5193](https://github.com/apache/kyuubi/pull/5193)] Make Spark hive connector plugin compilable on Scala 2.13 +[[KYUUBI #5195](https://github.com/apache/kyuubi/pull/5195)] Make Spark TPC-H connector plugin compilable on Scala 2.13 +[[KYUUBI #5196](https://github.com/apache/kyuubi/pull/5196)] Enable CI Test on Scala 2.13 and support custom or spark-core extracted Scala version for Spark's engine +[[KYUUBI #5196](https://github.com/apache/kyuubi/pull/5196)] Extract spark core scala version lazily and respect engine env +[[KYUUBI #5199](https://github.com/apache/kyuubi/pull/5199)] Read all columns of metadata to prevent column missing +[[KYUUBI #5202](https://github.com/apache/kyuubi/pull/5202)] Switch to Spark 3.4 by default +[[KYUUBI #5224](https://github.com/apache/kyuubi/pull/5224)] Modify the README file of authz to make spark 3.4 the default version +[[KYUUBI #5225](https://github.com/apache/kyuubi/pull/5225)] [KSHC] Unify the exception handling of v1 and v2 during dropDatabase +[[KYUUBI #5232](https://github.com/apache/kyuubi/pull/5232)] In SparkOperation#cleanup always calls cancelJobGroup even though it's in the completed state +[[KYUUBI #5257](https://github.com/apache/kyuubi/pull/5257)] [LINEAGE] Adapt `CreateTableAsSelect` plan to Spark 3.5 changes +[[KYUUBI #5268](https://github.com/apache/kyuubi/pull/5268)] [AUTHZ] Extract method for enabling authorization in single call mode +[[KYUUBI #5269](https://github.com/apache/kyuubi/pull/5269)] Duplicated AnalyzesTablesCommand in kyuubi authz +[[KYUUBI #5271](https://github.com/apache/kyuubi/pull/5271)] AnalyzeTableCommand should also add table write privilege +[[KYUUBI #5291](https://github.com/apache/kyuubi/pull/5291)] Verify Spark engine is compatible with Spark 3.5.0 +[[KYUUBI #5317](https://github.com/apache/kyuubi/pull/5317)] [KSHC] Hive Connector throws NotSerializableException on reading Hive Avro partitioned table +[[KYUUBI #5336](https://github.com/apache/kyuubi/pull/5336)] Spark extension supports Spark 3.5 +[[KYUUBI #5339](https://github.com/apache/kyuubi/pull/5339)] Avoid using `kyuubi.session.engine.initialize.timeout` string +[[KYUUBI #5414](https://github.com/apache/kyuubi/pull/5414)] [KSHC] Reader should not pollute the global hiveConf instance +[[KYUUBI #5426](https://github.com/apache/kyuubi/pull/5426)] [KSHC] Avoid use class.newInstance directly +[[KYUUBI #5550](https://github.com/apache/kyuubi/pull/5550)] Optimizing TPC-DS dataset generation for 10x speedup + +### Flink Engine + +[[KYUUBI #1652](https://github.com/apache/kyuubi/pull/1652)] Support Flink YARN application mode +[[KYUUBI #4367](https://github.com/apache/kyuubi/pull/4367)] Support Flink 1.17 +[[KYUUBI #4387](https://github.com/apache/kyuubi/pull/4387)] Remove support for Flink 1.14 +[[KYUUBI #4446](https://github.com/apache/kyuubi/pull/4446)] Fix connections blocked by Flink insert statements +[[KYUUBI #4495](https://github.com/apache/kyuubi/pull/4495)] Support Flink job management statements +[[KYUUBI #4517](https://github.com/apache/kyuubi/pull/4517)] Fix multiple executions lead to abnormal results on Flink 1.14 +[[KYUUBI #4635](https://github.com/apache/kyuubi/pull/4635)] Support flink time type in query operation +[[KYUUBI #4666](https://github.com/apache/kyuubi/pull/4666)] Support flink varbinary type in query operation +[[KYUUBI #4745](https://github.com/apache/kyuubi/pull/4745)] Support Flink's LocalZonedTimestamp DataType +[[KYUUBI #4806](https://github.com/apache/kyuubi/pull/4806)] Support time-out incremental result fetch for Flink engine +[[KYUUBI #4861](https://github.com/apache/kyuubi/pull/4861)] Support Flink session idleness +[[KYUUBI #4938](https://github.com/apache/kyuubi/pull/4938)] Implement Kyuubi UDF in Flink engine +[[KYUUBI #5080](https://github.com/apache/kyuubi/pull/5080)] Fix EmbeddedExecutorFactory not thread-safe during bootstrap +[[KYUUBI #5106](https://github.com/apache/kyuubi/pull/5106)] Improve logs for fatal errors +[[KYUUBI #5108](https://github.com/apache/kyuubi/pull/5108)] Fix FileNotFoundException during Flink engine bootstrap +[[KYUUBI #5160](https://github.com/apache/kyuubi/pull/5160)] Refactor getNextRowSetInternal to support fetch streaming data +[[KYUUBI #5190](https://github.com/apache/kyuubi/pull/5190)] Explicitly name Flink bootstrap SQL in application mode +[[KYUUBI #5200](https://github.com/apache/kyuubi/pull/5200)] Optimize Flink application name generating +[[KYUUBI #5222](https://github.com/apache/kyuubi/pull/5222)] Remove unused provided dependencies in Flink SQL engine +[[KYUUBI #5238](https://github.com/apache/kyuubi/pull/5238)] Fix credentials may break Flink engine launch command +[[KYUUBI #5315](https://github.com/apache/kyuubi/pull/5315)] Propagate HIVE_CONF_DIR on launching Flink engine +[[KYUUBI #5405](https://github.com/apache/kyuubi/pull/5405)] Support Flink 1.18 +[[KYUUBI #5531](https://github.com/apache/kyuubi/pull/5531)] Fix flaky FlinkOperationOnYarnSuite by enlarging the max rows setting + +### Other Engines + +[[KYUUBI #3374](https://github.com/apache/kyuubi/pull/3374)] Support password authentication for Trino engine +[[KYUUBI #4412](https://github.com/apache/kyuubi/pull/4412)] Align the server/engine session handle for flink/hive/trino/jdbc engines +[[KYUUBI #4491](https://github.com/apache/kyuubi/pull/4491)] Fix Trino typo +[[KYUUBI #4522](https://github.com/apache/kyuubi/pull/4522)] `use:catalog` should execute before than `use:database` +[[KYUUBI #4544](https://github.com/apache/kyuubi/pull/4544)] Initial implement Kyuubi Chat Engine +[[KYUUBI #4548](https://github.com/apache/kyuubi/pull/4548)] Kyuubi Chat Engine supports Chinese questions and HTTP proxy +[[KYUUBI #4554](https://github.com/apache/kyuubi/pull/4554)] Code improvement in ChatGPTProvider +[[KYUUBI #4556](https://github.com/apache/kyuubi/pull/4556)] Refactor ChatGPTProvider to use `openai-java` client +[[KYUUBI #4558](https://github.com/apache/kyuubi/pull/4558)] Make ChatGPT model ID configurable +[[KYUUBI #4559](https://github.com/apache/kyuubi/pull/4559)] Pretty chat engine launch command +[[KYUUBI #4711](https://github.com/apache/kyuubi/pull/4711)] JDBC client should catch task failed exception instead of NPE in the incremental mode +[[KYUUBI #4809](https://github.com/apache/kyuubi/pull/4809)] Set session user in ChatGPT request +[[KYUUBI #4810](https://github.com/apache/kyuubi/pull/4810)] Request and use a single choice for chat completion +[[KYUUBI #4881](https://github.com/apache/kyuubi/pull/4881)] JDBCEngine performs initialization sql +[[KYUUBI #4940](https://github.com/apache/kyuubi/pull/4940)] Implement Kyuubi UDF in Hive engine +[[KYUUBI #5120](https://github.com/apache/kyuubi/pull/5120)] Allow using session's user and password for connecting database in JDBC engine +[[KYUUBI #5282](https://github.com/apache/kyuubi/pull/5282)] Support configure Trino session conf in `kyuubi-default.conf` +[[KYUUBI #5333](https://github.com/apache/kyuubi/pull/5333)] Adaptive call fetchLaunchEngineResult +[[KYUUBI #5570](https://github.com/apache/kyuubi/pull/5570)] Fix memory leak when using incremental collect mode in JDBC engine + +### Server + +[[KYUUBI #3653](https://github.com/apache/kyuubi/pull/3653)] AdminResource add list Kyuubi server api +[[KYUUBI #3739](https://github.com/apache/kyuubi/pull/3739)] Remove unused parameters in SessionsResource rest docs +[[KYUUBI #3872](https://github.com/apache/kyuubi/pull/3872)] Skip setting proxy user when keytab is provided on batch session +[[KYUUBI #3876](https://github.com/apache/kyuubi/pull/3876)] Update the rest api docs for open session response +[[KYUUBI #3920](https://github.com/apache/kyuubi/pull/3920)] Fix flaky test - distribute lock +[[KYUUBI #3957](https://github.com/apache/kyuubi/pull/3957)] Rename the config prefix from kyuubi.server.batch.limit to kyuubi.server.limit.batch +[[KYUUBI #4325](https://github.com/apache/kyuubi/pull/4325)] Support replace preparedStatement for Trino-jdbc +[[KYUUBI #4344](https://github.com/apache/kyuubi/pull/4344)] Expose exec pool work queue size metrics +[[KYUUBI #4364](https://github.com/apache/kyuubi/pull/4364)] Add metrics for user opened connections with session type +[[KYUUBI #4376](https://github.com/apache/kyuubi/pull/4376)] Support to config the Kyuubi service administrator with Kyuubi conf +[[KYUUBI #4388](https://github.com/apache/kyuubi/pull/4388)] Limit the max rows for get nextRowSet api +[[KYUUBI #4404](https://github.com/apache/kyuubi/pull/4404)] Support to list/close sessions in AdminResource +[[KYUUBI #4407](https://github.com/apache/kyuubi/pull/4407)] Adapt SLF4J2 +[[KYUUBI #4424](https://github.com/apache/kyuubi/pull/4424)] Catch No Node Exception, when list kyuubi engines +[[KYUUBI #4439](https://github.com/apache/kyuubi/pull/4439)] Add list/close operation method for AdminResource +[[KYUUBI #4462](https://github.com/apache/kyuubi/pull/4462)] Fix variable usage issue in `SessionManager#stop` +[[KYUUBI #4464](https://github.com/apache/kyuubi/pull/4464)] Simplify and improve log for JDBCMetadataStore +[[KYUUBI #4467](https://github.com/apache/kyuubi/pull/4467)] Tolerate Driver Pod ephemerally invisible after submitting +[[KYUUBI #4472](https://github.com/apache/kyuubi/pull/4472)] add session/operation methods in AdminRestApi +[[KYUUBI #4480](https://github.com/apache/kyuubi/pull/4480)] Engine alive probe should close thrift connection on engine lost +[[KYUUBI #4482](https://github.com/apache/kyuubi/pull/4482)] Support to disable batch resource upload function +[[KYUUBI #4483](https://github.com/apache/kyuubi/pull/4483)] Enable REST frontend protocol by default +[[KYUUBI #4496](https://github.com/apache/kyuubi/pull/4496)] Call `super.clusterManager()` when `batchConf` does not have `spark.master` +[[KYUUBI #4498](https://github.com/apache/kyuubi/pull/4498)] Support to list sessions/operations with conditions in AdminRestApi +[[KYUUBI #4501](https://github.com/apache/kyuubi/pull/4501)] Allow administrator to specify the user for engine admin +[[KYUUBI #4506](https://github.com/apache/kyuubi/pull/4506)] Return sessionType in session data +[[KYUUBI #4516](https://github.com/apache/kyuubi/pull/4516)] Return kyuubi instance in SessionData and OperationData +[[KYUUBI #4518](https://github.com/apache/kyuubi/pull/4518)] Return engine id in SessionData and post kyuubi instance in KyuubiOperationEvent +[[KYUUBI #4519](https://github.com/apache/kyuubi/pull/4519)] Update metadata when batch application state changed +[[KYUUBI #4523](https://github.com/apache/kyuubi/pull/4523)] support close engine session gracefully +[[KYUUBI #4526](https://github.com/apache/kyuubi/pull/4526)] Defer evaluation for GroupProvider +[[KYUUBI #4540](https://github.com/apache/kyuubi/pull/4540)] Support to deny some users to make connection +[[KYUUBI #4540](https://github.com/apache/kyuubi/pull/4540)] Create session limiter if user unlimited list or deny list nonempty +[[KYUUBI #4563](https://github.com/apache/kyuubi/pull/4563)] Format error log output in case of metrics json file not found +[[KYUUBI #4575](https://github.com/apache/kyuubi/pull/4575)] Fix the empty last line may causes the session to exit directly +[[KYUUBI #4579](https://github.com/apache/kyuubi/pull/4579)] Fix the unstable situation of test cases for Trino-Fronted-Service +[[KYUUBI #4584](https://github.com/apache/kyuubi/pull/4584)] Post the session exception to EventBus in time +[[KYUUBI #4594](https://github.com/apache/kyuubi/pull/4594)] Support PUT method in REST client +[[KYUUBI #4611](https://github.com/apache/kyuubi/pull/4611)] support all sessions api in SessionRestApi +[[KYUUBI #4618](https://github.com/apache/kyuubi/pull/4618)] Admin Resource list operations with sessionHandle filter +[[KYUUBI #4623](https://github.com/apache/kyuubi/pull/4623)] KubernetesApplicationOperation uses Informer instead of list +[[KYUUBI #4639](https://github.com/apache/kyuubi/pull/4639)] Support to specify confOverlay when executing statement with RESTful API +[[KYUUBI #4652](https://github.com/apache/kyuubi/pull/4652)] Fix JaasConfiguration ClassNotFoundException for Hadoop 3.3.4 and previous +[[KYUUBI #4653](https://github.com/apache/kyuubi/pull/4653)] [KYUUBI 4650] LogDivertAppender supports reading RollingFileAppender pattern +[[KYUUBI #4669](https://github.com/apache/kyuubi/pull/4669)] LDAP authentication allows auth user contains domain when bind.dn/pw enabled +[[KYUUBI #4688](https://github.com/apache/kyuubi/pull/4688)] Fix the failure to read the operation log after executing catalog and database operation +[[KYUUBI #4691](https://github.com/apache/kyuubi/pull/4691)] Configure FAIL_ON_UNKNOWN_PROPERTIES to false for KyuubiScalaObjectMapper +[[KYUUBI #4713](https://github.com/apache/kyuubi/pull/4713)] Fix false positive result in SchedulerPoolSuite +[[KYUUBI #4719](https://github.com/apache/kyuubi/pull/4719)] Support submission timeout for yarn application manager and get the applicationInfo in-memory +[[KYUUBI #4731](https://github.com/apache/kyuubi/pull/4731)] Support batch session conf advisor +[[KYUUBI #4733](https://github.com/apache/kyuubi/pull/4733)] Introduce Kafka event logger for server events +[[KYUUBI #4738](https://github.com/apache/kyuubi/pull/4738)] `AdminResource.getEngineSpace` should use primary group name on GROUP share level +[[KYUUBI #4739](https://github.com/apache/kyuubi/pull/4739)] Add operation lock instead of locking state Enumeration +[[KYUUBI #4746](https://github.com/apache/kyuubi/pull/4746)] Do not recreate async request executor if has been shutdown +[[KYUUBI #4749](https://github.com/apache/kyuubi/pull/4749)] Fix flaky test issues in SchedulerPoolSuite +[[KYUUBI #4753](https://github.com/apache/kyuubi/pull/4753)] KyuubiConf.unset should not log deprecation warning +[[KYUUBI #4767](https://github.com/apache/kyuubi/pull/4767)] Correct the submit time for BatchJobSubmission and check applicationInfo if submitted application +[[KYUUBI #4777](https://github.com/apache/kyuubi/pull/4777)] Deregister event handlers when stopping server with event handler made auto-closeable +[[KYUUBI #4780](https://github.com/apache/kyuubi/pull/4780)] Get engine application info with interval to prevent frequent call to resource manager +[[KYUUBI #4786](https://github.com/apache/kyuubi/pull/4786)] Support yarn-client and yarn-cluster for YarnApplicationOperation +[[KYUUBI #4790](https://github.com/apache/kyuubi/pull/4790)] Initial implement Batch V2 +[[KYUUBI #4791](https://github.com/apache/kyuubi/pull/4791)] Add helper method to simplify REST enabled judgment +[[KYUUBI #4792](https://github.com/apache/kyuubi/pull/4792)] Enhance hardcode session keywords and remove unused code +[[KYUUBI #4796](https://github.com/apache/kyuubi/pull/4796)] Expose JVM attributes to metrics +[[KYUUBI #4798](https://github.com/apache/kyuubi/pull/4798)] Allows BatchJobSubmission to run in sync mode +[[KYUUBI #4801](https://github.com/apache/kyuubi/pull/4801)] Using different engine submit timeout config for kubernetes and yarn +[[KYUUBI #4807](https://github.com/apache/kyuubi/pull/4807)] Fix flaky test "basic batch rest client" +[[KYUUBI #4807](https://github.com/apache/kyuubi/pull/4807)] Retry more times to fix flaky test "spnego batch rest client" +[[KYUUBI #4811](https://github.com/apache/kyuubi/pull/4811)] Do not update app info after batch or application terminated +[[KYUUBI #4813](https://github.com/apache/kyuubi/pull/4813)] Add the fetch orientation parameter to the /v1/operations/:operationId/log interface +[[KYUUBI #4814](https://github.com/apache/kyuubi/pull/4814)] Introduce Apache Atlas hook support in lineage plugin +[[KYUUBI #4816](https://github.com/apache/kyuubi/pull/4816)] Correct the implementation of cleanup K8s terminated appInfo +[[KYUUBI #4829](https://github.com/apache/kyuubi/pull/4829)] Support to expose operation metrics +[[KYUUBI #4830](https://github.com/apache/kyuubi/pull/4830)] Fix flaky test "support to interrupt the thrift request if remote engine is broken" +[[KYUUBI #4835](https://github.com/apache/kyuubi/pull/4835)] Using hive conf to check whether to apply HIVE_DELEGATION_TOKEN +[[KYUUBI #4836](https://github.com/apache/kyuubi/pull/4836)] Set UncaughtExceptionHandler for thread to log exception +[[KYUUBI #4840](https://github.com/apache/kyuubi/pull/4840)] Return cached appInfo iif both op and app are terminated +[[KYUUBI #4841](https://github.com/apache/kyuubi/pull/4841)] Log more info for closing session +[[KYUUBI #4842](https://github.com/apache/kyuubi/pull/4842)] Fix typo in etcd client config +[[KYUUBI #4843](https://github.com/apache/kyuubi/pull/4843)] Support multiple kubernetes contexts and namespaces +[[KYUUBI #4847](https://github.com/apache/kyuubi/pull/4847)] Close the session immediately when engine corrupt +[[KYUUBI #4847](https://github.com/apache/kyuubi/pull/4847)] Close the session immediately when engine connection closed +[[KYUUBI #4850](https://github.com/apache/kyuubi/pull/4850)] Change default EC spec to secp521r1 for internal session variable signature +[[KYUUBI #4855](https://github.com/apache/kyuubi/pull/4855)] Change socket "connection refused" message match pattern +[[KYUUBI #4857](https://github.com/apache/kyuubi/pull/4857)] Fix flaky test TFrontendServiceSuite +[[KYUUBI #4858](https://github.com/apache/kyuubi/pull/4858)] Relax test message assertion in "sync query causes engine crash" +[[KYUUBI #4859](https://github.com/apache/kyuubi/pull/4859)] HttpException message should not be null +[[KYUUBI #4865](https://github.com/apache/kyuubi/pull/4865)] Fix flaky test BackendServiceMetricSuite +[[KYUUBI #4876](https://github.com/apache/kyuubi/pull/4876)] fix Flaky test BackendServiceMetricSuite (#4924) +[[KYUUBI #4889](https://github.com/apache/kyuubi/pull/4889)] Admin command line supports list server command +[[KYUUBI #4890](https://github.com/apache/kyuubi/pull/4890)] Return operation metrics in OperationData and fix typo +[[KYUUBI #4893](https://github.com/apache/kyuubi/pull/4893)] Prevent null collection for rest dto +[[KYUUBI #4898](https://github.com/apache/kyuubi/pull/4898)] Fix logOperation multiple read with missing line +[[KYUUBI #4919](https://github.com/apache/kyuubi/pull/4919)] Fix the flaky tests in the "open batch session" functionality within the "BatchesResourceSuite" +[[KYUUBI #4920](https://github.com/apache/kyuubi/pull/4920)] Add OperationRestApi +[[KYUUBI #4946](https://github.com/apache/kyuubi/pull/4946)] Alter the order of initLoggerEventHandler +[[KYUUBI #4949](https://github.com/apache/kyuubi/pull/4949)] For operation getNextRowSet method, with operation lock required +[[KYUUBI #4950](https://github.com/apache/kyuubi/pull/4950)] Migrate Kyuubi embedded database from Derby to SQLite +[[KYUUBI #4950](https://github.com/apache/kyuubi/pull/4950)] Misc improvement for usage of SQLite +[[KYUUBI #4950](https://github.com/apache/kyuubi/pull/4950)] SQLite schema files should start from 1.8.0 +[[KYUUBI #4951](https://github.com/apache/kyuubi/pull/4951)] Add reload4j support +[[KYUUBI #4977](https://github.com/apache/kyuubi/pull/4977)] Log error message when REST API invocation error occurs +[[KYUUBI #4978](https://github.com/apache/kyuubi/pull/4978)] Fix flaky test: close expired operations +[[KYUUBI #4978](https://github.com/apache/kyuubi/pull/4978)] Fix flaky test: close expired operations +[[KYUUBI #4979](https://github.com/apache/kyuubi/pull/4979)] Fix flaky test: support to interrupt the thrift request if remote engine is broken +[[KYUUBI #4981](https://github.com/apache/kyuubi/pull/4981)] Refactor code of closeBatchSession +[[KYUUBI #4985](https://github.com/apache/kyuubi/pull/4985)] Fix log level restore behavior of `KyuubiFunSuite#withLogAppender` +[[KYUUBI #4986](https://github.com/apache/kyuubi/pull/4986)] Always use Files#deleteIfExists +[[KYUUBI #4994](https://github.com/apache/kyuubi/pull/4994)] Respect the engine type and share level for listing all engines +[[KYUUBI #4994](https://github.com/apache/kyuubi/pull/4994)] Support listing all engines +[[KYUUBI #4996](https://github.com/apache/kyuubi/pull/4996)] Support to refresh kubernetes configs dynamically +[[KYUUBI #5004](https://github.com/apache/kyuubi/pull/5004)] Fix typo for kubernetes allowed context and namespace check +[[KYUUBI #5010](https://github.com/apache/kyuubi/pull/5010)] Make Kyuubi server's connection URL configurable +[[KYUUBI #5029](https://github.com/apache/kyuubi/pull/5029)] Close the alive probe session after engine session closed +[[KYUUBI #5036](https://github.com/apache/kyuubi/pull/5036)] Fix Operation.close not update complete timestamp +[[KYUUBI #5043](https://github.com/apache/kyuubi/pull/5043)] Destroy the build process when waitCompletion is false and the engine is running in cluster mode +[[KYUUBI #5050](https://github.com/apache/kyuubi/pull/5050)] Add admin checking in `getProxyUser` +[[KYUUBI #5056](https://github.com/apache/kyuubi/pull/5056)] Return more helpful error message to http client +[[KYUUBI #5063](https://github.com/apache/kyuubi/pull/5063)] Support to filter batch with batch name +[[KYUUBI #5065](https://github.com/apache/kyuubi/pull/5065)] Call destroy first on killing Spark startup process to allows it release temp files +[[KYUUBI #5065](https://github.com/apache/kyuubi/pull/5065)] Graceful close the process when launch engine timeout +[[KYUUBI #5078](https://github.com/apache/kyuubi/pull/5078)] Make `kyuubi_instance` nullable in metadata table schema +[[KYUUBI #5081](https://github.com/apache/kyuubi/pull/5081)] Minor refactor JDBCMetadataStore +[[KYUUBI #5089](https://github.com/apache/kyuubi/pull/5089)] Limit maximum engine startup concurrency of kyuubi server +[[KYUUBI #5130](https://github.com/apache/kyuubi/pull/5130)] Allow setting Zookeeper SPN using Kyuubi configuration +[[KYUUBI #5131](https://github.com/apache/kyuubi/pull/5131)] Create index on `metastore`.`create_time` +[[KYUUBI #5138](https://github.com/apache/kyuubi/pull/5138)] Fix flaky test BatchesResourceSuite - get batch session list +[[KYUUBI #5145](https://github.com/apache/kyuubi/pull/5145)] Change embedded Zookeeper server to method-local variable +[[KYUUBI #5158](https://github.com/apache/kyuubi/pull/5158)] Allow embedded Zookeeper binding IP address +[[KYUUBI #5165](https://github.com/apache/kyuubi/pull/5165)] Build Spark Driver/Executor Pod Name(Prefix) in process +[[KYUUBI #5185](https://github.com/apache/kyuubi/pull/5185)] Use Set collection for order-insensitive configs +[[KYUUBI #5187](https://github.com/apache/kyuubi/pull/5187)] Fix info message in UserFilter +[[KYUUBI #5196](https://github.com/apache/kyuubi/pull/5196)] Extract spark core scala version lazily and respect engine env +[[KYUUBI #5201](https://github.com/apache/kyuubi/pull/5201)] Allow disabling batch impl v2 on globally +[[KYUUBI #5204](https://github.com/apache/kyuubi/pull/5204)] Thrift HTTP FE checks req conf nullable before evaluating proxy user +[[KYUUBI #5206](https://github.com/apache/kyuubi/pull/5206)] Try to kill pod with label if no ApplicationInfo found to prevent pod leak +[[KYUUBI #5210](https://github.com/apache/kyuubi/pull/5210)] Cancel operation will cause the log file node to leak +[[KYUUBI #5213](https://github.com/apache/kyuubi/pull/5213)] Check config value by enum values +[[KYUUBI #5216](https://github.com/apache/kyuubi/pull/5216)] Workaround for negative counter in SessionLimiter +[[KYUUBI #5218](https://github.com/apache/kyuubi/pull/5218)] Improve logs of KubernetesApplicationOperation +[[KYUUBI #5220](https://github.com/apache/kyuubi/pull/5220)] Batch submitter should only block submitting stage +[[KYUUBI #5220](https://github.com/apache/kyuubi/pull/5220)] Batch submitted considers application state +[[KYUUBI #5243](https://github.com/apache/kyuubi/pull/5243)] Distinguish metadata between batch impl v2 and recovery +[[KYUUBI #5289](https://github.com/apache/kyuubi/pull/5289)] RESTful API should always print audit log +[[KYUUBI #5290](https://github.com/apache/kyuubi/pull/5290)] Batch impl v2 should pick batch jobs in FIFO +[[KYUUBI #5300](https://github.com/apache/kyuubi/pull/5300)] Batch v2 should not update kyuubi_instance when open session +[[KYUUBI #5305](https://github.com/apache/kyuubi/pull/5305)] Return dummy log when batch job before being added to session manager +[[KYUUBI #5306](https://github.com/apache/kyuubi/pull/5306)] YarnApplicationOperation supports proxy user +[[KYUUBI #5310](https://github.com/apache/kyuubi/pull/5310)] Batch session recovery should start after HTTP server getting started +[[KYUUBI #5327](https://github.com/apache/kyuubi/pull/5327)] Add priority field in batch session metadata table definition +[[KYUUBI #5328](https://github.com/apache/kyuubi/pull/5328)] Batch supports priority scheduling +[[KYUUBI #5340](https://github.com/apache/kyuubi/pull/5340)] AdminResourceSuite make engine idle time 3min +[[KYUUBI #5380](https://github.com/apache/kyuubi/pull/5380)] Create PySpark batch jobs tests for RESTful API +[[KYUUBI #5381](https://github.com/apache/kyuubi/pull/5381)] Change the default metrics reporter to Prometheus +[[KYUUBI #5391](https://github.com/apache/kyuubi/pull/5391)] Close batch session after batch operation completion for idle timeout +[[KYUUBI #5392](https://github.com/apache/kyuubi/pull/5392)] Add query timeout monitor on server-side in ExecuteStatement +[[KYUUBI #5408](https://github.com/apache/kyuubi/pull/5408)] MetadataManager tries MySQL 8 driver class first +[[KYUUBI #5412](https://github.com/apache/kyuubi/pull/5412)] Resolve the relative zk configuration dir based on KYUUBI_HOME +[[KYUUBI #5441](https://github.com/apache/kyuubi/pull/5441)] Make the configuration kyuubi.zookeeper.embedded.data.log.dir effective +[[KYUUBI #5451](https://github.com/apache/kyuubi/pull/5451)] Ignore NoSuchFileException during OperationLog.close() +[[KYUUBI #5499](https://github.com/apache/kyuubi/pull/5499)] [[KYUUBI #2503](https://github.com/apache/kyuubi/pull/2503)] Catch any exception when closing idle session +[[KYUUBI #5513](https://github.com/apache/kyuubi/pull/5513)] Always redirect delete batch request to Kyuubi instance that owns batch session +[[KYUUBI #5522](https://github.com/apache/kyuubi/pull/5522)] Ignore main class for PySpark batch job submission +[[KYUUBI #5528](https://github.com/apache/kyuubi/pull/5528)] Remove duplicate properties tag and respect KYUUBI_LOG_DIR +[[KYUUBI #5566](https://github.com/apache/kyuubi/pull/5566)] InternalRestClient respects `kyuubi.engine.security.enabled` to add HTTP auth header +[[KYUUBI #5566](https://github.com/apache/kyuubi/pull/5566)] Check InternalSecurityAccessor is initialized only when `kyuubi.engine.security.enabled` is true +[[KYUUBI #5568](https://github.com/apache/kyuubi/pull/5568)] Check administrator only when security is enabled +[[KYUUBI #5568](https://github.com/apache/kyuubi/pull/5568)] Fix security enabled judgement +[[KYUUBI #5606](https://github.com/apache/kyuubi/pull/5606)] Handle engine listing request properly for users who have not created engine + +### Client + +[[KYUUBI #3887](https://github.com/apache/kyuubi/pull/3887)] Fix kyuubiServerPrincipal logic in KyuubiCommands +[[KYUUBI #4305](https://github.com/apache/kyuubi/pull/4305)] Backport HIVE-15820: comment at the head of beeline -e +[[KYUUBI #4438](https://github.com/apache/kyuubi/pull/4438)] Render the batch info map in kyuubi-ctl +[[KYUUBI #4439](https://github.com/apache/kyuubi/pull/4439)] Add dto class for operation data +[[KYUUBI #4452](https://github.com/apache/kyuubi/pull/4452)] Strip the redundant leading and tailing slash of getZooKeeperNamespace. +[[KYUUBI #4479](https://github.com/apache/kyuubi/pull/4479)] Restore JDBC Kerberos authentication behavior for UGI.doAs +[[KYUUBI #4489](https://github.com/apache/kyuubi/pull/4489)] Fix missing `org.apache.commons.logging` by including `jcl-over-slf4j` dependency in REST client +[[KYUUBI #4492](https://github.com/apache/kyuubi/pull/4492)] Correct engine subdomain calculation in `kyuubi-ctl` +[[KYUUBI #4508](https://github.com/apache/kyuubi/pull/4508)] Beeline should reset stdin after consuming init SQL file +[[KYUUBI #4581](https://github.com/apache/kyuubi/pull/4581)] Add KyuubiCommands parse python snippets unit test +[[KYUUBI #4609](https://github.com/apache/kyuubi/pull/4609)] get engineRefId in KyuubiConnection +[[KYUUBI #4619](https://github.com/apache/kyuubi/pull/4619)] Fix beeline with -e when there are other SQL statements before the source statement, the source statement cannot be executed normally +[[KYUUBI #4657](https://github.com/apache/kyuubi/pull/4657)] Building rest client to kyuubi instance including original host urls +[[KYUUBI #4962](https://github.com/apache/kyuubi/pull/4962)] Backport HIVE-19048: init script errors are ignored +[[KYUUBI #4965](https://github.com/apache/kyuubi/pull/4965)] Beeline supports `--python-mode` option and remove comments for non-python mode +[[KYUUBI #4969](https://github.com/apache/kyuubi/pull/4969)] Run JUnit tests on beeline module +[[KYUUBI #5054](https://github.com/apache/kyuubi/pull/5054)] Refine the kyuubi client version properties file to prevent conflicts +[[KYUUBI #5100](https://github.com/apache/kyuubi/pull/5100)] Fix unpaired quote in message pattern +[[KYUUBI #5101](https://github.com/apache/kyuubi/pull/5101)] Remove unused StringBuilders in ResultSets +[[KYUUBI #5129](https://github.com/apache/kyuubi/pull/5129)] KyuubiBeeline should redirect JDK logging +[[KYUUBI #5179](https://github.com/apache/kyuubi/pull/5179)] Use Iterable instead of Seq in kyuubi-ctl commands +[[KYUUBI #5297](https://github.com/apache/kyuubi/pull/5297)] New RetryableClient get http://null server uri from metadata.kyuubiInstance +[[KYUUBI #5480](https://github.com/apache/kyuubi/pull/5480)] Support setting kyuubi hive jdbc client protocol version +[[KYUUBI #5564](https://github.com/apache/kyuubi/pull/5564)] Support to config ticket cache location in JDBC URL + +### Web UI + +[[KYUUBI #3420](https://github.com/apache/kyuubi/pull/3420)] Kyuubi Server Proxy Engine UI +[[KYUUBI #3646](https://github.com/apache/kyuubi/pull/3646)] Front-end style should bracket same line +[[KYUUBI #3646](https://github.com/apache/kyuubi/pull/3646)] Init Session Statistic Page +[[KYUUBI #3648](https://github.com/apache/kyuubi/pull/3648)] Add Session Detail Page +[[KYUUBI #3650](https://github.com/apache/kyuubi/pull/3650)] Add Operation Statistics Page +[[KYUUBI #3652](https://github.com/apache/kyuubi/pull/3652)] Add Kyuubi Server Management page +[[KYUUBI #3654](https://github.com/apache/kyuubi/pull/3654)] Add Engine Manager Page +[[KYUUBI #4537](https://github.com/apache/kyuubi/pull/4537)] Enable Vite proxy server for web ui development +[[KYUUBI #4565](https://github.com/apache/kyuubi/pull/4565)] Fix out-date `README.md` about `Development Project` +[[KYUUBI #4622](https://github.com/apache/kyuubi/pull/4622)] Session api use admin api for security +[[KYUUBI #4626](https://github.com/apache/kyuubi/pull/4626)] Fix Web UI CI check fail with pnpm the lockfile needs updates +[[KYUUBI #4757](https://github.com/apache/kyuubi/pull/4757)] Move Statistics to Management Menu +[[KYUUBI #4982](https://github.com/apache/kyuubi/pull/4982)] Add query string to forward requests to Engine UI +[[KYUUBI #5259](https://github.com/apache/kyuubi/pull/5259)] Web UI supports Swagger pages +[[KYUUBI #5463](https://github.com/apache/kyuubi/pull/5463)] Adjust the file directory structure +[[KYUUBI #5484](https://github.com/apache/kyuubi/pull/5484)] Remove legacy Web UI +[[KYUUBI #5573](https://github.com/apache/kyuubi/pull/5573)] Delete parts of the Kyuubi Web UI that are not useful +[[KYUUBI #5586](https://github.com/apache/kyuubi/pull/5586)] Kyuubi Web UI show current version +[[KYUUBI #5608](https://github.com/apache/kyuubi/pull/5608)] Kyuubi version should always use the light color +[[KYUUBI #5620](https://github.com/apache/kyuubi/pull/5620)] Remove dummy information card from Overview page + +### Documentation + +[[KYUUBI #4308](https://github.com/apache/kyuubi/pull/4308)] Make README more welcoming +[[KYUUBI #4312](https://github.com/apache/kyuubi/pull/4312)] Fix over-matched README pattern for markdown styling +[[KYUUBI #4367](https://github.com/apache/kyuubi/pull/4367)] Kyuubi requires Flink 1.16/1.17 +[[KYUUBI #4387](https://github.com/apache/kyuubi/pull/4387)] Update Flink version requirements +[[KYUUBI #4568](https://github.com/apache/kyuubi/pull/4568)] Make kyuubi-ctl doc enable variable 'release' automatic substitution +[[KYUUBI #4583](https://github.com/apache/kyuubi/pull/4583)] Rename Flink Table Store to Apache Paimon (Incubating) +[[KYUUBI #4593](https://github.com/apache/kyuubi/pull/4593)] Keep promote the release step update-to-date +[[KYUUBI #4599](https://github.com/apache/kyuubi/pull/4599)] Simplify project version evaluation +[[KYUUBI #4607](https://github.com/apache/kyuubi/pull/4607)] Rename Flink Table Store to Apache Paimon (Incubating) in docs `Connectors for Hive SQL Query Engine` +[[KYUUBI #4608](https://github.com/apache/kyuubi/pull/4608)] Rename Flink Table Store to Apache Paimon (Incubating) in docs `Connectors for Trino SQL Query Engine` +[[KYUUBI #4614](https://github.com/apache/kyuubi/pull/4614)] Rename Flink Table Store to Apache Paimon (Incubating) in docs `Connectors for Flink SQL Query Engine` +[[KYUUBI #4624](https://github.com/apache/kyuubi/pull/4624)] Fix table headers in kyuubi_vs_hive.md +[[KYUUBI #4627](https://github.com/apache/kyuubi/pull/4627)] Fix a typo in rest_api.md. +[[KYUUBI #4655](https://github.com/apache/kyuubi/pull/4655)] Enrich docs for Kyuubi Hive JDBC driver +[[KYUUBI #4655](https://github.com/apache/kyuubi/pull/4655)] Recover the missing JDBC URL +[[KYUUBI #4722](https://github.com/apache/kyuubi/pull/4722)] Kyuubi requires Spark 3.1 and above now +[[KYUUBI #4727](https://github.com/apache/kyuubi/pull/4727)] kyuubi-spark-lineage has no transitive deps +[[KYUUBI #4734](https://github.com/apache/kyuubi/pull/4734)] Fix typo in docs of custom event handler +[[KYUUBI #4742](https://github.com/apache/kyuubi/pull/4742)] Add docs for Flink application mode +[[KYUUBI #4763](https://github.com/apache/kyuubi/pull/4763)] Fix the Kyuubi JDBC kerberos parameters +[[KYUUBI #4800](https://github.com/apache/kyuubi/pull/4800)] Update readthedocs.yaml +[[KYUUBI #4864](https://github.com/apache/kyuubi/pull/4864)] Fix grammar in HA section +[[KYUUBI #4933](https://github.com/apache/kyuubi/pull/4933)] Mark `spark.sql.optimizer.insertRepartitionNum` config for Spark 3.1 only +[[KYUUBI #4954](https://github.com/apache/kyuubi/pull/4954)] Fix variable reference in quick_start_with_jdbc +[[KYUUBI #4960](https://github.com/apache/kyuubi/pull/4960)] Fix doc for pyhive client. +[[KYUUBI #5049](https://github.com/apache/kyuubi/pull/5049)] PyHive Kerberos usage doc +[[KYUUBI #5083](https://github.com/apache/kyuubi/pull/5083)] Add LDAP document +[[KYUUBI #5085](https://github.com/apache/kyuubi/pull/5085)] Update config section of deploy on kubernetes +[[KYUUBI #5094](https://github.com/apache/kyuubi/pull/5094)] Add Python JayDeBeApi doc +[[KYUUBI #5099](https://github.com/apache/kyuubi/pull/5099)] Document link failure +[[KYUUBI #5118](https://github.com/apache/kyuubi/pull/5118)] Add Flink KDF doc +[[KYUUBI #5122](https://github.com/apache/kyuubi/pull/5122)] Hive KDF usage +[[KYUUBI #5127](https://github.com/apache/kyuubi/pull/5127)] Improvements for markdown builder +[[KYUUBI #5146](https://github.com/apache/kyuubi/pull/5146)] Fix link of IntelliJ IDEA Setup Guide +[[KYUUBI #5153](https://github.com/apache/kyuubi/pull/5153)] Minor change w/ to with +[[KYUUBI #5154](https://github.com/apache/kyuubi/pull/5154)] Move configuration docs to the top level +[[KYUUBI #5162](https://github.com/apache/kyuubi/pull/5162)] Add statements about contributions co-authored with AIGC in developer guide page +[[KYUUBI #5178](https://github.com/apache/kyuubi/pull/5178)] Make MarkdownBuilder not extending Growable +[[KYUUBI #5227](https://github.com/apache/kyuubi/pull/5227)] Added deploy mode info +[[KYUUBI #5247](https://github.com/apache/kyuubi/pull/5247)] Correct building command for Spark authz/lineage plugins +[[KYUUBI #5273](https://github.com/apache/kyuubi/pull/5273)] Update release docs +[[KYUUBI #5275](https://github.com/apache/kyuubi/pull/5275)] Improve and fix comparison and regeneration for golden files +[[KYUUBI #5294](https://github.com/apache/kyuubi/pull/5294)] Update supported dialects for JDBC engine +[[KYUUBI #5335](https://github.com/apache/kyuubi/pull/5335)] Set markdown file EOL +[[KYUUBI #5345](https://github.com/apache/kyuubi/pull/5345)] Fix non-kerberized code description in quick_start_with_jdbc +[[KYUUBI #5347](https://github.com/apache/kyuubi/pull/5347)] Fix java code errors in quick_start_with_jdbc +[[KYUUBI #5409](https://github.com/apache/kyuubi/pull/5409)] Update config docs regeneration hints in developer guide +[[KYUUBI #5432](https://github.com/apache/kyuubi/pull/5432)] Fix typo in README.md +[[KYUUBI #5523](https://github.com/apache/kyuubi/pull/5523)] Update the Kyuubi supported components version + +### Building, Infrastructure and Dependency + +[[KYUUBI #4078](https://github.com/apache/kyuubi/pull/4078)] Fix shellcheck violations in scripts of /bin folder +[[KYUUBI #4187](https://github.com/apache/kyuubi/pull/4187)] Windows build-info script supports revision time +[[KYUUBI #4278](https://github.com/apache/kyuubi/pull/4278)] Use new Apache 'closer.lua' syntax to obtain Maven +[[KYUUBI #4295](https://github.com/apache/kyuubi/pull/4295)] Introduce `super-linter` action for linting JSON, XML, ENV files and bash_exec +[[KYUUBI #4299](https://github.com/apache/kyuubi/pull/4299)] Bump JUnit to 4.13.2 +[[KYUUBI #4299](https://github.com/apache/kyuubi/pull/4299)] Bump MySQL JDBC driver to 8.0.32 +[[KYUUBI #4315](https://github.com/apache/kyuubi/pull/4315)] Check modules available before build step in dependency workflow +[[KYUUBI #4347](https://github.com/apache/kyuubi/pull/4347)] Bump maven download plugin from 1.6.6 to 1.6.8 +[[KYUUBI #4348](https://github.com/apache/kyuubi/pull/4348)] Cache engine archives in CI jobs for maven download plugin +[[KYUUBI #4397](https://github.com/apache/kyuubi/pull/4397)] `build/dist` supports `--web-ui` +[[KYUUBI #4413](https://github.com/apache/kyuubi/pull/4413)] Build docker image should include web UI +[[KYUUBI #4447](https://github.com/apache/kyuubi/pull/4447)] Bump log4j from 2.19.0 to 2.20.0 +[[KYUUBI #4448](https://github.com/apache/kyuubi/pull/4448)] Bump Netty from 4.1.87.Final to 4.1.89.Final +[[KYUUBI #4451](https://github.com/apache/kyuubi/pull/4451)] Skip etcd test if no docker env +[[KYUUBI #4453](https://github.com/apache/kyuubi/pull/4453)] Bump Kubernetes Client Version to 6.4.1 +[[KYUUBI #4461](https://github.com/apache/kyuubi/pull/4461)] Add `Kyuubi Kubernetes IT` Module to CI style check +[[KYUUBI #4466](https://github.com/apache/kyuubi/pull/4466)] Use bitnami/minio docker image +[[KYUUBI #4471](https://github.com/apache/kyuubi/pull/4471)] Prompted command for docs generation should skip Java tests +[[KYUUBI #4484](https://github.com/apache/kyuubi/pull/4484)] [K8S][HELM] Set specific Kyuubi version to the chart appVersion +[[KYUUBI #4494](https://github.com/apache/kyuubi/pull/4494)] `bin/kyuubi` should use `exec` to run Kyuubi server +[[KYUUBI #4513](https://github.com/apache/kyuubi/pull/4513)] Bump Kyuubi 1.7.0 in Playground +[[KYUUBI #4546](https://github.com/apache/kyuubi/pull/4546)] Fully exclude `metrics` dir from git +[[KYUUBI #4557](https://github.com/apache/kyuubi/pull/4557)] [K8S][HELM] Kyuubi server should bind Pod IP by default +[[KYUUBI #4572](https://github.com/apache/kyuubi/pull/4572)] Bump Iceberg from 1.1.0 to 1.2.0 +[[KYUUBI #4576](https://github.com/apache/kyuubi/pull/4576)] Multi arch build for ARM support in Docker Hub +[[KYUUBI #4589](https://github.com/apache/kyuubi/pull/4589)] [K8S][HELM] Update template on port-forward usage guide NOTES.txt +[[KYUUBI #4590](https://github.com/apache/kyuubi/pull/4590)] Bump delta from 2.2.0 to 2.3.0 +[[KYUUBI #4598](https://github.com/apache/kyuubi/pull/4598)] Bump Flink-1.15 from 1.15.3 to 1.15.4 +[[KYUUBI #4600](https://github.com/apache/kyuubi/pull/4600)] Bump Iceberg 1.2.0 for playground +[[KYUUBI #4601](https://github.com/apache/kyuubi/pull/4601)] Bump Hadoop 3.3.5 for playground +[[KYUUBI #4605](https://github.com/apache/kyuubi/pull/4605)] [K8S][HELM] Set IfNotPresent pullPolicy by default +[[KYUUBI #4606](https://github.com/apache/kyuubi/pull/4606)] [K8S][HELM] Add command and args configuration support +[[KYUUBI #4615](https://github.com/apache/kyuubi/pull/4615)] Bump Ranger from 2.3.0 to 2.4.0 +[[KYUUBI #4628](https://github.com/apache/kyuubi/pull/4628)] Add Dependency management ISSUE_TEMPLATE +[[KYUUBI #4632](https://github.com/apache/kyuubi/pull/4632)] Add rule for labeler to tag `module:ui` +[[KYUUBI #4638](https://github.com/apache/kyuubi/pull/4638)] Update dependency.yml as label in not permitted in markdown block +[[KYUUBI #4652](https://github.com/apache/kyuubi/pull/4652)] Upgrade Hadoop from 3.3.4 to 3.3.5 +[[KYUUBI #4695](https://github.com/apache/kyuubi/pull/4695)] Bump super-linter action from v4 to v5 +[[KYUUBI #4696](https://github.com/apache/kyuubi/pull/4696)] Upgrade scalafmt from 3.7.1 to 3.7.3 +[[KYUUBI #4697](https://github.com/apache/kyuubi/pull/4697)] [K8S][HELM] Add template rendering info to README +[[KYUUBI #4698](https://github.com/apache/kyuubi/pull/4698)] [K8S][HELM] Centralize Kyuubi labels definition +[[KYUUBI #4706](https://github.com/apache/kyuubi/pull/4706)] [K8S][HELM] Use template comments for the chart license header +[[KYUUBI #4707](https://github.com/apache/kyuubi/pull/4707)] [K8S][HELM] Fix README typo and minor reformatting +[[KYUUBI #4712](https://github.com/apache/kyuubi/pull/4712)] Bump Spark from 3.2.3 to 3.2.4 +[[KYUUBI #4737](https://github.com/apache/kyuubi/pull/4737)] Restore Project & Community Status in README.md +[[KYUUBI #4744](https://github.com/apache/kyuubi/pull/4744)] Remove Hudi integration tests +[[KYUUBI #4747](https://github.com/apache/kyuubi/pull/4747)] Install and cache build/mvn in CI builds +[[KYUUBI #4768](https://github.com/apache/kyuubi/pull/4768)] Guidelines for Document and Code contributions +[[KYUUBI #4772](https://github.com/apache/kyuubi/pull/4772)] Bump Jersey from 2.39 to 2.39.1 +[[KYUUBI #4783](https://github.com/apache/kyuubi/pull/4783)] Use pnpm v8 and update pnpm lockfile to v6 schema +[[KYUUBI #4784](https://github.com/apache/kyuubi/pull/4784)] Use nodejs v18 in CI builds and maven web-ui builds +[[KYUUBI #4788](https://github.com/apache/kyuubi/pull/4788)] [K8S][HELM] Use StatefulSet instead of Deployment +[[KYUUBI #4805](https://github.com/apache/kyuubi/pull/4805)] Bump openai-java from 0.11.1 to 0.12.0 +[[KYUUBI #4808](https://github.com/apache/kyuubi/pull/4808)] Bump Jetty from 9.4.50 to 9.4.51 +[[KYUUBI #4818](https://github.com/apache/kyuubi/pull/4818)] [K8S][HELM] Update default Kyuubi version to 1.7.1 +[[KYUUBI #4819](https://github.com/apache/kyuubi/pull/4819)] [K8S][HELM] Refactor liveliness and readiness probes +[[KYUUBI #4824](https://github.com/apache/kyuubi/pull/4824)] Bump Jackson from 2.14.2 to 2.15.0 +[[KYUUBI #4827](https://github.com/apache/kyuubi/pull/4827)] Bump Apache Arrow from 11.0.0 to 12.0.0 +[[KYUUBI #4828](https://github.com/apache/kyuubi/pull/4828)] Exclude macOS tar extended metadata in build/dist +[[KYUUBI #4849](https://github.com/apache/kyuubi/pull/4849)] Open modules to enable JDK 17 support +[[KYUUBI #4851](https://github.com/apache/kyuubi/pull/4851)] Bump Maven 3.8.8 +[[KYUUBI #4852](https://github.com/apache/kyuubi/pull/4852)] Switch to Kyuubi Shaded Zookeeper +[[KYUUBI #4856](https://github.com/apache/kyuubi/pull/4856)] Add `restore-keys` for `action/cache` +[[KYUUBI #4870](https://github.com/apache/kyuubi/pull/4870)] Add kyuubi-util and kyuubi-util-scala modules +[[KYUUBI #4877](https://github.com/apache/kyuubi/pull/4877)] Bump Flink 1.17 to 1.17.1 +[[KYUUBI #4879](https://github.com/apache/kyuubi/pull/4879)] Refactor and promote reflection utils and cleanup similar reflection methods +[[KYUUBI #4883](https://github.com/apache/kyuubi/pull/4883)] Bump Iceberg 1.3.0 +[[KYUUBI #4887](https://github.com/apache/kyuubi/pull/4887)] Refactor and add ut for ClassUtils +[[KYUUBI #4891](https://github.com/apache/kyuubi/pull/4891)] Bump Flink 1.16.2 +[[KYUUBI #4894](https://github.com/apache/kyuubi/pull/4894)] Bump Mockito from 4.9.0 to 4.11.0 and ScalaTest from 3.2.15 to 3.2.16 +[[KYUUBI #4904](https://github.com/apache/kyuubi/pull/4904)] Move AssertionUtils to kyuubi-util-scala module +[[KYUUBI #4905](https://github.com/apache/kyuubi/pull/4905)] Generalize util method for loading class from service loader +[[KYUUBI #4907](https://github.com/apache/kyuubi/pull/4907)] Add a maven profile 'remote-debug' +[[KYUUBI #4908](https://github.com/apache/kyuubi/pull/4908)] Bump scalafmt from 3.7.3 to 3.7.4 +[[KYUUBI #4911](https://github.com/apache/kyuubi/pull/4911)] Remove duplicate test annotations `org.apache.kyuubi.tags.*` in kyuubi-common +[[KYUUBI #4912](https://github.com/apache/kyuubi/pull/4912)] Replace Scala's assert in tests with scalatest's for prettified error message +[[KYUUBI #4931](https://github.com/apache/kyuubi/pull/4931)] Bump vite from 3.0.0 to 4.2.3 and vitest from 0.22.0 to 0.32.0 +[[KYUUBI #4937](https://github.com/apache/kyuubi/pull/4937)] Remove redundant quoteIfNeeded method +[[KYUUBI #4943](https://github.com/apache/kyuubi/pull/4943)] Bump Guava from 31.1 to 32.0.1 +[[KYUUBI #4944](https://github.com/apache/kyuubi/pull/4944)] Code improvement for Java +[[KYUUBI #4947](https://github.com/apache/kyuubi/pull/4947)] Bump pnpm version in maven's frontend plugin to 8.6.1 for lockfileVersion 6.1 +[[KYUUBI #4948](https://github.com/apache/kyuubi/pull/4948)] Reuse versions of NodeJS and pnpm from maven properties +[[KYUUBI #4959](https://github.com/apache/kyuubi/pull/4959)] Code improvements for Scala +[[KYUUBI #4966](https://github.com/apache/kyuubi/pull/4966)] Bump Apache parent pom from 29 to 30 +[[KYUUBI #4968](https://github.com/apache/kyuubi/pull/4968)] Simplify Option conversion +[[KYUUBI #4970](https://github.com/apache/kyuubi/pull/4970)] Unified reflection methods invokeAs and getField +[[KYUUBI #4974](https://github.com/apache/kyuubi/pull/4974)] Eliminate raw-types compilation warning in KyuubiBeeLine +[[KYUUBI #4975](https://github.com/apache/kyuubi/pull/4975)] Fixed run test link in PR template +[[KYUUBI #4990](https://github.com/apache/kyuubi/pull/4990)] Bump Spark 3.4.1 +[[KYUUBI #4992](https://github.com/apache/kyuubi/pull/4992)] Bump Netty 4.1.93.Final +[[KYUUBI #5001](https://github.com/apache/kyuubi/pull/5001)] Bump Scala from 2.12.17 to 2.12.18 +[[KYUUBI #5003](https://github.com/apache/kyuubi/pull/5003)] [K8S][HELM] Add doc strings to the chart values.yaml +[[KYUUBI #5007](https://github.com/apache/kyuubi/pull/5007)] Bump scalafmt from 3.7.4 to 3.7.5 +[[KYUUBI #5019](https://github.com/apache/kyuubi/pull/5019)] Shows details of compilation info in SparkUI's engine tab +[[KYUUBI #5024](https://github.com/apache/kyuubi/pull/5024)] Bump grpc-protobuf from 1.48.0 to 1.53.0 +[[KYUUBI #5034](https://github.com/apache/kyuubi/pull/5034)] Bump tough-cookie from 4.0.0 to 4.1.3 in /kyuubi-server/web-ui +[[KYUUBI #5039](https://github.com/apache/kyuubi/pull/5039)] Use semantic versions and remove redundant version comparison methods +[[KYUUBI #5058](https://github.com/apache/kyuubi/pull/5058)] [K8S][HELM] Allow the user to set the priorityClass used by Kyuubi server Pod +[[KYUUBI #5059](https://github.com/apache/kyuubi/pull/5059)] Upgrade kubernetes client version to 6.7.2 +[[KYUUBI #5064](https://github.com/apache/kyuubi/pull/5064)] Audit kubernetes application state change in separate log files +[[KYUUBI #5068](https://github.com/apache/kyuubi/pull/5068)] Bump Spark in spark-master profile from 3.5.0-SNAPSHOT to 4.0.0-SNAPSHOT +[[KYUUBI #5070](https://github.com/apache/kyuubi/pull/5070)] Bump word-wrap from 1.2.3 to 1.2.4 in web UI +[[KYUUBI #5072](https://github.com/apache/kyuubi/pull/5072)] Fix KyuubiOperationWithEngineSecuritySuite and related issues +[[KYUUBI #5090](https://github.com/apache/kyuubi/pull/5090)] Fix AllKyuubiConfiguration to generate redundant blank lines in Windows +[[KYUUBI #5113](https://github.com/apache/kyuubi/pull/5113)] Remove dummy Spark Kudu connector module +[[KYUUBI #5116](https://github.com/apache/kyuubi/pull/5116)] Bump Hadoop 3.3.6 +[[KYUUBI #5117](https://github.com/apache/kyuubi/pull/5117)] Code improvements for Java and Scala code +[[KYUUBI #5121](https://github.com/apache/kyuubi/pull/5121)] Extract dev scripts for regenerating the golden files +[[KYUUBI #5132](https://github.com/apache/kyuubi/pull/5132)] [K8S][HELM] Kyuubi add server metrics monitor +[[KYUUBI #5135](https://github.com/apache/kyuubi/pull/5135)] Exclude web-ui coverage reports in rat plugins +[[KYUUBI #5149](https://github.com/apache/kyuubi/pull/5149)] Correct error message of ReflectUtils's invokeAs when method not found +[[KYUUBI #5150](https://github.com/apache/kyuubi/pull/5150)] Bump Spark 3.3.3 +[[KYUUBI #5152](https://github.com/apache/kyuubi/pull/5152)] Check milestone and assignees when merging pull request +[[KYUUBI #5177](https://github.com/apache/kyuubi/pull/5177)] Use Scala binary version placeholder in Maven module's artifactId suffix +[[KYUUBI #5183](https://github.com/apache/kyuubi/pull/5183)] Introducing profile scala-2.13 for development use +[[KYUUBI #5196](https://github.com/apache/kyuubi/pull/5196)] Enable CI Test on Scala 2.13 and support custom or spark-core extracted Scala version for Spark's engine +[[KYUUBI #5202](https://github.com/apache/kyuubi/pull/5202)] Update upload_nexus_staging +[[KYUUBI #5205](https://github.com/apache/kyuubi/pull/5205)] Upgrade `kubernetes-client` to 6.8.1 +[[KYUUBI #5209](https://github.com/apache/kyuubi/pull/5209)] Reformat Scala source code crossing versions +[[KYUUBI #5212](https://github.com/apache/kyuubi/pull/5212)] Fix configuration errors causing by helm charts of prometheus services +[[KYUUBI #5214](https://github.com/apache/kyuubi/pull/5214)] Correct file path for scala tests in util-scala module +[[KYUUBI #5234](https://github.com/apache/kyuubi/pull/5234)] Upgrade Jetty version from 9.4.51.v20230217 to 9.4.52.v20230823 +[[KYUUBI #5235](https://github.com/apache/kyuubi/pull/5235)] Daily deploy snapshot version on master and branch-1.7 +[[KYUUBI #5250](https://github.com/apache/kyuubi/pull/5250)] Bump Iceberg from 1.3.0 to 1.3.1 +[[KYUUBI #5252](https://github.com/apache/kyuubi/pull/5252)] Remove incubator link +[[KYUUBI #5253](https://github.com/apache/kyuubi/pull/5253)] Upgrade testcontainers-scala from 0.40.12 to 0.41.0 +[[KYUUBI #5253](https://github.com/apache/kyuubi/pull/5253)] Supply testcontainers-scala-scalatest deps +[[KYUUBI #5293](https://github.com/apache/kyuubi/pull/5293)] Upgrade snakeyaml from 1.33 to 2.2 +[[KYUUBI #5305](https://github.com/apache/kyuubi/pull/5305)] Fix get local log UT +[[KYUUBI #5308](https://github.com/apache/kyuubi/pull/5308)] Ensure release using Java 8 +[[KYUUBI #5318](https://github.com/apache/kyuubi/pull/5318)] Pin maven-source-plugin 3.2.1 +[[KYUUBI #5358](https://github.com/apache/kyuubi/pull/5358)] Bump iceberg 1.4.0 +[[KYUUBI #5365](https://github.com/apache/kyuubi/pull/5365)] Don't use Log4j2's extended throwable conversion pattern in default logging configurations +[[KYUUBI #5425](https://github.com/apache/kyuubi/pull/5425)] Add thread name and ID to log4j2-defaults.xml +[[KYUUBI #5435](https://github.com/apache/kyuubi/pull/5435)] Improve Kyuubi On Kubernetes IT +[[KYUUBI #5443](https://github.com/apache/kyuubi/pull/5443)] Add kyuubi-kubernetes-it module to reformat script +[[KYUUBI #5444](https://github.com/apache/kyuubi/pull/5444)] Check style with profiles of Spark 3.4 and 3.5 +[[KYUUBI #5483](https://github.com/apache/kyuubi/pull/5483)] Fix deploy command for Scala 2.13 TPC-DS/H connectors +[[KYUUBI #5483](https://github.com/apache/kyuubi/pull/5483)] Release Spark TPC-H/DS Connectors with Scala 2.13 +[[KYUUBI #5486](https://github.com/apache/kyuubi/pull/5486)] Bump Kafka client version from 3.4.0 to 3.5.1 +[[KYUUBI #5508](https://github.com/apache/kyuubi/pull/5508)] Upgrade download-maven-plugin from 1.6.8 to 1.7.1 +[[KYUUBI #5544](https://github.com/apache/kyuubi/pull/5544)] Release contains Spark 3.5 extension +[[KYUUBI #5561](https://github.com/apache/kyuubi/pull/5561)] Binary distribution artifact ships database schema scripts +[[KYUUBI #5561](https://github.com/apache/kyuubi/pull/5561)] Rename folder to db-scripts +[[KYUUBI #5569](https://github.com/apache/kyuubi/pull/5569)] Fix pnpm-lock file version + +### Credits + +Last but not least, this release would not have been possible without the following contributors: + +* Alex Zou +* Anurag Rajawat +* Binjie Yang +* Bowen Liang +* Bruce Wong +* CavemanIV +* Chao Chen +* Cheng Pan +* Chenliang Lu +* David Yuan +* Deng An +* Denis Krivenko +* Derick Cypho +* Drake Youngkun Min +* Fei Wang +* FourSpaces +* Fu Chen +* GitHub Bot +* Guanhua Li +* Hanna Liashchuk +* He Zhao +* Junjie Zu +* Kaifei Yi +* Kang Wang +* Karsonnel +* Kent Yao +* Kyofin +* Miao Wang +* Min Zhao +* Naijie Liu +* Omkar Makhare +* Paul Lin +* Peiyue Liu +* Pengqi Li +* Qiang Zhang +* QianyongY +* Remzi Yang +* Rojin Raju +* Ruguo Yu +* Ruilei Ma +* Senmiao Liu +* Shaoyun Chen +* Sidhant Nagpal +* Tianlin Liao +* Xianxun Ye +* Xiduo You +* Xieming Li +* XorSum +* Xu Guo +* Yang Du +* Yangming Zhao +* Yaodong Zhang +* Yaogai Zhu +* Yi Zhu +* Yifan Zhou +* Zhen Wang +* biangjuang +* camper42 +* fantasticKe +* huangzhir +* huapan123456 +* liaoyt +* mans2singh +* mrtisttt +* phyyou +* rhh777 +* runzhliu +* touchida +* uicosp +* wangyepeng2 +* weixi62961 +* yehere
