This is an automated email from the ASF dual-hosted git repository.
bowenliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new f74a70550 [KYUUBI #5127] [DOC] Improvements for markdown builder
f74a70550 is described below
commit f74a70550e05e59213a92285d6bc9f18ae1ae198
Author: liangbowen <[email protected]>
AuthorDate: Fri Aug 4 14:09:11 2023 +0800
[KYUUBI #5127] [DOC] Improvements for markdown builder
### _Why are the changes needed?_
- use `ListBuffer` instead of `ArrayBuffer` for inner string buffer, to
minimize allocations for resizing
- handy `+=` operator in chaining style without explicit quotes, to make
the user focus on content assembly and less distraction of quoting
- make `MarkdownBuilder` extending `Growable`, to utilize semantic
operators like `+=` and `++=` which is unified inside for single or batch
operation
- use `this +=` rather than `line(...)` , to reflect side effects in
semantic way
- change list to stream for output comparison
- remove unused methods
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run
test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests)
locally before make a pull request
Closes #5127 from bowenliang123/md-buff.
Closes #5127
458e18c3d [liangbowen] Improvements for markdown builder
Authored-by: liangbowen <[email protected]>
Signed-off-by: liangbowen <[email protected]>
---
.../hive/udf/KyuubiDefinedFunctionSuite.scala | 13 +-
.../spark/udf/KyuubiDefinedFunctionSuite.scala | 13 +-
.../scala/org/apache/kyuubi/MarkdownUtils.scala | 88 ++++-------
.../kyuubi/config/AllKyuubiConfiguration.scala | 161 ++++++++++-----------
4 files changed, 117 insertions(+), 158 deletions(-)
diff --git
a/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/udf/KyuubiDefinedFunctionSuite.scala
b/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/udf/KyuubiDefinedFunctionSuite.scala
index 2a7bfdfc2..f76eefe84 100644
---
a/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/udf/KyuubiDefinedFunctionSuite.scala
+++
b/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/udf/KyuubiDefinedFunctionSuite.scala
@@ -48,17 +48,16 @@ class KyuubiDefinedFunctionSuite extends KyuubiFunSuite {
test("verify or update kyuubi hive sql functions") {
val builder = MarkdownBuilder(licenced = true, getClass.getName)
- builder
- .line("# Auxiliary SQL Functions")
- .line("""Kyuubi provides several auxiliary SQL functions as supplement
to Hive's
+ builder += "# Auxiliary SQL Functions" +=
+ """Kyuubi provides several auxiliary SQL functions as supplement to
Hive's
| [Built-in
Functions](https://cwiki.apache.org/confluence/display/hive/languagemanual+udf#
- |LanguageManualUDF-Built-inFunctions)""")
- .lines("""
+ |LanguageManualUDF-Built-inFunctions)""" ++=
+ """
| Name | Description | Return Type | Since
| --- | --- | --- | ---
- |""")
+ |"""
KDFRegistry.registeredFunctions.foreach { func =>
- builder.line(s"${func.name} | ${func.description} | ${func.returnType} |
${func.since}")
+ builder += s"${func.name} | ${func.description} | ${func.returnType} |
${func.since}"
}
MarkdownUtils.verifyOutput(
diff --git
a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunctionSuite.scala
b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunctionSuite.scala
index 566126bb4..72f0a198b 100644
---
a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunctionSuite.scala
+++
b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunctionSuite.scala
@@ -48,17 +48,16 @@ class KyuubiDefinedFunctionSuite extends KyuubiFunSuite {
test("verify or update kyuubi spark sql functions") {
val builder = MarkdownBuilder(licenced = true, getClass.getName)
- builder
- .line("# Auxiliary SQL Functions")
- .line("""Kyuubi provides several auxiliary SQL functions as supplement
to Spark's
+ builder += "# Auxiliary SQL Functions" +=
+ """Kyuubi provides several auxiliary SQL functions as supplement to
Spark's
| [Built-in
Functions](https://spark.apache.org/docs/latest/api/sql/index.html#
- |built-in-functions)""")
- .lines("""
+ |built-in-functions)""" ++=
+ """
| Name | Description | Return Type | Since
| --- | --- | --- | ---
- |""")
+ |"""
KDFRegistry.registeredFunctions.foreach { func =>
- builder.line(s"${func.name} | ${func.description} | ${func.returnType} |
${func.since}")
+ builder += s"${func.name} | ${func.description} | ${func.returnType} |
${func.since}"
}
MarkdownUtils.verifyOutput(
diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/MarkdownUtils.scala
b/kyuubi-common/src/test/scala/org/apache/kyuubi/MarkdownUtils.scala
index 805913343..06c948903 100644
--- a/kyuubi-common/src/test/scala/org/apache/kyuubi/MarkdownUtils.scala
+++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/MarkdownUtils.scala
@@ -21,7 +21,8 @@ import java.nio.charset.StandardCharsets
import java.nio.file.{Files, Path, StandardOpenOption}
import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
+import scala.collection.generic.Growable
+import scala.collection.mutable.ListBuffer
import com.vladsch.flexmark.formatter.Formatter
import com.vladsch.flexmark.parser.{Parser, ParserEmulationProfile,
PegdownExtensions}
@@ -37,7 +38,7 @@ object MarkdownUtils {
newOutput: MarkdownBuilder,
agent: String,
module: String): Unit = {
- val formatted = newOutput.formatMarkdown()
+ val formatted = newOutput.toMarkdown
if (System.getenv("KYUUBI_UPDATE") == "1") {
Files.write(
markdown,
@@ -45,41 +46,33 @@ object MarkdownUtils {
StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING)
} else {
- val linesInFile = Files.readAllLines(markdown, StandardCharsets.UTF_8)
- linesInFile.asScala.zipWithIndex.zip(formatted).foreach { case ((str1,
index), str2) =>
- withClue(s"$markdown out of date, as line ${index + 1} is not
expected." +
- " Please update doc with KYUUBI_UPDATE=1 build/mvn clean test" +
- s" -pl $module -am -Pflink-provided,spark-provided,hive-provided" +
- s" -Dtest=none -DwildcardSuites=$agent ") {
- assertResult(str2)(str1)
+ Files.readAllLines(markdown, StandardCharsets.UTF_8).asScala.toStream
+ .zipWithIndex.zip(formatted).foreach { case ((lineInFile, lineIndex),
formattedLine) =>
+ withClue(
+ s""" The markdown file ($markdown:${lineIndex + 1}) is out of date.
+ | Please update doc with KYUUBI_UPDATE=1 build/mvn clean test
+ | -pl $module -am -Pflink-provided,spark-provided,hive-provided
+ | -Dtest=none -DwildcardSuites=$agent \n""".stripMargin) {
+ assertResult(formattedLine)(lineInFile)
+ }
}
- }
}
}
-
- def line(str: String): String = {
- str.stripMargin.linesIterator.mkString
- }
-
- def appendBlankLine(buffer: ArrayBuffer[String]): Unit = buffer += ""
-
- def appendFileContent(buffer: ArrayBuffer[String], path: Path): Unit = {
- buffer += "```bash"
- buffer ++= Files.readAllLines(path).asScala
- buffer += "```"
- }
}
-class MarkdownBuilder {
- private val buffer = new ArrayBuffer[String]()
+class MarkdownBuilder extends Growable[String] {
+ private val buffer = new ListBuffer[String]
+
+ override def clear: Unit = buffer.clear
/**
* append a single line
* with replacing EOL to empty string
+ *
* @param str single line
* @return
*/
- def line(str: String = ""): MarkdownBuilder = {
+ override def +=(str: String): MarkdownBuilder.this.type = {
buffer += str.stripMargin.linesIterator.mkString
this
}
@@ -87,12 +80,13 @@ class MarkdownBuilder {
/**
* append the multiline
* with splitting EOL into single lines
- * @param multiline multiline with default line margin "|"
+ *
+ * @param multiline multiline with line margin char
+ * @param marginChar margin char, default to "|"
* @return
*/
- def lines(multiline: String): MarkdownBuilder = {
- buffer ++= multiline.stripMargin.linesIterator
- this
+ def ++=(multiline: String, marginChar: Char = '|'): MarkdownBuilder = {
+ this ++= multiline.stripMargin(marginChar).linesIterator
}
/**
@@ -100,7 +94,7 @@ class MarkdownBuilder {
* @return
*/
def licence(): MarkdownBuilder = {
- lines("""
+ this ++= """
|<!--
|- Licensed to the Apache Software Foundation (ASF) under one or more
|- contributor license agreements. See the NOTICE file distributed
with
@@ -117,7 +111,7 @@ class MarkdownBuilder {
|- See the License for the specific language governing permissions and
|- limitations under the License.
|-->
- |""")
+ |"""
}
/**
@@ -126,36 +120,14 @@ class MarkdownBuilder {
* @return
*/
def generationHint(className: String): MarkdownBuilder = {
- lines(s"""
+ this ++=
+ s"""
|<!-- DO NOT MODIFY THIS FILE DIRECTLY, IT IS AUTO-GENERATED BY
[$className] -->
|
- |""")
- }
-
- /**
- * append file content
- * @param path file path
- * @return
- */
- def file(path: Path): MarkdownBuilder = {
- buffer ++= Files.readAllLines(path).asScala
- this
- }
-
- /**
- * append file content with code block quote
- * @param path path to file
- * @param language language of codeblock
- * @return
- */
- def fileWithBlock(path: Path, language: String = "bash"): MarkdownBuilder = {
- buffer += s"```$language"
- file(path)
- buffer += "```"
- this
+ |"""
}
- def formatMarkdown(): Stream[String] = {
+ def toMarkdown: Stream[String] = {
def createParserOptions(emulationProfile: ParserEmulationProfile):
MutableDataHolder = {
PegdownOptionsAdapter.flexmarkOptions(PegdownExtensions.ALL).toMutable
.set(Parser.PARSER_EMULATION_PROFILE, emulationProfile)
@@ -175,7 +147,7 @@ class MarkdownBuilder {
val parser = Parser.builder(parserOptions).build
val renderer = Formatter.builder(formatterOptions).build
val document = parser.parse(buffer.mkString(EOL))
- val formattedLines = new ArrayBuffer[String](buffer.length)
+ val formattedLines = new ListBuffer[String]
val formattedLinesAppendable = new Appendable {
override def append(csq: CharSequence): Appendable = {
if (csq.length() > 0) {
diff --git
a/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala
b/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala
index 9fff482d4..5dde9f935 100644
---
a/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala
+++
b/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala
@@ -62,165 +62,154 @@ class AllKyuubiConfiguration extends KyuubiFunSuite {
val builder = MarkdownBuilder(licenced = true, getClass.getName)
- builder
- .lines(s"""
+ builder ++=
+ s"""
|# Introduction to the Kyuubi Configurations System
|
|Kyuubi provides several ways to configure the system and
corresponding engines.
|
|## Environments
|
- |""")
- .line("""You can configure the environment variables in
`$KYUUBI_HOME/conf/kyuubi-env.sh`,
+ |""" +=
+ """You can configure the environment variables in
`$KYUUBI_HOME/conf/kyuubi-env.sh`,
| e.g, `JAVA_HOME`, then this java runtime will be used both for
Kyuubi server instance and
| the applications it launches. You can also change the variable in
the subprocess's env
| configuration file, e.g.`$SPARK_HOME/conf/spark-env.sh` to use more
specific ENV for
| SQL engine applications. see
`$KYUUBI_HOME/conf/kyuubi-env.sh.template` as an example.
- | """)
- .line(
- """
- | For the environment variables that only needed to be transferred
into engine
+ | """ +=
+ """ For the environment variables that only needed to be transferred
into engine
| side, you can set it with a Kyuubi configuration item formatted
| `kyuubi.engineEnv.VAR_NAME`. For example, with
`kyuubi.engineEnv.SPARK_DRIVER_MEMORY=4g`,
| the environment variable `SPARK_DRIVER_MEMORY` with value `4g` would
be transferred into
| engine side. With
`kyuubi.engineEnv.SPARK_CONF_DIR=/apache/confs/spark/conf`, the
| value of `SPARK_CONF_DIR` on the engine side is set to
`/apache/confs/spark/conf`.
- | """)
- .line("## Kyuubi Configurations")
- .line(""" You can configure the Kyuubi properties in
+ | """ += "## Kyuubi Configurations" +=
+ """ You can configure the Kyuubi properties in
| `$KYUUBI_HOME/conf/kyuubi-defaults.conf`, see
| `$KYUUBI_HOME/conf/kyuubi-defaults.conf.template` as an example.
- | """)
+ | """
KyuubiConf.getConfigEntries().asScala
.toStream
.filterNot(_.internal)
.groupBy(_.key.split("\\.")(1))
.toSeq.sortBy(_._1).foreach { case (category, entries) =>
- builder.lines(
+ builder ++=
s"""### ${category.capitalize}
| Key | Default | Meaning | Type | Since
| --- | --- | --- | --- | ---
- |""")
+ |"""
entries.sortBy(_.key).foreach { c =>
val dft = c.defaultValStr.replace("<", "<").replace(">", ">")
- builder.line(Seq(
+ builder += Seq(
s"${c.key}",
s"$dft",
s"${c.doc}",
s"${c.typ}",
- s"${c.version}").mkString("|"))
+ s"${c.version}").mkString("|")
}
}
- builder
- .lines("""
- |## Spark Configurations
+ builder ++=
+ """## Spark Configurations
|### Via spark-defaults.conf
- |""")
- .line("""
- | Setting them in `$SPARK_HOME/conf/spark-defaults.conf`
+ |""" +=
+ """ Setting them in `$SPARK_HOME/conf/spark-defaults.conf`
| supplies with default values for SQL engine application. Available
properties can be
| found at Spark official online documentation for
| [Spark
Configurations](https://spark.apache.org/docs/latest/configuration.html)
- | """)
- .line("### Via kyuubi-defaults.conf")
- .line("""
- | Setting them in `$KYUUBI_HOME/conf/kyuubi-defaults.conf`
+ | """ +=
+ "### Via kyuubi-defaults.conf" +=
+ """ Setting them in `$KYUUBI_HOME/conf/kyuubi-defaults.conf`
| supplies with default values for SQL engine application too. These
properties will
- | override all settings in `$SPARK_HOME/conf/spark-defaults.conf`""")
- .line("### Via JDBC Connection URL")
- .line("""
- | Setting them in the JDBC Connection URL
+ | override all settings in `$SPARK_HOME/conf/spark-defaults.conf`""" +=
+ "### Via JDBC Connection URL" +=
+ """ Setting them in the JDBC Connection URL
| supplies session-specific for each SQL engine. For example:
| ```
|jdbc:hive2://localhost:10009/default;#
|spark.sql.shuffle.partitions=2;spark.executor.memory=5g
- |```""")
- .line()
- .line("- **Runtime SQL Configuration**")
- .line(""" - For [Runtime SQL Configurations](
+ |```
+ |""" +=
+ "" +=
+ "- **Runtime SQL Configuration**" +=
+ """ - For [Runtime SQL Configurations](
|https://spark.apache.org/docs/latest/configuration.html#runtime-sql-configuration),
they
- | will take affect every time""")
- .line("- **Static SQL and Spark Core Configuration**")
- .line(""" - For [Static SQL Configurations](
+ | will take affect every time""" +=
+ "- **Static SQL and Spark Core Configuration**" +=
+ """ - For [Static SQL Configurations](
|https://spark.apache.org/docs/latest/configuration.html#static-sql-configuration)
and
| other spark core configs, e.g. `spark.executor.memory`, they will
take effect if there
- | is no existing SQL engine application. Otherwise, they will just be
ignored""")
- .line("### Via SET Syntax")
- .line("""Please refer to the Spark official online documentation for
+ | is no existing SQL engine application. Otherwise, they will just be
ignored
+ | """ +=
+ "### Via SET Syntax" +=
+ """Please refer to the Spark official online documentation for
| [SET
Command](https://spark.apache.org/docs/latest/sql-ref-syntax-aux-conf-mgmt-set.html)
- |""")
+ |"""
- builder
- .lines("""
- |## Flink Configurations
- |### Via flink-conf.yaml""")
- .line("""Setting them in `$FLINK_HOME/conf/flink-conf.yaml`
+ builder ++=
+ """## Flink Configurations
+ |### Via flink-conf.yaml""" += """Setting them in
`$FLINK_HOME/conf/flink-conf.yaml`
+ |
| supplies with default values for SQL engine application.
| Available properties can be found at Flink official online
documentation for
| [Flink Configurations]
-
|(https://nightlies.apache.org/flink/flink-docs-stable/docs/deployment/config/)""")
- .line("### Via kyuubi-defaults.conf")
- .line("""Setting them in `$KYUUBI_HOME/conf/kyuubi-defaults.conf`
+
|(https://nightlies.apache.org/flink/flink-docs-stable/docs/deployment/config/)"""
+=
+ "### Via kyuubi-defaults.conf" +=
+ """Setting them in `$KYUUBI_HOME/conf/kyuubi-defaults.conf`
| supplies with default values for SQL engine application too.
| You can use properties with the additional prefix `flink.` to
override settings in
- | `$FLINK_HOME/conf/flink-conf.yaml`.""")
- .lines("""
- |
+ | `$FLINK_HOME/conf/flink-conf.yaml`.""" ++=
+ """
|For example:
|```
|flink.parallelism.default 2
|flink.taskmanager.memory.process.size 5g
- |```""")
- .line("""The below options in `kyuubi-defaults.conf` will set
`parallelism.default: 2`
- | and `taskmanager.memory.process.size: 5g` into flink
configurations.""")
- .line("### Via JDBC Connection URL")
- .line("""Setting them in the JDBC Connection URL supplies
session-specific
+ |```""" +=
+ """The below options in `kyuubi-defaults.conf` will set
`parallelism.default: 2`
+ | and `taskmanager.memory.process.size: 5g` into flink
configurations.""" +=
+ "### Via JDBC Connection URL" +=
+ """Setting them in the JDBC Connection URL supplies session-specific
| for each SQL engine. For example:
```jdbc:hive2://localhost:10009/default;
|#parallelism.default=2;taskmanager.memory.process.size=5g```
- |""")
- .line("### Via SET Statements")
- .line("""Please refer to the Flink official online documentation for
[SET Statements]
-
|(https://nightlies.apache.org/flink/flink-docs-stable/docs/dev/table/sql/set/)""")
-
- builder
- .line("## Logging")
- .line("""Kyuubi uses [log4j](https://logging.apache.org/log4j/2.x/) for
logging.
+ |""" +=
+ "### Via SET Statements" +=
+ """Please refer to the Flink official online documentation for [SET
Statements]
+
|(https://nightlies.apache.org/flink/flink-docs-stable/docs/dev/table/sql/set/)"""
+
+ builder += "## Logging" +=
+ """Kyuubi uses [log4j](https://logging.apache.org/log4j/2.x/) for
logging.
| You can configure it using `$KYUUBI_HOME/conf/log4j2.xml`, see
| `$KYUUBI_HOME/conf/log4j2.xml.template` as an example.
- | """)
+ | """
- builder
- .lines("""
- |## Other Configurations
+ builder ++=
+ """## Other Configurations
|### Hadoop Configurations
- |""")
- .line("""Specifying `HADOOP_CONF_DIR` to the directory containing Hadoop
configuration
+ |""" +=
+ """Specifying `HADOOP_CONF_DIR` to the directory containing Hadoop
configuration
| files or treating them as Spark properties with a `spark.hadoop.`
prefix.
| Please refer to the Spark official online documentation for
| [Inheriting Hadoop Cluster
Configuration](https://spark.apache.org/docs/latest/
|configuration.html#inheriting-hadoop-cluster-configuration).
| Also, please refer to the [Apache
Hadoop](https://hadoop.apache.org)'s
- | online documentation for an overview on how to configure Hadoop.""")
- .line("### Hive Configurations")
- .line("""These configurations are used for SQL engine application to
talk to
+ | online documentation for an overview on how to configure Hadoop."""
+=
+ "### Hive Configurations" +=
+ """These configurations are used for SQL engine application to talk to
| Hive MetaStore and could be configured in a `hive-site.xml`.
| Placed it in `$SPARK_HOME/conf` directory, or treat them as Spark
properties with
- | a `spark.hadoop.` prefix.""")
+ | a `spark.hadoop.` prefix."""
- builder
- .line("## User Defaults")
- .line("""In Kyuubi, we can configure user default settings to meet
separate needs.
+ builder += "## User Defaults" +=
+ """In Kyuubi, we can configure user default settings to meet separate
needs.
| These user defaults override system defaults, but will be overridden
by those from
| [JDBC Connection URL](#via-jdbc-connection-url) or [Set
Command](#via-set-syntax)
- | if could be. They will take effect when creating the SQL engine
application ONLY.""")
- .line("""User default settings are in the form of
`___{username}___.{config key}`.
+ | if could be. They will take effect when creating the SQL engine
application ONLY.""" +=
+ """User default settings are in the form of `___{username}___.{config
key}`.
| There are three continuous underscores(`_`) at both sides of the
`username` and
- | a dot(`.`) that separates the config key and the prefix. For
example:""")
- .lines("""
- |```bash
+ | a dot(`.`) that separates the config key and the prefix. For
example:""" ++=
+ """```bash
|# For system defaults
|spark.master=local
|spark.sql.adaptive.enabled=true
@@ -232,13 +221,13 @@ class AllKyuubiConfiguration extends KyuubiFunSuite {
|___bob___.spark.executor.memory=8g
|```
|
- |""")
- .line("""In the above case, if there are related configurations from
+ |""" +=
+ """In the above case, if there are related configurations from
| [JDBC Connection URL](#via-jdbc-connection-url), `kent` will run his
SQL engine
| application on YARN and prefer the Spark AQE to be off, while `bob`
will activate
| his SQL engine application on a Spark standalone cluster with 8g
heap memory for each
| executor and obey the Spark AQE behavior of Kyuubi system default.
On the other hand,
- | for those users who do not have custom configurations will use
system defaults.""")
+ | for those users who do not have custom configurations will use
system defaults."""
MarkdownUtils.verifyOutput(markdown, builder, getClass.getCanonicalName,
"kyuubi-server")
}
