This is an automated email from the ASF dual-hosted git repository.
casion pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/linkis.git
The following commit(s) were added to refs/heads/master by this push:
new 2fdf9d3e2e [EnginePlugin][Spark]Turn off use secure random by default
(#5197)
2fdf9d3e2e is described below
commit 2fdf9d3e2ed2ed29a4c6a2382249286883818acb
Author: peacewong <[email protected]>
AuthorDate: Tue Nov 12 10:40:11 2024 +0800
[EnginePlugin][Spark]Turn off use secure random by default (#5197)
* Turn off use secure random by default close #5196
* Update Notification Mailing List
* Fix ds meta service build
---
.asf.yaml | 4 ++--
.../engineplugin/spark/config/SparkConfiguration.scala | 5 ++++-
.../engineplugin/spark/executor/SparkPythonExecutor.scala | 9 +++++++--
linkis-public-enhancements/distribution.xml | 2 +-
.../server/src/main/assembly/distribution.xml | 12 ++++++------
5 files changed, 20 insertions(+), 12 deletions(-)
diff --git a/.asf.yaml b/.asf.yaml
index 79396eba55..08ec821f75 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -67,6 +67,6 @@ github:
required_approving_review_count: 1
notifications:
commits: [email protected]
- issues: [email protected]
- pullrequests: [email protected]
+ issues: [email protected]
+ pullrequests: [email protected]
diff --git
a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/config/SparkConfiguration.scala
b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/config/SparkConfiguration.scala
index a493c5ff37..429048c77f 100644
---
a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/config/SparkConfiguration.scala
+++
b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/config/SparkConfiguration.scala
@@ -156,7 +156,7 @@ object SparkConfiguration extends Logging {
CommonVars("wds.linkis.spark.engineconn.fatal.log", "error writing
class;OutOfMemoryError")
val PYSPARK_PYTHON3_PATH =
- CommonVars[String]("pyspark.python3.path",
"/appcom/Install/anaconda3/bin/python")
+ CommonVars[String]("pyspark.python3.path", "python3")
val ENABLE_REPLACE_PACKAGE_NAME =
CommonVars("wds.linkis.spark.engine.scala.replace_package_header.enable",
true)
@@ -182,6 +182,9 @@ object SparkConfiguration extends Logging {
val LINKIS_SPARK_ETL_SUPPORT_HUDI =
CommonVars[Boolean]("linkis.spark.etl.support.hudi", false)
+ val LINKIS_PYSPARK_USE_SECURE_RANDOM =
+ CommonVars[Boolean]("linkis.pyspark.use.secure.random", false).getValue
+
val SCALA_PARSE_APPEND_CODE =
CommonVars("linkis.scala.parse.append.code", "val linkisVar=1").getValue
diff --git
a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkPythonExecutor.scala
b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkPythonExecutor.scala
index 5d4305c67c..f947db9338 100644
---
a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkPythonExecutor.scala
+++
b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkPythonExecutor.scala
@@ -46,6 +46,7 @@ import org.apache.spark.sql.execution.datasources.csv.UDF
import java.io._
import java.net.InetAddress
+import java.security.SecureRandom
import java.util
import scala.collection.JavaConverters._
@@ -76,7 +77,12 @@ class SparkPythonExecutor(val sparkEngineSession:
SparkEngineSession, val id: In
private val lineOutputStream = new RsOutputStream
val sqlContext = sparkEngineSession.sqlContext
val SUCCESS = "success"
- private lazy val py4jToken: String =
SecureRandomStringUtils.randomAlphanumeric(256)
+
+ private lazy val py4jToken: String = if
(SparkConfiguration.LINKIS_PYSPARK_USE_SECURE_RANDOM) {
+ SecureRandomStringUtils.randomAlphanumeric(256)
+ } else {
+ SecureRandom.getInstance("SHA1PRNG").nextInt(100000).toString
+ }
private lazy val gwBuilder: GatewayServerBuilder = {
val builder = new GatewayServerBuilder()
@@ -152,7 +158,6 @@ class SparkPythonExecutor(val sparkEngineSession:
SparkEngineSession, val id: In
)
val userDefinePythonVersion = engineCreationContext.getOptions
.getOrDefault("spark.python.version", "python")
- .toString
.toLowerCase()
val sparkPythonVersion =
if (
diff --git a/linkis-public-enhancements/distribution.xml
b/linkis-public-enhancements/distribution.xml
index 3e3baf6396..b4cc88582e 100644
--- a/linkis-public-enhancements/distribution.xml
+++ b/linkis-public-enhancements/distribution.xml
@@ -290,7 +290,7 @@
<fileSets>
<fileSet>
<directory>
-
./linkis-datasource/linkis-datasource-manager/server/target/out/lib/service
+
./linkis-datasource/linkis-datasource-manager/server/target/out/service
</directory>
<outputDirectory>
lib/metadataquery-service
diff --git
a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/assembly/distribution.xml
b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/assembly/distribution.xml
index 26130980b3..e124c84fc4 100644
---
a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/assembly/distribution.xml
+++
b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/assembly/distribution.xml
@@ -187,7 +187,7 @@
<fileSet>
<directory>${basedir}/../service/elasticsearch/target/out/lib</directory>
<fileMode>0755</fileMode>
- <outputDirectory>lib/service/elasticsearch</outputDirectory>
+ <outputDirectory>service/elasticsearch</outputDirectory>
<includes>
<include>*.jar</include>
</includes>
@@ -198,7 +198,7 @@
<fileSet>
<directory>${basedir}/../service/hive/target/out/lib</directory>
<fileMode>0755</fileMode>
- <outputDirectory>lib/service/hive</outputDirectory>
+ <outputDirectory>service/hive</outputDirectory>
<includes>
<include>*.jar</include>
</includes>
@@ -209,7 +209,7 @@
<fileSet>
<directory>${basedir}/../service/kafka/target/out/lib</directory>
<fileMode>0755</fileMode>
- <outputDirectory>lib/service/kafka</outputDirectory>
+ <outputDirectory>service/kafka</outputDirectory>
<includes>
<include>*.jar</include>
</includes>
@@ -220,7 +220,7 @@
<fileSet>
<directory>${basedir}/../service/jdbc/target/out/lib</directory>
<fileMode>0755</fileMode>
- <outputDirectory>lib/service/jdbc</outputDirectory>
+ <outputDirectory>service/jdbc</outputDirectory>
<includes>
<include>*.jar</include>
</includes>
@@ -231,7 +231,7 @@
<fileSet>
<directory>${basedir}/../service/hdfs/target/out/lib</directory>
<fileMode>0755</fileMode>
- <outputDirectory>lib/service/hdfs</outputDirectory>
+ <outputDirectory>service/hdfs</outputDirectory>
<includes>
<include>*.jar</include>
</includes>
@@ -242,7 +242,7 @@
<fileSet>
<directory>${basedir}/../service/mongodb/target/out/lib</directory>
<fileMode>0755</fileMode>
- <outputDirectory>lib/service/mongodb</outputDirectory>
+ <outputDirectory>service/mongodb</outputDirectory>
<includes>
<include>*.jar</include>
</includes>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
