[GitHub] [linkis] peacewong commented on a diff in pull request #4784: feat: "spark-submit jar on k8s" related label and resource expansion

[via GitHub]

peacewong commented on code in PR #4784:
URL: https://github.com/apache/linkis/pull/4784#discussion_r1284070297


##########
linkis-computation-governance/linkis-manager/linkis-application-manager/pom.xml:
##########
@@ -104,7 +104,31 @@
       <version>${gson.version}</version>
       <scope>provided</scope>
     </dependency>
-
+    <dependency>
+      <groupId>io.fabric8</groupId>
+      <artifactId>kubernetes-client</artifactId>
+      <version>${kubernetes-client.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>io.fabric8</groupId>
+          <artifactId>kubernetes-model-common</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>io.fabric8</groupId>
+          <artifactId>kubernetes-model-core</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>io.fabric8</groupId>
+      <artifactId>kubernetes-model-common</artifactId>
+      <version>${kubernetes-client.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.fabric8</groupId>
+      <artifactId>kubernetes-model-core</artifactId>
+      <version>5.4.1</version>

Review Comment:
   can use props value?



##########
linkis-engineconn-plugins/spark/pom.xml:
##########
@@ -418,7 +418,6 @@
       <artifactId>kubernetes-model-core</artifactId>
       <version>5.4.1</version>

Review Comment:
   can use props value?



##########
linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkEngineConnResourceFactory.scala:
##########
@@ -20,19 +20,54 @@ package org.apache.linkis.engineplugin.spark.factory
 import org.apache.linkis.common.utils.{ByteTimeUtils, Logging}
 import org.apache.linkis.engineplugin.spark.config.SparkResourceConfiguration._
 import org.apache.linkis.manager.common.entity.resource.{
+  DriverAndKubernetesResource,
   DriverAndYarnResource,
+  KubernetesResource,
   LoadInstanceResource,
   Resource,
   YarnResource
 }
-import 
org.apache.linkis.manager.engineplugin.common.resource.AbstractEngineResourceFactory
+import org.apache.linkis.manager.engineplugin.common.resource.{
+  AbstractEngineResourceFactory,
+  EngineResourceRequest
+}
+import org.apache.linkis.manager.label.entity.cluster.ClusterLabel
+import org.apache.linkis.manager.label.utils.LabelUtil
 
 import org.apache.commons.lang3.StringUtils
 
 import java.util
 
+import io.fabric8.kubernetes.api.model.Quantity
+
 class SparkEngineConnResourceFactory extends AbstractEngineResourceFactory 
with Logging {
 
+  override protected def getMinRequestResource(
+      engineResourceRequest: EngineResourceRequest
+  ): Resource = {
+    val clusterLabel = 
LabelUtil.getLabelFromList[ClusterLabel](engineResourceRequest.labels)
+    if (
+        clusterLabel != null && 
StringUtils.equals(clusterLabel.getClusterType.toUpperCase(), "K8S")

Review Comment:
   "k8s" should defined constant



##########
linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_ddl.sql:
##########
@@ -17,4 +17,5 @@
 
 ALTER TABLE `linkis_cg_manager_service_instance` ADD COLUMN `identifier` 
varchar(32) COLLATE utf8_bin DEFAULT NULL;
 ALTER TABLE `linkis_cg_manager_service_instance` ADD COLUMN `ticketId` 
varchar(255) COLLATE utf8_bin DEFAULT NULL;
-ALTER TABLE `linkis_cg_ec_resource_info_record` MODIFY COLUMN metrics TEXT 
DEFAULT NULL COMMENT 'ec metrics';
\ No newline at end of file
+ALTER TABLE `linkis_cg_ec_resource_info_record` MODIFY COLUMN metrics TEXT 
DEFAULT NULL COMMENT 'ec metrics';
+ALTER TABLE `linkis_cg_manager_label` MODIFY COLUMN label_key varchar(50);

Review Comment:
   need to move 1.5.0_schema



##########
linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/config/SparkResourceConfiguration.scala:
##########
@@ -31,4 +31,22 @@ object SparkResourceConfiguration {
   val LINKIS_SPARK_EXECUTOR_INSTANCES = 
CommonVars[Int]("spark.executor.instances", 3)
   val LINKIS_QUEUE_NAME = CommonVars[String]("wds.linkis.rm.yarnqueue", 
"default")
 
+  val LINKIS_SPARK_KUBERNETES_NAMESPACE =
+    CommonVars[String]("spark.kubernetes.namespace", "default")

Review Comment:
   need to be consistent
   
![image](https://github.com/apache/linkis/assets/11496700/3854c8a6-4915-4518-8f5b-cd577d2d4bf4)
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: notifications-unsubscr...@linkis.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscr...@linkis.apache.org
For additional commands, e-mail: notifications-h...@linkis.apache.org