This is an automated email from the ASF dual-hosted git repository.
peacewong pushed a commit to branch dev-1.3.2
in repository https://gitbox.apache.org/repos/asf/linkis.git
The following commit(s) were added to refs/heads/dev-1.3.2 by this push:
new ff4a38b66 Optimize scriptis code with @set module distinguish between
scriptContent/metadata by annotations (#4303)
ff4a38b66 is described below
commit ff4a38b6678e7ec8601df039032c4be543784384
Author: Casion <[email protected]>
AuthorDate: Mon Mar 6 14:53:28 2023 +0800
Optimize scriptis code with @set module distinguish between
scriptContent/metadata by annotations (#4303)
* optimize scriptis code with @set module
distinguish between scriptContent/metadata by annotations
---
LICENSE | 1 +
.../linkis/storage/script/ScriptFsWriter.scala | 4 +
.../script/compaction/CommonScriptCompaction.scala | 2 +
.../storage/script/parser/CommonScriptParser.scala | 2 +
.../script/reader/StorageScriptFsReader.scala | 25 +++-
.../script/writer/StorageScriptFsWriter.scala | 34 ++++-
.../apache/linkis/storage/source/FileSplit.scala | 23 +++-
.../script/writer/StorageScriptFsWriterTest.java | 151 +++++++++++++++++++++
.../src/test/resources/scritpis-test.sql | 10 ++
pom.xml | 2 +
10 files changed, 240 insertions(+), 14 deletions(-)
diff --git a/LICENSE b/LICENSE
index b9d493e97..549d75747 100644
--- a/LICENSE
+++ b/LICENSE
@@ -236,6 +236,7 @@ The following file are provided under the Apache 2.0
License.
linkis-web/src/config.json
linkis-web/public/favicon.ico
linkis-engineconn-plugins/seatunnel/src/main/java/org/apache/seatunnel/*
+ linkis-commons/linkis-storage/src/test/resources/scritpis-test.sql
The files:
.mvn/wrapper/MavenWrapperDownloader.java
diff --git
a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/ScriptFsWriter.scala
b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/ScriptFsWriter.scala
index 2de8687a2..dbcbfc126 100644
---
a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/ScriptFsWriter.scala
+++
b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/ScriptFsWriter.scala
@@ -81,6 +81,8 @@ trait Parser {
def belongTo(suffix: String): Boolean
def parse(line: String): Variable
+
+ def getAnnotationSymbol(): String
}
trait Compaction {
@@ -92,6 +94,8 @@ trait Compaction {
def belongTo(suffix: String): Boolean
def compact(variable: Variable): String
+
+ def getAnnotationSymbol(): String
}
class ScriptMetaData(var variables: Array[Variable]) extends MetaData {
diff --git
a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/CommonScriptCompaction.scala
b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/CommonScriptCompaction.scala
index 153d4c900..e085d2aac 100644
---
a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/CommonScriptCompaction.scala
+++
b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/CommonScriptCompaction.scala
@@ -29,4 +29,6 @@ abstract class CommonScriptCompaction extends Compaction {
}
}
+ override def getAnnotationSymbol(): String = prefix.split('@')(0)
+
}
diff --git
a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/CommonScriptParser.scala
b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/CommonScriptParser.scala
index 7a7a7f64d..b23a521cd 100644
---
a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/CommonScriptParser.scala
+++
b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/CommonScriptParser.scala
@@ -58,4 +58,6 @@ abstract class CommonScriptParser extends Parser {
}
}
+ override def getAnnotationSymbol(): String = prefix.split('@')(0)
+
}
diff --git
a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/reader/StorageScriptFsReader.scala
b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/reader/StorageScriptFsReader.scala
index 3f0613783..7b7b85ceb 100644
---
a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/reader/StorageScriptFsReader.scala
+++
b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/reader/StorageScriptFsReader.scala
@@ -50,13 +50,10 @@ class StorageScriptFsReader(val path: FsPath, val charset:
String, val inputStre
@scala.throws[IOException]
override def getMetaData: MetaData = {
if (metadata == null) init()
- val parser =
- ParserFactory.listParsers().filter(p =>
p.belongTo(StorageUtils.pathToSuffix(path.getPath)))
+ val parser = getScriptParser()
lineText = bufferedReader.readLine()
- while (
- hasNext && parser.length > 0 && isMetadata(lineText, parser(0).prefix,
parser(0).prefixConf)
- ) {
- variables += parser(0).parse(lineText)
+ while (hasNext && parser != null && isMetadata(lineText, parser.prefix,
parser.prefixConf)) {
+ variables += parser.parse(lineText)
lineText = bufferedReader.readLine()
}
metadata = new ScriptMetaData(variables.toArray)
@@ -110,4 +107,20 @@ class StorageScriptFsReader(val path: FsPath, val charset:
String, val inputStre
}
}
+ /**
+ * get the script parser according to the path(根据文件路径 获取对应的script parser )
+ * @return
+ * Scripts Parser
+ */
+
+ def getScriptParser(): Parser = {
+ val parsers =
+ ParserFactory.listParsers().filter(p =>
p.belongTo(StorageUtils.pathToSuffix(path.getPath)))
+ if (parsers.length > 0) {
+ parsers(0)
+ } else {
+ null
+ }
+ }
+
}
diff --git
a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.scala
b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.scala
index bd64d24f0..cdb9186da 100644
---
a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.scala
+++
b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.scala
@@ -40,16 +40,20 @@ class StorageScriptFsWriter(
@scala.throws[IOException]
override def addMetaData(metaData: MetaData): Unit = {
- val compactions = Compaction
- .listCompactions()
- .filter(p => p.belongTo(StorageUtils.pathToSuffix(path.getPath)))
+
val metadataLine = new util.ArrayList[String]()
- if (compactions.length > 0) {
+ val compaction = getScriptCompaction()
+ if (compaction != null) {
+
metaData
.asInstanceOf[ScriptMetaData]
.getMetaData
- .map(compactions(0).compact)
+ .map(compaction.compact)
.foreach(metadataLine.add)
+ // add annotition symbol
+ if (metadataLine.size() > 0) {
+ metadataLine.add(compaction.getAnnotationSymbol())
+ }
if (outputStream != null) {
IOUtils.writeLines(metadataLine, "\n", outputStream, charset)
} else {
@@ -57,6 +61,7 @@ class StorageScriptFsWriter(
metadataLine.asScala.foreach(m => stringBuilder.append(s"$m\n"))
}
}
+
}
@scala.throws[IOException]
@@ -93,4 +98,23 @@ class StorageScriptFsWriter(
)
}
+ /**
+ * get the script compaction according to the path(根据文件路径 获取对应的script
Compaction )
+ * @return
+ * Scripts Compaction
+ */
+
+ def getScriptCompaction(): Compaction = {
+
+ val compactions = Compaction
+ .listCompactions()
+ .filter(p => p.belongTo(StorageUtils.pathToSuffix(path.getPath)))
+
+ if (compactions.length > 0) {
+ compactions(0)
+ } else {
+ null
+ }
+ }
+
}
diff --git
a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala
b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala
index 358206357..467fbca33 100644
---
a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala
+++
b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala
@@ -22,6 +22,7 @@ import org.apache.linkis.storage.{LineMetaData, LineRecord}
import org.apache.linkis.storage.domain.{Column, DataType}
import org.apache.linkis.storage.resultset.table.{TableMetaData, TableRecord}
import org.apache.linkis.storage.script.{ScriptMetaData, VariableParser}
+import org.apache.linkis.storage.script.reader.StorageScriptFsReader
import org.apache.commons.io.IOUtils
import org.apache.commons.math3.util.Pair
@@ -73,10 +74,26 @@ class FileSplit(
fsReader.skip(start)
}
count = start
+ var hasRemovedFlag = false
while (fsReader.hasNext && ifContinueRead) {
- r(shuffler(fsReader.getRecord))
- totalLine += 1
- count += 1
+ val record = fsReader.getRecord
+ var needRemoveFlag = false
+ if (hasRemovedFlag == false &&
fsReader.isInstanceOf[StorageScriptFsReader]) {
+ val parser =
fsReader.asInstanceOf[StorageScriptFsReader].getScriptParser()
+ val meta = metaData.asInstanceOf[ScriptMetaData].getMetaData
+ if (
+ meta != null && meta.length > 0
+ && parser != null &&
parser.getAnnotationSymbol().equals(record.toString)
+ ) {
+ needRemoveFlag = true
+ hasRemovedFlag = true
+ }
+ }
+ if (needRemoveFlag == false) {
+ r(shuffler(record))
+ totalLine += 1
+ count += 1
+ }
}
t
}
diff --git
a/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriterTest.java
b/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriterTest.java
new file mode 100644
index 000000000..60a593665
--- /dev/null
+++
b/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriterTest.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.storage.script.writer;
+
+import org.apache.linkis.common.io.FsPath;
+import org.apache.linkis.common.io.MetaData;
+import org.apache.linkis.storage.script.ScriptFsWriter;
+import org.apache.linkis.storage.script.ScriptMetaData;
+import org.apache.linkis.storage.script.ScriptRecord;
+import org.apache.linkis.storage.script.Variable;
+import org.apache.linkis.storage.script.VariableParser;
+import org.apache.linkis.storage.source.FileSource;
+import org.apache.linkis.storage.source.FileSource$;
+
+import org.apache.commons.math3.util.Pair;
+
+import org.springframework.util.StringUtils;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.gson.Gson;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class StorageScriptFsWriterTest {
+
+ //
{"fileName":"229cf765-6839-4c82-829d-1907c2ccf668.sql","scriptContent":"select
+ // ${qq};\n--@set test=123\nselect ${test};\n--@set qq=222\nselect
+ //
${qq};\n","projectName":"test1122a1","metadata":{"variable":{"qq":"123"},"configuration":{"special":{},"runtime":{},"startup":{}}},"resourceId":"c9755cad-619b-4c1c-9204-cc4bb9836194"}
+ //
+
+ String scriptContent =
+ ""
+ + "select ${qq};\n"
+ + "--@set test=123\n"
+ + "select ${test};\n"
+ + "--@set qq=222\n"
+ + "select ${qq};\n"
+ + "--\n"
+ + "--\n"
+ + "select 1;";
+ String metaData =
+
"{\"variable\":{\"qq\":\"123\"},\"configuration\":{\"special\":{},\"runtime\":{},\"startup\":{}}}";
+
+ String resultMetaData = "{\"variable\":{\"qq\":\"123\"}}";
+
+ String resultString =
+ ""
+ + "--@set qq=123\n"
+ + "--\n"
+ + "select ${qq};\n"
+ + "--@set test=123\n"
+ + "select ${test};\n"
+ + "--@set qq=222\n"
+ + "select ${qq};\n"
+ + "--\n"
+ + "--\n"
+ + "select 1;";
+
+ Map<String, Object> params;
+
+ {
+ try {
+ params = new ObjectMapper().readValue(metaData, HashMap.class);
+ } catch (JsonProcessingException e) {
+ e.printStackTrace();
+ }
+ }
+
+ String fileName = "229cf765-6839-4c82-829d-1907c2ccf668.sql";
+
+ // return Message.ok().data("resourceId", resourceId).data("version",
version);
+
+ @Test
+ void TestSave() {
+ ScriptFsWriter writer =
+ StorageScriptFsWriter.getScriptFsWriter(new FsPath(fileName), "UTF-8",
null);
+ Variable[] v = VariableParser.getVariables(params);
+ List<Variable> variableList =
+ Arrays.stream(v)
+ .filter(var -> !StringUtils.isEmpty(var.value()))
+ .collect(Collectors.toList());
+ try {
+
+ MetaData metaData = new ScriptMetaData(variableList.toArray(new
Variable[0]));
+ writer.addMetaData(metaData);
+ writer.addRecord(new ScriptRecord(scriptContent));
+ InputStream inputStream = writer.getInputStream();
+
+ String text =
+ new BufferedReader(new InputStreamReader(inputStream,
StandardCharsets.UTF_8))
+ .lines()
+ .collect(Collectors.joining("\n"));
+ Assertions.assertEquals(text, resultString);
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ void TestOpen() throws FileNotFoundException {
+
+ //
/api/rest_j/v1/filesystem/openScriptFromBML?fileName=229cf765-6839-4c82-829d-1907c2ccf668.sql&resourceId=c9755cad-619b-4c1c-9204-cc4bb9836194&version=v000008&creator=&projectName=test1122a1
+
+ String filePath =
this.getClass().getResource("/scritpis-test.sql").getFile().toString();
+
+ File file = new File(filePath);
+
+ InputStream inputStream = new FileInputStream(file);
+
+ FileSource fileSource = FileSource$.MODULE$.create(new FsPath(fileName),
inputStream);
+ Pair<Object, ArrayList<String[]>> collect = fileSource.collect()[0];
+
+ String scriptRes = collect.getSecond().get(0)[0];
+ String metadataRes = new Gson().toJson(collect.getFirst());
+
+ Assertions.assertEquals(scriptRes, scriptContent + "\n");
+
+ Assertions.assertEquals(metadataRes, resultMetaData);
+ }
+}
diff --git a/linkis-commons/linkis-storage/src/test/resources/scritpis-test.sql
b/linkis-commons/linkis-storage/src/test/resources/scritpis-test.sql
new file mode 100644
index 000000000..8c3483a0c
--- /dev/null
+++ b/linkis-commons/linkis-storage/src/test/resources/scritpis-test.sql
@@ -0,0 +1,10 @@
+--@set qq=123
+--
+select ${qq};
+--@set test=123
+select ${test};
+--@set qq=222
+select ${qq};
+--
+--
+select 1;
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 98a61cd0b..c5e9e5b51 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1183,6 +1183,8 @@
<exclude>.github/**</exclude>
<exclude>compiler/**</exclude>
<exclude>**/generated/**</exclude>
+ <exclude>**/scritpis-test.sql</exclude>
+
</excludes>
</configuration>
<executions>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
