This is an automated email from the ASF dual-hosted git repository.
peacewong pushed a commit to branch dev-1.2.0
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
The following commit(s) were added to refs/heads/dev-1.2.0 by this push:
new 523d2f747 support Variable Operation (#2415)
523d2f747 is described below
commit 523d2f747f41a3ff84a8827c0d7a86a684371448
Author: 野鹿 <[email protected]>
AuthorDate: Thu Jul 7 20:56:35 2022 +0800
support Variable Operation (#2415)
* Variable Operation
---
.../VariableOperationFailedException.java | 30 ++
.../common/utils/VariableOperationUtils.java | 317 +++++++++++++++++++++
.../apache/linkis/common/conf/Configuration.scala | 2 +
.../apache/linkis/common/utils/VariableUtils.scala | 19 +-
.../common/variable/VariableOperationTest.java | 52 ++++
5 files changed, 418 insertions(+), 2 deletions(-)
diff --git
a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/exception/VariableOperationFailedException.java
b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/exception/VariableOperationFailedException.java
new file mode 100644
index 000000000..ed2ff207c
--- /dev/null
+++
b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/exception/VariableOperationFailedException.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.common.exception;
+
+public class VariableOperationFailedException extends ErrorException {
+
+ public VariableOperationFailedException(int errCode, String desc) {
+ super(errCode, desc);
+ }
+
+ public VariableOperationFailedException(int errCode, String desc,
Exception e) {
+ super(errCode, desc);
+ this.initCause(e);
+ }
+}
diff --git
a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/VariableOperationUtils.java
b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/VariableOperationUtils.java
new file mode 100644
index 000000000..39b98d134
--- /dev/null
+++
b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/VariableOperationUtils.java
@@ -0,0 +1,317 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.linkis.common.utils;
+
+import org.apache.linkis.common.exception.VariableOperationFailedException;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.Map;
+
+/** support variable operation ${yyyyMMdd%-1d}/${yyyy-MM-01%-2M} Date:
2021/5/7 11:10 */
+public class VariableOperationUtils {
+
+ private static final String DOLLAR = "$";
+ private static final String PLACEHOLDER_SPLIT = "%";
+ private static final String PLACEHOLDER_LEFT = "{";
+ private static final String PLACEHOLDER_RIGHT = "}";
+ private static final String CYCLE_YEAR = "y";
+ private static final String CYCLE_MONTH = "M";
+ private static final String CYCLE_DAY = "d";
+ private static final String CYCLE_HOUR = "H";
+ private static final String CYCLE_MINUTE = "m";
+ private static final String CYCLE_SECOND = "s";
+ private static final String[] CYCLES =
+ new String[] {
+ CYCLE_YEAR, CYCLE_MONTH, CYCLE_DAY, CYCLE_HOUR, CYCLE_MINUTE,
CYCLE_SECOND
+ };
+
+ /**
+ * yyyy-MM-dd HH:mm:ss
+ *
+ * @param date
+ * @return
+ */
+ public static ZonedDateTime toZonedDateTime(Date date) {
+ Instant instant = date.toInstant();
+ ZoneId zoneId = ZoneId.systemDefault();
+ LocalDateTime localDateTime = instant.atZone(zoneId).toLocalDateTime();
+ return ZonedDateTime.of(localDateTime, zoneId);
+ }
+
+ /**
+ * json support variable operation
+ *
+ * @param dateTime
+ * @param str
+ * @return
+ */
+ public static String replaces(ZonedDateTime dateTime, String str)
+ throws VariableOperationFailedException {
+ return replaces(dateTime, str, true);
+ }
+
+ /**
+ * json support variable operation
+ *
+ * @param dateTime
+ * @param str
+ * @param format
+ * @return
+ */
+ public static String replaces(ZonedDateTime dateTime, String str, boolean
format)
+ throws VariableOperationFailedException {
+ try {
+ JsonNode rootNode = JsonUtils.jackson().readTree(str);
+ if (rootNode.isArray() || rootNode.isObject()) {
+ replaceJson(dateTime, rootNode);
+ return rootNode.toString();
+ }
+ } catch (Exception e) {
+ return replace(dateTime, str);
+ }
+ return replace(dateTime, str);
+ }
+
+ /**
+ * @param dateTime
+ * @param str
+ * @return
+ */
+ private static String replace(ZonedDateTime dateTime, String str)
+ throws VariableOperationFailedException {
+ StringBuilder buffer = new StringBuilder(str);
+ int startIndex = str.indexOf(PLACEHOLDER_LEFT);
+
+ while (startIndex != -1) {
+ int endIndex = buffer.indexOf(PLACEHOLDER_RIGHT, startIndex);
+ if (endIndex != -1) {
+ String placeHolder = buffer.substring(startIndex, endIndex +
1);
+ String content =
+ placeHolder
+ .replace(PLACEHOLDER_LEFT, "")
+ .replace(PLACEHOLDER_RIGHT, "")
+ .trim();
+ String[] parts = content.split(PLACEHOLDER_SPLIT);
+ try {
+ ZonedDateTime ndt = dateTime;
+ for (int i = 1; i < parts.length; i++) {
+ ndt = changeDateTime(ndt, parts[i]);
+ }
+
+ String newContent =
ndt.format(DateTimeFormatter.ofPattern(parts[0]));
+ if (buffer.substring(startIndex - 1, endIndex +
1).contains(DOLLAR)) {
+ buffer.replace(startIndex - 1, endIndex + 1,
newContent);
+ } else {
+ buffer.replace(startIndex, endIndex + 1, newContent);
+ }
+ startIndex = buffer.indexOf(PLACEHOLDER_LEFT, startIndex +
newContent.length());
+ } catch (IllegalArgumentException e1) {
+ startIndex = buffer.indexOf(PLACEHOLDER_LEFT, endIndex);
+ } catch (Exception e2) {
+ throw new VariableOperationFailedException(
+ 20050, "variable operation expression" +
e2.getMessage(), e2);
+ }
+ } else {
+ startIndex = -1; // leave while
+ }
+ }
+ return buffer.toString();
+ }
+
+ /**
+ * @param dateTime
+ * @param str
+ * @return
+ */
+ private static ZonedDateTime changeDateTime(ZonedDateTime dateTime, String
str) {
+ if (str == null || str.isEmpty()) {
+ return dateTime;
+ }
+
+ for (String cycle : CYCLES) {
+ if (str.contains(cycle)) {
+ switch (cycle) {
+ case CYCLE_DAY:
+ return
dateTime.plusDays(Integer.parseInt(str.replace(CYCLE_DAY, "")));
+ case CYCLE_HOUR:
+ return
dateTime.plusHours(Integer.parseInt(str.replace(CYCLE_HOUR, "")));
+ case CYCLE_MINUTE:
+ return dateTime.plusMinutes(
+ Integer.parseInt(str.replace(CYCLE_MINUTE,
"")));
+ case CYCLE_MONTH:
+ return
dateTime.plusMonths(Integer.parseInt(str.replace(CYCLE_MONTH, "")));
+ case CYCLE_SECOND:
+ return dateTime.plusSeconds(
+ Integer.parseInt(str.replace(CYCLE_SECOND,
"")));
+ case CYCLE_YEAR:
+ return
dateTime.plusYears(Integer.parseInt(str.replace(CYCLE_YEAR, "")));
+ default:
+ break;
+ }
+ }
+ }
+
+ return dateTime;
+ }
+
+ /**
+ * @param keyValue
+ * @param str
+ * @return
+ */
+ private static String replace(Map<String, String> keyValue, String str)
+ throws VariableOperationFailedException {
+ StringBuilder buffer = new StringBuilder(str);
+ int startIndex = str.indexOf(PLACEHOLDER_LEFT);
+
+ while (startIndex != -1) {
+ int endIndex = buffer.indexOf(PLACEHOLDER_RIGHT, startIndex);
+ if (endIndex != -1) {
+ String placeHolder = buffer.substring(startIndex, endIndex +
1);
+ String content =
+ placeHolder
+ .replace(PLACEHOLDER_LEFT, "")
+ .replace(PLACEHOLDER_RIGHT, "")
+ .trim();
+ try {
+ String newContent = keyValue.get(content);
+ if (newContent != null) {
+ if (buffer.substring(startIndex - 1, endIndex +
1).contains(DOLLAR)) {
+ buffer.replace(startIndex - 1, endIndex + 1,
newContent);
+ } else {
+ buffer.replace(startIndex, endIndex + 1,
newContent);
+ }
+ startIndex =
+ buffer.indexOf(PLACEHOLDER_LEFT, startIndex +
newContent.length());
+ } else {
+ startIndex = buffer.indexOf(PLACEHOLDER_LEFT,
endIndex);
+ }
+ } catch (Exception e2) {
+ throw new VariableOperationFailedException(
+ 20050, "variable operation expression" +
e2.getMessage(), e2);
+ }
+ } else {
+ startIndex = -1; // leave while
+ }
+ }
+ return buffer.toString();
+ }
+
+ /**
+ * json support variable operation
+ *
+ * @param dateTime
+ * @param object
+ */
+ @SuppressWarnings("DuplicatedCode")
+ private static void replaceJson(ZonedDateTime dateTime, JsonNode object)
+ throws VariableOperationFailedException {
+ if (object.isArray()) {
+ ArrayNode arrayNode = (ArrayNode) object;
+ for (int i = 0; i < arrayNode.size(); i++) {
+ final JsonNode temp = arrayNode.get(i);
+ if (temp.isArray()) {
+ replaceJson(dateTime, temp);
+ } else if (temp.isObject()) {
+ replaceJson(dateTime, temp);
+ } else {
+ arrayNode.insert(i, replace(dateTime, temp.toString()));
+ }
+ }
+ } else if (object.isObject()) {
+ ObjectNode objectNode = (ObjectNode) object;
+ final Iterator<Map.Entry<String, JsonNode>> fields =
object.fields();
+ while (fields.hasNext()) {
+ final Map.Entry<String, JsonNode> field = fields.next();
+ final JsonNode temp = field.getValue();
+ if (temp.isArray()) {
+ replaceJson(dateTime, temp);
+ } else if (temp.isObject()) {
+ replaceJson(dateTime, temp);
+ } else {
+ objectNode.put(field.getKey(), replace(dateTime,
temp.toString()));
+ }
+ }
+ }
+ }
+
+ /**
+ * @param template
+ * @param map
+ * @return
+ */
+ public static String format(CharSequence template, Map<?, ?> map) {
+ return VariableOperationUtils.format(template, map, "${", "}", true);
+ }
+
+ /**
+ * map = {a: "aValue", b: "bValue"} format("{a} and {b}", map) ---=》
aValue and bValue
+ *
+ * @param template
+ * @param map
+ * @param leftStr
+ * @param rightStr
+ * @param ignoreNull
+ * @return
+ */
+ public static String format(
+ CharSequence template,
+ Map<?, ?> map,
+ CharSequence leftStr,
+ CharSequence rightStr,
+ boolean ignoreNull) {
+ if (null == template) {
+ return null;
+ }
+ if (null == map || map.isEmpty()) {
+ return template.toString();
+ }
+
+ if (StringUtils.isBlank(leftStr)) {
+ leftStr = "";
+ }
+
+ if (StringUtils.isBlank(rightStr)) {
+ rightStr = "";
+ }
+
+ String template2 = template.toString();
+ String value;
+ for (Map.Entry<?, ?> entry : map.entrySet()) {
+ value = entry.getValue().toString();
+ if (null == value && ignoreNull) {
+ continue;
+ }
+ template2 =
+ StringUtils.replace(
+ template2, leftStr.toString() + entry.getKey() +
rightStr, value);
+ }
+ return template2;
+ }
+}
diff --git
a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala
b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala
index f2696fab3..1b1a0528d 100644
---
a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala
+++
b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala
@@ -51,6 +51,8 @@ object Configuration extends Logging {
val GOVERNANCE_STATION_ADMIN =
CommonVars("wds.linkis.governance.station.admin", "hadoop")
+ val VARIABLE_OPERATION: Boolean =
CommonVars("wds.linkis.variable.operation", true).getValue
+
private val adminUsers = GOVERNANCE_STATION_ADMIN.getValue.split(",")
def isAdmin(username: String): Boolean = {
diff --git
a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala
b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala
index 4c7de924b..ac2a35bb3 100644
---
a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala
+++
b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala
@@ -18,11 +18,13 @@
package org.apache.linkis.common.utils
import org.apache.commons.lang3.StringUtils
+import org.apache.linkis.common.conf.Configuration
import org.apache.linkis.common.exception.LinkisCommonErrorException
import org.apache.linkis.common.variable
import org.apache.linkis.common.variable.DateTypeUtils.{getCurHour,
getMonthDay, getToday, getYesterday}
import org.apache.linkis.common.variable._
+import java.time.ZonedDateTime
import java.util
import scala.collection.JavaConverters.mapAsScalaMapConverter
import scala.collection.convert.WrapAsScala._
@@ -62,8 +64,10 @@ object VariableUtils extends Logging {
run_date = new CustomDateType(getYesterday(false), false)
nameAndType(RUN_DATE) = variable.DateType(new
CustomDateType(run_date.toString, false))
}
+
initAllDateVars(run_date, nameAndType)
- parserVar(replaceStr, nameAndType)
+ val codeOperation = parserDate(replaceStr, run_date)
+ parserVar(codeOperation, nameAndType)
}
def replace(code: String, runtType: String, variables: util.Map[String,
String]): String = {
@@ -116,9 +120,20 @@ object VariableUtils extends Logging {
}
initAllDateVars(run_date, nameAndType)
- parserVar(code, nameAndType)
+ val codeOperation = parserDate(code, run_date)
+ parserVar(codeOperation, nameAndType)
}
+ private def parserDate(code: String, run_date: CustomDateType) : String = {
+ if (Configuration.VARIABLE_OPERATION) {
+ val zonedDateTime: ZonedDateTime =
VariableOperationUtils.toZonedDateTime(run_date.getDate)
+ VariableOperationUtils.replaces(zonedDateTime, code)
+ } else {
+ code
+ }
+ }
+
+
private def initAllDateVars(run_date: CustomDateType, nameAndType:
mutable.Map[String, variable.VariableType]): Unit = {
val run_date_str = run_date.toString
nameAndType("run_date_std") = variable.DateType(new
CustomDateType(run_date.getStdDate))
diff --git
a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/variable/VariableOperationTest.java
b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/variable/VariableOperationTest.java
new file mode 100644
index 000000000..e8485aa46
--- /dev/null
+++
b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/variable/VariableOperationTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.linkis.common.variable;
+
+import org.apache.linkis.common.exception.VariableOperationFailedException;
+import org.apache.linkis.common.utils.VariableOperationUtils;
+
+import org.junit.jupiter.api.Test;
+
+import java.time.ZonedDateTime;
+import java.util.Date;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class VariableOperationTest {
+
+ private static final Date date = new Date(1648892107169L);
+ private static final ZonedDateTime zonedDateTime =
VariableOperationUtils.toZonedDateTime(date);
+
+ @Test
+ public void testJsonFormat() throws VariableOperationFailedException {
+ String jsonOld =
+
"{\"name\":\"${yyyyMMdd%-1d}\",\"address\":{\"street\":\"${yyyyMMdd%-1y}\"},\"links\":[{\"name\":\"${yyyyMMdd%-1M}\"}]}";
+ String jsonNew = VariableOperationUtils.replaces(zonedDateTime,
jsonOld);
+ System.out.println(jsonOld + "\n" + jsonNew);
+ assertEquals(
+ jsonNew,
+
"{\"name\":\"\\\"20220401\\\"\",\"address\":{\"street\":\"\\\"20210402\\\"\"},\"links\":[{\"name\":\"\\\"20220302\\\"\"}]}");
+ }
+
+ @Test
+ public void testTextFormat() throws VariableOperationFailedException {
+ String strOld = "abc${yyyyMMdd%-1d}def";
+ String strNew = VariableOperationUtils.replaces(zonedDateTime, strOld);
+ System.out.println(strOld + "\n" + strNew);
+ assertEquals(strNew, "abc20220401def");
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
