This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new ffbb28a38bb [To dev/1.3] Supprt trusted_uri_pattern in config file
ffbb28a38bb is described below
commit ffbb28a38bb020a4fb9c5fdfaa95e282a9d68237
Author: Jackie Tien <[email protected]>
AuthorDate: Thu Dec 12 18:20:53 2024 +0800
[To dev/1.3] Supprt trusted_uri_pattern in config file
---
.../org/apache/iotdb/ainode/it/AINodeBasicIT.java | 1 +
.../db/it/trigger/IoTDBTriggerManagementIT.java | 1 +
.../iotdb/db/it/udaf/IoTDBUDAFManagementIT.java | 1 +
.../iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java | 12 ++++++
.../iotdb/db/it/udf/IoTDBUDFManagementIT.java | 1 +
.../pipe/it/autocreate/IoTDBPipeLifeCycleIT.java | 2 +-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 2 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 23 +++++++++++
.../plan/execution/config/ConfigTaskVisitor.java | 44 ++++++++++++++++++++--
.../config/executor/ClusterConfigTaskExecutor.java | 18 ++++-----
.../db/queryengine/plan/parser/ASTVisitor.java | 2 +-
.../conf/iotdb-system.properties.template | 6 +++
.../apache/iotdb/commons/conf/CommonConfig.java | 11 ++++++
.../commons/executable/ExecutableManager.java | 5 +++
14 files changed, 113 insertions(+), 16 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeBasicIT.java
b/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeBasicIT.java
index 0306b1a5225..265fbf4e301 100644
---
a/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeBasicIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeBasicIT.java
@@ -56,6 +56,7 @@ public class AINodeBasicIT {
static String[] sqls =
new String[] {
+ "set configuration \"trusted_uri_pattern\"='.*'",
"create model identity using uri \"" + MODEL_PATH + "\"",
"CREATE DATABASE root.AI.data",
"CREATE TIMESERIES root.AI.data.s0 WITH DATATYPE=FLOAT, ENCODING=RLE",
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/trigger/IoTDBTriggerManagementIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/trigger/IoTDBTriggerManagementIT.java
index cb21faa628d..a36a9c048aa 100644
---
a/integration-test/src/test/java/org/apache/iotdb/db/it/trigger/IoTDBTriggerManagementIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/trigger/IoTDBTriggerManagementIT.java
@@ -109,6 +109,7 @@ public class IoTDBTriggerManagementIT {
"CREATE TIMESERIES root.test.stateful.b with
datatype=INT32,encoding=PLAIN");
statement.execute(
"CREATE TIMESERIES root.test.stateful.c with
datatype=INT32,encoding=PLAIN");
+ statement.execute("set configuration \"trusted_uri_pattern\"='.*'");
} catch (SQLException throwable) {
fail(throwable.getMessage());
}
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFManagementIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFManagementIT.java
index 75dc0c1f5f7..eff19645f63 100644
---
a/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFManagementIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFManagementIT.java
@@ -235,6 +235,7 @@ public class IoTDBUDAFManagementIT {
public void createFunctionWithInvalidURITest() {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
+ statement.execute("set configuration \"trusted_uri_pattern\"='.*'");
try {
statement.execute(
String.format(
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java
index 5de421ebd2a..71555bce0e3 100644
---
a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java
@@ -152,4 +152,16 @@ public class IoTDBUDFBlockQueryIT {
fail(throwable.getMessage());
}
}
+
+ @Test
+ public void testUntrustedUri() {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute(
+ "CREATE FUNCTION two_sum AS
'org.apache.iotdb.db.query.udf.example.TwoSum' USING URI
'https://alioss.timecho.com/upload/library-udf.jar'");
+ fail("should fail");
+ } catch (SQLException throwable) {
+ assertTrue(throwable.getMessage().contains("701: Untrusted uri "));
+ }
+ }
}
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFManagementIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFManagementIT.java
index 798256e5a16..261675a3c53 100644
---
a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFManagementIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFManagementIT.java
@@ -262,6 +262,7 @@ public class IoTDBUDFManagementIT {
public void testCreateFunctionWithInvalidURI() {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
+ statement.execute("set configuration \"trusted_uri_pattern\"='.*'");
try {
statement.execute(
String.format(
diff --git
a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java
b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java
index 02abcde0a12..7ad604e01d8 100644
---
a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java
@@ -846,7 +846,7 @@ public class IoTDBPipeLifeCycleIT extends
AbstractPipeDualAutoIT {
assertNonQueryTestFail(
senderEnv,
"create pipePlugin TestProcessor as
'org.apache.iotdb.db.pipe.example.TestProcessor' USING URI 'xxx'",
- "1603: The scheme of URI is not set, please specify the scheme of
URI.",
+ "701: Untrusted uri xxx",
"test",
"test123");
tryExecuteNonQueryWithRetry(senderEnv, "drop pipePlugin TestProcessor",
"test", "test123");
diff --git
a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
index 25bff7e5499..d0bffc17ad3 100644
---
a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
+++
b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
@@ -672,7 +672,7 @@ showSubscriptions
// AI Model
=========================================================================================
// ---- Create Model
createModel
- : CREATE MODEL modelName=identifier USING URI modelUri=STRING_LITERAL
+ : CREATE MODEL modelName=identifier uriClause
;
windowFunction
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index b5e29e5c901..0052cc82140 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -86,6 +86,7 @@ import java.util.Optional;
import java.util.Properties;
import java.util.ServiceLoader;
import java.util.Set;
+import java.util.regex.Pattern;
public class IoTDBDescriptor {
@@ -2003,6 +2004,28 @@ public class IoTDBDescriptor {
} else {
BinaryAllocator.getInstance().close(true);
}
+
+ // update trusted_uri_pattern
+ String trustedUriPattern =
+ Optional.ofNullable(
+ properties.getProperty(
+ "trusted_uri_pattern",
+
ConfigurationFileUtils.getConfigurationDefaultValue("trusted_uri_pattern")))
+ .map(String::trim)
+
.orElse(ConfigurationFileUtils.getConfigurationDefaultValue("trusted_uri_pattern"));
+ Pattern pattern;
+ if (trustedUriPattern != null) {
+ try {
+ pattern = Pattern.compile(trustedUriPattern);
+ } catch (Exception e) {
+ LOGGER.warn("Failed to parse trusted_uri_pattern {}",
trustedUriPattern);
+ pattern = commonDescriptor.getConfig().getTrustedUriPattern();
+ }
+ } else {
+ pattern = commonDescriptor.getConfig().getTrustedUriPattern();
+ }
+ commonDescriptor.getConfig().setTrustedUriPattern(pattern);
+
} catch (Exception e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java
index b928d472234..2f3b758e77a 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.queryengine.plan.execution.config;
+import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import
org.apache.iotdb.db.queryengine.plan.execution.config.metadata.CountDatabaseTask;
import
org.apache.iotdb.db.queryengine.plan.execution.config.metadata.CountTimeSlotListTask;
@@ -171,6 +172,8 @@ import
org.apache.iotdb.db.queryengine.plan.statement.sys.quota.ShowThrottleQuot
import org.apache.tsfile.exception.NotImplementedException;
+import static
org.apache.iotdb.commons.executable.ExecutableManager.isUriTrusted;
+
public class ConfigTaskVisitor extends StatementVisitor<IConfigTask,
MPPQueryContext> {
@Override
@@ -316,7 +319,16 @@ public class ConfigTaskVisitor extends
StatementVisitor<IConfigTask, MPPQueryCon
@Override
public IConfigTask visitCreateFunction(
CreateFunctionStatement createFunctionStatement, MPPQueryContext
context) {
- return new CreateFunctionTask(createFunctionStatement);
+ if (!createFunctionStatement.isUsingURI()
+ || (createFunctionStatement.getUriString() != null
+ && isUriTrusted(createFunctionStatement.getUriString()))) {
+ // 1. user specified uri and that uri is trusted
+ // 2. user doesn't specify uri
+ return new CreateFunctionTask(createFunctionStatement);
+ } else {
+ // user specified uri and that uri is not trusted
+ throw new SemanticException("Untrusted uri " +
createFunctionStatement.getUriString());
+ }
}
@Override
@@ -334,7 +346,16 @@ public class ConfigTaskVisitor extends
StatementVisitor<IConfigTask, MPPQueryCon
@Override
public IConfigTask visitCreateTrigger(
CreateTriggerStatement createTriggerStatement, MPPQueryContext context) {
- return new CreateTriggerTask(createTriggerStatement);
+ if (!createTriggerStatement.isUsingURI()
+ || (createTriggerStatement.getUriString() != null
+ && isUriTrusted(createTriggerStatement.getUriString()))) {
+ // 1. user specified uri and that uri is trusted
+ // 2. user doesn't specify uri
+ return new CreateTriggerTask(createTriggerStatement);
+ } else {
+ // user specified uri and that uri is not trusted
+ throw new SemanticException("Untrusted uri " +
createTriggerStatement.getUriString());
+ }
}
@Override
@@ -352,7 +373,15 @@ public class ConfigTaskVisitor extends
StatementVisitor<IConfigTask, MPPQueryCon
@Override
public IConfigTask visitCreatePipePlugin(
CreatePipePluginStatement createPipePluginStatement, MPPQueryContext
context) {
- return new CreatePipePluginTask(createPipePluginStatement);
+ if (createPipePluginStatement.getUriString() != null
+ && isUriTrusted(createPipePluginStatement.getUriString())) {
+ // 1. user specified uri and that uri is trusted
+ // 2. user doesn't specify uri
+ return new CreatePipePluginTask(createPipePluginStatement);
+ } else {
+ // user specified uri and that uri is not trusted
+ throw new SemanticException("Untrusted uri " +
createPipePluginStatement.getUriString());
+ }
}
@Override
@@ -598,7 +627,14 @@ public class ConfigTaskVisitor extends
StatementVisitor<IConfigTask, MPPQueryCon
@Override
public IConfigTask visitCreateModel(
CreateModelStatement createModelStatement, MPPQueryContext context) {
- return new CreateModelTask(createModelStatement, context);
+ if (createModelStatement.getUri() != null &&
isUriTrusted(createModelStatement.getUri())) {
+ // 1. user specified uri and that uri is trusted
+ // 2. user doesn't specify uri
+ return new CreateModelTask(createModelStatement, context);
+ } else {
+ // user specified uri and that uri is not trusted
+ throw new SemanticException("Untrusted uri " +
createModelStatement.getUri());
+ }
}
@Override
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
index 618a9cd53dc..f9e824a87b6 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
@@ -473,7 +473,6 @@ public class ClusterConfigTaskExecutor implements
IConfigTaskExecutor {
String jarFilePathUnderTempDir =
UDFExecutableManager.getInstance()
.getDirStringUnderTempRootByRequestId(resource.getRequestId())
- + File.separator
+ jarFileName;
// libRoot should be the path of the specified jar
libRoot = jarFilePathUnderTempDir;
@@ -506,12 +505,15 @@ public class ClusterConfigTaskExecutor implements
IConfigTaskExecutor {
tCreateFunctionReq.setJarFile(jarFile);
tCreateFunctionReq.setJarMD5(jarMd5);
tCreateFunctionReq.setIsUsingURI(true);
- tCreateFunctionReq.setJarName(
- String.format(
- "%s-%s.%s",
- jarFileName.substring(0, jarFileName.lastIndexOf(".")),
- jarMd5,
- jarFileName.substring(jarFileName.lastIndexOf(".") + 1)));
+ int index = jarFileName.lastIndexOf(".");
+ if (index < 0) {
+ tCreateFunctionReq.setJarName(String.format("%s-%s", jarFileName,
jarMd5));
+ } else {
+ tCreateFunctionReq.setJarName(
+ String.format(
+ "%s-%s.%s",
+ jarFileName.substring(0, index), jarMd5,
jarFileName.substring(index + 1)));
+ }
}
// try to create instance, this request will fail if creation is not
successful
@@ -645,7 +647,6 @@ public class ClusterConfigTaskExecutor implements
IConfigTaskExecutor {
String jarFilePathUnderTempDir =
TriggerExecutableManager.getInstance()
.getDirStringUnderTempRootByRequestId(resource.getRequestId())
- + File.separator
+ jarFileName;
// libRoot should be the path of the specified jar
libRoot = jarFilePathUnderTempDir;
@@ -811,7 +812,6 @@ public class ClusterConfigTaskExecutor implements
IConfigTaskExecutor {
final String jarFilePathUnderTempDir =
PipePluginExecutableManager.getInstance()
.getDirStringUnderTempRootByRequestId(resource.getRequestId())
- + File.separator
+ jarFileName;
// libRoot should be the path of the specified jar
libRoot = jarFilePathUnderTempDir;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
index 9b38d4856be..a498efbac47 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
@@ -1327,7 +1327,7 @@ public class ASTVisitor extends
IoTDBSqlParserBaseVisitor<Statement> {
String modelName = ctx.modelName.getText();
validateModelName(modelName);
createModelStatement.setModelName(parseIdentifier(modelName));
- createModelStatement.setUri(ctx.modelUri.getText());
+ createModelStatement.setUri(ctx.uriClause().uri().getText());
return createModelStatement;
}
diff --git
a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
index 4cde7f8eff2..6253e50c85c 100644
---
a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
+++
b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
@@ -1587,6 +1587,12 @@ author_cache_size=1000
# Datatype: int
author_cache_expire_time=30
+# A regex pattern representing trusted uri by system
+# effectiveMode: hot_reload
+# Datatype: regex
+# If you want to allow all URIs, you can specify it as .*
+trusted_uri_pattern=file:.*
+
####################
### UDF Configuration
####################
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
index 2d1585d6525..8d30d0c6d80 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
@@ -36,6 +36,7 @@ import java.io.File;
import java.io.IOException;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
import static org.apache.iotdb.commons.conf.IoTDBConstant.MB;
@@ -343,6 +344,8 @@ public class CommonConfig {
private volatile long remoteWriteMaxRetryDurationInMs = 60000;
+ private volatile Pattern trustedUriPattern = Pattern.compile("file:.*");
+
CommonConfig() {
// Empty constructor
}
@@ -1529,4 +1532,12 @@ public class CommonConfig {
public void setLog2SizeClassGroup(int log2SizeClassGroup) {
this.log2SizeClassGroup = log2SizeClassGroup;
}
+
+ public Pattern getTrustedUriPattern() {
+ return trustedUriPattern;
+ }
+
+ public void setTrustedUriPattern(Pattern trustedUriPattern) {
+ this.trustedUriPattern = trustedUriPattern;
+ }
}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ExecutableManager.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ExecutableManager.java
index 435ee489234..b00276db38a 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ExecutableManager.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ExecutableManager.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.commons.executable;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.trigger.exception.TriggerJarTooLargeException;
import org.apache.commons.io.FileUtils;
@@ -282,4 +283,8 @@ public class ExecutableManager {
public String getInstallDir() {
return libRoot + File.separator + INSTALL_DIR;
}
+
+ public static boolean isUriTrusted(String uri) {
+ return
CommonDescriptor.getInstance().getConfig().getTrustedUriPattern().matcher(uri).matches();
+ }
}
