This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch TrustedUri-1.3 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit cc335c0d1bb8b192f8c0760609ea4e7f17b16a2b Author: Jackie Tien <[email protected]> AuthorDate: Thu Dec 12 14:33:40 2024 +0800 Supprt trusted_uri_pattern in config file (cherry picked from commit 37450ec160b3f66e46c2015eb758b549877753b1) --- .../org/apache/iotdb/ainode/it/AINodeBasicIT.java | 1 + .../iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java | 12 ++++++ .../pipe/it/autocreate/IoTDBPipeLifeCycleIT.java | 2 +- .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 2 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 23 +++++++++++ .../plan/execution/config/ConfigTaskVisitor.java | 44 ++++++++++++++++++++-- .../config/executor/ClusterConfigTaskExecutor.java | 18 ++++----- .../db/queryengine/plan/parser/ASTVisitor.java | 2 +- .../conf/iotdb-system.properties.template | 6 +++ .../apache/iotdb/commons/conf/CommonConfig.java | 11 ++++++ .../commons/executable/ExecutableManager.java | 5 +++ 11 files changed, 110 insertions(+), 16 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeBasicIT.java b/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeBasicIT.java index 0306b1a5225..265fbf4e301 100644 --- a/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeBasicIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeBasicIT.java @@ -56,6 +56,7 @@ public class AINodeBasicIT { static String[] sqls = new String[] { + "set configuration \"trusted_uri_pattern\"='.*'", "create model identity using uri \"" + MODEL_PATH + "\"", "CREATE DATABASE root.AI.data", "CREATE TIMESERIES root.AI.data.s0 WITH DATATYPE=FLOAT, ENCODING=RLE", diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java index 5de421ebd2a..71555bce0e3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java @@ -152,4 +152,16 @@ public class IoTDBUDFBlockQueryIT { fail(throwable.getMessage()); } } + + @Test + public void testUntrustedUri() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute( + "CREATE FUNCTION two_sum AS 'org.apache.iotdb.db.query.udf.example.TwoSum' USING URI 'https://alioss.timecho.com/upload/library-udf.jar'"); + fail("should fail"); + } catch (SQLException throwable) { + assertTrue(throwable.getMessage().contains("701: Untrusted uri ")); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java index 02abcde0a12..7ad604e01d8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java @@ -846,7 +846,7 @@ public class IoTDBPipeLifeCycleIT extends AbstractPipeDualAutoIT { assertNonQueryTestFail( senderEnv, "create pipePlugin TestProcessor as 'org.apache.iotdb.db.pipe.example.TestProcessor' USING URI 'xxx'", - "1603: The scheme of URI is not set, please specify the scheme of URI.", + "701: Untrusted uri xxx", "test", "test123"); tryExecuteNonQueryWithRetry(senderEnv, "drop pipePlugin TestProcessor", "test", "test123"); diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 index 25bff7e5499..d0bffc17ad3 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 @@ -672,7 +672,7 @@ showSubscriptions // AI Model ========================================================================================= // ---- Create Model createModel - : CREATE MODEL modelName=identifier USING URI modelUri=STRING_LITERAL + : CREATE MODEL modelName=identifier uriClause ; windowFunction diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index b5e29e5c901..0052cc82140 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -86,6 +86,7 @@ import java.util.Optional; import java.util.Properties; import java.util.ServiceLoader; import java.util.Set; +import java.util.regex.Pattern; public class IoTDBDescriptor { @@ -2003,6 +2004,28 @@ public class IoTDBDescriptor { } else { BinaryAllocator.getInstance().close(true); } + + // update trusted_uri_pattern + String trustedUriPattern = + Optional.ofNullable( + properties.getProperty( + "trusted_uri_pattern", + ConfigurationFileUtils.getConfigurationDefaultValue("trusted_uri_pattern"))) + .map(String::trim) + .orElse(ConfigurationFileUtils.getConfigurationDefaultValue("trusted_uri_pattern")); + Pattern pattern; + if (trustedUriPattern != null) { + try { + pattern = Pattern.compile(trustedUriPattern); + } catch (Exception e) { + LOGGER.warn("Failed to parse trusted_uri_pattern {}", trustedUriPattern); + pattern = commonDescriptor.getConfig().getTrustedUriPattern(); + } + } else { + pattern = commonDescriptor.getConfig().getTrustedUriPattern(); + } + commonDescriptor.getConfig().setTrustedUriPattern(pattern); + } catch (Exception e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java index b928d472234..2f3b758e77a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.plan.execution.config; +import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.CountDatabaseTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.CountTimeSlotListTask; @@ -171,6 +172,8 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.quota.ShowThrottleQuot import org.apache.tsfile.exception.NotImplementedException; +import static org.apache.iotdb.commons.executable.ExecutableManager.isUriTrusted; + public class ConfigTaskVisitor extends StatementVisitor<IConfigTask, MPPQueryContext> { @Override @@ -316,7 +319,16 @@ public class ConfigTaskVisitor extends StatementVisitor<IConfigTask, MPPQueryCon @Override public IConfigTask visitCreateFunction( CreateFunctionStatement createFunctionStatement, MPPQueryContext context) { - return new CreateFunctionTask(createFunctionStatement); + if (!createFunctionStatement.isUsingURI() + || (createFunctionStatement.getUriString() != null + && isUriTrusted(createFunctionStatement.getUriString()))) { + // 1. user specified uri and that uri is trusted + // 2. user doesn't specify uri + return new CreateFunctionTask(createFunctionStatement); + } else { + // user specified uri and that uri is not trusted + throw new SemanticException("Untrusted uri " + createFunctionStatement.getUriString()); + } } @Override @@ -334,7 +346,16 @@ public class ConfigTaskVisitor extends StatementVisitor<IConfigTask, MPPQueryCon @Override public IConfigTask visitCreateTrigger( CreateTriggerStatement createTriggerStatement, MPPQueryContext context) { - return new CreateTriggerTask(createTriggerStatement); + if (!createTriggerStatement.isUsingURI() + || (createTriggerStatement.getUriString() != null + && isUriTrusted(createTriggerStatement.getUriString()))) { + // 1. user specified uri and that uri is trusted + // 2. user doesn't specify uri + return new CreateTriggerTask(createTriggerStatement); + } else { + // user specified uri and that uri is not trusted + throw new SemanticException("Untrusted uri " + createTriggerStatement.getUriString()); + } } @Override @@ -352,7 +373,15 @@ public class ConfigTaskVisitor extends StatementVisitor<IConfigTask, MPPQueryCon @Override public IConfigTask visitCreatePipePlugin( CreatePipePluginStatement createPipePluginStatement, MPPQueryContext context) { - return new CreatePipePluginTask(createPipePluginStatement); + if (createPipePluginStatement.getUriString() != null + && isUriTrusted(createPipePluginStatement.getUriString())) { + // 1. user specified uri and that uri is trusted + // 2. user doesn't specify uri + return new CreatePipePluginTask(createPipePluginStatement); + } else { + // user specified uri and that uri is not trusted + throw new SemanticException("Untrusted uri " + createPipePluginStatement.getUriString()); + } } @Override @@ -598,7 +627,14 @@ public class ConfigTaskVisitor extends StatementVisitor<IConfigTask, MPPQueryCon @Override public IConfigTask visitCreateModel( CreateModelStatement createModelStatement, MPPQueryContext context) { - return new CreateModelTask(createModelStatement, context); + if (createModelStatement.getUri() != null && isUriTrusted(createModelStatement.getUri())) { + // 1. user specified uri and that uri is trusted + // 2. user doesn't specify uri + return new CreateModelTask(createModelStatement, context); + } else { + // user specified uri and that uri is not trusted + throw new SemanticException("Untrusted uri " + createModelStatement.getUri()); + } } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 618a9cd53dc..f9e824a87b6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -473,7 +473,6 @@ public class ClusterConfigTaskExecutor implements IConfigTaskExecutor { String jarFilePathUnderTempDir = UDFExecutableManager.getInstance() .getDirStringUnderTempRootByRequestId(resource.getRequestId()) - + File.separator + jarFileName; // libRoot should be the path of the specified jar libRoot = jarFilePathUnderTempDir; @@ -506,12 +505,15 @@ public class ClusterConfigTaskExecutor implements IConfigTaskExecutor { tCreateFunctionReq.setJarFile(jarFile); tCreateFunctionReq.setJarMD5(jarMd5); tCreateFunctionReq.setIsUsingURI(true); - tCreateFunctionReq.setJarName( - String.format( - "%s-%s.%s", - jarFileName.substring(0, jarFileName.lastIndexOf(".")), - jarMd5, - jarFileName.substring(jarFileName.lastIndexOf(".") + 1))); + int index = jarFileName.lastIndexOf("."); + if (index < 0) { + tCreateFunctionReq.setJarName(String.format("%s-%s", jarFileName, jarMd5)); + } else { + tCreateFunctionReq.setJarName( + String.format( + "%s-%s.%s", + jarFileName.substring(0, index), jarMd5, jarFileName.substring(index + 1))); + } } // try to create instance, this request will fail if creation is not successful @@ -645,7 +647,6 @@ public class ClusterConfigTaskExecutor implements IConfigTaskExecutor { String jarFilePathUnderTempDir = TriggerExecutableManager.getInstance() .getDirStringUnderTempRootByRequestId(resource.getRequestId()) - + File.separator + jarFileName; // libRoot should be the path of the specified jar libRoot = jarFilePathUnderTempDir; @@ -811,7 +812,6 @@ public class ClusterConfigTaskExecutor implements IConfigTaskExecutor { final String jarFilePathUnderTempDir = PipePluginExecutableManager.getInstance() .getDirStringUnderTempRootByRequestId(resource.getRequestId()) - + File.separator + jarFileName; // libRoot should be the path of the specified jar libRoot = jarFilePathUnderTempDir; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index 8796c4e8b65..3c2e18fad17 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -1327,7 +1327,7 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> { String modelName = ctx.modelName.getText(); validateModelName(modelName); createModelStatement.setModelName(parseIdentifier(modelName)); - createModelStatement.setUri(ctx.modelUri.getText()); + createModelStatement.setUri(ctx.uriClause().uri().getText()); return createModelStatement; } diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 4cde7f8eff2..6253e50c85c 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1587,6 +1587,12 @@ author_cache_size=1000 # Datatype: int author_cache_expire_time=30 +# A regex pattern representing trusted uri by system +# effectiveMode: hot_reload +# Datatype: regex +# If you want to allow all URIs, you can specify it as .* +trusted_uri_pattern=file:.* + #################### ### UDF Configuration #################### diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index 2d1585d6525..8d30d0c6d80 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -36,6 +36,7 @@ import java.io.File; import java.io.IOException; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; import static org.apache.iotdb.commons.conf.IoTDBConstant.MB; @@ -343,6 +344,8 @@ public class CommonConfig { private volatile long remoteWriteMaxRetryDurationInMs = 60000; + private volatile Pattern trustedUriPattern = Pattern.compile("file:.*"); + CommonConfig() { // Empty constructor } @@ -1529,4 +1532,12 @@ public class CommonConfig { public void setLog2SizeClassGroup(int log2SizeClassGroup) { this.log2SizeClassGroup = log2SizeClassGroup; } + + public Pattern getTrustedUriPattern() { + return trustedUriPattern; + } + + public void setTrustedUriPattern(Pattern trustedUriPattern) { + this.trustedUriPattern = trustedUriPattern; + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ExecutableManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ExecutableManager.java index 435ee489234..b00276db38a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ExecutableManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ExecutableManager.java @@ -19,6 +19,7 @@ package org.apache.iotdb.commons.executable; +import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.trigger.exception.TriggerJarTooLargeException; import org.apache.commons.io.FileUtils; @@ -282,4 +283,8 @@ public class ExecutableManager { public String getInstallDir() { return libRoot + File.separator + INSTALL_DIR; } + + public static boolean isUriTrusted(String uri) { + return CommonDescriptor.getInstance().getConfig().getTrustedUriPattern().matcher(uri).matches(); + } }
