This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch fixCopyToSecurityIssue in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b3ed39d067c86eb030e5562212079f7cfc4786ea Author: shuwenwei <[email protected]> AuthorDate: Thu Sep 3 09:34:26 2026 +0800 Restrict COPY TO export paths --- .../query/recent/copyto/IoTDBCopyToTsFileIT.java | 24 +++++++++ .../apache/iotdb/db/i18n/DataNodeMiscMessages.java | 4 ++ .../iotdb/db/i18n/DataNodeQueryMessages.java | 3 ++ .../apache/iotdb/db/i18n/DataNodeMiscMessages.java | 4 ++ .../iotdb/db/i18n/DataNodeQueryMessages.java | 2 + .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 63 ++++++++++++++++++++++ .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 14 +++++ .../relational/analyzer/StatementAnalyzer.java | 22 ++++++++ .../conf/iotdb-system.properties.template | 10 ++++ .../org/apache/iotdb/commons/utils/FileUtils.java | 33 ++++++++++++ 10 files changed, 179 insertions(+) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/copyto/IoTDBCopyToTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/copyto/IoTDBCopyToTsFileIT.java index e25c3f58299..e170605b022 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/copyto/IoTDBCopyToTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/copyto/IoTDBCopyToTsFileIT.java @@ -141,6 +141,30 @@ public class IoTDBCopyToTsFileIT { } } + @Test + public void testCopyToRejectsClientSuppliedAbsolutePath() + throws IoTDBConnectionException, IOException { + File targetDirectory = Files.createTempDirectory("iotdb-copy-to-security").toFile(); + File targetFile = new File(targetDirectory, "result.tsfile"); + String targetPath = targetFile.getAbsolutePath().replace("\\", "\\\\").replace("'", "''"); + + try (ITableSession session = + EnvFactory.getEnv().getTableSessionConnectionWithDB(DATABASE_NAME)) { + try { + session.executeQueryStatement( + "copy table1 to '" + targetPath + "' (memory_threshold 1000000)"); + Assert.fail("COPY TO should reject a client-supplied absolute path"); + } catch (StatementExecutionException e) { + Assert.assertTrue( + e.getMessage(), e.getMessage().contains("COPY TO target path is outside")); + } + Assert.assertFalse(targetFile.exists()); + } finally { + Files.deleteIfExists(targetFile.toPath()); + Files.deleteIfExists(targetDirectory.toPath()); + } + } + @Test public void testCopySelectAllColumns() throws IoTDBConnectionException, StatementExecutionException, IOException { diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index 4aef9cd74f0..1e79e3079a3 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -1383,6 +1383,10 @@ public final class DataNodeMiscMessages { + "is your token hint info: %s"; public static final String MISC_EXCEPTION_FAIL_TO_RELOAD_CONFIG_FILE_S_BECAUSE_S_93CCAB8D = "Fail to reload config file %s because %s"; + public static final String MISC_EXCEPTION_FAILED_TO_CANONICALIZE_COPY_TO_ALLOWED_EXPORT_DIR_70EB7BF1 = + "Failed to canonicalize COPY TO allowed export directory: %s"; + public static final String MISC_EXCEPTION_COPY_TO_ALLOWED_EXPORT_DIR_OVERLAPS_ACTIVE_LOAD_DIR_DAACF086 = + "COPY TO allowed export directory overlaps active load directory: %s"; public static final String MISC_EXCEPTION_EACH_SUBSECTION_OF_CONFIGURATION_ITEM_UDF_READER_TRANSFORMER_97CA8962 = "Each subsection of configuration item udf_reader_transformer_collector_memory_proportion " + "should be an integer, which is %s"; diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 3feefd72a6f..91a1a03a3f0 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -190,6 +190,9 @@ public final class DataNodeQueryMessages { "Target file already exists: "; public static final String FAILED_TO_CREATE_FILE = "Failed to create file: "; + public static final String COPY_TO_TARGET_PATH_NOT_ALLOWED = + "COPY TO target path is outside the allowed export directories" + + " (configure copy_to_allowed_export_dirs to permit it): "; public static final String DATA_TYPE_OF_TARGET_TIME_COLUMN_IS_NOT = "Data type of target time column is not TIMESTAMP"; public static final String DUPLICATE_COLUMN_NAMES_IN_QUERY_DATASET = diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index bf909f27745..8556b3ade0c 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -1355,6 +1355,10 @@ public final class DataNodeMiscMessages { "首次启动后不允许修改 TsFile 加密类型或密钥。token hint 信息：%s"; public static final String MISC_EXCEPTION_FAIL_TO_RELOAD_CONFIG_FILE_S_BECAUSE_S_93CCAB8D = "无法重新加载配置文件 %s，原因：%s"; + public static final String MISC_EXCEPTION_FAILED_TO_CANONICALIZE_COPY_TO_ALLOWED_EXPORT_DIR_70EB7BF1 = + "无法获取 COPY TO 允许导出目录的规范路径：%s"; + public static final String MISC_EXCEPTION_COPY_TO_ALLOWED_EXPORT_DIR_OVERLAPS_ACTIVE_LOAD_DIR_DAACF086 = + "COPY TO 允许导出目录与 active load 目录重叠：%s"; public static final String MISC_EXCEPTION_EACH_SUBSECTION_OF_CONFIGURATION_ITEM_UDF_READER_TRANSFORMER_97CA8962 = "配置项 udf_reader_transformer_collector_memory_proportion 的每个子项都应为整数，当前为 %s"; public static final String diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 0d4e7cc0071..dc6e4af5cfd 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -179,6 +179,8 @@ public final class DataNodeQueryMessages { "目标文件已存在："; public static final String FAILED_TO_CREATE_FILE = "创建文件失败："; + public static final String COPY_TO_TARGET_PATH_NOT_ALLOWED = + "COPY TO 目标路径不在允许的导出目录内（可通过 copy_to_allowed_export_dirs 配置）："; public static final String DATA_TYPE_OF_TARGET_TIME_COLUMN_IS_NOT = "目标时间列的数据类型不是 TIMESTAMP"; public static final String DUPLICATE_COLUMN_NAMES_IN_QUERY_DATASET = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 5e0ff2dec6a..4885d5300be 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.client.property.ClientPoolProperty.DefaultProper import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.enums.ReadConsistencyLevel; +import org.apache.iotdb.commons.exception.IoTDBRuntimeException; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.consensus.ConsensusFactory; @@ -47,6 +48,7 @@ import org.apache.iotdb.metrics.config.MetricConfigDescriptor; import org.apache.iotdb.metrics.metricsets.system.SystemMetrics; import org.apache.iotdb.rpc.BaseRpcTransportFactory; import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.rpc.ZeroCopyRpcTransportFactory; import org.apache.tsfile.common.conf.TSFileDescriptor; @@ -1191,6 +1193,13 @@ public class IoTDBConfig { + IoTDBConstant.LOAD_TSFILE_ACTIVE_LISTENING_PENDING_FOLDER_NAME }; + /** + * Directories into which COPY ... TO may export when the client supplies a target path with a + * parent component. Empty (the default) rejects such paths; bare file names always land in the + * TierManager-managed copyto folders. + */ + private String[] copyToAllowedExportDirs = new String[0]; + private String loadActiveListeningPipeDir = IoTDBConstant.EXT_FOLDER_NAME + File.separator @@ -1419,6 +1428,9 @@ public class IoTDBConfig { loadTsFileAllowedDirs[i] = addDataHomeDir(loadTsFileAllowedDirs[i]); } loadTsFileAllowedDirCanonicalPaths = canonicalPaths(loadTsFileAllowedDirs); + for (int i = 0; i < copyToAllowedExportDirs.length; i++) { + copyToAllowedExportDirs[i] = addDataHomeDir(copyToAllowedExportDirs[i]); + } loadActiveListeningPipeDir = addDataHomeDir(loadActiveListeningPipeDir); loadActiveListeningFailDir = addDataHomeDir(loadActiveListeningFailDir); udfDir = addDataHomeDir(udfDir); @@ -4337,6 +4349,57 @@ public class IoTDBConfig { this.loadActiveListeningDirs = normalizedDirs; } + public String[] getCopyToAllowedExportDirs() { + return copyToAllowedExportDirs; + } + + public void setCopyToAllowedExportDirs(final String[] copyToAllowedExportDirs) { + if (copyToAllowedExportDirs == null) { + this.copyToAllowedExportDirs = new String[0]; + return; + } + this.copyToAllowedExportDirs = + Arrays.stream(copyToAllowedExportDirs) + .filter(Objects::nonNull) + .map(String::trim) + .filter(dir -> !dir.isEmpty()) + .map(IoTDBConfig::addDataHomeDir) + .toArray(String[]::new); + } + + public void validateCopyToAllowedExportDirs() { + try { + for (final String allowedExportDir : copyToAllowedExportDirs) { + final Path allowedExportPath = new File(allowedExportDir).getCanonicalFile().toPath(); + for (final String activeLoadDir : getLoadActiveListeningDirs()) { + validateCopyToDirectoryOverlap(allowedExportPath, activeLoadDir); + } + validateCopyToDirectoryOverlap(allowedExportPath, getLoadActiveListeningPipeDir()); + } + } catch (IOException e) { + throw new IoTDBRuntimeException( + DataNodeMiscMessages + .MISC_EXCEPTION_FAILED_TO_CANONICALIZE_COPY_TO_ALLOWED_EXPORT_DIR_70EB7BF1, + e, + TSStatusCode.CONFIGURATION_ERROR.getStatusCode()); + } + } + + private void validateCopyToDirectoryOverlap( + final Path allowedExportPath, final String activeLoadDir) throws IOException { + final Path activeLoadPath = new File(activeLoadDir).getCanonicalFile().toPath(); + if (allowedExportPath.startsWith(activeLoadPath) + || activeLoadPath.startsWith(allowedExportPath)) { + throw new IoTDBRuntimeException( + String.format( + DataNodeMiscMessages + .MISC_EXCEPTION_COPY_TO_ALLOWED_EXPORT_DIR_OVERLAPS_ACTIVE_LOAD_DIR_DAACF086, + allowedExportPath), + TSStatusCode.CONFIGURATION_ERROR.getStatusCode(), + true); + } + } + public boolean getLoadActiveListeningEnable() { return loadActiveListeningEnable; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index c60db041223..f6591857863 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -509,6 +509,18 @@ public class IoTDBDescriptor { properties.getProperty( "query_timeout_threshold", Long.toString(conf.getQueryTimeoutThreshold())))); + conf.setCopyToAllowedExportDirs( + Arrays.stream( + properties + .getProperty( + "copy_to_allowed_export_dirs", + String.join(",", conf.getCopyToAllowedExportDirs())) + .trim() + .split(",")) + .map(String::trim) + .filter(dir -> !dir.isEmpty()) + .toArray(String[]::new)); + conf.setSessionTimeoutThreshold( Integer.parseInt( properties.getProperty( @@ -2617,6 +2629,7 @@ public class IoTDBDescriptor { conf.setLoadActiveListeningPipeDir( properties.getProperty( "load_active_listening_pipe_dir", conf.getLoadActiveListeningPipeDir())); + conf.validateCopyToAllowedExportDirs(); final long loadActiveListeningCheckIntervalSeconds = Long.parseLong( @@ -2754,6 +2767,7 @@ public class IoTDBDescriptor { conf.setLoadActiveListeningPipeDir( properties.getProperty( "load_active_listening_pipe_dir", conf.getLoadActiveListeningPipeDir())); + conf.validateCopyToAllowedExportDirs(); conf.setLoadTsFileSpiltPartitionMaxSize( Integer.parseInt( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index b5c394539f2..5b7f5bb65f2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -21,6 +21,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.analyzer; import org.apache.iotdb.calc.plan.relational.metadata.CommonMetadataUtils; import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.exception.IoTDBRuntimeException; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.i18n.QueryMessages; import org.apache.iotdb.commons.queryengine.common.SessionInfo; @@ -124,6 +125,9 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.udf.builtin.relational.tvf.FFTTableFunction; import org.apache.iotdb.commons.udf.builtin.relational.tvf.M4TableFunction; import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer; +import org.apache.iotdb.commons.utils.FileUtils; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.MPPQueryContext.ExplainType; @@ -246,7 +250,9 @@ import org.apache.tsfile.read.common.type.UnknownType; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; +import java.io.File; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -1314,6 +1320,22 @@ public class StatementAnalyzer { @Override public Scope visitCopyTo(CopyTo node, Optional<Scope> context) { accessControl.checkUserGlobalSysPrivilege(queryContext); + final String targetFilePath = node.getTargetFileName(); + final File targetFile = new File(targetFilePath); + if (targetFile.getParent() != null) { + final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + final String[] activeLoadDirectories = config.getLoadActiveListeningDirs(); + final String[] forbiddenDirectories = + Arrays.copyOf(activeLoadDirectories, activeLoadDirectories.length + 1); + forbiddenDirectories[activeLoadDirectories.length] = config.getLoadActiveListeningPipeDir(); + if (!FileUtils.isFilePathAllowed( + targetFilePath, config.getCopyToAllowedExportDirs(), forbiddenDirectories)) { + throw new IoTDBRuntimeException( + DataNodeQueryMessages.COPY_TO_TARGET_PATH_NOT_ALLOWED + targetFilePath, + TSStatusCode.COPY_TO_WRITE_ERROR.getStatusCode(), + true); + } + } Scope innerQueryScope = visitQuery((Query) node.getQueryStatement(), context); analysis.setScope(node, innerQueryScope); return innerQueryScope; diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index e325ff42573..1248b13cef6 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -2411,6 +2411,16 @@ load_active_listening_max_thread_num=0 # Datatype: int load_active_listening_check_interval_seconds=5 +# The directories into which COPY ... TO statements may export files when the client supplies +# a target path containing a parent component (absolute or relative). Multiple directories +# should be separated by a ','. +# Empty (the default) rejects such paths; a target consisting of a bare file name is always +# accepted and lands in the managed 'copyto' folder under the data directories. The configured +# directories must not overlap load_active_listening_dirs or load_active_listening_pipe_dir. +# effectiveMode: restart +# Datatype: String +# copy_to_allowed_export_dirs= + # The operation performed to LastCache when a TsFile is successfully loaded. # UPDATE: use the data in the TsFile to update LastCache; # UPDATE_NO_BLOB: similar to UPDATE, but will invalidate LastCache for blob series; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java index d5b1b76a7eb..4058f1e41fe 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java @@ -329,6 +329,39 @@ public class FileUtils { return file; } + /** + * Checks whether a target path is under one of the allowed directories and none of the forbidden + * directories after canonicalization. + * + * <p>The method returns {@code false} if any path cannot be canonicalized. + */ + public static boolean isFilePathAllowed( + String targetFilePath, String[] allowedDirectories, String[] forbiddenDirectories) { + if (targetFilePath == null || allowedDirectories == null || forbiddenDirectories == null) { + return false; + } + try { + final Path targetPath = new File(targetFilePath).getCanonicalFile().toPath(); + for (String forbiddenDirectory : forbiddenDirectories) { + if (forbiddenDirectory != null + && !forbiddenDirectory.isEmpty() + && targetPath.startsWith(new File(forbiddenDirectory).getCanonicalFile().toPath())) { + return false; + } + } + for (String allowedDirectory : allowedDirectories) { + if (allowedDirectory != null + && !allowedDirectory.isEmpty() + && targetPath.startsWith(new File(allowedDirectory).getCanonicalFile().toPath())) { + return true; + } + } + return false; + } catch (IOException e) { + return false; + } + } + /** * Move source file to target file. The move will be divided into three steps: 1. Copy the source * file to the "target.unfinished" location 2. Rename the "target.unfinished" to "target" 3.
