This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch dev-1.0.0 in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 3f64a9355cf5e95847cdc1d13e509d96855d2a12 Author: Mingyu Chen <[email protected]> AuthorDate: Fri Mar 11 17:20:09 2022 +0800 [feature](user-property) Support user level exec_mem_limit and load_mem_limit (#8365) ``` SET PROPERTY FOR 'jack' 'exec_mem_limit' = '2147483648'; SET PROPERTY FOR 'jack' 'load_mem_limit' = '2147483648'; ``` The user level property will overwrite the value in session variables. --- .../Account Management/SET PROPERTY.md | 10 ++++++ .../Account Management/SET PROPERTY.md | 12 ++++++- .../mysql/privilege/CommonUserProperties.java | 22 ++++++++++++ .../org/apache/doris/mysql/privilege/PaloAuth.java | 18 ++++++++++ .../apache/doris/mysql/privilege/UserProperty.java | 40 ++++++++++++++++++++++ .../doris/mysql/privilege/UserPropertyMgr.java | 22 ++++++++++-- .../main/java/org/apache/doris/qe/Coordinator.java | 23 ++++++++++--- 7 files changed, 139 insertions(+), 8 deletions(-) diff --git a/docs/en/sql-reference/sql-statements/Account Management/SET PROPERTY.md b/docs/en/sql-reference/sql-statements/Account Management/SET PROPERTY.md index f2be712..d0ac580 100644 --- a/docs/en/sql-reference/sql-statements/Account Management/SET PROPERTY.md +++ b/docs/en/sql-reference/sql-statements/Account Management/SET PROPERTY.md @@ -42,10 +42,14 @@ max_user_connections: Maximum number of connections. max_query_instances: Maximum number of query instance user can use when query. sql_block_rules: set sql block rules。After setting, if the query user execute match the rules, it will be rejected. cpu_resource_limit: limit the cpu resource usage of a query. See session variable `cpu_resource_limit`. +exec_mem_limit: Limit the memory usage of the query. See the description of the session variable `exec_mem_limit` for details. -1 means not set. +load_mem_limit: Limit memory usage for imports. See the introduction of the session variable `load_mem_limit` for details. -1 means not set. resource.cpu_share: cpu resource assignment.(Derepcated) Load_cluster. {cluster_name}. priority: assigns priority to a specified cluster, which can be HIGH or NORMAL resource_tags: Specify the user's resource tag permissions. +> Notice: The `cpu_resource_limit`, `exec_mem_limit`, and `load_mem_limit` properties default to the values in the session variables if they are not set. + Ordinary user rights: Quota.normal: Resource allocation at the normal level. Quota.high: Resource allocation at the high level. @@ -93,6 +97,12 @@ SET PROPERTY FOR 'jack' 'cpu_resource_limit' = '2'; 11. Modify user's resource tag permission SET PROPERTY FOR 'jack' 'resource_tags.location' = 'group_a, group_b'; +12. modify the user's query memory usage limit in bytes +SET PROPERTY FOR 'jack' 'exec_mem_limit' = '2147483648'; + +13. modify the user's import memory usage limit in bytes +SET PROPERTY FOR 'jack' 'load_mem_limit' = '2147483648'; + ## keyword SET, PROPERTY diff --git a/docs/zh-CN/sql-reference/sql-statements/Account Management/SET PROPERTY.md b/docs/zh-CN/sql-reference/sql-statements/Account Management/SET PROPERTY.md index 3885d57..fb9d34e 100644 --- a/docs/zh-CN/sql-reference/sql-statements/Account Management/SET PROPERTY.md +++ b/docs/zh-CN/sql-reference/sql-statements/Account Management/SET PROPERTY.md @@ -41,11 +41,15 @@ under the License. max_user_connections: 最大连接数。 max_query_instances: 用户同一时间点执行查询可以使用的instance个数。 sql_block_rules: 设置 sql block rules。设置后，该用户发送的查询如果匹配规则，则会被拒绝。 - cpu_resource_limit: 限制查询的cpu资源。详见会话变量 `cpu_resource_limit` 的介绍。 + cpu_resource_limit: 限制查询的cpu资源。详见会话变量 `cpu_resource_limit` 的介绍。-1 表示未设置。 + exec_mem_limit: 限制查询的内存使用。详见会话变量 `exec_mem_limit` 的介绍。-1 表示未设置。 + load_mem_limit: 限制导入的内存使用。详见会话变量 `load_mem_limit` 的介绍。-1 表示未设置。 resource.cpu_share: cpu资源分配。（已废弃） load_cluster.{cluster_name}.priority: 为指定的cluster分配优先级，可以为 HIGH 或 NORMAL resource_tags：指定用户的资源标签权限。 + 注：`cpu_resource_limit`, `exec_mem_limit`, `load_mem_limit` 三个属性如果未设置，则默认使用会话变量中值。 + 普通用户权限： quota.normal: normal级别的资源分配。 quota.high: high级别的资源分配。 @@ -93,6 +97,12 @@ under the License. 11. 修改用户的资源标签权限 SET PROPERTY FOR 'jack' 'resource_tags.location' = 'group_a, group_b'; + 12. 修改用户的查询内存使用限制，单位字节 + SET PROPERTY FOR 'jack' 'exec_mem_limit' = '2147483648'; + + 13. 修改用户的导入内存使用限制，单位字节 + SET PROPERTY FOR 'jack' 'load_mem_limit' = '2147483648'; + ## keyword SET, PROPERTY diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java index 2b88909..ecaf436 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java @@ -47,6 +47,12 @@ public class CommonUserProperties implements Writable { // The tag of the resource that the user is allowed to use @SerializedName("resourceTags") private Set<Tag> resourceTags = Sets.newHashSet(); + // user level exec_mem_limit, if > 0, will overwrite the exec_mem_limit in session variable + @SerializedName("execMemLimit") + private long execMemLimit = -1; + // user level load_mem_limit, if > 0, will overwrite the load_mem_limit in session variable + @SerializedName("loadMemLimit") + private long loadMemLimit = -1; private String[] sqlBlockRulesSplit = {}; @@ -100,6 +106,22 @@ public class CommonUserProperties implements Writable { return resourceTags; } + public long getExecMemLimit() { + return execMemLimit; + } + + public void setExecMemLimit(long execMemLimit) { + this.execMemLimit = execMemLimit; + } + + public long getLoadMemLimit() { + return loadMemLimit; + } + + public void setLoadMemLimit(long loadMemLimit) { + this.loadMemLimit = loadMemLimit; + } + public static CommonUserProperties read(DataInput in) throws IOException { String json = Text.readString(in); CommonUserProperties commonUserProperties = GsonUtils.GSON.fromJson(json, CommonUserProperties.class); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java index f739fa0..a210dfd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java @@ -1172,6 +1172,24 @@ public class PaloAuth implements Writable { } } + public long getExecMemLimit(String qualifiedUser) { + readLock(); + try { + return propertyMgr.getExecMemLimit(qualifiedUser); + } finally { + readUnlock(); + } + } + + public long getLoadMemLimit(String qualifiedUser) { + readLock(); + try { + return propertyMgr.getLoadMemLimit(qualifiedUser); + } finally { + readUnlock(); + } + } + public void getAllDomains(Set<String> allDomains) { readLock(); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java index 08e5af8..04c5b0a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java @@ -66,6 +66,8 @@ public class UserProperty implements Writable { private static final String PROP_RESOURCE = "resource"; private static final String PROP_SQL_BLOCK_RULES = "sql_block_rules"; private static final String PROP_CPU_RESOURCE_LIMIT = "cpu_resource_limit"; + private static final String PROP_EXEC_MEM_LIMIT = "exec_mem_limit"; + private static final String PROP_LOAD_MEM_LIMIT = "load_mem_limit"; // advanced properties end private static final String PROP_LOAD_CLUSTER = "load_cluster"; @@ -110,6 +112,8 @@ public class UserProperty implements Writable { ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_SQL_BLOCK_RULES + "$", Pattern.CASE_INSENSITIVE)); ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_CPU_RESOURCE_LIMIT + "$", Pattern.CASE_INSENSITIVE)); ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_RESOURCE_TAGS + "$", Pattern.CASE_INSENSITIVE)); + ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_EXEC_MEM_LIMIT + "$", Pattern.CASE_INSENSITIVE)); + ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_LOAD_MEM_LIMIT + "$", Pattern.CASE_INSENSITIVE)); COMMON_PROPERTIES.add(Pattern.compile("^" + PROP_QUOTA + ".", Pattern.CASE_INSENSITIVE)); COMMON_PROPERTIES.add(Pattern.compile("^" + PROP_DEFAULT_LOAD_CLUSTER + "$", Pattern.CASE_INSENSITIVE)); @@ -152,6 +156,14 @@ public class UserProperty implements Writable { return Sets.newHashSet(this.commonProperties.getResourceTags()); } + public long getExecMemLimit() { + return commonProperties.getExecMemLimit(); + } + + public long getLoadMemLimit() { + return commonProperties.getLoadMemLimit(); + } + public void setPasswordForDomain(String domain, byte[] password, boolean errOnExist) throws DdlException { if (errOnExist && whiteList.containsDomain(domain)) { throw new DdlException("Domain " + domain + " of user " + qualifiedUser + " already exists"); @@ -173,6 +185,8 @@ public class UserProperty implements Writable { String sqlBlockRules = this.commonProperties.getSqlBlockRules(); int cpuResourceLimit = this.commonProperties.getCpuResourceLimit(); Set<Tag> resourceTags = this.commonProperties.getResourceTags(); + long execMemLimit = this.commonProperties.getExecMemLimit(); + long loadMemLimit = this.commonProperties.getLoadMemLimit(); UserResource newResource = resource.getCopiedUserResource(); String newDefaultLoadCluster = defaultLoadCluster; @@ -301,6 +315,11 @@ public class UserProperty implements Writable { throw new DdlException(PROP_RESOURCE_TAGS + " parse failed: " + e.getMessage()); } } + } else if (keyArr[0].equalsIgnoreCase(PROP_EXEC_MEM_LIMIT)) { + // set property "exec_mem_limit" = "2147483648"; + execMemLimit = getLongProperty(key, value, keyArr, PROP_EXEC_MEM_LIMIT); + } else if (keyArr[0].equalsIgnoreCase(PROP_LOAD_MEM_LIMIT)) { + loadMemLimit = getLongProperty(key, value, keyArr, PROP_LOAD_MEM_LIMIT); } else { throw new DdlException("Unknown user property(" + key + ")"); } @@ -312,6 +331,8 @@ public class UserProperty implements Writable { this.commonProperties.setSqlBlockRules(sqlBlockRules); this.commonProperties.setCpuResourceLimit(cpuResourceLimit); this.commonProperties.setResourceTags(resourceTags); + this.commonProperties.setExecMemLimit(execMemLimit); + this.commonProperties.setExecMemLimit(loadMemLimit); resource = newResource; if (newDppConfigs.containsKey(newDefaultLoadCluster)) { defaultLoadCluster = newDefaultLoadCluster; @@ -321,6 +342,25 @@ public class UserProperty implements Writable { clusterToDppConfig = newDppConfigs; } + private long getLongProperty(String key, String value, String[] keyArr, String propName) throws DdlException { + // eg: set property "load_mem_limit" = "2147483648"; + if (keyArr.length != 1) { + throw new DdlException(propName + " format error"); + } + long limit = -1; + try { + limit = Long.parseLong(value); + } catch (NumberFormatException e) { + throw new DdlException(key + " is not number"); + } + + // -1 means unlimited + if (limit <= 0 && limit != -1) { + throw new DdlException(key + " is not valid. Should not larger than 0 or equal to -1"); + } + return limit; + } + private Set<Tag> parseLocationResoureTags(String value) throws AnalysisException { Set<Tag> tags = Sets.newHashSet(); String[] parts = value.replaceAll(" ", "").split(","); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java index 18551d9..21070f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java @@ -29,12 +29,12 @@ import org.apache.doris.resource.Tag; import org.apache.doris.thrift.TAgentServiceVersion; import org.apache.doris.thrift.TFetchResourceResult; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import com.google.common.base.Preconditions; import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -250,6 +250,22 @@ public class UserPropertyMgr implements Writable { return propertyMap.get(qualifiedUserName); } + public long getExecMemLimit(String qualifiedUser) { + UserProperty existProperty = propertyMap.get(qualifiedUser); + if (existProperty == null) { + return -1; + } + return existProperty.getExecMemLimit(); + } + + public long getLoadMemLimit(String qualifiedUser) { + UserProperty existProperty = propertyMap.get(qualifiedUser); + if (existProperty == null) { + return -1; + } + return existProperty.getLoadMemLimit(); + } + public static UserPropertyMgr read(DataInput in) throws IOException { UserPropertyMgr userPropertyMgr = new UserPropertyMgr(); userPropertyMgr.readFields(in); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index a64853a..015344c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -273,15 +273,30 @@ public class Coordinator { } private void setFromUserProperty(Analyzer analyzer) { - // set cpu resource limit String qualifiedUser = analyzer.getQualifiedUser(); - int limit = Catalog.getCurrentCatalog().getAuth().getCpuResourceLimit(qualifiedUser); - if (limit > 0) { + // set cpu resource limit + int cpuLimit = Catalog.getCurrentCatalog().getAuth().getCpuResourceLimit(qualifiedUser); + if (cpuLimit > 0) { // overwrite the cpu resource limit from session variable; TResourceLimit resourceLimit = new TResourceLimit(); - resourceLimit.setCpuLimit(limit); + resourceLimit.setCpuLimit(cpuLimit); this.queryOptions.setResourceLimit(resourceLimit); } + // set exec mem limit + long memLimit = Catalog.getCurrentCatalog().getAuth().getExecMemLimit(qualifiedUser); + if (memLimit > 0) { + // overwrite the exec_mem_limit from session variable; + this.queryOptions.setMemLimit(memLimit); + this.queryOptions.setMaxReservation(memLimit); + this.queryOptions.setInitialReservationTotalClaims(memLimit); + this.queryOptions.setBufferPoolLimit(memLimit); + } + // set load mem limit + memLimit = Catalog.getCurrentCatalog().getAuth().getLoadMemLimit(qualifiedUser); + if (memLimit > 0) { + // overwrite the load_mem_limit from session variable; + this.queryOptions.setLoadMemLimit(memLimit); + } } public long getJobId() { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
