This is an automated email from the ASF dual-hosted git repository.
wyxxxcat pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new bc9686578a5 [Fix](ai) Fix some AI Resours Issues (#68023)
bc9686578a5 is described below
commit bc9686578a5a3675b8d04b0a0b4fc9ff6a983d96
Author: linrrarity <[email protected]>
AuthorDate: Thu Nov 5 17:11:40 2026 +0800
[Fix](ai) Fix some AI Resours Issues (#68023)
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
- **Centralize property handling:** Move binding, defaults, validation,
and Thrift conversion into `AIResourceProperties`, reusing the existing
`ConnectorProperty` framework. Keep `AIProperties` as constants.
- **Strengthen validation:** Reject blank values and invalid numeric
ranges, and apply standard string trimming.
- **Ensure ALTER consistency:** Validate the complete configuration
before replacement and safely publish updates to concurrent readers.
- **Preserve metadata compatibility:** Keep the persisted map format and
defer validation of loaded metadata until use.
- **Correct resource usage:** Update AI function resource checks and add
regression coverage.
---
.../java/org/apache/doris/catalog/AIResource.java | 126 +----
.../datasource/property/AIResourceProperties.java | 295 ++++++++++++
.../property/constants/AIProperties.java | 105 -----
.../nereids/rules/analysis/ExpressionAnalyzer.java | 18 +-
.../trees/expressions/functions/agg/AIAgg.java | 7 +
.../trees/expressions/functions/ai/AIFunction.java | 16 +
.../trees/expressions/functions/ai/Embed.java | 1 +
.../doris/catalog/AIResourceConcurrencyTest.java | 134 ++++++
.../doris/catalog/AIResourcePropertiesTest.java | 516 +++++++++++++++++++++
.../org/apache/doris/catalog/AIResourceTest.java | 251 +++++++---
.../data/ai_p0/test_create_ai_resource.out | 37 ++
.../suites/ai_p0/test_ai_resource_privilege.groovy | 114 +++++
.../suites/ai_p0/test_create_ai_resource.groovy | 109 ++++-
13 files changed, 1445 insertions(+), 284 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/AIResource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/AIResource.java
index 1709236b2a0..c18e9e9a365 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/AIResource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/AIResource.java
@@ -19,18 +19,17 @@ package org.apache.doris.catalog;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.proc.BaseProcResult;
+import org.apache.doris.datasource.property.AIResourceProperties;
import org.apache.doris.datasource.property.constants.AIProperties;
import org.apache.doris.thrift.TAIResource;
import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
/**
@@ -55,7 +54,7 @@ import java.util.Map;
public class AIResource extends Resource {
@SerializedName(value = "properties")
- private Map<String, String> properties;
+ private volatile Map<String, String> properties;
@SerializedName(value = "createdByRoot")
private boolean createdByRoot;
@@ -79,10 +78,7 @@ public class AIResource extends Resource {
@Override
protected void setProperties(ImmutableMap<String, String> newProperties)
throws DdlException {
Preconditions.checkState(newProperties != null);
- this.properties = Maps.newHashMap(newProperties);
-
- AIProperties.requiredAIProperties(properties);
- AIProperties.optionalAIProperties(this.properties);
+ this.properties = parseProperties(newProperties).toMap();
}
public String getProperty(String propertyKey) {
@@ -90,28 +86,15 @@ public class AIResource extends Resource {
}
public boolean hasCompleteGeneralProperties() {
- return AIProperties.REQUIRED_FIELDS.stream()
- .allMatch(field ->
!Strings.isNullOrEmpty(properties.get(field)))
- &&
("LOCAL".equalsIgnoreCase(properties.get(AIProperties.PROVIDER_TYPE))
- ||
!Strings.isNullOrEmpty(properties.get(AIProperties.API_KEY)));
+ return AIResourceProperties.hasCompleteGeneralProperties(properties);
}
public boolean hasCompleteEmbedProperties() {
- return hasCompleteProperties(AIProperties.EMBED_REQUIRED_FIELDS,
- AIProperties.EMBED_PROVIDER_TYPE, AIProperties.EMBED_API_KEY);
+ return AIResourceProperties.hasCompleteEmbedProperties(properties);
}
public boolean hasCompleteMultimodalEmbedProperties() {
- return
hasCompleteProperties(AIProperties.MULTIMODAL_EMBED_REQUIRED_FIELDS,
- AIProperties.MULTIMODAL_EMBED_PROVIDER_TYPE,
AIProperties.MULTIMODAL_EMBED_API_KEY);
- }
-
- private boolean hasCompleteProperties(List<String> requiredFields,
- String providerTypeField, String apiKeyField) {
- return requiredFields.stream()
- .allMatch(field ->
!Strings.isNullOrEmpty(properties.get(field)))
- && ("LOCAL".equalsIgnoreCase(properties.get(providerTypeField))
- ||
!Strings.isNullOrEmpty(properties.get(apiKeyField)));
+ return
AIResourceProperties.hasCompleteMultimodalEmbedProperties(properties);
}
@Override
@@ -119,20 +102,8 @@ public class AIResource extends Resource {
writeLock();
try {
Map<String, String> changedProperties = new
HashMap<>(this.properties);
- for (Map.Entry<String, String> kv : properties.entrySet()) {
- replaceIfEffectiveValue(changedProperties, kv.getKey(),
kv.getValue());
- if (kv.getKey().equals(AIProperties.API_KEY)
- || kv.getKey().equals(AIProperties.EMBED_API_KEY)
- ||
kv.getKey().equals(AIProperties.MULTIMODAL_EMBED_API_KEY)) {
- changedProperties.put(kv.getKey(), kv.getValue());
- } else if (kv.getKey().equals(AIProperties.EFFORT)
- && Strings.isNullOrEmpty(kv.getValue())) {
- changedProperties.remove(kv.getKey());
- }
- }
- AIProperties.requiredAIProperties(changedProperties);
-
- this.properties = changedProperties;
+ changedProperties.putAll(properties);
+ this.properties = parseProperties(changedProperties).toMap();
++version;
} finally {
writeUnlock();
@@ -163,82 +134,15 @@ public class AIResource extends Resource {
readUnlock();
}
- public TAIResource toThrift() throws NumberFormatException {
- TAIResource tAIResource = new TAIResource();
-
tAIResource.setEnableStructuredOutput(Boolean.parseBoolean(properties.getOrDefault(
- AIProperties.ENABLE_STRUCTURED_OUTPUT,
AIProperties.DEFAULT_ENABLE_STRUCTURED_OUTPUT)));
- if (properties.containsKey(AIProperties.PROVIDER_TYPE)) {
-
tAIResource.setProviderType(properties.get(AIProperties.PROVIDER_TYPE));
- }
- if (properties.containsKey(AIProperties.ENDPOINT)) {
- tAIResource.setEndpoint(properties.get(AIProperties.ENDPOINT));
- }
- if (properties.containsKey(AIProperties.API_KEY)) {
- tAIResource.setApiKey(properties.get(AIProperties.API_KEY));
- }
- if (properties.containsKey(AIProperties.MODEL_NAME)) {
- tAIResource.setModelName(properties.get(AIProperties.MODEL_NAME));
- }
- if (properties.containsKey(AIProperties.EMBED_PROVIDER_TYPE)) {
-
tAIResource.setEmbedProviderType(properties.get(AIProperties.EMBED_PROVIDER_TYPE));
- }
- if (properties.containsKey(AIProperties.EMBED_ENDPOINT)) {
-
tAIResource.setEmbedEndpoint(properties.get(AIProperties.EMBED_ENDPOINT));
- }
- if (properties.containsKey(AIProperties.EMBED_API_KEY)) {
-
tAIResource.setEmbedApiKey(properties.get(AIProperties.EMBED_API_KEY));
- }
- if (properties.containsKey(AIProperties.EMBED_MODEL_NAME)) {
-
tAIResource.setEmbedModelName(properties.get(AIProperties.EMBED_MODEL_NAME));
- }
- if
(properties.containsKey(AIProperties.MULTIMODAL_EMBED_PROVIDER_TYPE)) {
-
tAIResource.setEmbedMmProviderType(properties.get(AIProperties.MULTIMODAL_EMBED_PROVIDER_TYPE));
- }
- if (properties.containsKey(AIProperties.MULTIMODAL_EMBED_ENDPOINT)) {
-
tAIResource.setEmbedMmEndpoint(properties.get(AIProperties.MULTIMODAL_EMBED_ENDPOINT));
- }
- if (properties.containsKey(AIProperties.MULTIMODAL_EMBED_API_KEY)) {
-
tAIResource.setEmbedMmApiKey(properties.get(AIProperties.MULTIMODAL_EMBED_API_KEY));
- }
- if (properties.containsKey(AIProperties.MULTIMODAL_EMBED_MODEL_NAME)) {
-
tAIResource.setEmbedMmModelName(properties.get(AIProperties.MULTIMODAL_EMBED_MODEL_NAME));
- }
- if (!Strings.isNullOrEmpty(properties.get(AIProperties.EFFORT))) {
- tAIResource.setEffort(properties.get(AIProperties.EFFORT));
- }
-
tAIResource.setAnthropicVersion(properties.get(AIProperties.ANTHROPIC_VERSION));
+ public TAIResource toThrift() {
+ return AIResourceProperties.of(properties).toThrift();
+ }
+ private static AIResourceProperties parseProperties(Map<String, String>
properties) throws DdlException {
try {
-
tAIResource.setTemperature(Double.parseDouble(properties.get(AIProperties.TEMPERATURE)));
- } catch (NumberFormatException e) {
- throw new NumberFormatException("Failed to parse temperature: "
- +
properties.get(AIProperties.TEMPERATURE));
- }
- try {
-
tAIResource.setMaxTokens(Long.parseLong(properties.get(AIProperties.MAX_TOKEN)));
- } catch (NumberFormatException e) {
- throw new NumberFormatException("Failed to parse max_token: "
- +
properties.get(AIProperties.MAX_TOKEN));
+ return AIResourceProperties.of(properties);
+ } catch (IllegalArgumentException e) {
+ throw new DdlException(e.getMessage(), e);
}
- try {
-
tAIResource.setMaxRetries(Integer.parseInt(properties.get(AIProperties.MAX_RETRIES)));
- } catch (NumberFormatException e) {
- throw new NumberFormatException("Failed to parse max_retries: "
- +
properties.get(AIProperties.MAX_RETRIES));
- }
- try {
-
tAIResource.setRetryDelaySecond(Integer.parseInt(properties.get(AIProperties.RETRY_DELAY_SECOND)));
- } catch (NumberFormatException e) {
- throw new NumberFormatException("Failed to parse
retry_delay_second: "
- +
properties.get(AIProperties.RETRY_DELAY_SECOND));
- }
- try {
-
tAIResource.setDimensions(Integer.parseInt(properties.get(AIProperties.DIMENSIONS)));
- } catch (NumberFormatException e) {
- throw new NumberFormatException("Failed to parse dimensions: "
- +
properties.get(AIProperties.DIMENSIONS));
- }
-
- return tAIResource;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/AIResourceProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/AIResourceProperties.java
new file mode 100644
index 00000000000..83776af54c5
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/AIResourceProperties.java
@@ -0,0 +1,295 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property;
+
+import org.apache.doris.datasource.property.constants.AIProperties;
+import org.apache.doris.foundation.property.ConnectorPropertiesUtils;
+import org.apache.doris.foundation.property.ConnectorProperty;
+import org.apache.doris.foundation.property.ConnectorPropertyValidator;
+import org.apache.doris.foundation.property.ParamRules;
+import org.apache.doris.thrift.TAIResource;
+
+import com.google.common.base.Strings;
+import org.apache.commons.lang3.StringUtils;
+
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+public final class AIResourceProperties {
+ private static final List<String> ALL_EFFORT_LEVELS =
+ Arrays.asList("none", "minimal", "low", "medium", "high", "xhigh",
"max");
+ private static final List<String> ANTHROPIC_EFFORT_LEVELS =
+ Arrays.asList("low", "medium", "high", "xhigh", "max");
+ private static final List<String> GEMINI_EFFORT_LEVELS =
+ Arrays.asList("minimal", "low", "medium", "high");
+
+ @ConnectorProperty(names = {AIProperties.ENDPOINT}, required = false)
+ private String endpoint;
+ @ConnectorProperty(names = {AIProperties.PROVIDER_TYPE}, required = false)
+ private String providerType;
+ @ConnectorProperty(names = {AIProperties.MODEL_NAME}, required = false)
+ private String modelName;
+ @ConnectorProperty(names = {AIProperties.API_KEY}, required = false,
sensitive = true)
+ private String apiKey;
+ @ConnectorProperty(names = {AIProperties.EMBED_ENDPOINT}, required = false)
+ private String embedEndpoint;
+ @ConnectorProperty(names = {AIProperties.EMBED_PROVIDER_TYPE}, required =
false)
+ private String embedProviderType;
+ @ConnectorProperty(names = {AIProperties.EMBED_MODEL_NAME}, required =
false)
+ private String embedModelName;
+ @ConnectorProperty(names = {AIProperties.EMBED_API_KEY}, required = false,
sensitive = true)
+ private String embedApiKey;
+ @ConnectorProperty(names = {AIProperties.MULTIMODAL_EMBED_ENDPOINT},
required = false)
+ private String multimodalEmbedEndpoint;
+ @ConnectorProperty(names = {AIProperties.MULTIMODAL_EMBED_PROVIDER_TYPE},
required = false)
+ private String multimodalEmbedProviderType;
+ @ConnectorProperty(names = {AIProperties.MULTIMODAL_EMBED_MODEL_NAME},
required = false)
+ private String multimodalEmbedModelName;
+ @ConnectorProperty(names = {AIProperties.MULTIMODAL_EMBED_API_KEY},
required = false, sensitive = true)
+ private String multimodalEmbedApiKey;
+ @ConnectorProperty(names = {AIProperties.TEMPERATURE}, required = false,
validator = TemperatureValidator.class)
+ private double temperature;
+ @ConnectorProperty(names = {AIProperties.MAX_TOKEN}, required = false,
validator = MaxTokenValidator.class)
+ private long maxTokens;
+ @ConnectorProperty(names = {AIProperties.MAX_RETRIES}, required = false,
+ validator = NonNegativeIntegerValidator.class)
+ private int maxRetries;
+ @ConnectorProperty(names = {AIProperties.RETRY_DELAY_SECOND}, required =
false,
+ validator = NonNegativeIntegerValidator.class)
+ private int retryDelaySecond;
+ @ConnectorProperty(names = {AIProperties.DIMENSIONS}, required = false,
validator = DimensionsValidator.class)
+ private int dimensions;
+ @ConnectorProperty(names = {AIProperties.ANTHROPIC_VERSION}, required =
false)
+ private String anthropicVersion;
+ @ConnectorProperty(names = {AIProperties.EFFORT}, required = false)
+ private String effort;
+ @ConnectorProperty(names = {AIProperties.ENABLE_STRUCTURED_OUTPUT},
required = false,
+ validator = StructuredOutputValidator.class)
+ private boolean enableStructuredOutput;
+
+ private final Map<String, String> properties;
+
+ private AIResourceProperties(Map<String, String> properties) {
+ this.properties = new HashMap<>(properties);
+ this.properties.putIfAbsent(AIProperties.ENABLE_STRUCTURED_OUTPUT,
+ AIProperties.DEFAULT_ENABLE_STRUCTURED_OUTPUT);
+ this.properties.putIfAbsent(AIProperties.TEMPERATURE,
AIProperties.DEFAULT_TEMPERATURE);
+ this.properties.putIfAbsent(AIProperties.MAX_TOKEN,
AIProperties.DEFAULT_MAX_TOKEN);
+ this.properties.putIfAbsent(AIProperties.MAX_RETRIES,
AIProperties.DEFAULT_MAX_RETRIES);
+ this.properties.putIfAbsent(AIProperties.RETRY_DELAY_SECOND,
AIProperties.DEFAULT_RETRY_DELAY_SECOND);
+ this.properties.putIfAbsent(AIProperties.DIMENSIONS,
AIProperties.DEFAULT_DIMENSIONS);
+ this.properties.putIfAbsent(AIProperties.ANTHROPIC_VERSION,
AIProperties.DEFAULT_ANTHROPIC_VERSION);
+ }
+
+ public static AIResourceProperties of(Map<String, String> properties) {
+ ParamRules rules = new ParamRules();
+ properties.forEach((key, value) -> rules.require(value, "[" + key + "]
must not be empty"));
+ rules.validate();
+ AIResourceProperties result = new AIResourceProperties(properties);
+ ConnectorPropertiesUtils.bindConnectorProperties(result,
result.properties);
+ result.providerType =
result.normalizeProvider(AIProperties.PROVIDER_TYPE, result.providerType);
+ result.embedProviderType =
result.normalizeProvider(AIProperties.EMBED_PROVIDER_TYPE,
result.embedProviderType);
+ result.multimodalEmbedProviderType = result.normalizeProvider(
+ AIProperties.MULTIMODAL_EMBED_PROVIDER_TYPE,
result.multimodalEmbedProviderType);
+ result.validate();
+ return result;
+ }
+
+ private String normalizeProvider(String key, String value) {
+ if (value == null) {
+ return null;
+ }
+ String normalized = value.toUpperCase(Locale.ROOT);
+ properties.put(key, normalized);
+ return normalized;
+ }
+
+ private void validate() {
+ boolean hasGeneral = hasAnyProperty(AIProperties.REQUIRED_FIELDS,
AIProperties.API_KEY);
+ boolean hasEmbed = hasAnyProperty(AIProperties.EMBED_REQUIRED_FIELDS,
AIProperties.EMBED_API_KEY);
+ boolean hasMultimodal = hasAnyProperty(
+ AIProperties.MULTIMODAL_EMBED_REQUIRED_FIELDS,
AIProperties.MULTIMODAL_EMBED_API_KEY);
+ new ParamRules().check(() -> !hasGeneral && !hasEmbed &&
!hasMultimodal,
+ "At least one complete AI property group must be
configured.").validate();
+ if (hasGeneral) {
+ validateGroup(AIProperties.REQUIRED_FIELDS, providerType,
AIProperties.API_KEY, apiKey);
+ }
+ if (hasEmbed) {
+ validateGroup(AIProperties.EMBED_REQUIRED_FIELDS,
embedProviderType,
+ AIProperties.EMBED_API_KEY, embedApiKey);
+ }
+ if (hasMultimodal) {
+ validateGroup(AIProperties.MULTIMODAL_EMBED_REQUIRED_FIELDS,
multimodalEmbedProviderType,
+ AIProperties.MULTIMODAL_EMBED_API_KEY,
multimodalEmbedApiKey);
+ }
+ List<String> effortLevels = "ANTHROPIC".equals(providerType) ?
ANTHROPIC_EFFORT_LEVELS
+ : "GEMINI".equals(providerType) ? GEMINI_EFFORT_LEVELS :
ALL_EFFORT_LEVELS;
+ new ParamRules().check(() -> !Strings.isNullOrEmpty(effort) &&
!effortLevels.contains(effort),
+ "[" + AIProperties.EFFORT + "] must be one of " + effortLevels
+ + " for provider: " + providerType).validate();
+ }
+
+ private boolean hasAnyProperty(List<String> requiredFields, String
apiKeyField) {
+ return properties.containsKey(apiKeyField) ||
requiredFields.stream().anyMatch(properties::containsKey);
+ }
+
+ private void validateGroup(List<String> requiredFields, String provider,
String apiKeyField, String key) {
+ ParamRules rules = new ParamRules();
+ for (String field : requiredFields) {
+ rules.check(() -> Strings.isNullOrEmpty(properties.get(field)),
+ "Missing [" + field + "] in properties.");
+ }
+ rules.check(() -> !AIProperties.PROVIDERS.contains(provider),
+ "Provider must be one of " + AIProperties.PROVIDERS);
+ if (!"LOCAL".equals(provider)) {
+ rules.check(() -> Strings.isNullOrEmpty(key),
+ "Missing [" + apiKeyField + "] in properties for provider:
" + provider);
+ }
+ rules.validate();
+ }
+
+ public boolean hasCompleteGeneralProperties() {
+ return hasCompleteGeneralProperties(properties);
+ }
+
+ public boolean hasCompleteEmbedProperties() {
+ return hasCompleteEmbedProperties(properties);
+ }
+
+ public boolean hasCompleteMultimodalEmbedProperties() {
+ return hasCompleteMultimodalEmbedProperties(properties);
+ }
+
+ public static boolean hasCompleteGeneralProperties(Map<String, String>
properties) {
+ return hasCompleteGroup(properties, AIProperties.REQUIRED_FIELDS,
+ AIProperties.PROVIDER_TYPE, AIProperties.API_KEY);
+ }
+
+ public static boolean hasCompleteEmbedProperties(Map<String, String>
properties) {
+ return hasCompleteGroup(properties, AIProperties.EMBED_REQUIRED_FIELDS,
+ AIProperties.EMBED_PROVIDER_TYPE, AIProperties.EMBED_API_KEY);
+ }
+
+ public static boolean hasCompleteMultimodalEmbedProperties(Map<String,
String> properties) {
+ return hasCompleteGroup(properties,
AIProperties.MULTIMODAL_EMBED_REQUIRED_FIELDS,
+ AIProperties.MULTIMODAL_EMBED_PROVIDER_TYPE,
AIProperties.MULTIMODAL_EMBED_API_KEY);
+ }
+
+ private static boolean hasCompleteGroup(Map<String, String> properties,
List<String> requiredFields,
+ String providerField, String apiKeyField) {
+ return requiredFields.stream()
+ .allMatch(field ->
StringUtils.isNotEmpty(StringUtils.trim(properties.get(field))))
+ &&
("LOCAL".equalsIgnoreCase(StringUtils.trim(properties.get(providerField)))
+ ||
StringUtils.isNotEmpty(StringUtils.trim(properties.get(apiKeyField))));
+ }
+
+ public Map<String, String> toMap() {
+ Map<String, String> normalized = new HashMap<>(properties);
+ for (Field field :
ConnectorPropertiesUtils.getConnectorProperties(AIResourceProperties.class)) {
+ for (String name :
field.getAnnotation(ConnectorProperty.class).names()) {
+ normalized.computeIfPresent(name, (key, value) ->
value.trim());
+ }
+ }
+ return normalized;
+ }
+
+ public TAIResource toThrift() {
+ TAIResource result = new TAIResource();
+ result.setEnableStructuredOutput(enableStructuredOutput);
+ result.setEndpoint(endpoint);
+ result.setProviderType(providerType);
+ result.setModelName(modelName);
+ result.setApiKey(apiKey);
+ result.setEmbedEndpoint(embedEndpoint);
+ result.setEmbedProviderType(embedProviderType);
+ result.setEmbedModelName(embedModelName);
+ result.setEmbedApiKey(embedApiKey);
+ result.setEmbedMmEndpoint(multimodalEmbedEndpoint);
+ result.setEmbedMmProviderType(multimodalEmbedProviderType);
+ result.setEmbedMmModelName(multimodalEmbedModelName);
+ result.setEmbedMmApiKey(multimodalEmbedApiKey);
+ result.setTemperature(temperature);
+ result.setMaxTokens(maxTokens);
+ result.setMaxRetries(maxRetries);
+ result.setRetryDelaySecond(retryDelaySecond);
+ result.setDimensions(dimensions);
+ result.setAnthropicVersion(anthropicVersion);
+ if (!Strings.isNullOrEmpty(effort)) {
+ result.setEffort(effort);
+ }
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return ConnectorPropertiesUtils.toMaskedString(this);
+ }
+
+ public static final class StructuredOutputValidator implements
ConnectorPropertyValidator {
+ @Override
+ public void validate(Object target, Field field, String propertyName,
Object value,
+ Map<String, String> properties) {
+ String text = properties.get(propertyName).trim();
+ new ParamRules().check(() -> !"true".equalsIgnoreCase(text) &&
!"false".equalsIgnoreCase(text),
+ "[" + propertyName + "] must be true or false").validate();
+ }
+ }
+
+ public static final class TemperatureValidator implements
ConnectorPropertyValidator {
+ @Override
+ public void validate(Object target, Field field, String propertyName,
Object value,
+ Map<String, String> properties) {
+ double temperature = (Double) value;
+ new ParamRules().check(() -> temperature != -1 && !(temperature >=
0 && temperature <= 1),
+ "Temperature must be -1 or a double between 0 and
1").validate();
+ }
+ }
+
+ public static final class MaxTokenValidator implements
ConnectorPropertyValidator {
+ @Override
+ public void validate(Object target, Field field, String propertyName,
Object value,
+ Map<String, String> properties) {
+ long maxTokens = (Long) value;
+ new ParamRules().check(() -> maxTokens != -1 && maxTokens <= 0,
+ "Max token must be a positive integer or -1").validate();
+ }
+ }
+
+ public static final class NonNegativeIntegerValidator implements
ConnectorPropertyValidator {
+ @Override
+ public void validate(Object target, Field field, String propertyName,
Object value,
+ Map<String, String> properties) {
+ new ParamRules().check(() -> (Integer) value < 0,
+ "[" + propertyName + "] must be a non-negative
integer").validate();
+ }
+ }
+
+ public static final class DimensionsValidator implements
ConnectorPropertyValidator {
+ @Override
+ public void validate(Object target, Field field, String propertyName,
Object value,
+ Map<String, String> properties) {
+ int dimensions = (Integer) value;
+ new ParamRules().check(() -> dimensions != -1 && dimensions <= 0,
+ "Dimensions must be a positive integer or -1").validate();
+ }
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/AIProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/AIProperties.java
index e8c6a35d692..7aa4983f728 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/AIProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/AIProperties.java
@@ -17,13 +17,8 @@
package org.apache.doris.datasource.property.constants;
-import org.apache.doris.common.DdlException;
-
-import com.google.common.base.Strings;
-
import java.util.Arrays;
import java.util.List;
-import java.util.Map;
public class AIProperties extends BaseProperties {
public static final String AI_PREFIX = "ai.";
@@ -73,104 +68,4 @@ public class AIProperties extends BaseProperties {
public static final List<String> PROVIDERS
= Arrays.asList("OPENAI", "LOCAL", "GEMINI", "DEEPSEEK",
"ANTHROPIC",
"MOONSHOT", "QWEN", "MINIMAX", "ZHIPU", "BAICHUAN", "VOYAGEAI",
"JINA");
- private static final List<String> ALL_EFFORT_LEVELS =
- Arrays.asList("none", "minimal", "low", "medium", "high", "xhigh",
"max");
- private static final List<String> ANTHROPIC_EFFORT_LEVELS =
- Arrays.asList("low", "medium", "high", "xhigh", "max");
- private static final List<String> GEMINI_EFFORT_LEVELS =
- Arrays.asList("minimal", "low", "medium", "high");
-
- public static void requiredAIProperties(Map<String, String> properties)
throws DdlException {
- boolean hasGeneralProperties = hasAnyProperty(properties,
REQUIRED_FIELDS, API_KEY);
- boolean hasEmbedProperties = hasAnyProperty(properties,
EMBED_REQUIRED_FIELDS, EMBED_API_KEY);
- boolean hasMultimodalEmbedProperties = hasAnyProperty(properties,
- MULTIMODAL_EMBED_REQUIRED_FIELDS, MULTIMODAL_EMBED_API_KEY);
- if (!hasGeneralProperties && !hasEmbedProperties &&
!hasMultimodalEmbedProperties) {
- throw new DdlException("At least one complete AI property group
must be configured.");
- }
-
- if (hasGeneralProperties) {
- validatePropertyGroup(properties, REQUIRED_FIELDS, PROVIDER_TYPE,
API_KEY);
- }
- if (hasEmbedProperties) {
- validatePropertyGroup(properties, EMBED_REQUIRED_FIELDS,
EMBED_PROVIDER_TYPE, EMBED_API_KEY);
- }
- if (hasMultimodalEmbedProperties) {
- validatePropertyGroup(properties, MULTIMODAL_EMBED_REQUIRED_FIELDS,
- MULTIMODAL_EMBED_PROVIDER_TYPE, MULTIMODAL_EMBED_API_KEY);
- }
-
- String enableStructuredOutput =
properties.get(ENABLE_STRUCTURED_OUTPUT);
- if (enableStructuredOutput != null &&
!"true".equalsIgnoreCase(enableStructuredOutput)
- && !"false".equalsIgnoreCase(enableStructuredOutput)) {
- throw new DdlException("[" + ENABLE_STRUCTURED_OUTPUT + "] must be
true or false");
- }
-
- String effort = properties.get(EFFORT);
- if (!Strings.isNullOrEmpty(effort)) {
- String providerType = properties.get(PROVIDER_TYPE);
- List<String> effortLevels = ALL_EFFORT_LEVELS;
- if ("ANTHROPIC".equals(providerType)) {
- effortLevels = ANTHROPIC_EFFORT_LEVELS;
- } else if ("GEMINI".equals(providerType)) {
- effortLevels = GEMINI_EFFORT_LEVELS;
- }
- if (!effortLevels.contains(effort)) {
- throw new DdlException("[" + EFFORT + "] must be one of " +
effortLevels
- + " for provider: " + providerType);
- }
- }
-
- // Check weather the 'temperature' is valid
- String temp = properties.get(AIProperties.TEMPERATURE);
- if (!Strings.isNullOrEmpty(temp) && !temp.equals("-1")) {
- double tempVal = Double.parseDouble(temp);
- if (!(tempVal >= 0 && tempVal <= 1)) {
- throw new DdlException("Temperature must be a double between 0
and 1");
- }
- }
-
- // Check 'dimensions'
- temp = properties.get(AIProperties.DIMENSIONS);
- if (!Strings.isNullOrEmpty(temp) && !temp.equals("-1")) {
- int tempVal = Integer.parseInt(temp);
- if (tempVal <= 0) {
- throw new DdlException("Dimensions must be a positive
integer");
- }
- }
- }
-
- private static boolean hasAnyProperty(Map<String, String> properties,
List<String> requiredFields,
- String apiKeyField) {
- return properties.containsKey(apiKeyField) ||
requiredFields.stream().anyMatch(properties::containsKey);
- }
-
- private static void validatePropertyGroup(Map<String, String> properties,
List<String> requiredFields,
- String providerTypeField, String apiKeyField) throws DdlException {
- for (String field : requiredFields) {
- if (Strings.isNullOrEmpty(properties.get(field))) {
- throw new DdlException("Missing [" + field + "] in
properties.");
- }
- }
-
- String providerType = properties.get(providerTypeField).toUpperCase();
- properties.put(providerTypeField, providerType);
- if (!PROVIDERS.contains(providerType)) {
- throw new DdlException("Provider must be one of " + PROVIDERS);
- }
-
- if (!"LOCAL".equals(providerType) &&
Strings.isNullOrEmpty(properties.get(apiKeyField))) {
- throw new DdlException("Missing [" + apiKeyField + "] in
properties for provider: " + providerType);
- }
- }
-
- public static void optionalAIProperties(Map<String, String> properties) {
- properties.putIfAbsent(ENABLE_STRUCTURED_OUTPUT,
DEFAULT_ENABLE_STRUCTURED_OUTPUT);
- properties.putIfAbsent(AIProperties.TEMPERATURE,
AIProperties.DEFAULT_TEMPERATURE);
- properties.putIfAbsent(AIProperties.MAX_TOKEN,
AIProperties.DEFAULT_MAX_TOKEN);
- properties.putIfAbsent(AIProperties.MAX_RETRIES,
AIProperties.DEFAULT_MAX_RETRIES);
- properties.putIfAbsent(AIProperties.RETRY_DELAY_SECOND,
AIProperties.DEFAULT_RETRY_DELAY_SECOND);
- properties.putIfAbsent(AIProperties.ANTHROPIC_VERSION,
AIProperties.DEFAULT_ANTHROPIC_VERSION);
- properties.putIfAbsent(AIProperties.DIMENSIONS,
AIProperties.DEFAULT_DIMENSIONS);
- }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
index 746afaae700..f67ac12251a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
@@ -76,9 +76,11 @@ import
org.apache.doris.nereids.trees.expressions.functions.LambdaBinding;
import org.apache.doris.nereids.trees.expressions.functions.LambdaBindingSpec;
import org.apache.doris.nereids.trees.expressions.functions.RewriteWhenAnalyze;
import org.apache.doris.nereids.trees.expressions.functions.Udf;
+import org.apache.doris.nereids.trees.expressions.functions.agg.AIAgg;
import
org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import
org.apache.doris.nereids.trees.expressions.functions.agg.NullableAggregateFunction;
import
org.apache.doris.nereids.trees.expressions.functions.agg.SupportMultiDistinct;
+import org.apache.doris.nereids.trees.expressions.functions.ai.AIFunction;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda;
import
org.apache.doris.nereids.trees.expressions.functions.udf.AliasUdfBuilder;
@@ -638,13 +640,8 @@ public class ExpressionAnalyzer extends
SubExprAnalyzer<ExpressionRewriteContext
if (wantToParseSqlFromSqlCache) {
sqlCacheContext =
context.cascadesContext.getStatementContext().getSqlCacheContext();
- if (builder instanceof AliasUdfBuilder
- || buildResult.second instanceof Udf) {
- if (sqlCacheContext.isPresent()) {
- sqlCacheContext.get().setCannotProcessExpression(true);
- }
- }
}
+ disableSqlCacheForUnsupportedFunction(buildResult.second);
if (unboundFunction.isSkew() && unboundFunction.isDistinct() &&
buildResult.first instanceof AggregateFunction
&& buildResult.first instanceof SupportMultiDistinct) {
Expression bound = ((AggregateFunction)
buildResult.first).withIsSkew(true);
@@ -674,9 +671,18 @@ public class ExpressionAnalyzer extends
SubExprAnalyzer<ExpressionRewriteContext
@Override
public Expression visitBoundFunction(BoundFunction boundFunction,
ExpressionRewriteContext context) {
boundFunction = (BoundFunction)
super.visitBoundFunction(boundFunction, context);
+ disableSqlCacheForUnsupportedFunction(boundFunction);
return TypeCoercionUtils.processBoundFunction(boundFunction);
}
+ private void disableSqlCacheForUnsupportedFunction(BoundFunction function)
{
+ if (wantToParseSqlFromSqlCache
+ && (function instanceof Udf || function instanceof AIFunction
|| function instanceof AIAgg)) {
+ getCascadesContext().getStatementContext().getSqlCacheContext()
+ .ifPresent(cache ->
cache.setCannotProcessExpression(true));
+ }
+ }
+
private ElementAt canonicalizeStructSelector(ElementAt elementAt) {
Expression left = elementAt.left();
Expression right = elementAt.right();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AIAgg.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AIAgg.java
index 7caedc19869..ecb33286e25 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AIAgg.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AIAgg.java
@@ -24,6 +24,7 @@ import org.apache.doris.catalog.Resource;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
+import org.apache.doris.nereids.trees.expressions.functions.ai.AIFunction;
import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.StringType;
@@ -73,6 +74,11 @@ public class AIAgg extends NullableAggregateFunction
super(functionParams);
}
+ @Override
+ public boolean isDeterministic() {
+ return false;
+ }
+
@Override
public void checkLegalityAfterRewrite() {
// The rewrite has folded a constant task and resource name, unless
constant folding is skipped
@@ -113,6 +119,7 @@ public class AIAgg extends NullableAggregateFunction
private void checkResource(Expression resourceNameArgument) {
//Check if the resource is valid
String resourceName =
resourceNameArgument.toString().replaceAll("^['\"]|['\"]$", "");
+ AIFunction.checkResourceUsagePrivilege(resourceName);
Resource resource =
Env.getCurrentEnv().getResourceMgr().getResource(resourceName);
if (!(resource instanceof AIResource)) {
throw new AnalysisException("AI resource '" + resourceName + "'
does not exist");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ai/AIFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ai/AIFunction.java
index 64180adec7b..ddccffadba7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ai/AIFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ai/AIFunction.java
@@ -20,6 +20,8 @@ package
org.apache.doris.nereids.trees.expressions.functions.ai;
import org.apache.doris.catalog.AIResource;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Resource;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
@@ -48,6 +50,11 @@ public abstract class AIFunction extends ScalarFunction
public abstract int getMaxArgsNum();
+ @Override
+ public boolean isDeterministic() {
+ return false;
+ }
+
@Override
public void checkLegalityAfterRewrite() {
// The rewrite has folded a constant resource name, unless constant
folding is skipped
@@ -66,6 +73,7 @@ public abstract class AIFunction extends ScalarFunction
//Check if the resource is valid
String resourceName =
resourceNameArgument.toString().replaceAll("^['\"]|['\"]$", "");
+ checkResourceUsagePrivilege(resourceName);
Resource resource =
Env.getCurrentEnv().getResourceMgr().getResource(resourceName);
if (!(resource instanceof AIResource)) {
throw new AnalysisException("AI resource '" + resourceName + "'
does not exist");
@@ -77,6 +85,14 @@ public abstract class AIFunction extends ScalarFunction
Resource.registerUsedAIResourceName(resourceName);
}
+ public static void checkResourceUsagePrivilege(String resourceName) {
+ if (!Env.getCurrentEnv().getAccessManager()
+ .checkResourcePriv(ConnectContext.get(), resourceName,
PrivPredicate.USAGE)) {
+ throw new
AnalysisException(ErrorCode.ERR_RESOURCE_ACCESS_DENIED_ERROR.formatErrorMsg(
+ PrivPredicate.USAGE.getPrivs().toString(), resourceName));
+ }
+ }
+
/**
* If a user doesn't specify which resource to use when calling a function
* the current session variable will automatically allocate a resource.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ai/Embed.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ai/Embed.java
index b561de799f6..e75b80f63a4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ai/Embed.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ai/Embed.java
@@ -132,6 +132,7 @@ public class Embed extends AIFunction {
}
private static void validateAIResource(String resourceName, boolean
isMultimodal) {
+ checkResourceUsagePrivilege(resourceName);
Resource resource =
Env.getCurrentEnv().getResourceMgr().getResource(resourceName);
if (!(resource instanceof AIResource)) {
throw new AnalysisException("AI resource '" + resourceName + "'
does not exist");
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/AIResourceConcurrencyTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/AIResourceConcurrencyTest.java
new file mode 100644
index 00000000000..3ffad214708
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/catalog/AIResourceConcurrencyTest.java
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog;
+
+import org.apache.doris.common.proc.BaseProcResult;
+import org.apache.doris.persist.gson.GsonUtils;
+import org.apache.doris.thrift.TAIResource;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+public class AIResourceConcurrencyTest {
+ private ImmutableMap<String, String> properties(String suffix) {
+ return ImmutableMap.<String, String>builder()
+ .put("ai.provider_type", "LOCAL")
+ .put("ai.endpoint", "http://localhost:8000/"; + suffix)
+ .put("ai.model_name", "model-" + suffix)
+ .put("ai.api_key", "key-" + suffix)
+ .put("ai.embed.provider_type", "LOCAL")
+ .put("ai.embed.endpoint", "http://localhost:8001/"; + suffix)
+ .put("ai.embed.model_name", "embed-" + suffix)
+ .put("ai.embed.api_key", "embed-key-" + suffix)
+ .put("ai.embed.mm.provider_type", "LOCAL")
+ .put("ai.embed.mm.endpoint", "http://localhost:8002/"; + suffix)
+ .put("ai.embed.mm.model_name", "multimodal-" + suffix)
+ .put("ai.embed.mm.api_key", "multimodal-key-" + suffix)
+ .build();
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ public void testConcurrentAlterKeepsCompleteSnapshots(boolean restored)
throws Exception {
+ AIResource created = new AIResource("concurrent-snapshots");
+ created.setProperties(properties("0"));
+ AIResource resource = restored
+ ? (AIResource)
GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(created), Resource.class) :
created;
+ Map<String, String> original = resource.getCopiedProperties();
+ TAIResource originalThrift = resource.toThrift();
+ AIResource alternate = new AIResource("alternate-snapshot");
+ alternate.setProperties(properties("1"));
+ Map<String, String> changed = alternate.getCopiedProperties();
+ TAIResource changedThrift = alternate.toThrift();
+ ImmutableMap<String, String> zero = properties("0");
+ ImmutableMap<String, String> one = properties("1");
+ CountDownLatch ready = new CountDownLatch(3);
+ CountDownLatch start = new CountDownLatch(1);
+ ExecutorService executor = Executors.newFixedThreadPool(3);
+ try {
+ Future<?> writer = executor.submit(() -> {
+ ready.countDown();
+ Assertions.assertTrue(start.await(10, TimeUnit.SECONDS));
+ for (int i = 1; i <= 1000; ++i) {
+ resource.modifyProperties(i % 2 == 0 ? zero : one);
+ }
+ return null;
+ });
+ Future<?> reader = executor.submit(() -> {
+ ready.countDown();
+ Assertions.assertTrue(start.await(10, TimeUnit.SECONDS));
+ for (int i = 0; i < 1000; ++i) {
+ Map<String, String> snapshot =
resource.getCopiedProperties();
+ Assertions.assertTrue(snapshot.equals(original) ||
snapshot.equals(changed));
+ TAIResource thrift = resource.toThrift();
+ Assertions.assertTrue(thrift.equals(originalThrift) ||
thrift.equals(changedThrift));
+
Assertions.assertTrue(resource.hasCompleteGeneralProperties());
+
Assertions.assertTrue(resource.hasCompleteEmbedProperties());
+
Assertions.assertTrue(resource.hasCompleteMultimodalEmbedProperties());
+ }
+ return null;
+ });
+ Future<?> procReader = executor.submit(() -> {
+ ready.countDown();
+ Assertions.assertTrue(start.await(10, TimeUnit.SECONDS));
+ for (int i = 0; i < 1000; ++i) {
+ BaseProcResult result = new BaseProcResult();
+ resource.getProcNodeData(result);
+ long version = Long.parseLong(procValue(result.getRows(),
"version"));
+ String suffix = Long.toString((version + 1) % 2);
+ Assertions.assertEquals("model-" + suffix,
procValue(result.getRows(), "ai.model_name"));
+ Assertions.assertEquals("embed-" + suffix,
procValue(result.getRows(), "ai.embed.model_name"));
+ Assertions.assertEquals("multimodal-" + suffix,
+ procValue(result.getRows(),
"ai.embed.mm.model_name"));
+ Assertions.assertEquals("******",
procValue(result.getRows(), "ai.api_key"));
+ }
+ return null;
+ });
+ Assertions.assertTrue(ready.await(10, TimeUnit.SECONDS));
+ start.countDown();
+ writer.get(30, TimeUnit.SECONDS);
+ reader.get(30, TimeUnit.SECONDS);
+ procReader.get(30, TimeUnit.SECONDS);
+ Assertions.assertEquals(999, resource.getVersion());
+ Assertions.assertEquals(original, resource.getCopiedProperties());
+ Assertions.assertEquals("model-0",
resource.getProperty("ai.model_name"));
+ original.put("ai.model_name", "external-change");
+ Assertions.assertEquals("model-0",
resource.getProperty("ai.model_name"));
+ Assertions.assertEquals("model-0", originalThrift.getModelName());
+ } finally {
+ start.countDown();
+ executor.shutdownNow();
+ Assertions.assertTrue(executor.awaitTermination(10,
TimeUnit.SECONDS));
+ }
+ }
+
+ private String procValue(List<List<String>> rows, String key) {
+ return rows.stream().filter(row -> key.equals(row.get(2)))
+ .map(row ->
row.get(3)).findFirst().orElseThrow(AssertionError::new);
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/AIResourcePropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/AIResourcePropertiesTest.java
new file mode 100644
index 00000000000..cec999c0912
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/catalog/AIResourcePropertiesTest.java
@@ -0,0 +1,516 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog;
+
+import org.apache.doris.common.DdlException;
+import org.apache.doris.datasource.property.AIResourceProperties;
+import org.apache.doris.persist.gson.GsonUtils;
+import org.apache.doris.thrift.TAIResource;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Stream;
+
+public class AIResourcePropertiesTest {
+ private Map<String, String> generalProperties() {
+ return new HashMap<>(ImmutableMap.of(
+ "ai.endpoint", "http://localhost:8000/v1/chat/completions";,
+ "ai.provider_type", "local",
+ "ai.model_name", "chat-model"));
+ }
+
+ @Test
+ public void testTypedDefaultsAndPropertyOwnership() {
+ Map<String, String> raw = generalProperties();
+ raw.put("ai.validity_check", "false");
+ raw.put("custom.property", "retained");
+ AIResourceProperties properties = AIResourceProperties.of(raw);
+ raw.put("ai.model_name", "changed-model");
+
+ TAIResource thrift = properties.toThrift();
+ Assertions.assertEquals("LOCAL", thrift.getProviderType());
+ Assertions.assertEquals("chat-model", thrift.getModelName());
+ Assertions.assertFalse(thrift.isSetApiKey());
+ Assertions.assertEquals(-1.0, thrift.getTemperature());
+ Assertions.assertEquals(-1L, thrift.getMaxTokens());
+ Assertions.assertEquals(3, thrift.getMaxRetries());
+ Assertions.assertEquals(1, thrift.getRetryDelaySecond());
+ Assertions.assertEquals(-1, thrift.getDimensions());
+ Assertions.assertEquals("2023-06-01", thrift.getAnthropicVersion());
+ Assertions.assertTrue(properties.hasCompleteGeneralProperties());
+ Assertions.assertFalse(properties.hasCompleteEmbedProperties());
+
Assertions.assertFalse(properties.hasCompleteMultimodalEmbedProperties());
+ Assertions.assertEquals("false",
properties.toMap().get("ai.validity_check"));
+ Assertions.assertEquals("retained",
properties.toMap().get("custom.property"));
+ properties.toMap().put("ai.model_name", "another-model");
+ Assertions.assertEquals("chat-model",
properties.toThrift().getModelName());
+ }
+
+ @Test
+ public void testIndependentEmbedGroups() {
+ AIResourceProperties embed = AIResourceProperties.of(ImmutableMap.of(
+ "ai.embed.endpoint", "http://localhost:8000/v1/embeddings";,
+ "ai.embed.provider_type", "local",
+ "ai.embed.model_name", "text-model"));
+ Assertions.assertFalse(embed.hasCompleteGeneralProperties());
+ Assertions.assertTrue(embed.hasCompleteEmbedProperties());
+ Assertions.assertFalse(embed.hasCompleteMultimodalEmbedProperties());
+ Assertions.assertFalse(embed.toThrift().isSetProviderType());
+ Assertions.assertEquals("LOCAL",
embed.toThrift().getEmbedProviderType());
+ Assertions.assertEquals("text-model",
embed.toThrift().getEmbedModelName());
+
+ AIResourceProperties multimodal =
AIResourceProperties.of(ImmutableMap.of(
+ "ai.embed.mm.endpoint", "http://localhost:8000/v1/embeddings";,
+ "ai.embed.mm.provider_type", "local",
+ "ai.embed.mm.model_name", "image-model"));
+ Assertions.assertFalse(multimodal.hasCompleteGeneralProperties());
+ Assertions.assertFalse(multimodal.hasCompleteEmbedProperties());
+
Assertions.assertTrue(multimodal.hasCompleteMultimodalEmbedProperties());
+ Assertions.assertEquals("LOCAL",
multimodal.toThrift().getEmbedMmProviderType());
+ Assertions.assertEquals("image-model",
multimodal.toThrift().getEmbedMmModelName());
+ }
+
+ @Test
+ public void testExplicitValuesAndCombinedGroups() {
+ Map<String, String> raw = generalProperties();
+ raw.put("ai.api_key", "general-key");
+ raw.put("ai.embed.endpoint", "http://localhost:8001/embeddings";);
+ raw.put("ai.embed.provider_type", "openai");
+ raw.put("ai.embed.model_name", "text-model");
+ raw.put("ai.embed.api_key", "text-key");
+ raw.put("ai.embed.mm.endpoint", "http://localhost:8002/embeddings";);
+ raw.put("ai.embed.mm.provider_type", "gemini");
+ raw.put("ai.embed.mm.model_name", "image-model");
+ raw.put("ai.embed.mm.api_key", "image-key");
+ raw.put("ai.temperature", "0.75");
+ raw.put("ai.max_token", "4294967296");
+ raw.put("ai.max_retries", "0");
+ raw.put("ai.retry_delay_second", "0");
+ raw.put("ai.dimensions", "768");
+ raw.put("ai.anthropic_version", "2024-01-01");
+ raw.put("ai.effort", "high");
+ AIResourceProperties properties = AIResourceProperties.of(raw);
+ TAIResource thrift = properties.toThrift();
+ Assertions.assertTrue(properties.hasCompleteGeneralProperties());
+ Assertions.assertTrue(properties.hasCompleteEmbedProperties());
+
Assertions.assertTrue(properties.hasCompleteMultimodalEmbedProperties());
+ Assertions.assertEquals("general-key", thrift.getApiKey());
+ Assertions.assertEquals("text-key", thrift.getEmbedApiKey());
+ Assertions.assertEquals("image-key", thrift.getEmbedMmApiKey());
+ Assertions.assertEquals("http://localhost:8001/embeddings";,
thrift.getEmbedEndpoint());
+ Assertions.assertEquals("http://localhost:8002/embeddings";,
thrift.getEmbedMmEndpoint());
+ Assertions.assertEquals("OPENAI", thrift.getEmbedProviderType());
+ Assertions.assertEquals("GEMINI", thrift.getEmbedMmProviderType());
+ Assertions.assertEquals(0.75, thrift.getTemperature());
+ Assertions.assertEquals(4294967296L, thrift.getMaxTokens());
+ Assertions.assertEquals(0, thrift.getMaxRetries());
+ Assertions.assertEquals(0, thrift.getRetryDelaySecond());
+ Assertions.assertEquals(768, thrift.getDimensions());
+ Assertions.assertEquals("2024-01-01", thrift.getAnthropicVersion());
+ Assertions.assertEquals("high", thrift.getEffort());
+ Assertions.assertFalse(properties.toString().contains("general-key"));
+ Assertions.assertFalse(properties.toString().contains("text-key"));
+ Assertions.assertFalse(properties.toString().contains("image-key"));
+ }
+
+ @Test
+ public void testExistingGroupAndValueRules() {
+ Assertions.assertThrows(IllegalArgumentException.class, () ->
AIResourceProperties.of(ImmutableMap.of()));
+ Map<String, String> raw = generalProperties();
+ raw.put("ai.embed.model_name", "partial-model");
+ Assertions.assertThrows(IllegalArgumentException.class, () ->
AIResourceProperties.of(raw));
+ raw.remove("ai.embed.model_name");
+ raw.put("ai.provider_type", "openai");
+ Assertions.assertThrows(IllegalArgumentException.class, () ->
AIResourceProperties.of(raw));
+ raw.put("ai.api_key", "key");
+ raw.put("ai.temperature", "1.5");
+ Assertions.assertThrows(IllegalArgumentException.class, () ->
AIResourceProperties.of(raw));
+ raw.put("ai.temperature", "-1");
+ raw.put("ai.dimensions", "0");
+ Assertions.assertThrows(IllegalArgumentException.class, () ->
AIResourceProperties.of(raw));
+ raw.put("ai.dimensions", "-1");
+ raw.put("ai.provider_type", "anthropic");
+ raw.put("ai.effort", "none");
+ Assertions.assertThrows(IllegalArgumentException.class, () ->
AIResourceProperties.of(raw));
+ raw.put("ai.effort", "high");
+ Assertions.assertEquals("high",
AIResourceProperties.of(raw).toThrift().getEffort());
+ }
+
+ @Test
+ public void testInvalidNumericBinding() {
+ for (String key : new String[] {"ai.temperature", "ai.max_token",
"ai.max_retries",
+ "ai.retry_delay_second", "ai.dimensions"}) {
+ for (String value : new String[] {"", " ", "not-a-number"}) {
+ Map<String, String> raw = generalProperties();
+ raw.put(key, value);
+ IllegalArgumentException failure =
Assertions.assertThrows(IllegalArgumentException.class,
+ () -> AIResourceProperties.of(raw));
+ Assertions.assertTrue(failure.getMessage().contains(key));
+ }
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("invalidNumericProperties")
+ public void testInvalidNumericRangesOnCreateAndAlter(String key, String
value) throws Exception {
+ Map<String, String> invalid = generalProperties();
+ invalid.put(key, value);
+ AIResource created = new AIResource("invalid-numeric-resource");
+ DdlException createFailure =
Assertions.assertThrows(DdlException.class,
+ () -> created.setProperties(ImmutableMap.copyOf(invalid)));
+ Assertions.assertTrue(createFailure.getMessage().contains(key));
+
+ AIResource altered = new AIResource("valid-numeric-resource");
+ altered.setProperties(ImmutableMap.copyOf(generalProperties()));
+ Map<String, String> before = altered.getCopiedProperties();
+ long version = altered.version;
+ DdlException alterFailure = Assertions.assertThrows(DdlException.class,
+ () -> altered.modifyProperties(ImmutableMap.of(key, value)));
+ Assertions.assertTrue(alterFailure.getMessage().contains(key));
+ Assertions.assertEquals(before, altered.getCopiedProperties());
+ Assertions.assertEquals(version, altered.version);
+ }
+
+ private static Stream<Arguments> invalidNumericProperties() {
+ return Stream.of(
+ Arguments.of("ai.temperature", "NaN"),
+ Arguments.of("ai.temperature", "Infinity"),
+ Arguments.of("ai.temperature", "-Infinity"),
+ Arguments.of("ai.temperature", "1e309"),
+ Arguments.of("ai.temperature", "-0.1"),
+ Arguments.of("ai.temperature", "1.1"),
+ Arguments.of("ai.temperature", "-2"),
+ Arguments.of("ai.max_token", "0"),
+ Arguments.of("ai.max_token", "-2"),
+ Arguments.of("ai.max_token", "-9223372036854775808"),
+ Arguments.of("ai.max_token", "9223372036854775808"),
+ Arguments.of("ai.max_token", "1.5"),
+ Arguments.of("ai.max_retries", "-1"),
+ Arguments.of("ai.max_retries", "-2147483648"),
+ Arguments.of("ai.max_retries", "2147483648"),
+ Arguments.of("ai.max_retries", "1.5"),
+ Arguments.of("ai.retry_delay_second", "-1"),
+ Arguments.of("ai.retry_delay_second", "-2147483648"),
+ Arguments.of("ai.retry_delay_second", "2147483648"),
+ Arguments.of("ai.retry_delay_second", "1.5"),
+ Arguments.of("ai.dimensions", "0"),
+ Arguments.of("ai.dimensions", "-2"),
+ Arguments.of("ai.dimensions", "-2147483648"),
+ Arguments.of("ai.dimensions", "2147483648"),
+ Arguments.of("ai.dimensions", "1.5"));
+ }
+
+ @Test
+ public void testNumericBoundaryValues() throws Exception {
+ Map<String, String> raw = generalProperties();
+ raw.putAll(ImmutableMap.of("ai.temperature", "1", "ai.max_token",
"9223372036854775807",
+ "ai.max_retries", "2147483647", "ai.retry_delay_second",
"2147483647",
+ "ai.dimensions", "2147483647"));
+ AIResource resource = new AIResource("numeric-boundaries");
+ resource.setProperties(ImmutableMap.copyOf(raw));
+ TAIResource maximum = resource.toThrift();
+ Assertions.assertEquals(1.0, maximum.getTemperature());
+ Assertions.assertEquals(Long.MAX_VALUE, maximum.getMaxTokens());
+ Assertions.assertEquals(Integer.MAX_VALUE, maximum.getMaxRetries());
+ Assertions.assertEquals(Integer.MAX_VALUE,
maximum.getRetryDelaySecond());
+ Assertions.assertEquals(Integer.MAX_VALUE, maximum.getDimensions());
+
+ resource.modifyProperties(ImmutableMap.of("ai.temperature", "0",
"ai.max_token", "1",
+ "ai.max_retries", "0", "ai.retry_delay_second", "0",
"ai.dimensions", "1"));
+ TAIResource minimum = resource.toThrift();
+ Assertions.assertEquals(0.0, minimum.getTemperature());
+ Assertions.assertEquals(1L, minimum.getMaxTokens());
+ Assertions.assertEquals(0, minimum.getMaxRetries());
+ Assertions.assertEquals(0, minimum.getRetryDelaySecond());
+ Assertions.assertEquals(1, minimum.getDimensions());
+
+ resource.modifyProperties(ImmutableMap.of("ai.temperature", "-1",
"ai.max_token", "-1",
+ "ai.dimensions", "-1"));
+ Assertions.assertEquals(-1.0, resource.toThrift().getTemperature());
+ Assertions.assertEquals(-1L, resource.toThrift().getMaxTokens());
+ Assertions.assertEquals(-1, resource.toThrift().getDimensions());
+ }
+
+ @Test
+ public void testProviderNormalizationIsLocaleIndependent() throws
Exception {
+ Locale original = Locale.getDefault();
+ try {
+ Locale.setDefault(Locale.forLanguageTag("tr-TR"));
+ Map<String, String> raw = generalProperties();
+ raw.put("ai.provider_type", "minimax");
+ raw.put("ai.api_key", "general-key");
+ raw.put("ai.embed.endpoint", "http://localhost:8001/embeddings";);
+ raw.put("ai.embed.provider_type", "gemini");
+ raw.put("ai.embed.model_name", "text-model");
+ raw.put("ai.embed.api_key", "text-key");
+ raw.put("ai.embed.mm.endpoint",
"http://localhost:8002/embeddings";);
+ raw.put("ai.embed.mm.provider_type", "jina");
+ raw.put("ai.embed.mm.model_name", "image-model");
+ raw.put("ai.embed.mm.api_key", "image-key");
+ AIResource resource = new
AIResource("locale-independent-resource");
+ resource.setProperties(ImmutableMap.copyOf(raw));
+ Assertions.assertEquals("MINIMAX",
resource.toThrift().getProviderType());
+ Assertions.assertEquals("GEMINI",
resource.toThrift().getEmbedProviderType());
+ Assertions.assertEquals("JINA",
resource.toThrift().getEmbedMmProviderType());
+ resource.modifyProperties(ImmutableMap.of("ai.provider_type",
"jina",
+ "ai.embed.provider_type", "minimax",
"ai.embed.mm.provider_type", "gemini"));
+ Assertions.assertEquals("JINA",
resource.getProperty("ai.provider_type"));
+ Assertions.assertEquals("MINIMAX",
resource.getProperty("ai.embed.provider_type"));
+ Assertions.assertEquals("GEMINI",
resource.getProperty("ai.embed.mm.provider_type"));
+ } finally {
+ Locale.setDefault(original);
+ }
+ }
+
+ @Test
+ public void testLegacyOutOfRangeMetadataLoadsAndCanBeRepaired() throws
Exception {
+ Map<String, String> invalid = ImmutableMap.of(
+ "ai.max_token", "0", "ai.max_retries", "-1",
"ai.retry_delay_second", "-1");
+ for (Map.Entry<String, String> entry : invalid.entrySet()) {
+ Map<String, String> raw = generalProperties();
+ raw.put(entry.getKey(), entry.getValue());
+ AIResource loaded = readMetadata(raw);
+ Assertions.assertEquals(entry.getValue(),
loaded.getProperty(entry.getKey()));
+ Assertions.assertTrue(loaded.hasCompleteGeneralProperties());
+ IllegalArgumentException failure =
Assertions.assertThrows(IllegalArgumentException.class,
+ loaded::toThrift);
+
Assertions.assertTrue(failure.getMessage().contains(entry.getKey()));
+ loaded.modifyProperties(ImmutableMap.of(entry.getKey(), "1"));
+ Assertions.assertNotNull(loaded.toThrift());
+ }
+ }
+
+ @Test
+ public void testResourceAlterAndPersistence() throws Exception {
+ AIResource resource = new AIResource("typed-resource");
+ resource.setProperties(ImmutableMap.copyOf(generalProperties()));
+ resource.modifyProperties(ImmutableMap.of("ai.provider_type",
"openai", "ai.api_key", "key",
+ "ai.effort", "high", "ai.max_token", "2048"));
+ Assertions.assertEquals("OPENAI",
resource.getProperty("ai.provider_type"));
+ Assertions.assertEquals(2048L, resource.toThrift().getMaxTokens());
+ Map<String, String> before = resource.getCopiedProperties();
+ long version = resource.version;
+ Assertions.assertThrows(DdlException.class,
+ () -> resource.modifyProperties(ImmutableMap.of("ai.effort",
"invalid")));
+ Assertions.assertEquals(before, resource.getCopiedProperties());
+ Assertions.assertEquals(version, resource.version);
+ Assertions.assertThrows(DdlException.class,
+ () -> resource.modifyProperties(ImmutableMap.of("ai.effort",
"")));
+ Assertions.assertEquals("high", resource.toThrift().getEffort());
+
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ try (DataOutputStream output = new DataOutputStream(bytes)) {
+ resource.write(output);
+ }
+ try (DataInputStream input = new DataInputStream(new
ByteArrayInputStream(bytes.toByteArray()))) {
+ AIResource loaded = (AIResource) Resource.read(input);
+ Assertions.assertEquals(resource.getCopiedProperties(),
loaded.getCopiedProperties());
+ Assertions.assertEquals(resource.toThrift(), loaded.toThrift());
+ }
+ }
+
+ @Test
+ public void testMetadataLoadingDoesNotValidateProperties() {
+ String json =
"{\"clazz\":\"AIResource\",\"name\":\"legacy-ai\",\"type\":\"AI\","
+ +
"\"properties\":{\"ai.provider_type\":\"LOCAL\",\"ai.endpoint\":\"http://localhost:8000\",";
+ +
"\"ai.model_name\":\"legacy-model\",\"ai.max_retries\":\"not-an-integer\","
+ + "\"ai.validity_check\":\"false\"}}";
+ AIResource loaded = (AIResource) GsonUtils.GSON.fromJson(json,
Resource.class);
+ Assertions.assertEquals("not-an-integer",
loaded.getProperty("ai.max_retries"));
+ Assertions.assertEquals("false",
loaded.getProperty("ai.validity_check"));
+ Assertions.assertTrue(loaded.hasCompleteGeneralProperties());
+ Assertions.assertFalse(loaded.hasCompleteEmbedProperties());
+ Assertions.assertFalse(loaded.hasCompleteMultimodalEmbedProperties());
+ Assertions.assertThrows(IllegalArgumentException.class,
loaded::toThrift);
+ }
+
+ @Test
+ public void testStringValuesAreTrimmed() {
+ Map<String, String> raw = generalProperties();
+ raw.put("ai.model_name", " model ");
+ raw.put("ai.endpoint", " http://localhost:8000 ");
+ raw.put("ai.api_key", " key ");
+ raw.put("ai.anthropic_version", " 2023-06-01 ");
+ AIResourceProperties properties = AIResourceProperties.of(raw);
+ TAIResource thrift = properties.toThrift();
+ Assertions.assertEquals("model", thrift.getModelName());
+ Assertions.assertEquals("http://localhost:8000";, thrift.getEndpoint());
+ Assertions.assertEquals("key", thrift.getApiKey());
+ Assertions.assertEquals("2023-06-01", thrift.getAnthropicVersion());
+ Assertions.assertEquals("model",
properties.toMap().get("ai.model_name"));
+ Assertions.assertEquals("http://localhost:8000";,
properties.toMap().get("ai.endpoint"));
+ Assertions.assertEquals("key", properties.toMap().get("ai.api_key"));
+ Assertions.assertEquals("2023-06-01",
properties.toMap().get("ai.anthropic_version"));
+ }
+
+ @Test
+ public void testCreateAndAlterPersistNormalizedProperties() throws
Exception {
+ Map<String, String> expected = new HashMap<>(ImmutableMap.<String,
String>builder()
+ .put("ai.provider_type", "LOCAL")
+ .put("ai.endpoint", "http://localhost:8000/chat";)
+ .put("ai.model_name", "chat-model")
+ .put("ai.api_key", "chat-key")
+ .put("ai.embed.provider_type", "OPENAI")
+ .put("ai.embed.endpoint", "http://localhost:8001/embeddings";)
+ .put("ai.embed.model_name", "text-model")
+ .put("ai.embed.api_key", "text-key")
+ .put("ai.embed.mm.provider_type", "GEMINI")
+ .put("ai.embed.mm.endpoint",
"http://localhost:8002/embeddings";)
+ .put("ai.embed.mm.model_name", "image-model")
+ .put("ai.embed.mm.api_key", "image-key")
+ .put("ai.temperature", "0.75")
+ .put("ai.max_token", "4294967296")
+ .put("ai.max_retries", "3")
+ .put("ai.retry_delay_second", "2")
+ .put("ai.dimensions", "768")
+ .put("ai.anthropic_version", "2023-06-01")
+ .put("ai.effort", "high")
+ .put("ai.enable_structured_output", "true")
+ .build());
+ Map<String, String> raw = new HashMap<>();
+ expected.forEach((key, value) -> raw.put(key, " \t" + value + "\t "));
+ raw.put("custom.property", " retained ");
+ raw.put("ai.validity_check", " false ");
+ expected.put("custom.property", " retained ");
+ expected.put("ai.validity_check", " false ");
+ Assertions.assertEquals(expected,
AIResourceProperties.of(raw).toMap());
+ Assertions.assertEquals(" \t768\t ", raw.get("ai.dimensions"));
+
+ AIResource resource = new AIResource("normalized-resource");
+ resource.setProperties(ImmutableMap.copyOf(raw));
+ assertNormalizedMetadata(resource, expected);
+
+ resource.modifyProperties(ImmutableMap.of("ai.dimensions", " 1024 ",
+ "ai.max_retries", " 4 ", "ai.model_name", " updated-model ",
+ "ai.embed.api_key", " updated-key ",
"ai.enable_structured_output", " false "));
+ expected.putAll(ImmutableMap.of("ai.dimensions", "1024",
"ai.max_retries", "4",
+ "ai.model_name", "updated-model", "ai.embed.api_key",
"updated-key",
+ "ai.enable_structured_output", "false"));
+ assertNormalizedMetadata(resource, expected);
+ }
+
+ private void assertNormalizedMetadata(AIResource resource, Map<String,
String> expected) throws Exception {
+ Assertions.assertEquals(expected, resource.getCopiedProperties());
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ try (DataOutputStream output = new DataOutputStream(bytes)) {
+ resource.write(output);
+ }
+ try (DataInputStream input = new DataInputStream(new
ByteArrayInputStream(bytes.toByteArray()))) {
+ AIResource loaded = (AIResource) Resource.read(input);
+ Assertions.assertEquals(expected, loaded.getCopiedProperties());
+ Assertions.assertEquals(resource.toThrift(), loaded.toThrift());
+ Assertions.assertDoesNotThrow(() ->
Double.parseDouble(loaded.getProperty("ai.temperature")));
+ Assertions.assertDoesNotThrow(() ->
Long.parseLong(loaded.getProperty("ai.max_token")));
+ Assertions.assertDoesNotThrow(() ->
Integer.parseInt(loaded.getProperty("ai.max_retries")));
+ Assertions.assertDoesNotThrow(() ->
Integer.parseInt(loaded.getProperty("ai.retry_delay_second")));
+ Assertions.assertDoesNotThrow(() ->
Integer.parseInt(loaded.getProperty("ai.dimensions")));
+
Assertions.assertEquals(resource.toThrift().isEnableStructuredOutput(),
+
Boolean.parseBoolean(loaded.getProperty("ai.enable_structured_output")));
+ }
+ }
+
+ @Test
+ public void testRejectBlankCreateAndAlterProperties() throws Exception {
+ AIResource resource = new AIResource("non-empty-properties");
+ resource.setProperties(ImmutableMap.copyOf(generalProperties()));
+ Map<String, String> before = resource.getCopiedProperties();
+ long version = resource.version;
+ for (String key : new String[] {"ai.endpoint", "ai.provider_type",
"ai.model_name", "ai.api_key",
+ "ai.embed.endpoint", "ai.embed.provider_type",
"ai.embed.model_name", "ai.embed.api_key",
+ "ai.embed.mm.endpoint", "ai.embed.mm.provider_type",
"ai.embed.mm.model_name", "ai.embed.mm.api_key",
+ "ai.temperature", "ai.max_token", "ai.max_retries",
"ai.retry_delay_second", "ai.dimensions",
+ "ai.anthropic_version", "ai.effort"}) {
+ for (String value : new String[] {"", " \t ", null}) {
+ Map<String, String> raw = generalProperties();
+ raw.put(key, value);
+ IllegalArgumentException failure =
Assertions.assertThrows(IllegalArgumentException.class,
+ () -> AIResourceProperties.of(raw));
+ Assertions.assertTrue(failure.getMessage().contains(key));
+ Map<String, String> updates = new HashMap<>();
+ updates.put(key, value);
+ Assertions.assertThrows(DdlException.class, () ->
resource.modifyProperties(updates));
+ Assertions.assertEquals(before,
resource.getCopiedProperties());
+ Assertions.assertEquals(version, resource.version);
+ }
+ }
+ }
+
+ @Test
+ public void testLegacyPaddedMetadata() {
+ Map<String, String> raw = generalProperties();
+ raw.put("ai.provider_type", " local ");
+ raw.put("ai.model_name", " model ");
+ raw.put("ai.endpoint", " http://localhost:8000 ");
+ raw.put("ai.max_retries", " 3 ");
+ AIResource loaded = readMetadata(raw);
+ Assertions.assertTrue(loaded.hasCompleteGeneralProperties());
+ Assertions.assertEquals("LOCAL", loaded.toThrift().getProviderType());
+ Assertions.assertEquals("model", loaded.toThrift().getModelName());
+ Assertions.assertEquals("http://localhost:8000";,
loaded.toThrift().getEndpoint());
+ Assertions.assertEquals(3, loaded.toThrift().getMaxRetries());
+ Assertions.assertEquals(raw, loaded.getCopiedProperties());
+ }
+
+ @Test
+ public void testLegacyBlankMetadataLoadsAndCanBeRepaired() throws
Exception {
+ for (String key : new String[] {"ai.model_name",
"ai.anthropic_version", "ai.effort", "ai.max_retries"}) {
+ for (String value : new String[] {"", " \t "}) {
+ Map<String, String> raw = generalProperties();
+ raw.put(key, value);
+ AIResource loaded = readMetadata(raw);
+ Assertions.assertEquals(value, loaded.getProperty(key));
+ Assertions.assertEquals(!key.equals("ai.model_name"),
loaded.hasCompleteGeneralProperties());
+ IllegalArgumentException failure =
Assertions.assertThrows(IllegalArgumentException.class,
+ loaded::toThrift);
+ Assertions.assertTrue(failure.getMessage().contains(key));
+ loaded.modifyProperties(ImmutableMap.of(key,
key.equals("ai.max_retries") ? "3" : "high"));
+ Assertions.assertNotNull(loaded.toThrift());
+ }
+ }
+ }
+
+ private AIResource readMetadata(Map<String, String> properties) {
+ String json = GsonUtils.GSON.toJson(ImmutableMap.of(
+ "clazz", "AIResource", "name", "legacy", "type", "AI",
"properties", properties));
+ return (AIResource) GsonUtils.GSON.fromJson(json, Resource.class);
+ }
+
+ @Test
+ public void testIncompleteMetadataReturnsGroupCompleteness() {
+ AIResource loaded = (AIResource) GsonUtils.GSON.fromJson(
+
"{\"clazz\":\"AIResource\",\"name\":\"partial\",\"type\":\"AI\","
+ +
"\"properties\":{\"ai.endpoint\":\"http://localhost:8000\",";
+ +
"\"ai.embed.provider_type\":\"local\",\"ai.embed.model_name\":\"text-model\","
+ + "\"ai.embed.endpoint\":\"http://localhost:8001\"}}";,
Resource.class);
+ Assertions.assertFalse(loaded.hasCompleteGeneralProperties());
+ Assertions.assertTrue(loaded.hasCompleteEmbedProperties());
+ Assertions.assertFalse(loaded.hasCompleteMultimodalEmbedProperties());
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/AIResourceTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/AIResourceTest.java
index 7ac4c4cd7cf..4aeca1988f2 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/AIResourceTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/AIResourceTest.java
@@ -17,6 +17,9 @@
package org.apache.doris.catalog;
+import org.apache.doris.analysis.PasswordOptions;
+import org.apache.doris.analysis.ResourcePattern;
+import org.apache.doris.analysis.ResourceTypeEnum;
import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
@@ -27,19 +30,38 @@ import org.apache.doris.common.proc.BaseProcResult;
import org.apache.doris.datasource.property.constants.AIProperties;
import org.apache.doris.meta.MetaContext;
import org.apache.doris.mysql.privilege.AccessControllerManager;
+import org.apache.doris.mysql.privilege.Auth;
+import org.apache.doris.mysql.privilege.PrivBitSet;
import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.mysql.privilege.Privilege;
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.SqlCacheContext;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.analyzer.UnboundFunction;
import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
import org.apache.doris.nereids.trees.expressions.functions.agg.AIAgg;
+import org.apache.doris.nereids.trees.expressions.functions.ai.AIGenerate;
import org.apache.doris.nereids.trees.expressions.functions.ai.AISentiment;
import org.apache.doris.nereids.trees.expressions.functions.ai.Embed;
import org.apache.doris.nereids.trees.expressions.literal.JsonLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
+import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.commands.CreateResourceCommand;
import org.apache.doris.nereids.trees.plans.commands.info.CreateResourceInfo;
+import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
+import org.apache.doris.persist.DropResourceOperationLog;
import org.apache.doris.persist.EditLog;
+import org.apache.doris.persist.PrivInfo;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.OriginStatement;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
@@ -48,6 +70,8 @@ import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
@@ -359,82 +383,204 @@ public class AIResourceTest {
@Test
public void testRejectEmbedOnlyResourceForNonEmbedScalarFunction() throws
DdlException {
AIResource aiResource = createEmbedOnlyResource();
- try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
- Env env = Mockito.mock(Env.class);
- ResourceMgr resourceMgr = Mockito.mock(ResourceMgr.class);
- mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
- Mockito.when(env.getResourceMgr()).thenReturn(resourceMgr);
-
Mockito.when(resourceMgr.getResource("embed-only-resource")).thenReturn(aiResource);
-
+ withAIResource(aiResource, () -> {
AISentiment function = new AISentiment(new
StringLiteral("embed-only-resource"),
new StringLiteral("text"));
- Assertions.assertThrows(AnalysisException.class,
function::checkLegalityAfterRewrite);
- }
+ AnalysisException exception =
Assertions.assertThrows(AnalysisException.class,
+ function::checkLegalityAfterRewrite);
+ Assertions.assertTrue(exception.getMessage().contains("does not
contain complete general AI properties"));
+ });
}
@Test
public void testRejectEmbedOnlyResourceForAiAgg() throws DdlException {
AIResource aiResource = createEmbedOnlyResource();
- try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
- Env env = Mockito.mock(Env.class);
- ResourceMgr resourceMgr = Mockito.mock(ResourceMgr.class);
- mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
- Mockito.when(env.getResourceMgr()).thenReturn(resourceMgr);
-
Mockito.when(resourceMgr.getResource("embed-only-resource")).thenReturn(aiResource);
-
+ withAIResource(aiResource, () -> {
AIAgg function = new AIAgg(new
StringLiteral("embed-only-resource"),
new StringLiteral("text"), new StringLiteral("task"));
- Assertions.assertThrows(AnalysisException.class,
function::checkLegalityAfterRewrite);
- }
+ AnalysisException exception =
Assertions.assertThrows(AnalysisException.class,
+ function::checkLegalityAfterRewrite);
+ Assertions.assertTrue(exception.getMessage().contains("does not
contain complete general AI properties"));
+ });
}
@Test
public void testAcceptEmbedOnlyResourceForEmbedFunction() throws
DdlException {
AIResource aiResource = createEmbedOnlyResource();
- try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
- Env env = Mockito.mock(Env.class);
- ResourceMgr resourceMgr = Mockito.mock(ResourceMgr.class);
- mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
- Mockito.when(env.getResourceMgr()).thenReturn(resourceMgr);
-
Mockito.when(resourceMgr.getResource("embed-only-resource")).thenReturn(aiResource);
-
+ withAIResource(aiResource, () -> {
Embed function = new Embed(new
StringLiteral("embed-only-resource"),
new StringLiteral("text"));
Assertions.assertDoesNotThrow(function::checkLegalityBeforeTypeCoercion);
Assertions.assertDoesNotThrow(function::checkLegalityAfterRewrite);
- }
+ });
}
@Test
public void testAcceptMultimodalEmbedOnlyResourceForJsonEmbed() throws
DdlException {
AIResource aiResource = createMultimodalEmbedOnlyResource();
- try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
- Env env = Mockito.mock(Env.class);
- ResourceMgr resourceMgr = Mockito.mock(ResourceMgr.class);
- mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
- Mockito.when(env.getResourceMgr()).thenReturn(resourceMgr);
-
Mockito.when(resourceMgr.getResource("multimodal-embed-only-resource")).thenReturn(aiResource);
-
+ withAIResource(aiResource, () -> {
Embed function = new Embed(new
StringLiteral("multimodal-embed-only-resource"),
new JsonLiteral("{\"text\":\"hello\"}"));
Assertions.assertDoesNotThrow(function::checkLegalityBeforeTypeCoercion);
Assertions.assertDoesNotThrow(function::checkLegalityAfterRewrite);
- }
+ });
}
@Test
public void testRejectMultimodalEmbedOnlyResourceForTextEmbed() throws
DdlException {
AIResource aiResource = createMultimodalEmbedOnlyResource();
- try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
- Env env = Mockito.mock(Env.class);
- ResourceMgr resourceMgr = Mockito.mock(ResourceMgr.class);
- mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
- Mockito.when(env.getResourceMgr()).thenReturn(resourceMgr);
-
Mockito.when(resourceMgr.getResource("multimodal-embed-only-resource")).thenReturn(aiResource);
-
+ withAIResource(aiResource, () -> {
Embed function = new Embed(new
StringLiteral("multimodal-embed-only-resource"),
new StringLiteral("hello"));
- Assertions.assertThrows(AnalysisException.class,
function::checkLegalityBeforeTypeCoercion);
+ AnalysisException exception =
Assertions.assertThrows(AnalysisException.class,
+ function::checkLegalityBeforeTypeCoercion);
+ Assertions.assertTrue(exception.getMessage().contains("does not
have properties required by EMBED"));
+ });
+ }
+
+ private void withAIResource(AIResource resource, Runnable check) {
+ ConnectContext previousContext = ConnectContext.get();
+ ConnectContext context = new ConnectContext();
+ context.setCurrentUserIdentity(UserIdentity.ROOT);
+ context.setRemoteIP("127.0.0.1");
+ context.setThreadLocalInfo();
+ ResourceMgr resourceMgr = Env.getCurrentEnv().getResourceMgr();
+ resourceMgr.replayCreateResource(resource);
+ try {
+ check.run();
+ } finally {
+ resourceMgr.replayDropResource(new
DropResourceOperationLog(resource.getName()));
+ if (previousContext == null) {
+ ConnectContext.remove();
+ } else {
+ previousContext.setThreadLocalInfo();
+ }
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"generate", "aggregate", "embed", "multimodal",
"default_generate",
+ "default_aggregate", "default_embed", "default_multimodal",
"null_input"})
+ public void testAIResourceUsagePrivilege(String kind) throws Exception {
+ String resourceName = "ai_resource_privilege_test";
+ UserIdentity user =
UserIdentity.createAnalyzedUserIdentWithIp("ai_resource_privilege_user", "%");
+ AIResource resource = new AIResource(resourceName);
+ resource.setProperties(ImmutableMap.<String, String>builder()
+ .put("ai.provider_type", "LOCAL")
+ .put("ai.endpoint", "http://localhost:8000";)
+ .put("ai.model_name", "chat")
+ .put("ai.embed.provider_type", "LOCAL")
+ .put("ai.embed.endpoint", "http://localhost:8001";)
+ .put("ai.embed.model_name", "embed")
+ .put("ai.embed.mm.provider_type", "LOCAL")
+ .put("ai.embed.mm.endpoint", "http://localhost:8002";)
+ .put("ai.embed.mm.model_name", "multimodal")
+ .build());
+ Auth auth = Env.getCurrentEnv().getAuth();
+ ResourceMgr resourceMgr = Env.getCurrentEnv().getResourceMgr();
+ ConnectContext previousContext = ConnectContext.get();
+ ConnectContext context = new ConnectContext();
+ context.setCurrentUserIdentity(user);
+ context.setRemoteIP("127.0.0.1");
+ context.getSessionVariable().defaultAIResource = resourceName;
+ context.getSessionVariable().setEnableSqlCache(true);
+ context.setThreadLocalInfo();
+ try {
+ auth.replayCreateUser(new PrivInfo(user, null, new byte[0], null,
PasswordOptions.UNSET_OPTION));
+ Assertions.assertTrue(auth.doesUserExist(user));
+ resourceMgr.replayCreateResource(resource);
+ BoundFunction function = aiFunctionForPrivilegeCheck(kind,
resourceName);
+ Assertions.assertFalse(function.isDeterministic());
+ assertResourceAccessDenied(function, resourceName);
+
+ auth.replayGrant(usagePrivilege(user, "another_ai_resource"));
+ assertResourceAccessDenied(function, resourceName);
+
+ auth.replayGrant(usagePrivilege(user, resourceName));
+ Assertions.assertTrue(Env.getCurrentEnv().getAccessManager()
+ .checkResourcePriv(context, resourceName,
PrivPredicate.USAGE));
+ for (Expression expression : ImmutableList.of(function,
+ new UnboundFunction(function.getName(),
function.getArguments()))) {
+ StatementContext statementContext = new
StatementContext(context, new OriginStatement("SELECT 1", 0));
+ context.setStatementContext(statementContext);
+ SqlCacheContext sqlCacheContext =
statementContext.getSqlCacheContext().orElseThrow();
+ function.checkLegalityBeforeTypeCoercion();
+ function.checkLegalityAfterRewrite();
+ Assertions.assertTrue(sqlCacheContext.supportSqlCache());
+ LogicalEmptyRelation plan = new LogicalEmptyRelation(new
RelationId(0), ImmutableList.of());
+ CascadesContext cascadesContext =
CascadesContext.initContext(statementContext,
+ plan, PhysicalProperties.ANY);
+ Expression analyzed = ExpressionAnalyzer.analyzeFunction(plan,
cascadesContext, expression);
+ Assertions.assertTrue(analyzed.containsNondeterministic());
+ Assertions.assertTrue(statementContext.hasNondeterministic());
+ Assertions.assertFalse(sqlCacheContext.supportSqlCache());
+ }
+
+ auth.replayRevoke(usagePrivilege(user, resourceName));
+ Assertions.assertFalse(Env.getCurrentEnv().getAccessManager()
+ .checkResourcePriv(context, resourceName,
PrivPredicate.USAGE));
+ assertResourceAccessDenied(function, resourceName);
+
+ auth.replayGrant(usagePrivilege(user, "*"));
+ function.checkLegalityBeforeTypeCoercion();
+ function.checkLegalityAfterRewrite();
+ auth.replayRevoke(usagePrivilege(user, "*"));
+ assertResourceAccessDenied(function, resourceName);
+ } finally {
+ auth.replayDropUser(user);
+ resourceMgr.replayDropResource(new
DropResourceOperationLog(resourceName));
+ if (previousContext == null) {
+ ConnectContext.remove();
+ } else {
+ previousContext.setThreadLocalInfo();
+ }
+ }
+ }
+
+ private void assertResourceAccessDenied(BoundFunction function, String
resourceName) {
+ AnalysisException before =
Assertions.assertThrows(AnalysisException.class,
+ function::checkLegalityBeforeTypeCoercion);
+ AnalysisException after =
Assertions.assertThrows(AnalysisException.class,
+ function::checkLegalityAfterRewrite);
+ for (AnalysisException exception : new AnalysisException[] {before,
after}) {
+ Assertions.assertTrue(exception.getMessage().contains("Access
denied"), exception.getMessage());
+
Assertions.assertTrue(exception.getMessage().contains("Usage_priv"),
exception.getMessage());
+
Assertions.assertTrue(exception.getMessage().contains(resourceName),
exception.getMessage());
+ }
+ }
+
+ private PrivInfo usagePrivilege(UserIdentity user, String resourceName)
throws Exception {
+ ResourcePattern pattern = new ResourcePattern(resourceName,
ResourceTypeEnum.GENERAL);
+ pattern.analyze();
+ return new PrivInfo(user, pattern,
PrivBitSet.of(Privilege.USAGE_PRIV), null, null);
+ }
+
+ private BoundFunction aiFunctionForPrivilegeCheck(String kind, String
resourceName) {
+ StringLiteral resource = new StringLiteral(resourceName);
+ StringLiteral input = new StringLiteral("input");
+ StringLiteral task = new StringLiteral("summarize");
+ JsonLiteral multimodal = new JsonLiteral("{\"text\":\"input\"}");
+ switch (kind) {
+ case "generate":
+ return new AIGenerate(resource, input);
+ case "aggregate":
+ return new AIAgg(resource, input, task);
+ case "embed":
+ return new Embed(resource, input);
+ case "multimodal":
+ return new Embed(resource, multimodal);
+ case "default_generate":
+ return new AIGenerate(input);
+ case "default_aggregate":
+ return new AIAgg(input, task);
+ case "default_embed":
+ return new Embed(input);
+ case "default_multimodal":
+ return new Embed(multimodal);
+ case "null_input":
+ return new AIGenerate(resource, NullLiteral.INSTANCE);
+ default:
+ throw new IllegalArgumentException(kind);
}
}
@@ -624,35 +770,34 @@ public class AIResourceTest {
}
@Test
- public void testEmptyEffortIsNotForwarded() throws DdlException {
+ public void testEmptyEffortIsRejected() throws DdlException {
Map<String, String> properties = new HashMap<>(aiProperties);
properties.put(AIProperties.EFFORT, "");
AIResource aiResource = new AIResource("empty-effort-resource");
- aiResource.setProperties(ImmutableMap.copyOf(properties));
-
- Assertions.assertFalse(aiResource.toThrift().isSetEffort());
+ Assertions.assertThrows(DdlException.class, () ->
aiResource.setProperties(ImmutableMap.copyOf(properties)));
}
@Test
- public void testClearEffortOnModifyAndPersistence() throws Exception {
+ public void testRejectEmptyEffortOnModifyAndPersistence() throws Exception
{
Map<String, String> properties = new HashMap<>(aiProperties);
properties.put(AIProperties.EFFORT, "high");
- AIResource aiResource = new AIResource("clear-effort-resource");
+ AIResource aiResource = new AIResource("non-empty-effort-resource");
aiResource.setProperties(ImmutableMap.copyOf(properties));
- aiResource.modifyProperties(ImmutableMap.of(AIProperties.EFFORT, ""));
+ Assertions.assertThrows(DdlException.class,
+ () ->
aiResource.modifyProperties(ImmutableMap.of(AIProperties.EFFORT, "")));
- Assertions.assertNull(aiResource.getProperty(AIProperties.EFFORT));
- Assertions.assertFalse(aiResource.toThrift().isSetEffort());
+ Assertions.assertEquals("high",
aiResource.getProperty(AIProperties.EFFORT));
+ Assertions.assertEquals("high", aiResource.toThrift().getEffort());
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
aiResource.write(new DataOutputStream(bytes));
AIResource restoredResource = (AIResource) Resource.read(
new DataInputStream(new
ByteArrayInputStream(bytes.toByteArray())));
-
Assertions.assertNull(restoredResource.getProperty(AIProperties.EFFORT));
- Assertions.assertFalse(restoredResource.toThrift().isSetEffort());
+ Assertions.assertEquals("high",
restoredResource.getProperty(AIProperties.EFFORT));
+ Assertions.assertEquals("high",
restoredResource.toThrift().getEffort());
}
@Test
diff --git a/regression-test/data/ai_p0/test_create_ai_resource.out
b/regression-test/data/ai_p0/test_create_ai_resource.out
new file mode 100644
index 00000000000..fb0d7675a2b
--- /dev/null
+++ b/regression-test/data/ai_p0/test_create_ai_resource.out
@@ -0,0 +1,37 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !normalized_create --
+ai.anthropic_version 2023-06-01
+ai.api_key ******
+ai.dimensions 768
+ai.effort high
+ai.enable_structured_output true
+ai.endpoint https://api.deepseek.com/chat/completions
+ai.max_retries 3
+ai.max_token 1024
+ai.model_name deepseek-chat
+ai.provider_type DEEPSEEK
+ai.retry_delay_second 1
+ai.temperature 0.7
+ai.validity_check false
+id <generated>
+type ai
+version <generated>
+
+-- !normalized_alter --
+ai.anthropic_version 2023-06-01
+ai.api_key ******
+ai.dimensions 1024
+ai.effort high
+ai.enable_structured_output false
+ai.endpoint https://api.deepseek.com/chat/completions
+ai.max_retries 4
+ai.max_token 2048
+ai.model_name updated-model
+ai.provider_type DEEPSEEK
+ai.retry_delay_second 2
+ai.temperature 0.5
+ai.validity_check false
+id <generated>
+type ai
+version <generated>
+
diff --git a/regression-test/suites/ai_p0/test_ai_resource_privilege.groovy
b/regression-test/suites/ai_p0/test_ai_resource_privilege.groovy
new file mode 100644
index 00000000000..707b30218ce
--- /dev/null
+++ b/regression-test/suites/ai_p0/test_ai_resource_privilege.groovy
@@ -0,0 +1,114 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_ai_resource_privilege", "p0,auth") {
+ String resourceName = "test_ai_resource_privilege"
+ String userName = "test_ai_resource_privilege_user"
+ String password = "AiResource_12345"
+ def adminConnection = context.getConnection()
+ def queries = [
+ "SELECT AI_GENERATE('${resourceName}', 'input')",
+ "SELECT AI_AGG('${resourceName}', 'input', 'summarize')",
+ "SELECT EMBED('${resourceName}', 'input')",
+ "SELECT EMBED('${resourceName}', CAST('{\"text\":\"input\"}' AS
JSON))",
+ "SELECT AI_GENERATE('input')",
+ "SELECT AI_AGG('input', 'summarize')",
+ "SELECT EMBED('input')",
+ "SELECT EMBED(CAST('{\"text\":\"input\"}' AS JSON))",
+ "SELECT AI_GENERATE('${resourceName}', NULL)",
+ "SELECT AI_GENERATE(CONCAT('${resourceName}', ''), 'input')",
+ "SELECT AI_AGG(CONCAT('${resourceName}', ''), 'input', 'summarize')",
+ "SELECT EMBED(CONCAT('${resourceName}', ''), 'input')"
+ ]
+ sql "DROP USER IF EXISTS '${userName}'"
+ sql "DROP RESOURCE IF EXISTS '${resourceName}'"
+ try {
+ sql """CREATE RESOURCE '${resourceName}' PROPERTIES (
+ 'type' = 'ai',
+ 'ai.provider_type' = 'local',
+ 'ai.endpoint' = 'http://localhost:8000',
+ 'ai.model_name' = 'chat',
+ 'ai.embed.provider_type' = 'local',
+ 'ai.embed.endpoint' = 'http://localhost:8001',
+ 'ai.embed.model_name' = 'embed',
+ 'ai.embed.mm.provider_type' = 'local',
+ 'ai.embed.mm.endpoint' = 'http://localhost:8002',
+ 'ai.embed.mm.model_name' = 'multimodal'
+ )"""
+ sql "CREATE USER '${userName}' IDENTIFIED BY '${password}'"
+ sql "GRANT SELECT_PRIV ON ${context.dbName}.* TO '${userName}'"
+ if (isCloudMode()) {
+ def clusters = sql "SHOW CLUSTERS"
+ assertTrue(!clusters.isEmpty())
+ sql "GRANT USAGE_PRIV ON CLUSTER `${clusters[0][0]}` TO
'${userName}'"
+ }
+ connect(userName, password, adminConnection.getMetaData().getURL()) {
+ sql "SET default_ai_resource = '${resourceName}'"
+ def assertDenied = {
+ queries.each { query ->
+ test {
+ sql "EXPLAIN ${query}"
+ exception "privilege(s) on resource ${resourceName}"
+ }
+ }
+ }
+ assertDenied()
+ sql_impl(adminConnection, "GRANT USAGE_PRIV ON RESOURCE
'${resourceName}' TO '${userName}'")
+ queries.each { query -> sql "EXPLAIN ${query}" }
+ sql_impl(adminConnection, "REVOKE USAGE_PRIV ON RESOURCE
'${resourceName}' FROM '${userName}'")
+ assertDenied()
+
+ sql "SET enable_sql_cache = true"
+ sql "SELECT 20261104"
+ explain {
+ sql "physical plan SELECT 20261104"
+ contains "PhysicalSqlCache"
+ }
+ def nullQueries = [
+ "SELECT AI_GENERATE('${resourceName}', NULL)",
+ "SELECT AI_GENERATE(NULL)",
+ "SELECT AI_GENERATE(CONCAT('${resourceName}', ''), NULL)",
+ "SELECT IF(TRUE, 'ok', AI_GENERATE('${resourceName}', NULL))",
+ "SELECT EMBED('${resourceName}', CAST(NULL AS STRING))",
+ "SELECT EMBED(CAST(NULL AS STRING))",
+ "SELECT EMBED('${resourceName}', CAST(NULL AS JSON))",
+ "SELECT EMBED(CAST(NULL AS JSON))"
+ ]
+ nullQueries.each { query ->
+ sql_impl(adminConnection, "GRANT USAGE_PRIV ON RESOURCE
'${resourceName}' TO '${userName}'")
+ sql query
+ sql query
+ explain {
+ sql "physical plan ${query}"
+ notContains "PhysicalSqlCache"
+ }
+ sql_impl(adminConnection, "REVOKE USAGE_PRIV ON RESOURCE
'${resourceName}' FROM '${userName}'")
+ test {
+ sql query
+ exception "privilege(s) on resource ${resourceName}"
+ }
+ test {
+ sql "EXPLAIN ${query}"
+ exception "privilege(s) on resource ${resourceName}"
+ }
+ }
+ }
+ } finally {
+ sql "DROP USER IF EXISTS '${userName}'"
+ sql "DROP RESOURCE IF EXISTS '${resourceName}'"
+ }
+}
diff --git a/regression-test/suites/ai_p0/test_create_ai_resource.groovy
b/regression-test/suites/ai_p0/test_create_ai_resource.groovy
index d9e9522ab6b..277265774dd 100644
--- a/regression-test/suites/ai_p0/test_create_ai_resource.groovy
+++ b/regression-test/suites/ai_p0/test_create_ai_resource.groovy
@@ -76,21 +76,112 @@ suite("test_create_ai_resource") {
exception "Missing [ai.api_key] in properties for provider: DEEPSEEK"
}
+ for (String key : ['ai.api_key', 'ai.max_retries', 'ai.anthropic_version',
'ai.effort']) {
+ for (String value : ['', ' ']) {
+ test {
+ sql """CREATE RESOURCE "${resourceName}" PROPERTIES(
+ 'type' = 'ai',
+ 'ai.provider_type' = 'local',
+ 'ai.endpoint' =
'http://localhost:8000/v1/chat/completions',
+ 'ai.model_name' = 'chat-model',
+ '${key}' = '${value}'
+ );"""
+ exception "[${key}] must not be empty"
+ }
+ }
+ }
+
+ def invalidNumericProperties = [
+ ['ai.temperature', 'NaN'],
+ ['ai.temperature', 'Infinity'],
+ ['ai.temperature', '-Infinity'],
+ ['ai.temperature', '-0.1'],
+ ['ai.temperature', '1.1'],
+ ['ai.max_token', '0'],
+ ['ai.max_token', '-2'],
+ ['ai.max_token', '9223372036854775808'],
+ ['ai.max_retries', '-1'],
+ ['ai.max_retries', '2147483648'],
+ ['ai.retry_delay_second', '-1'],
+ ['ai.retry_delay_second', '2147483648'],
+ ['ai.dimensions', '0'],
+ ['ai.dimensions', '-2'],
+ ['ai.dimensions', '2147483648']
+ ]
+ for (def entry : invalidNumericProperties) {
+ test {
+ sql """CREATE RESOURCE "${resourceName}" PROPERTIES(
+ 'type' = 'ai',
+ 'ai.provider_type' = 'local',
+ 'ai.endpoint' = 'http://localhost:8000/v1/chat/completions',
+ 'ai.model_name' = 'chat-model',
+ '${entry[0]}' = '${entry[1]}'
+ );"""
+ exception "Failed to set property '${entry[0]}'"
+ }
+ }
+
sql """CREATE RESOURCE IF NOT EXISTS "${resourceName}"
PROPERTIES(
'type' = 'ai',
- 'ai.provider_type' = 'deepseek',
- 'ai.endpoint' = 'https://api.deepseek.com/chat/completions',
- 'ai.model_name' = 'deepseek-chat',
- 'ai.api_key' = 'sk-xxx',
- 'ai.temperature' = '0.7',
- 'ai.max_token' = '1024',
- 'ai.max_retries' = '3',
- 'ai.retry_delay_second' = '1',
+ 'ai.provider_type' = ' deepseek ',
+ 'ai.endpoint' = ' https://api.deepseek.com/chat/completions ',
+ 'ai.model_name' = ' deepseek-chat ',
+ 'ai.api_key' = ' sk-xxx ',
+ 'ai.temperature' = ' 0.7 ',
+ 'ai.max_token' = ' 1024 ',
+ 'ai.max_retries' = ' 3 ',
+ 'ai.retry_delay_second' = ' 1 ',
+ 'ai.dimensions' = ' 768 ',
+ 'ai.anthropic_version' = ' 2023-06-01 ',
+ 'ai.effort' = ' high ',
+ 'ai.enable_structured_output' = ' true ',
'ai.validity_check' = 'false'
);"""
def res = sql """SHOW RESOURCES WHERE NAME = '${resourceName}'"""
assertTrue(res.size() > 0)
+ def checkStoredProperties = { tag ->
+ quickRunTest(tag, "SHOW RESOURCES WHERE NAME = '${resourceName}'",
true) { row ->
+ [row[2], row[2] in ['id', 'version'] ? '<generated>' : row[3]]
+ }
+ }
+ checkStoredProperties("normalized_create")
+
+ for (String key : ['ai.model_name', 'ai.api_key', 'ai.max_retries',
'ai.anthropic_version', 'ai.effort']) {
+ for (String value : ['', ' ']) {
+ test {
+ sql """ALTER RESOURCE "${resourceName}" PROPERTIES('${key}' =
'${value}');"""
+ exception "[${key}] must not be empty"
+ }
+ }
+ }
+
+ for (def entry : invalidNumericProperties) {
+ test {
+ sql """ALTER RESOURCE "${resourceName}" PROPERTIES('${entry[0]}' =
'${entry[1]}');"""
+ exception "Failed to set property '${entry[0]}'"
+ }
+ }
+
+ sql """ALTER RESOURCE "${resourceName}" PROPERTIES(
+ 'ai.temperature' = '1', 'ai.max_token' = '9223372036854775807',
+ 'ai.max_retries' = '2147483647', 'ai.retry_delay_second' =
'2147483647',
+ 'ai.dimensions' = '2147483647'
+ );"""
+ sql """ALTER RESOURCE "${resourceName}" PROPERTIES(
+ 'ai.temperature' = '0', 'ai.max_token' = '1',
+ 'ai.max_retries' = '0', 'ai.retry_delay_second' = '0', 'ai.dimensions'
= '1'
+ );"""
+ sql """ALTER RESOURCE "${resourceName}" PROPERTIES(
+ 'ai.temperature' = '-1', 'ai.max_token' = '-1', 'ai.dimensions' = '-1'
+ );"""
+ sql """ALTER RESOURCE "${resourceName}" PROPERTIES(
+ 'ai.temperature' = ' 0.5 ', 'ai.max_token' = ' 2048 ',
+ 'ai.max_retries' = ' 4 ', 'ai.retry_delay_second' = ' 2 ',
+ 'ai.dimensions' = ' 1024 ', 'ai.model_name' = ' updated-model ',
+ 'ai.enable_structured_output' = ' false '
+ );"""
+ checkStoredProperties("normalized_alter")
try_sql("""DROP RESOURCE '${resourceName}'""")
-}
\ No newline at end of file
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
