jerryshao commented on code in PR #13571: URL: https://github.com/apache/gravitino/pull/13571#discussion_r4228837721
########## core/src/main/java/org/apache/gravitino/semantic/OssieDocumentConverter.java: ########## @@ -0,0 +1,993 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.semantic; + +import static org.apache.gravitino.semantic.CustomExtension.GRAVITINO_PROPERTIES_VENDOR; +import static org.apache.gravitino.semantic.OssieVersion.DEFAULT_VERSION; +import static org.apache.gravitino.semantic.SemanticModel.PROPERTY_OSSIE_VERSION; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.StreamReadFeature; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; +import java.util.function.IntFunction; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.exceptions.IllegalSemanticModelException; + +/** Converts between standalone Apache Ossie documents and Gravitino Semantic Models. */ +public final class OssieDocumentConverter { + + private static final Set<String> ROOT_PROPERTIES = + Set.of( + "version", + "name", + "description", + "ai_context", + "datasets", + "relationships", + "metrics", + "custom_extensions"); + private static final Set<String> DATASET_PROPERTIES = + Set.of( + "name", + "source", + "primary_key", + "unique_keys", + "description", + "ai_context", + "fields", + "custom_extensions"); + private static final Set<String> FIELD_PROPERTIES = + Set.of( + "name", + "expression", + "dimension", + "label", + "description", + "datatype", + "ai_context", + "custom_extensions"); + private static final Set<String> RELATIONSHIP_PROPERTIES = + Set.of("name", "from", "to", "from_columns", "to_columns", "ai_context", "custom_extensions"); + private static final Set<String> METRIC_PROPERTIES = + Set.of("name", "expression", "description", "datatype", "ai_context", "custom_extensions"); + private static final Set<String> EXPRESSION_PROPERTIES = Set.of("dialects"); + private static final Set<String> DIALECT_EXPRESSION_PROPERTIES = Set.of("dialect", "expression"); + private static final Set<String> DIMENSION_PROPERTIES = Set.of("is_time"); + private static final Set<String> CUSTOM_EXTENSION_PROPERTIES = Set.of("vendor_name", "data"); + + private static final Set<String> AI_CONTEXT_PROPERTIES = + Set.of("instructions", "synonyms", "examples"); + private static final Map<DataType, String> DATA_TYPE_NAMES = + Map.of( + DataType.STRING, "String", + DataType.INTEGER, "Integer", + DataType.DECIMAL, "Decimal", + DataType.FLOAT, "Float", + DataType.BOOLEAN, "Boolean", + DataType.DATE, "Date", + DataType.TIME, "Time", + DataType.DATE_TIME, "DateTime", + DataType.DATE_TIME_TZ, "DateTimeTz", + DataType.OPAQUE, "Opaque"); + + private static final ObjectMapper JSON_MAPPER = createJsonMapper(); + private static final ObjectMapper YAML_MAPPER = createYamlMapper(); + + private OssieDocumentConverter() {} + + /** + * Converts one standalone Apache Ossie YAML or JSON document into native model values. + * + * @param document The standalone Ossie document. + * @return The model name, comment, native definition, and properties. + * @throws IllegalSemanticModelException If the document cannot be parsed or represented by + * Gravitino. + */ + public static ImportedSemanticModel importDocument(OssieDocument document) { + Objects.requireNonNull(document, "document must not be null"); + ObjectNode root = parseDocument(document); + return toImportedModel(root); + } + + /** + * Exports one Gravitino Semantic Model as a standalone Apache Ossie document. + * + * @param semanticModel The Semantic Model to export. + * @param format The requested serialization format. + * @return The serialized Ossie document. + * @throws IllegalSemanticModelException If the model cannot be represented as a valid Ossie + * document. + */ + public static OssieDocument exportDocument(SemanticModel semanticModel, OssieFormat format) { + Objects.requireNonNull(semanticModel, "semanticModel must not be null"); + Objects.requireNonNull(format, "format must not be null"); + + ObjectNode definition = writeDefinition(semanticModel.definition()); + + ObjectNode root = JSON_MAPPER.createObjectNode(); + root.put("version", ossieVersion(semanticModel.properties())); + root.put("name", semanticModel.name()); + if (semanticModel.comment() != null) { + root.put("description", semanticModel.comment()); + } + definition.fields().forEachRemaining(entry -> root.set(entry.getKey(), entry.getValue())); + stashProperties(root, semanticModel.properties()); + + // Validate the generated document through the same conversion path used for imports. + toImportedModel(root.deepCopy()); + + try { + if (format == OssieFormat.JSON) { + return OssieDocument.json( + JSON_MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(root) + "\n"); + } + return OssieDocument.yaml(YAML_MAPPER.writeValueAsString(root)); + } catch (JsonProcessingException e) { + throw new IllegalSemanticModelException( + e, "Cannot serialize Apache Ossie document: %s", e.getOriginalMessage()); + } + } + + /** Native model values parsed from an Ossie document, before the model is created. */ + public static final class ImportedSemanticModel { + + private final String name; + @Nullable private final String comment; + private final SemanticModelDefinition definition; + private final Map<String, String> properties; + + private ImportedSemanticModel( + String name, + @Nullable String comment, + SemanticModelDefinition definition, + Map<String, String> properties) { + this.name = name; + this.comment = comment; + this.definition = definition; + this.properties = Map.copyOf(properties); + } + + /** + * Returns the model name supplied by the document. + * + * @return The model name. + */ + public String name() { + return name; + } + + /** + * Returns the document description. + * + * @return The model comment, or {@code null} when absent. + */ + @Nullable + public String comment() { + return comment; + } + + /** + * Returns the parsed native definition. + * + * @return The immutable Semantic Model definition. + */ + public SemanticModelDefinition definition() { + return definition; + } + + /** + * Returns the Gravitino properties, including the Ossie version. + * + * @return The immutable properties. + */ + public Map<String, String> properties() { + return properties; + } + } + + private static ObjectMapper createJsonMapper() { + JsonFactory factory = + JsonFactory.builder().enable(StreamReadFeature.STRICT_DUPLICATE_DETECTION).build(); + return JsonMapper.builder(factory) + .enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS) + .enable(DeserializationFeature.USE_BIG_INTEGER_FOR_INTS) + .build(); + } + + private static ObjectMapper createYamlMapper() { + YAMLFactory factory = + YAMLFactory.builder() + .enable(StreamReadFeature.STRICT_DUPLICATE_DETECTION) + .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER) + .build(); + return new ObjectMapper(factory) + .enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS) + .enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS) + .enable(DeserializationFeature.USE_BIG_INTEGER_FOR_INTS); + } + + private static ObjectNode parseDocument(OssieDocument document) { + String content = document.content(); + try { + ObjectMapper mapper = document.format() == OssieFormat.JSON ? JSON_MAPPER : YAML_MAPPER; + // Reject trailing content after the standalone document. + JsonNode parsed = + mapper.reader().with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS).readTree(content); + if (!(parsed instanceof ObjectNode)) { + throw invalid("$", "document root must be an object"); + } + return (ObjectNode) parsed; + } catch (IOException e) { + throw new IllegalSemanticModelException( + e, "Cannot parse Apache Ossie %s: %s", document.format(), originalMessage(e)); + } + } + + private static ImportedSemanticModel toImportedModel(ObjectNode root) { + validateObject(root, "$", ROOT_PROPERTIES); + String name = readText(root, "name", "$"); + String comment = readText(root, "description", "$"); + String ossieVersion = validateVersion(root.get("version")); + + Map<String, String> properties = new LinkedHashMap<>(extractProperties(root)); + String extensionVersion = properties.get(PROPERTY_OSSIE_VERSION); + if (extensionVersion != null && !extensionVersion.equals(ossieVersion)) { + throw invalid( + "$.custom_extensions", + "Gravitino property '" + PROPERTY_OSSIE_VERSION + "' conflicts with $.version"); + } + properties.put(PROPERTY_OSSIE_VERSION, ossieVersion); + + try { + if (StringUtils.isBlank(name)) { + throw new IllegalArgumentException("\"name\" field is required and cannot be empty"); + } + return new ImportedSemanticModel(name, comment, readDefinition(root), properties); + } catch (IllegalArgumentException e) { Review Comment: [Nit] This `catch` also swallows `IllegalSemanticModelException`, so document-structure errors now carry an extra generic prefix -- but only the ones raised below this line. `IllegalSemanticModelException extends IllegalArgumentException` (`api/src/main/java/org/apache/gravitino/exceptions/IllegalSemanticModelException.java:25`), so every `invalid(path, detail)` thrown inside `readDefinition` is caught here and re-wrapped, giving `Cannot convert Apache Ossie document to a Gravitino Semantic Model: $.datasets[0].fields[0].datatype: must be a string`. The same class of error raised above the `try` -- `validateObject(root, "$", ...)`, `validateVersion`, `extractProperties` -- keeps the bare `$.description: must not be null` form. Before this commit the definition-level checks ran in `transformOssieDefinition`, outside the `try` in `toCreateRequest`, so they were never wrapped. Both shapes are still 400 and the `$.path` prefix survives, so nothing is broken; it is just two renderings of one failure class, and `assertInvalid` only does a `contains`, so no test notices either way. A `catch (IllegalSemanticModelException e) { throw e; }` ahead of this block, or narrowing the `try` to the native-builder calls, would keep one shape. Verified by: read the exception hierarchy; diffed this method against `git show 088ceac:core/src/main/java/org/apache/gravitino/semantic/OssieDocumentConverter.java` (`toCreateRequest`, :197-240) to confirm the old ordering; read `assertInvalid` (`TestOssieDocumentConverter.java:826-834`). ########## core/src/main/java/org/apache/gravitino/semantic/OssieDocumentConverter.java: ########## @@ -0,0 +1,993 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.semantic; + +import static org.apache.gravitino.semantic.CustomExtension.GRAVITINO_PROPERTIES_VENDOR; +import static org.apache.gravitino.semantic.OssieVersion.DEFAULT_VERSION; +import static org.apache.gravitino.semantic.SemanticModel.PROPERTY_OSSIE_VERSION; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.StreamReadFeature; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; +import java.util.function.IntFunction; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.exceptions.IllegalSemanticModelException; + +/** Converts between standalone Apache Ossie documents and Gravitino Semantic Models. */ +public final class OssieDocumentConverter { + + private static final Set<String> ROOT_PROPERTIES = + Set.of( + "version", + "name", + "description", + "ai_context", + "datasets", + "relationships", + "metrics", + "custom_extensions"); + private static final Set<String> DATASET_PROPERTIES = + Set.of( + "name", + "source", + "primary_key", + "unique_keys", + "description", + "ai_context", + "fields", + "custom_extensions"); + private static final Set<String> FIELD_PROPERTIES = + Set.of( + "name", + "expression", + "dimension", + "label", + "description", + "datatype", + "ai_context", + "custom_extensions"); + private static final Set<String> RELATIONSHIP_PROPERTIES = + Set.of("name", "from", "to", "from_columns", "to_columns", "ai_context", "custom_extensions"); + private static final Set<String> METRIC_PROPERTIES = + Set.of("name", "expression", "description", "datatype", "ai_context", "custom_extensions"); + private static final Set<String> EXPRESSION_PROPERTIES = Set.of("dialects"); + private static final Set<String> DIALECT_EXPRESSION_PROPERTIES = Set.of("dialect", "expression"); + private static final Set<String> DIMENSION_PROPERTIES = Set.of("is_time"); + private static final Set<String> CUSTOM_EXTENSION_PROPERTIES = Set.of("vendor_name", "data"); + + private static final Set<String> AI_CONTEXT_PROPERTIES = + Set.of("instructions", "synonyms", "examples"); + private static final Map<DataType, String> DATA_TYPE_NAMES = + Map.of( + DataType.STRING, "String", + DataType.INTEGER, "Integer", + DataType.DECIMAL, "Decimal", + DataType.FLOAT, "Float", + DataType.BOOLEAN, "Boolean", + DataType.DATE, "Date", + DataType.TIME, "Time", + DataType.DATE_TIME, "DateTime", + DataType.DATE_TIME_TZ, "DateTimeTz", + DataType.OPAQUE, "Opaque"); + + private static final ObjectMapper JSON_MAPPER = createJsonMapper(); + private static final ObjectMapper YAML_MAPPER = createYamlMapper(); + + private OssieDocumentConverter() {} + + /** + * Converts one standalone Apache Ossie YAML or JSON document into native model values. + * + * @param document The standalone Ossie document. + * @return The model name, comment, native definition, and properties. + * @throws IllegalSemanticModelException If the document cannot be parsed or represented by + * Gravitino. + */ + public static ImportedSemanticModel importDocument(OssieDocument document) { + Objects.requireNonNull(document, "document must not be null"); + ObjectNode root = parseDocument(document); + return toImportedModel(root); + } + + /** + * Exports one Gravitino Semantic Model as a standalone Apache Ossie document. + * + * @param semanticModel The Semantic Model to export. + * @param format The requested serialization format. + * @return The serialized Ossie document. + * @throws IllegalSemanticModelException If the model cannot be represented as a valid Ossie + * document. + */ + public static OssieDocument exportDocument(SemanticModel semanticModel, OssieFormat format) { + Objects.requireNonNull(semanticModel, "semanticModel must not be null"); + Objects.requireNonNull(format, "format must not be null"); + + ObjectNode definition = writeDefinition(semanticModel.definition()); + + ObjectNode root = JSON_MAPPER.createObjectNode(); + root.put("version", ossieVersion(semanticModel.properties())); + root.put("name", semanticModel.name()); + if (semanticModel.comment() != null) { + root.put("description", semanticModel.comment()); + } + definition.fields().forEachRemaining(entry -> root.set(entry.getKey(), entry.getValue())); + stashProperties(root, semanticModel.properties()); + + // Validate the generated document through the same conversion path used for imports. + toImportedModel(root.deepCopy()); + + try { + if (format == OssieFormat.JSON) { + return OssieDocument.json( + JSON_MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(root) + "\n"); + } + return OssieDocument.yaml(YAML_MAPPER.writeValueAsString(root)); + } catch (JsonProcessingException e) { + throw new IllegalSemanticModelException( + e, "Cannot serialize Apache Ossie document: %s", e.getOriginalMessage()); + } + } + + /** Native model values parsed from an Ossie document, before the model is created. */ + public static final class ImportedSemanticModel { + + private final String name; + @Nullable private final String comment; + private final SemanticModelDefinition definition; + private final Map<String, String> properties; + + private ImportedSemanticModel( + String name, + @Nullable String comment, + SemanticModelDefinition definition, + Map<String, String> properties) { + this.name = name; + this.comment = comment; + this.definition = definition; + this.properties = Map.copyOf(properties); + } + + /** + * Returns the model name supplied by the document. + * + * @return The model name. + */ + public String name() { + return name; + } + + /** + * Returns the document description. + * + * @return The model comment, or {@code null} when absent. + */ + @Nullable + public String comment() { + return comment; + } + + /** + * Returns the parsed native definition. + * + * @return The immutable Semantic Model definition. + */ + public SemanticModelDefinition definition() { + return definition; + } + + /** + * Returns the Gravitino properties, including the Ossie version. + * + * @return The immutable properties. + */ + public Map<String, String> properties() { + return properties; + } + } + + private static ObjectMapper createJsonMapper() { + JsonFactory factory = + JsonFactory.builder().enable(StreamReadFeature.STRICT_DUPLICATE_DETECTION).build(); + return JsonMapper.builder(factory) + .enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS) + .enable(DeserializationFeature.USE_BIG_INTEGER_FOR_INTS) + .build(); + } + + private static ObjectMapper createYamlMapper() { + YAMLFactory factory = + YAMLFactory.builder() + .enable(StreamReadFeature.STRICT_DUPLICATE_DETECTION) + .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER) + .build(); + return new ObjectMapper(factory) + .enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS) + .enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS) + .enable(DeserializationFeature.USE_BIG_INTEGER_FOR_INTS); + } + + private static ObjectNode parseDocument(OssieDocument document) { + String content = document.content(); + try { + ObjectMapper mapper = document.format() == OssieFormat.JSON ? JSON_MAPPER : YAML_MAPPER; + // Reject trailing content after the standalone document. + JsonNode parsed = + mapper.reader().with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS).readTree(content); + if (!(parsed instanceof ObjectNode)) { + throw invalid("$", "document root must be an object"); + } + return (ObjectNode) parsed; + } catch (IOException e) { + throw new IllegalSemanticModelException( + e, "Cannot parse Apache Ossie %s: %s", document.format(), originalMessage(e)); + } + } + + private static ImportedSemanticModel toImportedModel(ObjectNode root) { + validateObject(root, "$", ROOT_PROPERTIES); + String name = readText(root, "name", "$"); + String comment = readText(root, "description", "$"); + String ossieVersion = validateVersion(root.get("version")); + + Map<String, String> properties = new LinkedHashMap<>(extractProperties(root)); + String extensionVersion = properties.get(PROPERTY_OSSIE_VERSION); + if (extensionVersion != null && !extensionVersion.equals(ossieVersion)) { + throw invalid( + "$.custom_extensions", + "Gravitino property '" + PROPERTY_OSSIE_VERSION + "' conflicts with $.version"); + } + properties.put(PROPERTY_OSSIE_VERSION, ossieVersion); + + try { + if (StringUtils.isBlank(name)) { + throw new IllegalArgumentException("\"name\" field is required and cannot be empty"); + } + return new ImportedSemanticModel(name, comment, readDefinition(root), properties); + } catch (IllegalArgumentException e) { + throw new IllegalSemanticModelException( + e, + "Cannot convert Apache Ossie document to a Gravitino Semantic Model: %s", + originalMessage(e)); + } + } + + private static String validateVersion(@Nullable JsonNode version) { + if (version == null || !version.isTextual() || StringUtils.isBlank(version.textValue())) { + throw invalid("$.version", "must be a non-empty string"); + } + return version.textValue(); + } + + private static String ossieVersion(Map<String, String> properties) { + String version = properties.getOrDefault(PROPERTY_OSSIE_VERSION, DEFAULT_VERSION); + if (StringUtils.isBlank(version)) { + throw invalid( + "$.version", + "Semantic Model property '" + PROPERTY_OSSIE_VERSION + "' must not be blank"); + } + return version; + } + + private static SemanticModelDefinition readDefinition(ObjectNode root) { + return SemanticModelDefinition.builder() + .withAIContext(readAIContext(root.get("ai_context"), "$.ai_context")) + .withDatasets( + readObjectArray( + root.get("datasets"), + "$.datasets", + OssieDocumentConverter::readDataset, + Dataset[]::new)) + .withRelationships( + readObjectArray( + root.get("relationships"), + "$.relationships", + OssieDocumentConverter::readRelationship, + Relationship[]::new)) + .withMetrics( + readObjectArray( + root.get("metrics"), + "$.metrics", + OssieDocumentConverter::readMetric, + Metric[]::new)) + .withCustomExtensions( + readCustomExtensions(root.get("custom_extensions"), "$.custom_extensions")) + .build(); + } + + private static Dataset readDataset(ObjectNode dataset, String path) { + validateObject(dataset, path, DATASET_PROPERTIES); + String source = readText(dataset, "source", path); + return Dataset.builder() + .withName(readText(dataset, "name", path)) + .withSource( + source == null ? null : NameIdentifier.of(parseOssieSource(source, path + ".source"))) + .withPrimaryKey(readStringArray(dataset, "primary_key", path)) + .withUniqueKeys(readStringArrays(dataset, "unique_keys", path)) + .withDescription(readText(dataset, "description", path)) + .withAIContext(readAIContext(dataset.get("ai_context"), path + ".ai_context")) + .withFields( + readObjectArray( + dataset.get("fields"), + path + ".fields", + OssieDocumentConverter::readField, + Field[]::new)) + .withCustomExtensions( + readCustomExtensions(dataset.get("custom_extensions"), path + ".custom_extensions")) + .build(); + } + + private static String[] parseOssieSource(String source, String path) { + List<String> parts = new ArrayList<>(3); + int offset = 0; + while (offset < source.length()) { + StringBuilder part = new StringBuilder(); + if (source.charAt(offset) == '`') { + offset++; + boolean closed = false; + while (offset < source.length()) { + char current = source.charAt(offset); + if (current != '`') { + part.append(current); + offset++; + } else if (offset + 1 < source.length() && source.charAt(offset + 1) == '`') { + part.append('`'); + offset += 2; + } else { + closed = true; + offset++; + break; + } + } + if (!closed) { + throw invalid(path, "contains an unterminated quoted segment"); + } + if (offset < source.length() && source.charAt(offset) != '.') { + throw invalid(path, "quoted segments must be separated by dots"); + } + } else { + while (offset < source.length() && source.charAt(offset) != '.') { + char current = source.charAt(offset); + if (current == '`') { + throw invalid(path, "backticks must quote an entire segment"); + } + part.append(current); + offset++; + } + } + + if (StringUtils.isBlank(part)) { + throw invalid(path, "source segments must not be blank"); + } + parts.add(part.toString()); + if (offset < source.length()) { + offset++; + if (offset == source.length()) { + throw invalid(path, "source segments must not be blank"); + } + } + } + + if (parts.size() != 3) { + throw invalid( + path, + "must be a three-part catalog.schema.entity identifier; query sources are not supported"); + } + return parts.toArray(new String[0]); + } + + private static Field readField(ObjectNode field, String path) { + validateObject(field, path, FIELD_PROPERTIES); + return Field.builder() + .withName(readText(field, "name", path)) + .withExpression(readExpression(field.get("expression"), path + ".expression")) + .withDimension(readDimension(field.get("dimension"), path + ".dimension")) + .withLabel(readText(field, "label", path)) + .withDescription(readText(field, "description", path)) + .withDatatype(readDataType(field, path)) + .withAIContext(readAIContext(field.get("ai_context"), path + ".ai_context")) + .withCustomExtensions( + readCustomExtensions(field.get("custom_extensions"), path + ".custom_extensions")) + .build(); + } + + private static Relationship readRelationship(ObjectNode relationship, String path) { + validateObject(relationship, path, RELATIONSHIP_PROPERTIES); + return Relationship.builder() + .withName(readText(relationship, "name", path)) + .withFrom(readText(relationship, "from", path)) + .withTo(readText(relationship, "to", path)) + .withFromColumns(readStringArray(relationship, "from_columns", path)) + .withToColumns(readStringArray(relationship, "to_columns", path)) + .withAIContext(readAIContext(relationship.get("ai_context"), path + ".ai_context")) + .withCustomExtensions( + readCustomExtensions( + relationship.get("custom_extensions"), path + ".custom_extensions")) + .build(); + } + + private static Metric readMetric(ObjectNode metric, String path) { + validateObject(metric, path, METRIC_PROPERTIES); + return Metric.builder() + .withName(readText(metric, "name", path)) + .withExpression(readExpression(metric.get("expression"), path + ".expression")) + .withDescription(readText(metric, "description", path)) + .withDatatype(readDataType(metric, path)) + .withAIContext(readAIContext(metric.get("ai_context"), path + ".ai_context")) + .withCustomExtensions( + readCustomExtensions(metric.get("custom_extensions"), path + ".custom_extensions")) + .build(); + } + + @Nullable + private static Expression readExpression(@Nullable JsonNode node, String path) { + if (node == null) { + return null; + } + ObjectNode expression = requireObject(node, path); + validateObject(expression, path, EXPRESSION_PROPERTIES); + return Expression.builder() + .withDialects( + readObjectArray( + expression.get("dialects"), + path + ".dialects", + OssieDocumentConverter::readDialectExpression, + DialectExpression[]::new)) + .build(); + } + + private static DialectExpression readDialectExpression(ObjectNode dialect, String path) { + validateObject(dialect, path, DIALECT_EXPRESSION_PROPERTIES); + return DialectExpression.builder() + .withDialect(readText(dialect, "dialect", path)) + .withExpression(readText(dialect, "expression", path)) + .build(); + } + + @Nullable + private static Dimension readDimension(@Nullable JsonNode node, String path) { + if (node == null) { + return null; + } + ObjectNode dimension = requireObject(node, path); + validateObject(dimension, path, DIMENSION_PROPERTIES); + validateOptionalBoolean(dimension, "is_time", path); + JsonNode isTime = dimension.get("is_time"); + return Dimension.builder().withIsTime(isTime == null ? null : isTime.booleanValue()).build(); + } + + @Nullable + private static DataType readDataType(ObjectNode object, String path) { + String value = readText(object, "datatype", path); + if (value == null) { + return null; + } + for (DataType type : DataType.values()) { + if (value.equals(DATA_TYPE_NAMES.get(type))) { + return type; + } + } + throw invalid( + path + ".datatype", + "Unknown Semantic Model data type: " + + value + + ". Supported values: " + + Arrays.stream(DataType.values()) + .map(DATA_TYPE_NAMES::get) + .collect(Collectors.joining(", "))); + } + + @Nullable + private static CustomExtension[] readCustomExtensions(@Nullable JsonNode node, String path) { + return readObjectArray( + node, + path, + (extension, extensionPath) -> { + validateObject(extension, extensionPath, CUSTOM_EXTENSION_PROPERTIES); + return CustomExtension.builder() + .withVendorName(readText(extension, "vendor_name", extensionPath)) + .withData(readText(extension, "data", extensionPath)) + .build(); + }, + CustomExtension[]::new); + } + + @Nullable + private static AIContext readAIContext(@Nullable JsonNode node, String path) { + if (node == null) { + return null; + } + if (node.isTextual()) { + return AIContext.of(node.textValue()); + } + if (!(node instanceof ObjectNode)) { + throw invalid(path, "must be a string or object"); + } + ObjectNode object = (ObjectNode) node; + Map<String, Object> additionalProperties = new LinkedHashMap<>(); + object + .fields() + .forEachRemaining( + entry -> { + if (!AI_CONTEXT_PROPERTIES.contains(entry.getKey())) { + additionalProperties.put( + entry.getKey(), JSON_MAPPER.convertValue(entry.getValue(), Object.class)); + } + }); + return AIContext.of( + AIContextObject.builder() + .withInstructions(readText(object, "instructions", path)) + .withSynonyms(readStringArray(object, "synonyms", path)) + .withExamples(readStringArray(object, "examples", path)) + .withAdditionalProperties(additionalProperties) + .build()); + } + + private static ObjectNode writeDefinition(SemanticModelDefinition definition) { + ObjectNode node = JSON_MAPPER.createObjectNode(); + putOptional(node, "ai_context", writeAIContext(definition.aiContext())); + writeObjectArray(node, "datasets", definition.datasets(), OssieDocumentConverter::writeDataset); + writeObjectArray( + node, + "relationships", + definition.relationships(), + OssieDocumentConverter::writeRelationship); + writeObjectArray(node, "metrics", definition.metrics(), OssieDocumentConverter::writeMetric); + writeObjectArray( + node, + "custom_extensions", + definition.customExtensions(), + OssieDocumentConverter::writeCustomExtension); + return node; + } + + private static ObjectNode writeDataset(Dataset dataset) { + ObjectNode node = JSON_MAPPER.createObjectNode(); + node.put("name", dataset.name()); + NameIdentifier source = dataset.source(); + String[] namespace = source.namespace().levels(); + if (namespace.length != 2) { + throw invalid( + "$.datasets." + dataset.name() + ".source", "must contain exactly catalog.schema.name"); Review Comment: [Question] Is this branch reachable, and should the path read `$.datasets[0].source`? Every import-side path in this file is indexed (`$.datasets[0].source`, built by `readObjectArray`), and so is the native validator's: `SemanticModelValidator` formats `datasets[index].source` (`core/src/main/java/org/apache/gravitino/catalog/SemanticModelValidator.java:70,99-103`). This one keys by dataset name instead, so the same defect is reported at two different locations depending on which side detects it -- and a dataset name containing a dot makes the path ambiguous. It also looks unreachable: `validateSource` already rejects any source whose namespace is not exactly two levels on create and replaceDefinition, and it is wired into the create path at `SemanticModelOperationDispatcher.java:73-77`, so no persisted model can reach `writeDataset` with `namespace.length != 2`. If it is meant as defence in depth, passing the index down from `writeObjectArray` and using it here would make the two sides agree; if not, it could go. Verified by: read `writeDataset` and `writeObjectArray` in this file; read `SemanticModelValidator.validateDefinition`/`validateDataset`/`validateSource` and confirmed the validator is supplied to `ManagedSemanticModelOperations` in the `SemanticModelOperationDispatcher` constructor. ########## core/src/main/java/org/apache/gravitino/semantic/OssieDocumentConverter.java: ########## @@ -0,0 +1,993 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.semantic; + +import static org.apache.gravitino.semantic.CustomExtension.GRAVITINO_PROPERTIES_VENDOR; +import static org.apache.gravitino.semantic.OssieVersion.DEFAULT_VERSION; +import static org.apache.gravitino.semantic.SemanticModel.PROPERTY_OSSIE_VERSION; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.StreamReadFeature; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; +import java.util.function.IntFunction; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.exceptions.IllegalSemanticModelException; + +/** Converts between standalone Apache Ossie documents and Gravitino Semantic Models. */ +public final class OssieDocumentConverter { + + private static final Set<String> ROOT_PROPERTIES = + Set.of( + "version", + "name", + "description", + "ai_context", + "datasets", + "relationships", + "metrics", + "custom_extensions"); + private static final Set<String> DATASET_PROPERTIES = + Set.of( + "name", + "source", + "primary_key", + "unique_keys", + "description", + "ai_context", + "fields", + "custom_extensions"); + private static final Set<String> FIELD_PROPERTIES = + Set.of( + "name", + "expression", + "dimension", + "label", + "description", + "datatype", + "ai_context", + "custom_extensions"); + private static final Set<String> RELATIONSHIP_PROPERTIES = + Set.of("name", "from", "to", "from_columns", "to_columns", "ai_context", "custom_extensions"); + private static final Set<String> METRIC_PROPERTIES = + Set.of("name", "expression", "description", "datatype", "ai_context", "custom_extensions"); + private static final Set<String> EXPRESSION_PROPERTIES = Set.of("dialects"); + private static final Set<String> DIALECT_EXPRESSION_PROPERTIES = Set.of("dialect", "expression"); + private static final Set<String> DIMENSION_PROPERTIES = Set.of("is_time"); + private static final Set<String> CUSTOM_EXTENSION_PROPERTIES = Set.of("vendor_name", "data"); + + private static final Set<String> AI_CONTEXT_PROPERTIES = + Set.of("instructions", "synonyms", "examples"); + private static final Map<DataType, String> DATA_TYPE_NAMES = + Map.of( + DataType.STRING, "String", + DataType.INTEGER, "Integer", + DataType.DECIMAL, "Decimal", + DataType.FLOAT, "Float", + DataType.BOOLEAN, "Boolean", + DataType.DATE, "Date", + DataType.TIME, "Time", + DataType.DATE_TIME, "DateTime", + DataType.DATE_TIME_TZ, "DateTimeTz", + DataType.OPAQUE, "Opaque"); + + private static final ObjectMapper JSON_MAPPER = createJsonMapper(); + private static final ObjectMapper YAML_MAPPER = createYamlMapper(); + + private OssieDocumentConverter() {} + + /** + * Converts one standalone Apache Ossie YAML or JSON document into native model values. + * + * @param document The standalone Ossie document. + * @return The model name, comment, native definition, and properties. + * @throws IllegalSemanticModelException If the document cannot be parsed or represented by + * Gravitino. + */ + public static ImportedSemanticModel importDocument(OssieDocument document) { + Objects.requireNonNull(document, "document must not be null"); + ObjectNode root = parseDocument(document); + return toImportedModel(root); + } + + /** + * Exports one Gravitino Semantic Model as a standalone Apache Ossie document. + * + * @param semanticModel The Semantic Model to export. + * @param format The requested serialization format. + * @return The serialized Ossie document. + * @throws IllegalSemanticModelException If the model cannot be represented as a valid Ossie + * document. + */ + public static OssieDocument exportDocument(SemanticModel semanticModel, OssieFormat format) { + Objects.requireNonNull(semanticModel, "semanticModel must not be null"); + Objects.requireNonNull(format, "format must not be null"); + + ObjectNode definition = writeDefinition(semanticModel.definition()); + + ObjectNode root = JSON_MAPPER.createObjectNode(); + root.put("version", ossieVersion(semanticModel.properties())); + root.put("name", semanticModel.name()); + if (semanticModel.comment() != null) { + root.put("description", semanticModel.comment()); + } + definition.fields().forEachRemaining(entry -> root.set(entry.getKey(), entry.getValue())); + stashProperties(root, semanticModel.properties()); + + // Validate the generated document through the same conversion path used for imports. + toImportedModel(root.deepCopy()); + + try { + if (format == OssieFormat.JSON) { + return OssieDocument.json( + JSON_MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(root) + "\n"); + } + return OssieDocument.yaml(YAML_MAPPER.writeValueAsString(root)); + } catch (JsonProcessingException e) { + throw new IllegalSemanticModelException( + e, "Cannot serialize Apache Ossie document: %s", e.getOriginalMessage()); + } + } + + /** Native model values parsed from an Ossie document, before the model is created. */ + public static final class ImportedSemanticModel { + + private final String name; + @Nullable private final String comment; + private final SemanticModelDefinition definition; + private final Map<String, String> properties; + + private ImportedSemanticModel( + String name, + @Nullable String comment, + SemanticModelDefinition definition, + Map<String, String> properties) { + this.name = name; + this.comment = comment; + this.definition = definition; + this.properties = Map.copyOf(properties); + } + + /** + * Returns the model name supplied by the document. + * + * @return The model name. + */ + public String name() { + return name; + } + + /** + * Returns the document description. + * + * @return The model comment, or {@code null} when absent. + */ + @Nullable + public String comment() { + return comment; + } + + /** + * Returns the parsed native definition. + * + * @return The immutable Semantic Model definition. + */ + public SemanticModelDefinition definition() { + return definition; + } + + /** + * Returns the Gravitino properties, including the Ossie version. + * + * @return The immutable properties. + */ + public Map<String, String> properties() { + return properties; + } + } + + private static ObjectMapper createJsonMapper() { + JsonFactory factory = + JsonFactory.builder().enable(StreamReadFeature.STRICT_DUPLICATE_DETECTION).build(); + return JsonMapper.builder(factory) + .enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS) + .enable(DeserializationFeature.USE_BIG_INTEGER_FOR_INTS) + .build(); + } + + private static ObjectMapper createYamlMapper() { + YAMLFactory factory = + YAMLFactory.builder() + .enable(StreamReadFeature.STRICT_DUPLICATE_DETECTION) + .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER) + .build(); + return new ObjectMapper(factory) + .enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS) + .enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS) + .enable(DeserializationFeature.USE_BIG_INTEGER_FOR_INTS); + } + + private static ObjectNode parseDocument(OssieDocument document) { + String content = document.content(); + try { + ObjectMapper mapper = document.format() == OssieFormat.JSON ? JSON_MAPPER : YAML_MAPPER; + // Reject trailing content after the standalone document. + JsonNode parsed = + mapper.reader().with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS).readTree(content); + if (!(parsed instanceof ObjectNode)) { + throw invalid("$", "document root must be an object"); + } + return (ObjectNode) parsed; + } catch (IOException e) { + throw new IllegalSemanticModelException( + e, "Cannot parse Apache Ossie %s: %s", document.format(), originalMessage(e)); + } + } + + private static ImportedSemanticModel toImportedModel(ObjectNode root) { + validateObject(root, "$", ROOT_PROPERTIES); + String name = readText(root, "name", "$"); + String comment = readText(root, "description", "$"); + String ossieVersion = validateVersion(root.get("version")); + + Map<String, String> properties = new LinkedHashMap<>(extractProperties(root)); + String extensionVersion = properties.get(PROPERTY_OSSIE_VERSION); + if (extensionVersion != null && !extensionVersion.equals(ossieVersion)) { + throw invalid( + "$.custom_extensions", + "Gravitino property '" + PROPERTY_OSSIE_VERSION + "' conflicts with $.version"); + } + properties.put(PROPERTY_OSSIE_VERSION, ossieVersion); + + try { + if (StringUtils.isBlank(name)) { + throw new IllegalArgumentException("\"name\" field is required and cannot be empty"); + } + return new ImportedSemanticModel(name, comment, readDefinition(root), properties); + } catch (IllegalArgumentException e) { + throw new IllegalSemanticModelException( + e, + "Cannot convert Apache Ossie document to a Gravitino Semantic Model: %s", + originalMessage(e)); + } + } + + private static String validateVersion(@Nullable JsonNode version) { + if (version == null || !version.isTextual() || StringUtils.isBlank(version.textValue())) { + throw invalid("$.version", "must be a non-empty string"); + } + return version.textValue(); + } + + private static String ossieVersion(Map<String, String> properties) { + String version = properties.getOrDefault(PROPERTY_OSSIE_VERSION, DEFAULT_VERSION); + if (StringUtils.isBlank(version)) { + throw invalid( + "$.version", + "Semantic Model property '" + PROPERTY_OSSIE_VERSION + "' must not be blank"); + } + return version; + } + + private static SemanticModelDefinition readDefinition(ObjectNode root) { + return SemanticModelDefinition.builder() + .withAIContext(readAIContext(root.get("ai_context"), "$.ai_context")) + .withDatasets( + readObjectArray( + root.get("datasets"), + "$.datasets", + OssieDocumentConverter::readDataset, + Dataset[]::new)) + .withRelationships( + readObjectArray( + root.get("relationships"), + "$.relationships", + OssieDocumentConverter::readRelationship, + Relationship[]::new)) + .withMetrics( + readObjectArray( + root.get("metrics"), + "$.metrics", + OssieDocumentConverter::readMetric, + Metric[]::new)) + .withCustomExtensions( + readCustomExtensions(root.get("custom_extensions"), "$.custom_extensions")) + .build(); + } + + private static Dataset readDataset(ObjectNode dataset, String path) { + validateObject(dataset, path, DATASET_PROPERTIES); + String source = readText(dataset, "source", path); + return Dataset.builder() + .withName(readText(dataset, "name", path)) + .withSource( + source == null ? null : NameIdentifier.of(parseOssieSource(source, path + ".source"))) + .withPrimaryKey(readStringArray(dataset, "primary_key", path)) + .withUniqueKeys(readStringArrays(dataset, "unique_keys", path)) + .withDescription(readText(dataset, "description", path)) + .withAIContext(readAIContext(dataset.get("ai_context"), path + ".ai_context")) + .withFields( + readObjectArray( + dataset.get("fields"), + path + ".fields", + OssieDocumentConverter::readField, + Field[]::new)) + .withCustomExtensions( + readCustomExtensions(dataset.get("custom_extensions"), path + ".custom_extensions")) + .build(); + } + + private static String[] parseOssieSource(String source, String path) { + List<String> parts = new ArrayList<>(3); + int offset = 0; + while (offset < source.length()) { + StringBuilder part = new StringBuilder(); + if (source.charAt(offset) == '`') { + offset++; + boolean closed = false; + while (offset < source.length()) { + char current = source.charAt(offset); + if (current != '`') { + part.append(current); + offset++; + } else if (offset + 1 < source.length() && source.charAt(offset + 1) == '`') { + part.append('`'); + offset += 2; + } else { + closed = true; + offset++; + break; + } + } + if (!closed) { + throw invalid(path, "contains an unterminated quoted segment"); + } + if (offset < source.length() && source.charAt(offset) != '.') { + throw invalid(path, "quoted segments must be separated by dots"); + } + } else { + while (offset < source.length() && source.charAt(offset) != '.') { + char current = source.charAt(offset); + if (current == '`') { + throw invalid(path, "backticks must quote an entire segment"); + } + part.append(current); + offset++; + } + } + + if (StringUtils.isBlank(part)) { + throw invalid(path, "source segments must not be blank"); + } + parts.add(part.toString()); + if (offset < source.length()) { + offset++; + if (offset == source.length()) { + throw invalid(path, "source segments must not be blank"); + } + } + } + + if (parts.size() != 3) { + throw invalid( + path, + "must be a three-part catalog.schema.entity identifier; query sources are not supported"); + } + return parts.toArray(new String[0]); + } + + private static Field readField(ObjectNode field, String path) { + validateObject(field, path, FIELD_PROPERTIES); + return Field.builder() + .withName(readText(field, "name", path)) + .withExpression(readExpression(field.get("expression"), path + ".expression")) + .withDimension(readDimension(field.get("dimension"), path + ".dimension")) + .withLabel(readText(field, "label", path)) + .withDescription(readText(field, "description", path)) + .withDatatype(readDataType(field, path)) + .withAIContext(readAIContext(field.get("ai_context"), path + ".ai_context")) + .withCustomExtensions( + readCustomExtensions(field.get("custom_extensions"), path + ".custom_extensions")) + .build(); + } + + private static Relationship readRelationship(ObjectNode relationship, String path) { + validateObject(relationship, path, RELATIONSHIP_PROPERTIES); + return Relationship.builder() + .withName(readText(relationship, "name", path)) + .withFrom(readText(relationship, "from", path)) + .withTo(readText(relationship, "to", path)) + .withFromColumns(readStringArray(relationship, "from_columns", path)) + .withToColumns(readStringArray(relationship, "to_columns", path)) + .withAIContext(readAIContext(relationship.get("ai_context"), path + ".ai_context")) + .withCustomExtensions( + readCustomExtensions( + relationship.get("custom_extensions"), path + ".custom_extensions")) + .build(); + } + + private static Metric readMetric(ObjectNode metric, String path) { + validateObject(metric, path, METRIC_PROPERTIES); + return Metric.builder() + .withName(readText(metric, "name", path)) + .withExpression(readExpression(metric.get("expression"), path + ".expression")) + .withDescription(readText(metric, "description", path)) + .withDatatype(readDataType(metric, path)) + .withAIContext(readAIContext(metric.get("ai_context"), path + ".ai_context")) + .withCustomExtensions( + readCustomExtensions(metric.get("custom_extensions"), path + ".custom_extensions")) + .build(); + } + + @Nullable + private static Expression readExpression(@Nullable JsonNode node, String path) { + if (node == null) { + return null; + } + ObjectNode expression = requireObject(node, path); + validateObject(expression, path, EXPRESSION_PROPERTIES); + return Expression.builder() + .withDialects( + readObjectArray( + expression.get("dialects"), + path + ".dialects", + OssieDocumentConverter::readDialectExpression, + DialectExpression[]::new)) + .build(); + } + + private static DialectExpression readDialectExpression(ObjectNode dialect, String path) { + validateObject(dialect, path, DIALECT_EXPRESSION_PROPERTIES); + return DialectExpression.builder() + .withDialect(readText(dialect, "dialect", path)) + .withExpression(readText(dialect, "expression", path)) + .build(); + } + + @Nullable + private static Dimension readDimension(@Nullable JsonNode node, String path) { + if (node == null) { + return null; + } + ObjectNode dimension = requireObject(node, path); + validateObject(dimension, path, DIMENSION_PROPERTIES); + validateOptionalBoolean(dimension, "is_time", path); + JsonNode isTime = dimension.get("is_time"); + return Dimension.builder().withIsTime(isTime == null ? null : isTime.booleanValue()).build(); + } + + @Nullable + private static DataType readDataType(ObjectNode object, String path) { + String value = readText(object, "datatype", path); + if (value == null) { + return null; + } + for (DataType type : DataType.values()) { + if (value.equals(DATA_TYPE_NAMES.get(type))) { + return type; + } + } + throw invalid( + path + ".datatype", + "Unknown Semantic Model data type: " + + value + + ". Supported values: " + + Arrays.stream(DataType.values()) + .map(DATA_TYPE_NAMES::get) + .collect(Collectors.joining(", "))); + } + + @Nullable + private static CustomExtension[] readCustomExtensions(@Nullable JsonNode node, String path) { + return readObjectArray( + node, + path, + (extension, extensionPath) -> { + validateObject(extension, extensionPath, CUSTOM_EXTENSION_PROPERTIES); + return CustomExtension.builder() + .withVendorName(readText(extension, "vendor_name", extensionPath)) + .withData(readText(extension, "data", extensionPath)) + .build(); + }, + CustomExtension[]::new); + } + + @Nullable + private static AIContext readAIContext(@Nullable JsonNode node, String path) { + if (node == null) { + return null; + } + if (node.isTextual()) { + return AIContext.of(node.textValue()); + } + if (!(node instanceof ObjectNode)) { + throw invalid(path, "must be a string or object"); + } + ObjectNode object = (ObjectNode) node; + Map<String, Object> additionalProperties = new LinkedHashMap<>(); + object + .fields() + .forEachRemaining( + entry -> { + if (!AI_CONTEXT_PROPERTIES.contains(entry.getKey())) { + additionalProperties.put( + entry.getKey(), JSON_MAPPER.convertValue(entry.getValue(), Object.class)); + } + }); + return AIContext.of( + AIContextObject.builder() + .withInstructions(readText(object, "instructions", path)) + .withSynonyms(readStringArray(object, "synonyms", path)) + .withExamples(readStringArray(object, "examples", path)) + .withAdditionalProperties(additionalProperties) + .build()); + } + + private static ObjectNode writeDefinition(SemanticModelDefinition definition) { + ObjectNode node = JSON_MAPPER.createObjectNode(); + putOptional(node, "ai_context", writeAIContext(definition.aiContext())); + writeObjectArray(node, "datasets", definition.datasets(), OssieDocumentConverter::writeDataset); + writeObjectArray( + node, + "relationships", + definition.relationships(), + OssieDocumentConverter::writeRelationship); + writeObjectArray(node, "metrics", definition.metrics(), OssieDocumentConverter::writeMetric); + writeObjectArray( + node, + "custom_extensions", + definition.customExtensions(), + OssieDocumentConverter::writeCustomExtension); + return node; + } + + private static ObjectNode writeDataset(Dataset dataset) { + ObjectNode node = JSON_MAPPER.createObjectNode(); + node.put("name", dataset.name()); + NameIdentifier source = dataset.source(); + String[] namespace = source.namespace().levels(); + if (namespace.length != 2) { + throw invalid( + "$.datasets." + dataset.name() + ".source", "must contain exactly catalog.schema.name"); + } + node.put( + "source", + formatOssieSourceSegment(namespace[0]) + + "." + + formatOssieSourceSegment(namespace[1]) + + "." + + formatOssieSourceSegment(source.name())); + putOptional(node, "primary_key", dataset.primaryKey()); + putOptional(node, "unique_keys", dataset.uniqueKeys()); + putOptional(node, "description", dataset.description()); + putOptional(node, "ai_context", writeAIContext(dataset.aiContext())); + writeObjectArray(node, "fields", dataset.fields(), OssieDocumentConverter::writeField); + writeObjectArray( + node, + "custom_extensions", + dataset.customExtensions(), + OssieDocumentConverter::writeCustomExtension); + return node; + } + + private static String formatOssieSourceSegment(String segment) { + if (!segment.contains(".") && !segment.contains("`")) { + return segment; + } + return "`" + segment.replace("`", "``") + "`"; + } + + private static ObjectNode writeField(Field field) { + ObjectNode node = JSON_MAPPER.createObjectNode(); + node.put("name", field.name()); + node.set("expression", writeExpression(field.expression())); + if (field.dimension() != null) { + ObjectNode dimension = node.putObject("dimension"); + putOptional(dimension, "is_time", field.dimension().isTime()); + } + putOptional(node, "label", field.label()); + putOptional(node, "description", field.description()); + putOptional( + node, "datatype", field.datatype() == null ? null : DATA_TYPE_NAMES.get(field.datatype())); Review Comment: [Nit] An unmapped `DataType` is silently dropped from the exported document, where the DTO path threw. `DATA_TYPE_NAMES` (:97) is a hand-maintained second copy of `SemanticDTOUtils.dataTypeName` (`common/src/main/java/org/apache/gravitino/dto/semantic/SemanticDTOUtils.java:177-201`), which this commit stopped using. The two differ in the default case: the DTO serializer throws `IllegalArgumentException("Unsupported Semantic Model data type: ...")`, while `DATA_TYPE_NAMES.get()` returns null and `putOptional` (:972-976) then skips the key, so the field's `datatype` vanishes from the document instead of failing. The import side degrades too: the loop at :502 can never match such a value, and :512 renders the unmapped constant as `null` inside the "Supported values" list. `DataType` is `@Evolving` (`api/src/main/java/org/apache/gravitino/semantic/DataType.java:28-29`), so an eleventh constant is the trigger. `testAllDataTypesRoundTripWithoutDTOs` does catch that via `assertEquals(names.length, types.length)`, so this is not a live bug -- but a map lookup returning null on a path that should fail loudly is cheap to close: throw when the name is missing, or `switch` on the enum so the compiler flags the gap. Verified by: read `DATA_TYPE_NAMES`, `readDataType`, `writeField`, `writeMetric` and `putOptional` in this file at `c7a4ed2`; read `dataTypeName` and `DataTypeDeserializer` in `SemanticDTOUtils` and confirmed all ten names match exactly, including the throwing `default` branch; confirmed `putOptional` drops nulls without comment. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
