mneethiraj commented on code in PR #782:
URL: https://github.com/apache/atlas/pull/782#discussion_r4294682996
##########
addons/models/7000-Iceberg/7010-iceberg_model.json:
##########
@@ -0,0 +1,320 @@
+{
+ "enumDefs": [
+ {
+ "name": "iceberg_catalog_type",
+ "description": "Storage ownership model for an Iceberg REST catalog",
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "elementDefs": [
+ { "ordinal": 0, "value": "INTERNAL" },
+ { "ordinal": 1, "value": "EXTERNAL" }
+ ]
+ },
+ {
+ "name": "iceberg_table_format_version",
+ "description": "Iceberg table or view format version",
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "elementDefs": [
+ { "ordinal": 1, "value": "V1" },
+ { "ordinal": 2, "value": "V2" },
+ { "ordinal": 3, "value": "V3" }
+ ]
+ },
+ {
+ "name": "iceberg_operation_type",
Review Comment:
Since this enum captures only DDL operations, I suggest renaming
`iceberg_operation_type` to `iceberg_ddl_operation_type`.
##########
addons/models/7000-Iceberg/7010-iceberg_model.json:
##########
@@ -0,0 +1,320 @@
+{
+ "enumDefs": [
+ {
+ "name": "iceberg_catalog_type",
+ "description": "Storage ownership model for an Iceberg REST catalog",
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "elementDefs": [
+ { "ordinal": 0, "value": "INTERNAL" },
+ { "ordinal": 1, "value": "EXTERNAL" }
+ ]
+ },
+ {
+ "name": "iceberg_table_format_version",
+ "description": "Iceberg table or view format version",
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "elementDefs": [
+ { "ordinal": 1, "value": "V1" },
+ { "ordinal": 2, "value": "V2" },
+ { "ordinal": 3, "value": "V3" }
+ ]
+ },
+ {
+ "name": "iceberg_operation_type",
+ "description": "Catalog operation recorded by an iceberg_ddl audit
entity",
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "elementDefs": [
+ { "ordinal": 0, "value": "CREATE_CATALOG" },
+ { "ordinal": 1, "value": "UPDATE_CATALOG" },
+ { "ordinal": 2, "value": "DELETE_CATALOG" },
+ { "ordinal": 3, "value": "CREATE_NAMESPACE" },
+ { "ordinal": 4, "value": "DROP_NAMESPACE" },
+ { "ordinal": 5, "value": "UPDATE_NAMESPACE_PROPERTIES" },
+ { "ordinal": 6, "value": "CREATE_TABLE" },
+ { "ordinal": 7, "value": "UPDATE_TABLE" },
+ { "ordinal": 8, "value": "DROP_TABLE" },
+ { "ordinal": 9, "value": "REGISTER_TABLE" },
+ { "ordinal": 10, "value": "RENAME_TABLE" },
+ { "ordinal": 11, "value": "CREATE_VIEW" },
+ { "ordinal": 12, "value": "REPLACE_VIEW" },
+ { "ordinal": 13, "value": "DROP_VIEW" },
+ { "ordinal": 14, "value": "RENAME_VIEW" }
+ ]
+ }
+ ],
+ "entityDefs": [
+ {
+ "name": "iceberg_catalog",
+ "description": "A catalog exposed by an Iceberg REST catalog service",
+ "superTypes": [ "Asset" ],
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ { "name": "clusterName", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false, "isUnique":
false, "includeInNotification": true },
+ { "name": "realmId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false, "isUnique":
false, "includeInNotification": true },
+ { "name": "catalogType", "typeName": "iceberg_catalog_type",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique":
false },
+ { "name": "storageType", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique":
false },
+ { "name": "defaultBaseLocation", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true, "isUnique":
false },
+ { "name": "properties", "typeName": "map<string,string>",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true, "isUnique":
false }
+ ]
+ },
+ {
+ "name": "iceberg_namespace",
+ "description": "An Iceberg namespace within a catalog; nests
arbitrarily deep",
+ "superTypes": [ "Asset" ],
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ { "name": "clusterName", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false, "isUnique":
false, "includeInNotification": true },
+ { "name": "realmId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false, "isUnique":
false, "includeInNotification": true },
+ { "name": "levels", "typeName": "array<string>",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": false, "isUnique":
false },
+ { "name": "location", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true, "isUnique":
false },
+ { "name": "properties", "typeName": "map<string,string>",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true, "isUnique":
false }
+ ]
+ },
+ {
+ "name": "iceberg_table",
+ "description": "An Iceberg table or view within a namespace; tableType
discriminates. Mirrors Hive, where a view is a hive_table with
tableType=VIRTUAL_VIEW rather than a separate type.",
+ "superTypes": [ "DataSet" ],
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ { "name": "clusterName", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false,
"isUnique": false, "includeInNotification": true },
+ { "name": "realmId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false,
"isUnique": false, "includeInNotification": true },
+ { "name": "tableType", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false,
"isUnique": false, "includeInNotification": true },
+ { "name": "tableUuid", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": true,
"isUnique": false, "includeInNotification": true },
+ { "name": "formatVersion", "typeName":
"iceberg_table_format_version", "cardinality": "SINGLE", "isIndexable": true,
"isOptional": true, "isUnique": false },
+ { "name": "location", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "metadataLocation", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "currentSnapshotId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": true,
"isUnique": false },
+ { "name": "currentSequenceNumber", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "snapshotTimestampMs", "typeName": "date",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "snapshotCount", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "snapshotOperation", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "totalDataFiles", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "totalDeleteFiles", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "totalFilesSize", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "lastUpdatedMs", "typeName": "date",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "schemaId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "partitionSpec", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "sortOrder", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "viewDefinition", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "viewDialect", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": true,
"isUnique": false },
+ { "name": "viewDefaultCatalog", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "viewDefaultNamespace", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "viewVersionId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "properties", "typeName": "map<string,string>",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false }
+ ]
+ },
+ {
+ "name": "iceberg_column",
+ "description": "A field in the current Iceberg schema of an
iceberg_table (table or view)",
+ "superTypes": [ "DataSet" ],
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ { "name": "fieldId", "typeName": "int", "cardinality": "SINGLE",
"isIndexable": true, "isOptional": false, "isUnique": false,
"includeInNotification": true },
+ { "name": "type", "typeName": "string", "cardinality": "SINGLE",
"isIndexable": true, "isOptional": false, "isUnique": false,
"includeInNotification": true },
+ { "name": "required", "typeName": "boolean", "cardinality": "SINGLE",
"isIndexable": false, "isOptional": true, "isUnique": false },
+ { "name": "position", "typeName": "int", "cardinality": "SINGLE",
"isIndexable": false, "isOptional": true, "isUnique": false },
+ { "name": "doc", "typeName": "string", "cardinality": "SINGLE",
"isIndexable": false, "isOptional": true, "isUnique": false },
+ { "name": "isNested", "typeName": "boolean", "cardinality": "SINGLE",
"isIndexable": true, "isOptional": true, "isUnique": false }
+ ]
+ },
+ {
+ "name": "iceberg_ddl",
+ "description": "Supertype for Iceberg REST catalog operation audit
records; inherits queryText, execTime, userName and serviceType from the Atlas
ddl base type. Not instantiated directly - use the per-parent subtypes.",
+ "superTypes": [ "ddl" ],
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ { "name": "operationType", "typeName": "iceberg_operation_type",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false, "isUnique":
false, "includeInNotification": true },
+ { "name": "eventId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false, "isUnique":
false, "includeInNotification": true },
+ { "name": "realmId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false, "isUnique":
false },
+ { "name": "requestId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique":
false },
+ { "name": "traceId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique":
false },
+ { "name": "snapshotId", "typeName": "long",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true, "isUnique":
false }
+ ]
+ },
+ {
+ "name": "iceberg_catalog_ddl",
+ "description": "Audit record for one committed catalog-level
operation",
+ "superTypes": [ "iceberg_ddl" ],
+ "serviceType": "iceberg",
+ "typeVersion": "1.0"
+ },
+ {
+ "name": "iceberg_namespace_ddl",
+ "description": "Audit record for one committed namespace-level
operation",
+ "superTypes": [ "iceberg_ddl" ],
+ "serviceType": "iceberg",
+ "typeVersion": "1.0"
+ },
+ {
+ "name": "iceberg_table_ddl",
+ "description": "Audit record for one committed table-level or
view-level operation",
+ "superTypes": [ "iceberg_ddl" ],
+ "serviceType": "iceberg",
+ "typeVersion": "1.0"
+ }
+ ],
+ "relationshipDefs": [
+ {
+ "name": "iceberg_catalog_namespaces",
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "relationshipCategory": "AGGREGATION",
+ "relationshipLabel": "__iceberg_catalog.namespaces",
Review Comment:
`relationshipLabel` exists to support migration from very early version of
Atlas. I suggest removing this from the model; its value is auto generated by
Atlas.
##########
addons/models/7000-Iceberg/7010-iceberg_model.json:
##########
@@ -0,0 +1,320 @@
+{
+ "enumDefs": [
+ {
+ "name": "iceberg_catalog_type",
+ "description": "Storage ownership model for an Iceberg REST catalog",
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "elementDefs": [
+ { "ordinal": 0, "value": "INTERNAL" },
+ { "ordinal": 1, "value": "EXTERNAL" }
+ ]
+ },
+ {
+ "name": "iceberg_table_format_version",
+ "description": "Iceberg table or view format version",
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "elementDefs": [
+ { "ordinal": 1, "value": "V1" },
+ { "ordinal": 2, "value": "V2" },
+ { "ordinal": 3, "value": "V3" }
+ ]
+ },
+ {
+ "name": "iceberg_operation_type",
+ "description": "Catalog operation recorded by an iceberg_ddl audit
entity",
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "elementDefs": [
+ { "ordinal": 0, "value": "CREATE_CATALOG" },
+ { "ordinal": 1, "value": "UPDATE_CATALOG" },
+ { "ordinal": 2, "value": "DELETE_CATALOG" },
+ { "ordinal": 3, "value": "CREATE_NAMESPACE" },
+ { "ordinal": 4, "value": "DROP_NAMESPACE" },
+ { "ordinal": 5, "value": "UPDATE_NAMESPACE_PROPERTIES" },
+ { "ordinal": 6, "value": "CREATE_TABLE" },
+ { "ordinal": 7, "value": "UPDATE_TABLE" },
+ { "ordinal": 8, "value": "DROP_TABLE" },
+ { "ordinal": 9, "value": "REGISTER_TABLE" },
+ { "ordinal": 10, "value": "RENAME_TABLE" },
+ { "ordinal": 11, "value": "CREATE_VIEW" },
+ { "ordinal": 12, "value": "REPLACE_VIEW" },
+ { "ordinal": 13, "value": "DROP_VIEW" },
+ { "ordinal": 14, "value": "RENAME_VIEW" }
+ ]
+ }
+ ],
+ "entityDefs": [
+ {
+ "name": "iceberg_catalog",
+ "description": "A catalog exposed by an Iceberg REST catalog service",
+ "superTypes": [ "Asset" ],
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ { "name": "clusterName", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false, "isUnique":
false, "includeInNotification": true },
+ { "name": "realmId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false, "isUnique":
false, "includeInNotification": true },
+ { "name": "catalogType", "typeName": "iceberg_catalog_type",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique":
false },
+ { "name": "storageType", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique":
false },
+ { "name": "defaultBaseLocation", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true, "isUnique":
false },
+ { "name": "properties", "typeName": "map<string,string>",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true, "isUnique":
false }
+ ]
+ },
+ {
+ "name": "iceberg_namespace",
+ "description": "An Iceberg namespace within a catalog; nests
arbitrarily deep",
+ "superTypes": [ "Asset" ],
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ { "name": "clusterName", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false, "isUnique":
false, "includeInNotification": true },
+ { "name": "realmId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false, "isUnique":
false, "includeInNotification": true },
+ { "name": "levels", "typeName": "array<string>",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": false, "isUnique":
false },
+ { "name": "location", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true, "isUnique":
false },
+ { "name": "properties", "typeName": "map<string,string>",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true, "isUnique":
false }
+ ]
+ },
+ {
+ "name": "iceberg_table",
+ "description": "An Iceberg table or view within a namespace; tableType
discriminates. Mirrors Hive, where a view is a hive_table with
tableType=VIRTUAL_VIEW rather than a separate type.",
+ "superTypes": [ "DataSet" ],
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ { "name": "clusterName", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false,
"isUnique": false, "includeInNotification": true },
+ { "name": "realmId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false,
"isUnique": false, "includeInNotification": true },
+ { "name": "tableType", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false,
"isUnique": false, "includeInNotification": true },
+ { "name": "tableUuid", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": true,
"isUnique": false, "includeInNotification": true },
+ { "name": "formatVersion", "typeName":
"iceberg_table_format_version", "cardinality": "SINGLE", "isIndexable": true,
"isOptional": true, "isUnique": false },
+ { "name": "location", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "metadataLocation", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "currentSnapshotId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": true,
"isUnique": false },
+ { "name": "currentSequenceNumber", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "snapshotTimestampMs", "typeName": "date",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "snapshotCount", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "snapshotOperation", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "totalDataFiles", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "totalDeleteFiles", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "totalFilesSize", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "lastUpdatedMs", "typeName": "date",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "schemaId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "partitionSpec", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "sortOrder", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "viewDefinition", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "viewDialect", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": true,
"isUnique": false },
+ { "name": "viewDefaultCatalog", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "viewDefaultNamespace", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "viewVersionId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "properties", "typeName": "map<string,string>",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false }
+ ]
Review Comment:
I suggest capturing partition spec of a table with the following:
- add structDef: iceberg_partition_field(fieldId: int, name: string,
sourceIds: array<int>, sourceColumnNames: array<string>, transform: string)
- add entityDef: iceberg_partition_spec(specId: int, fields:
array<iceberg_partition_field>)
- add relationshipDef:
iceberg_table_partition_specs(end1(type=iceberg_table, name=partitionSpecs,
isContainer=true, cardinality=SET), end2(type=iceberg_partition_spec,
isContainer=false, name=table, cardinality=SINGLE)
- update entityDef: iceberg_table with addition of attribute defaultSpecId:
int
##########
addons/models/7000-Iceberg/7010-iceberg_model.json:
##########
@@ -0,0 +1,320 @@
+{
+ "enumDefs": [
+ {
+ "name": "iceberg_catalog_type",
+ "description": "Storage ownership model for an Iceberg REST catalog",
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "elementDefs": [
+ { "ordinal": 0, "value": "INTERNAL" },
+ { "ordinal": 1, "value": "EXTERNAL" }
+ ]
+ },
+ {
+ "name": "iceberg_table_format_version",
+ "description": "Iceberg table or view format version",
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "elementDefs": [
+ { "ordinal": 1, "value": "V1" },
+ { "ordinal": 2, "value": "V2" },
+ { "ordinal": 3, "value": "V3" }
+ ]
+ },
+ {
+ "name": "iceberg_operation_type",
+ "description": "Catalog operation recorded by an iceberg_ddl audit
entity",
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "elementDefs": [
+ { "ordinal": 0, "value": "CREATE_CATALOG" },
+ { "ordinal": 1, "value": "UPDATE_CATALOG" },
+ { "ordinal": 2, "value": "DELETE_CATALOG" },
+ { "ordinal": 3, "value": "CREATE_NAMESPACE" },
+ { "ordinal": 4, "value": "DROP_NAMESPACE" },
+ { "ordinal": 5, "value": "UPDATE_NAMESPACE_PROPERTIES" },
+ { "ordinal": 6, "value": "CREATE_TABLE" },
+ { "ordinal": 7, "value": "UPDATE_TABLE" },
+ { "ordinal": 8, "value": "DROP_TABLE" },
+ { "ordinal": 9, "value": "REGISTER_TABLE" },
+ { "ordinal": 10, "value": "RENAME_TABLE" },
+ { "ordinal": 11, "value": "CREATE_VIEW" },
+ { "ordinal": 12, "value": "REPLACE_VIEW" },
+ { "ordinal": 13, "value": "DROP_VIEW" },
+ { "ordinal": 14, "value": "RENAME_VIEW" }
+ ]
+ }
+ ],
+ "entityDefs": [
+ {
+ "name": "iceberg_catalog",
+ "description": "A catalog exposed by an Iceberg REST catalog service",
+ "superTypes": [ "Asset" ],
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ { "name": "clusterName", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false, "isUnique":
false, "includeInNotification": true },
+ { "name": "realmId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false, "isUnique":
false, "includeInNotification": true },
+ { "name": "catalogType", "typeName": "iceberg_catalog_type",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique":
false },
+ { "name": "storageType", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique":
false },
+ { "name": "defaultBaseLocation", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true, "isUnique":
false },
+ { "name": "properties", "typeName": "map<string,string>",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true, "isUnique":
false }
+ ]
+ },
+ {
+ "name": "iceberg_namespace",
+ "description": "An Iceberg namespace within a catalog; nests
arbitrarily deep",
+ "superTypes": [ "Asset" ],
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ { "name": "clusterName", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false, "isUnique":
false, "includeInNotification": true },
+ { "name": "realmId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false, "isUnique":
false, "includeInNotification": true },
+ { "name": "levels", "typeName": "array<string>",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": false, "isUnique":
false },
+ { "name": "location", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true, "isUnique":
false },
+ { "name": "properties", "typeName": "map<string,string>",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true, "isUnique":
false }
+ ]
+ },
+ {
+ "name": "iceberg_table",
+ "description": "An Iceberg table or view within a namespace; tableType
discriminates. Mirrors Hive, where a view is a hive_table with
tableType=VIRTUAL_VIEW rather than a separate type.",
+ "superTypes": [ "DataSet" ],
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ { "name": "clusterName", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false,
"isUnique": false, "includeInNotification": true },
+ { "name": "realmId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false,
"isUnique": false, "includeInNotification": true },
+ { "name": "tableType", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false,
"isUnique": false, "includeInNotification": true },
+ { "name": "tableUuid", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": true,
"isUnique": false, "includeInNotification": true },
+ { "name": "formatVersion", "typeName":
"iceberg_table_format_version", "cardinality": "SINGLE", "isIndexable": true,
"isOptional": true, "isUnique": false },
+ { "name": "location", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "metadataLocation", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "currentSnapshotId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": true,
"isUnique": false },
+ { "name": "currentSequenceNumber", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "snapshotTimestampMs", "typeName": "date",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "snapshotCount", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "snapshotOperation", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "totalDataFiles", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "totalDeleteFiles", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "totalFilesSize", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "lastUpdatedMs", "typeName": "date",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "schemaId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "partitionSpec", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "sortOrder", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "viewDefinition", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "viewDialect", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": true,
"isUnique": false },
+ { "name": "viewDefaultCatalog", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "viewDefaultNamespace", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "viewVersionId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false },
+ { "name": "properties", "typeName": "map<string,string>",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true,
"isUnique": false }
+ ]
+ },
+ {
+ "name": "iceberg_column",
+ "description": "A field in the current Iceberg schema of an
iceberg_table (table or view)",
+ "superTypes": [ "DataSet" ],
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ { "name": "fieldId", "typeName": "int", "cardinality": "SINGLE",
"isIndexable": true, "isOptional": false, "isUnique": false,
"includeInNotification": true },
+ { "name": "type", "typeName": "string", "cardinality": "SINGLE",
"isIndexable": true, "isOptional": false, "isUnique": false,
"includeInNotification": true },
+ { "name": "required", "typeName": "boolean", "cardinality": "SINGLE",
"isIndexable": false, "isOptional": true, "isUnique": false },
+ { "name": "position", "typeName": "int", "cardinality": "SINGLE",
"isIndexable": false, "isOptional": true, "isUnique": false },
+ { "name": "doc", "typeName": "string", "cardinality": "SINGLE",
"isIndexable": false, "isOptional": true, "isUnique": false },
+ { "name": "isNested", "typeName": "boolean", "cardinality": "SINGLE",
"isIndexable": true, "isOptional": true, "isUnique": false }
+ ]
+ },
+ {
+ "name": "iceberg_ddl",
+ "description": "Supertype for Iceberg REST catalog operation audit
records; inherits queryText, execTime, userName and serviceType from the Atlas
ddl base type. Not instantiated directly - use the per-parent subtypes.",
+ "superTypes": [ "ddl" ],
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ { "name": "operationType", "typeName": "iceberg_operation_type",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false, "isUnique":
false, "includeInNotification": true },
+ { "name": "eventId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false, "isUnique":
false, "includeInNotification": true },
+ { "name": "realmId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": false, "isUnique":
false },
+ { "name": "requestId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique":
false },
+ { "name": "traceId", "typeName": "string",
"cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique":
false },
+ { "name": "snapshotId", "typeName": "long",
"cardinality": "SINGLE", "isIndexable": false, "isOptional": true, "isUnique":
false }
+ ]
+ },
+ {
+ "name": "iceberg_catalog_ddl",
+ "description": "Audit record for one committed catalog-level
operation",
+ "superTypes": [ "iceberg_ddl" ],
+ "serviceType": "iceberg",
+ "typeVersion": "1.0"
+ },
+ {
+ "name": "iceberg_namespace_ddl",
+ "description": "Audit record for one committed namespace-level
operation",
+ "superTypes": [ "iceberg_ddl" ],
+ "serviceType": "iceberg",
+ "typeVersion": "1.0"
+ },
+ {
+ "name": "iceberg_table_ddl",
+ "description": "Audit record for one committed table-level or
view-level operation",
+ "superTypes": [ "iceberg_ddl" ],
+ "serviceType": "iceberg",
+ "typeVersion": "1.0"
+ }
+ ],
+ "relationshipDefs": [
+ {
+ "name": "iceberg_catalog_namespaces",
+ "serviceType": "iceberg",
+ "typeVersion": "1.0",
+ "relationshipCategory": "AGGREGATION",
+ "relationshipLabel": "__iceberg_catalog.namespaces",
+ "propagateTags": "NONE",
+ "endDef1": {
+ "type": "iceberg_catalog",
+ "name": "namespaces",
+ "isContainer": true,
+ "cardinality": "SET",
+ "isLegacyAttribute": false
Review Comment:
`isLegacyAttribute` exists to support migration from very early versions of
Atlas. I suggest removing this from the model.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
