[ 
https://issues.apache.org/jira/browse/ATLAS-2696?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16491199#comment-16491199
 ] 

Barbara Eckman commented on ATLAS-2696:
---------------------------------------

Notes on the attributes:

While any of these could be specified by the topic creator, many of them are by 
default calculated from the sizing params that are entered by the user.  They 
are labeled "#user input" or "# calculated" accordingly.

{
 "category": "ENTITY",
 "guid": "c268420a-86d1-498e-ae8c-451896ec0230",
 "createdBy": "atlas",
 "updatedBy": "admin",
 "createTime": 1525358038631,
 "updateTime": 1525447432952,
 "version": 2,
 "name": "kafka_topic",
 "description": "Atlas Type representing kafka topic",
 "typeVersion": "1.0",
 "attributeDefs": [
 {
 "name": "topic",       #user input
 "typeName": "string",
 "isOptional": false,
 "cardinality": "SINGLE",
 "valuesMinCount": 1,
 "valuesMaxCount": 1,
 "isUnique": true,
 "isIndexable": true
 },
 {
 "name": "uri",   # user input
 "typeName": "string",
 "isOptional": false,
 "cardinality": "SINGLE",
 "valuesMinCount": 1,
 "valuesMaxCount": 1,
 "isUnique": false,
 "isIndexable": false
 },
 {
 "name": "avroSchema",        # user input; avroSchemas associated with the 
topic. Can be multiple.
 "typeName": "array<avro_type>",
 "isOptional": true,
 "cardinality": "SINGLE",
 "valuesMinCount": 0,
 "valuesMaxCount": 2147483647,
 "isUnique": false,
 "isIndexable": true
 },
 {
 "name": "replicationFactorNational",    # calculated; We have National and 
Local kafka installations and may have different replication Factors, retention 
Bytes, partitionCounts, and segmentBytes for National vs Local.
 "typeName": "int",
 "isOptional": true,
 "cardinality": "SINGLE",
 "valuesMinCount": 0,
 "valuesMaxCount": 1,
 "isUnique": false,
 "isIndexable": false
 },
 {
 "name": "replicationFactorLocal",  # calculated
 "typeName": "int",
 "isOptional": true,
 "cardinality": "SINGLE",
 "valuesMinCount": 0,
 "valuesMaxCount": 1,
 "isUnique": false,
 "isIndexable": false
 },
 {
 "name": "retentionBytesNational",  # calculated
 "typeName": "long",
 "isOptional": true,
 "cardinality": "SINGLE",
 "valuesMinCount": 0,
 "valuesMaxCount": 1,
 "isUnique": false,
 "isIndexable": false
 },
 {
 "name": "segmentBytesNational",  # calculated
 "typeName": "long",
 "isOptional": true,
 "cardinality": "SINGLE",
 "valuesMinCount": 0,
 "valuesMaxCount": 1,
 "isUnique": false,
 "isIndexable": false
 },
 {
 "name": "segmentBytesLocal",  # calculated
 "typeName": "long",
 "isOptional": true,
 "cardinality": "SINGLE",
 "valuesMinCount": 0,
 "valuesMaxCount": 1,
 "isUnique": false,
 "isIndexable": false
 },
 {
 "name": "contactInfo",  #user input
 "typeName": "string",
 "isOptional": true,
 "cardinality": "SINGLE",
 "valuesMinCount": 0,
 "valuesMaxCount": 1,
 "isUnique": false,
 "isIndexable": false
 },
 {
 "name": "type",          # user input; type of data produced to the topic. 
E.g, "avro", "json", "csv", "raw"
 "typeName": "string",
 "isOptional": true,
 "cardinality": "SINGLE",
 "valuesMinCount": 0,
 "valuesMaxCount": 1,
 "isUnique": false,
 "isIndexable": false
 },
 {
 "name": "partitionCountLocal",  # calculated
 "typeName": "int",
 "isOptional": true,
 "cardinality": "SINGLE",
 "valuesMinCount": 0,
 "valuesMaxCount": 1,
 "isUnique": false,
 "isIndexable": false
 },
 {
 "name": "partitionCountNational",  # calculated
 "typeName": "int",
 "isOptional": true,
 "cardinality": "SINGLE",
 "valuesMinCount": 0,
 "valuesMaxCount": 1,
 "isUnique": false,
 "isIndexable": false
 },
 {
 "name": "retentionBytesLocal",  # calculated
 "typeName": "long",
 "isOptional": true,
 "cardinality": "SINGLE",
 "valuesMinCount": 0,
 "valuesMaxCount": 1,
 "isUnique": false,
 "isIndexable": false
 },
 {
 "name": "retentiontimeLocalInHrs",  # calculated
 "typeName": "int",
 "isOptional": true,
 "cardinality": "SINGLE",
 "valuesMinCount": 0,
 "valuesMaxCount": 1,
 "isUnique": false,
 "isIndexable": false
 },
 {
 "name": "retentiontimeNationalInHrs",  # calculated
 "typeName": "int",
 "isOptional": true,
 "cardinality": "SINGLE",
 "valuesMinCount": 0,
 "valuesMaxCount": 1,
 "isUnique": false,
 "isIndexable": false
 },
 {
 "name": "desiredRetentionInHrs",   # user input
 "typeName": "int",
 "isOptional": true,
 "cardinality": "SINGLE",
 "valuesMinCount": 0,
 "valuesMaxCount": 1,
 "isUnique": false,
 "isIndexable": false
 },
 {
 "name": "keyClassname",    # user input; for Cloudera-style avro schemas, 
which have keyClasses and avroClasses (here = avroSchema)
 "typeName": "string",
 "isOptional": true,
 "cardinality": "SINGLE",
 "valuesMinCount": 0,
 "valuesMaxCount": 1,
 "isUnique": false,
 "isIndexable": false
 },
 {
 "name": "numberOfEventsPerDay",  # user input
 "typeName": "long",
 "isOptional": true,
 "cardinality": "SINGLE",
 "valuesMinCount": 0,
 "valuesMaxCount": 1,
 "isUnique": false,
 "isIndexable": false
 },
 {
 "name": "maxThroughputPerSec",   # user input
 "typeName": "long",
 "isOptional": true,
 "cardinality": "SINGLE",
 "valuesMinCount": 0,
 "valuesMaxCount": 1,
 "isUnique": false,
 "isIndexable": false
 },
 {
 "name": "avgMessageSizeInBytes",  #user input
 "typeName": "long",
 "isOptional": true,
 "cardinality": "SINGLE",
 "valuesMinCount": 0,
 "valuesMaxCount": 1,
 "isUnique": false,
 "isIndexable": false
 }
 ],
 "superTypes": [
 "DataSet",
 "Asset"
 ]
}

> Typedef extensions for Kafka in Atlas
> -------------------------------------
>
>                 Key: ATLAS-2696
>                 URL: https://issues.apache.org/jira/browse/ATLAS-2696
>             Project: Atlas
>          Issue Type: Improvement
>            Reporter: Srikanth Venkat
>            Assignee: Barbara Eckman
>            Priority: Critical
>         Attachments: kafka_topic_typedef.json
>
>
> Kafka topic typedef in Atlas can be improved to include many useful 
> operational (config related) and contextual metadata properties:
>  * config inputs such as max throughput per sec, average message size, 
> messages per day, retention time
>  * Sizing outputs such as number and size of partitions, replication factor
>  * Contact information on the data producer
>  * Array of avro schemas that are associated with the topic (based on Avro 
> schema extensions outlined in ATLAS-2694)



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Reply via email to