Repository: atlas Updated Branches: refs/heads/master a2ccdf12c -> 19d67a132
ATLAS-2446: updated HBase Model to capture more attributes of table and column-family Signed-off-by: Madhan Neethiraj <mad...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/19d67a13 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/19d67a13 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/19d67a13 Branch: refs/heads/master Commit: 19d67a13206a68c49ce92926f9513e833c423c83 Parents: a2ccdf1 Author: rmani <rm...@hortonworks.com> Authored: Wed Feb 14 14:43:16 2018 -0800 Committer: Madhan Neethiraj <mad...@apache.org> Committed: Wed Feb 14 16:25:04 2018 -0800 ---------------------------------------------------------------------- .../atlas/hbase/bridge/HBaseAtlasHook.java | 56 ++++++ ..._column_family_add_additional_attribute.json | 183 +++++++++++++++++++ 2 files changed, 239 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/19d67a13/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseAtlasHook.java ---------------------------------------------------------------------- diff --git a/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseAtlasHook.java b/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseAtlasHook.java index f839f87..2074684 100644 --- a/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseAtlasHook.java +++ b/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseAtlasHook.java @@ -92,6 +92,30 @@ public class HBaseAtlasHook extends AtlasHook { public static final String ATTR_OWNER = "owner"; public static final String ATTR_NAME = "name"; + // column addition metadata + public static final String ATTR_TABLE_MAX_FILESIZE = "maxFileSize"; + public static final String ATTR_TABLE_ISREADONLY = "isReadOnly"; + public static final String ATTR_TABLE_ISCOMPACTION_ENABLED = "isCompactionEnabled"; + public static final String ATTR_TABLE_REPLICATION_PER_REGION = "replicasPerRegion"; + public static final String ATTR_TABLE_DURABLILITY = "durability"; + + // column family additional metadata + public static final String ATTR_CF_BLOOMFILTER_TYPE = "bloomFilterType"; + public static final String ATTR_CF_COMPRESSION_TYPE = "compressionType"; + public static final String ATTR_CF_COMPACTION_COMPRESSION_TYPE = "compactionCompressionType"; + public static final String ATTR_CF_ENCRYPTION_TYPE = "encryptionType"; + public static final String ATTR_CF_KEEP_DELETE_CELLS = "keepDeletedCells"; + public static final String ATTR_CF_MAX_VERSIONS = "maxVersions"; + public static final String ATTR_CF_MIN_VERSIONS = "minVersions"; + public static final String ATTR_CF_DATA_BLOCK_ENCODING = "dataBlockEncoding"; + public static final String ATTR_CF_TTL = "ttl"; + public static final String ATTR_CF_BLOCK_CACHE_ENABLED = "blockCacheEnabled"; + public static final String ATTR_CF_CACHED_BLOOM_ON_WRITE = "cacheBloomsOnWrite"; + public static final String ATTR_CF_CACHED_DATA_ON_WRITE = "cacheDataOnWrite"; + public static final String ATTR_CF_CACHED_INDEXES_ON_WRITE = "cacheIndexesOnWrite"; + public static final String ATTR_CF_EVICT_BLOCK_ONCLOSE = "evictBlocksOnClose"; + public static final String ATTR_CF_PREFETCH_BLOCK_ONOPEN = "prefetchBlocksOnOpen"; + private static final String REFERENCEABLE_ATTRIBUTE_NAME = "qualifiedName"; private String clusterName = null; @@ -422,12 +446,25 @@ public class HBaseAtlasHook extends AtlasHook { table.setAttribute(ATTR_PARAMETERS, hbaseOperationContext.getHbaseConf()); table.setAttribute(ATTR_NAMESPACE, AtlasTypeUtil.getAtlasObjectId(nameSpace)); + HTableDescriptor htableDescriptor = hbaseOperationContext.gethTableDescriptor(); + if (htableDescriptor != null) { + table.setAttribute(ATTR_TABLE_MAX_FILESIZE, htableDescriptor.getMaxFileSize()); + table.setAttribute(ATTR_TABLE_REPLICATION_PER_REGION, htableDescriptor.getRegionReplication()); + table.setAttribute(ATTR_TABLE_ISREADONLY, htableDescriptor.getMaxFileSize()); + table.setAttribute(ATTR_TABLE_ISCOMPACTION_ENABLED, htableDescriptor.isCompactionEnabled()); + table.setAttribute(ATTR_TABLE_DURABLILITY, (htableDescriptor.getDurability() != null ? htableDescriptor.getDurability().name() : null)); + } + switch (operation) { case CREATE_TABLE: table.setAttribute(ATTR_CREATE_TIME, now); table.setAttribute(ATTR_MODIFIED_TIME, now); break; + case CREATE_COLUMN_FAMILY: + table.setAttribute(ATTR_MODIFIED_TIME, now); + break; case ALTER_TABLE: + case ALTER_COLUMN_FAMILY: table.setAttribute(ATTR_MODIFIED_TIME, now); break; default: @@ -466,8 +503,27 @@ public class HBaseAtlasHook extends AtlasHook { columnFamily.setAttribute(ATTR_OWNER, hbaseOperationContext.getOwner()); columnFamily.setAttribute(ATTR_TABLE, AtlasTypeUtil.getAtlasObjectId(table)); + if (hColumnDescriptor!= null) { + columnFamily.setAttribute(ATTR_CF_BLOCK_CACHE_ENABLED, hColumnDescriptor.isBlockCacheEnabled()); + columnFamily.setAttribute(ATTR_CF_BLOOMFILTER_TYPE, (hColumnDescriptor.getBloomFilterType() != null ? hColumnDescriptor.getBloomFilterType().name():null)); + columnFamily.setAttribute(ATTR_CF_CACHED_BLOOM_ON_WRITE, hColumnDescriptor.isCacheBloomsOnWrite()); + columnFamily.setAttribute(ATTR_CF_CACHED_DATA_ON_WRITE, hColumnDescriptor.isCacheBloomsOnWrite()); + columnFamily.setAttribute(ATTR_CF_CACHED_INDEXES_ON_WRITE, hColumnDescriptor.isCacheIndexesOnWrite()); + columnFamily.setAttribute(ATTR_CF_COMPACTION_COMPRESSION_TYPE, (hColumnDescriptor.getCompactionCompressionType() != null ? hColumnDescriptor.getCompactionCompressionType().name():null)); + columnFamily.setAttribute(ATTR_CF_COMPRESSION_TYPE, (hColumnDescriptor.getCompressionType() != null ? hColumnDescriptor.getCompressionType().name():null)); + columnFamily.setAttribute(ATTR_CF_DATA_BLOCK_ENCODING, (hColumnDescriptor.getDataBlockEncoding() != null ? hColumnDescriptor.getDataBlockEncoding().name():null)); + columnFamily.setAttribute(ATTR_CF_ENCRYPTION_TYPE, hColumnDescriptor.getEncryptionType()); + columnFamily.setAttribute(ATTR_CF_EVICT_BLOCK_ONCLOSE, hColumnDescriptor.isEvictBlocksOnClose()); + columnFamily.setAttribute(ATTR_CF_KEEP_DELETE_CELLS, ( hColumnDescriptor.getKeepDeletedCells() != null ? hColumnDescriptor.getKeepDeletedCells().name():null)); + columnFamily.setAttribute(ATTR_CF_MAX_VERSIONS, hColumnDescriptor.getMaxVersions()); + columnFamily.setAttribute(ATTR_CF_MIN_VERSIONS, hColumnDescriptor.getMinVersions()); + columnFamily.setAttribute(ATTR_CF_PREFETCH_BLOCK_ONOPEN, hColumnDescriptor.isPrefetchBlocksOnOpen()); + columnFamily.setAttribute(ATTR_CF_TTL, hColumnDescriptor.getTimeToLive()); + } + switch (hbaseOperationContext.getOperation()) { case CREATE_COLUMN_FAMILY: + case CREATE_TABLE: columnFamily.setAttribute(ATTR_CREATE_TIME, now); columnFamily.setAttribute(ATTR_MODIFIED_TIME, now); break; http://git-wip-us.apache.org/repos/asf/atlas/blob/19d67a13/addons/models/1000-Hadoop/patches/005-hbase_table_column_family_add_additional_attribute.json ---------------------------------------------------------------------- diff --git a/addons/models/1000-Hadoop/patches/005-hbase_table_column_family_add_additional_attribute.json b/addons/models/1000-Hadoop/patches/005-hbase_table_column_family_add_additional_attribute.json new file mode 100644 index 0000000..0ee7908 --- /dev/null +++ b/addons/models/1000-Hadoop/patches/005-hbase_table_column_family_add_additional_attribute.json @@ -0,0 +1,183 @@ +{ + "patches": [ + { + "action": "ADD_ATTRIBUTE", + "typeName": "hbase_table", + "applyToVersion": "1.2", + "updateToVersion": "1.3", + "params": null, + "attributeDefs": [ + { + "name": "maxFileSize", + "typeName": "int", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "isReadOnly", + "typeName": "boolean", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "isCompactionEnabled", + "typeName": "boolean", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "replicasPerRegion", + "typeName": "int", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "durability", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + + ] + }, + { + "action": "ADD_ATTRIBUTE", + "typeName": "hbase_column_family", + "applyToVersion": "1.1", + "updateToVersion": "1.2", + "params": null, + "attributeDefs": [ + { + "name": "bloomFilterType", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "compressionType", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "compactionCompressionType", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "encryptionType", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "keepDeletedCells", + "typeName": "boolean", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "maxVersions", + "typeName": "int", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "minVersions", + "typeName": "int", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "dataBlockEncoding", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "ttl", + "typeName": "int", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "blockCacheEnabled", + "typeName": "boolean", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "cacheBloomsOnWrite", + "typeName": "boolean", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "cacheDataOnWrite", + "typeName": "boolean", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "cacheIndexesOnWrite", + "typeName": "boolean", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "evictBlocksOnClose", + "typeName": "boolean", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "prefetchBlocksOnOpen", + "typeName": "boolean", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + } + ] +} \ No newline at end of file
