Repository: atlas
Updated Branches:
  refs/heads/branch-0.8 9640a7c64 -> 043ab72b9


ATLAS-2446: updated HBase Model to capture more attributes of table and 
column-family

Signed-off-by: Madhan Neethiraj <mad...@apache.org>
(cherry picked from commit 19d67a13206a68c49ce92926f9513e833c423c83)


Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/043ab72b
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/043ab72b
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/043ab72b

Branch: refs/heads/branch-0.8
Commit: 043ab72b91755bd8b2b5fda112fbcdcfb0151ce8
Parents: 9640a7c
Author: rmani <rm...@hortonworks.com>
Authored: Wed Feb 14 14:43:16 2018 -0800
Committer: Madhan Neethiraj <mad...@apache.org>
Committed: Wed Feb 14 16:28:32 2018 -0800

----------------------------------------------------------------------
 .../atlas/hbase/bridge/HBaseAtlasHook.java      |  56 ++++++
 ..._column_family_add_additional_attribute.json | 183 +++++++++++++++++++
 2 files changed, 239 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/atlas/blob/043ab72b/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseAtlasHook.java
----------------------------------------------------------------------
diff --git 
a/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseAtlasHook.java
 
b/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseAtlasHook.java
index f544a8c..3a2a6ab 100644
--- 
a/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseAtlasHook.java
+++ 
b/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseAtlasHook.java
@@ -92,6 +92,30 @@ public class HBaseAtlasHook extends AtlasHook {
     public static final String ATTR_OWNER           = "owner";
     public static final String ATTR_NAME            = "name";
 
+    // column addition metadata
+    public static final String ATTR_TABLE_MAX_FILESIZE              = 
"maxFileSize";
+    public static final String ATTR_TABLE_ISREADONLY                = 
"isReadOnly";
+    public static final String ATTR_TABLE_ISCOMPACTION_ENABLED      = 
"isCompactionEnabled";
+    public static final String ATTR_TABLE_REPLICATION_PER_REGION    = 
"replicasPerRegion";
+    public static final String ATTR_TABLE_DURABLILITY               = 
"durability";
+
+    // column family additional metadata
+    public static final String ATTR_CF_BLOOMFILTER_TYPE             = 
"bloomFilterType";
+    public static final String ATTR_CF_COMPRESSION_TYPE             = 
"compressionType";
+    public static final String ATTR_CF_COMPACTION_COMPRESSION_TYPE  = 
"compactionCompressionType";
+    public static final String ATTR_CF_ENCRYPTION_TYPE              = 
"encryptionType";
+    public static final String ATTR_CF_KEEP_DELETE_CELLS            = 
"keepDeletedCells";
+    public static final String ATTR_CF_MAX_VERSIONS                 = 
"maxVersions";
+    public static final String ATTR_CF_MIN_VERSIONS                 = 
"minVersions";
+    public static final String ATTR_CF_DATA_BLOCK_ENCODING          = 
"dataBlockEncoding";
+    public static final String ATTR_CF_TTL                          = "ttl";
+    public static final String ATTR_CF_BLOCK_CACHE_ENABLED          = 
"blockCacheEnabled";
+    public static final String ATTR_CF_CACHED_BLOOM_ON_WRITE        = 
"cacheBloomsOnWrite";
+    public static final String ATTR_CF_CACHED_DATA_ON_WRITE         = 
"cacheDataOnWrite";
+    public static final String ATTR_CF_CACHED_INDEXES_ON_WRITE      = 
"cacheIndexesOnWrite";
+    public static final String ATTR_CF_EVICT_BLOCK_ONCLOSE          = 
"evictBlocksOnClose";
+    public static final String ATTR_CF_PREFETCH_BLOCK_ONOPEN        = 
"prefetchBlocksOnOpen";
+
     private static final String REFERENCEABLE_ATTRIBUTE_NAME = "qualifiedName";
     private              String clusterName                  = null;
 
@@ -422,12 +446,25 @@ public class HBaseAtlasHook extends AtlasHook {
         table.setAttribute(ATTR_PARAMETERS, 
hbaseOperationContext.getHbaseConf());
         table.setAttribute(ATTR_NAMESPACE, 
AtlasTypeUtil.getAtlasObjectId(nameSpace));
 
+        HTableDescriptor htableDescriptor = 
hbaseOperationContext.gethTableDescriptor();
+        if (htableDescriptor != null) {
+            table.setAttribute(ATTR_TABLE_MAX_FILESIZE, 
htableDescriptor.getMaxFileSize());
+            table.setAttribute(ATTR_TABLE_REPLICATION_PER_REGION, 
htableDescriptor.getRegionReplication());
+            table.setAttribute(ATTR_TABLE_ISREADONLY, 
htableDescriptor.getMaxFileSize());
+            table.setAttribute(ATTR_TABLE_ISCOMPACTION_ENABLED, 
htableDescriptor.isCompactionEnabled());
+            table.setAttribute(ATTR_TABLE_DURABLILITY, 
(htableDescriptor.getDurability() != null ? 
htableDescriptor.getDurability().name() : null));
+        }
+
         switch (operation) {
             case CREATE_TABLE:
                 table.setAttribute(ATTR_CREATE_TIME, now);
                 table.setAttribute(ATTR_MODIFIED_TIME, now);
                 break;
+            case CREATE_COLUMN_FAMILY:
+                table.setAttribute(ATTR_MODIFIED_TIME, now);
+                break;
             case ALTER_TABLE:
+            case ALTER_COLUMN_FAMILY:
                 table.setAttribute(ATTR_MODIFIED_TIME, now);
                 break;
             default:
@@ -466,8 +503,27 @@ public class HBaseAtlasHook extends AtlasHook {
         columnFamily.setAttribute(ATTR_OWNER, 
hbaseOperationContext.getOwner());
         columnFamily.setAttribute(ATTR_TABLE, 
AtlasTypeUtil.getAtlasObjectId(table));
 
+        if (hColumnDescriptor!= null) {
+            columnFamily.setAttribute(ATTR_CF_BLOCK_CACHE_ENABLED, 
hColumnDescriptor.isBlockCacheEnabled());
+            columnFamily.setAttribute(ATTR_CF_BLOOMFILTER_TYPE, 
(hColumnDescriptor.getBloomFilterType() != null ? 
hColumnDescriptor.getBloomFilterType().name():null));
+            columnFamily.setAttribute(ATTR_CF_CACHED_BLOOM_ON_WRITE, 
hColumnDescriptor.isCacheBloomsOnWrite());
+            columnFamily.setAttribute(ATTR_CF_CACHED_DATA_ON_WRITE, 
hColumnDescriptor.isCacheBloomsOnWrite());
+            columnFamily.setAttribute(ATTR_CF_CACHED_INDEXES_ON_WRITE, 
hColumnDescriptor.isCacheIndexesOnWrite());
+            columnFamily.setAttribute(ATTR_CF_COMPACTION_COMPRESSION_TYPE, 
(hColumnDescriptor.getCompactionCompressionType() != null ? 
hColumnDescriptor.getCompactionCompressionType().name():null));
+            columnFamily.setAttribute(ATTR_CF_COMPRESSION_TYPE, 
(hColumnDescriptor.getCompressionType() != null ? 
hColumnDescriptor.getCompressionType().name():null));
+            columnFamily.setAttribute(ATTR_CF_DATA_BLOCK_ENCODING, 
(hColumnDescriptor.getDataBlockEncoding() != null ? 
hColumnDescriptor.getDataBlockEncoding().name():null));
+            columnFamily.setAttribute(ATTR_CF_ENCRYPTION_TYPE, 
hColumnDescriptor.getEncryptionType());
+            columnFamily.setAttribute(ATTR_CF_EVICT_BLOCK_ONCLOSE, 
hColumnDescriptor.isEvictBlocksOnClose());
+            columnFamily.setAttribute(ATTR_CF_KEEP_DELETE_CELLS, ( 
hColumnDescriptor.getKeepDeletedCells() != null ? 
hColumnDescriptor.getKeepDeletedCells().name():null));
+            columnFamily.setAttribute(ATTR_CF_MAX_VERSIONS, 
hColumnDescriptor.getMaxVersions());
+            columnFamily.setAttribute(ATTR_CF_MIN_VERSIONS, 
hColumnDescriptor.getMinVersions());
+            columnFamily.setAttribute(ATTR_CF_PREFETCH_BLOCK_ONOPEN, 
hColumnDescriptor.isPrefetchBlocksOnOpen());
+            columnFamily.setAttribute(ATTR_CF_TTL, 
hColumnDescriptor.getTimeToLive());
+        }
+
         switch (hbaseOperationContext.getOperation()) {
             case CREATE_COLUMN_FAMILY:
+            case CREATE_TABLE:
                 columnFamily.setAttribute(ATTR_CREATE_TIME, now);
                 columnFamily.setAttribute(ATTR_MODIFIED_TIME, now);
                 break;

http://git-wip-us.apache.org/repos/asf/atlas/blob/043ab72b/addons/models/1000-Hadoop/patches/005-hbase_table_column_family_add_additional_attribute.json
----------------------------------------------------------------------
diff --git 
a/addons/models/1000-Hadoop/patches/005-hbase_table_column_family_add_additional_attribute.json
 
b/addons/models/1000-Hadoop/patches/005-hbase_table_column_family_add_additional_attribute.json
new file mode 100644
index 0000000..0ee7908
--- /dev/null
+++ 
b/addons/models/1000-Hadoop/patches/005-hbase_table_column_family_add_additional_attribute.json
@@ -0,0 +1,183 @@
+{
+  "patches": [
+    {
+      "action": "ADD_ATTRIBUTE",
+      "typeName": "hbase_table",
+      "applyToVersion": "1.2",
+      "updateToVersion": "1.3",
+      "params": null,
+      "attributeDefs": [
+        {
+          "name": "maxFileSize",
+          "typeName": "int",
+          "cardinality": "SINGLE",
+          "isIndexable": false,
+          "isOptional": true,
+          "isUnique": false
+        },
+        {
+        "name": "isReadOnly",
+        "typeName": "boolean",
+        "cardinality": "SINGLE",
+        "isIndexable": false,
+        "isOptional": true,
+        "isUnique": false
+        },
+        {
+          "name": "isCompactionEnabled",
+          "typeName": "boolean",
+          "cardinality": "SINGLE",
+          "isIndexable": false,
+          "isOptional": true,
+          "isUnique": false
+        },
+        {
+          "name": "replicasPerRegion",
+          "typeName": "int",
+          "cardinality": "SINGLE",
+          "isIndexable": false,
+          "isOptional": true,
+          "isUnique": false
+        },
+        {
+          "name": "durability",
+          "typeName": "string",
+          "cardinality": "SINGLE",
+          "isIndexable": false,
+          "isOptional": true,
+          "isUnique": false
+        }
+
+      ]
+    },
+    {
+      "action": "ADD_ATTRIBUTE",
+      "typeName": "hbase_column_family",
+      "applyToVersion": "1.1",
+      "updateToVersion": "1.2",
+      "params": null,
+      "attributeDefs": [
+        {
+          "name": "bloomFilterType",
+          "typeName": "string",
+          "cardinality": "SINGLE",
+          "isIndexable": false,
+          "isOptional": true,
+          "isUnique": false
+        },
+        {
+          "name": "compressionType",
+          "typeName": "string",
+          "cardinality": "SINGLE",
+          "isIndexable": false,
+          "isOptional": true,
+          "isUnique": false
+        },
+        {
+          "name": "compactionCompressionType",
+          "typeName": "string",
+          "cardinality": "SINGLE",
+          "isIndexable": false,
+          "isOptional": true,
+          "isUnique": false
+        },
+        {
+          "name": "encryptionType",
+          "typeName": "string",
+          "cardinality": "SINGLE",
+          "isIndexable": false,
+          "isOptional": true,
+          "isUnique": false
+        },
+        {
+          "name": "keepDeletedCells",
+          "typeName": "boolean",
+          "cardinality": "SINGLE",
+          "isIndexable": false,
+          "isOptional": true,
+          "isUnique": false
+        },
+        {
+          "name": "maxVersions",
+          "typeName": "int",
+          "cardinality": "SINGLE",
+          "isIndexable": false,
+          "isOptional": true,
+          "isUnique": false
+        },
+        {
+          "name": "minVersions",
+          "typeName": "int",
+          "cardinality": "SINGLE",
+          "isIndexable": false,
+          "isOptional": true,
+          "isUnique": false
+        },
+        {
+          "name": "dataBlockEncoding",
+          "typeName": "string",
+          "cardinality": "SINGLE",
+          "isIndexable": false,
+          "isOptional": true,
+          "isUnique": false
+        },
+        {
+          "name": "ttl",
+          "typeName": "int",
+          "cardinality": "SINGLE",
+          "isIndexable": false,
+          "isOptional": true,
+          "isUnique": false
+        },
+        {
+          "name": "blockCacheEnabled",
+          "typeName": "boolean",
+          "cardinality": "SINGLE",
+          "isIndexable": false,
+          "isOptional": true,
+          "isUnique": false
+        },
+        {
+          "name": "cacheBloomsOnWrite",
+          "typeName": "boolean",
+          "cardinality": "SINGLE",
+          "isIndexable": false,
+          "isOptional": true,
+          "isUnique": false
+        },
+        {
+          "name": "cacheDataOnWrite",
+          "typeName": "boolean",
+          "cardinality": "SINGLE",
+          "isIndexable": false,
+          "isOptional": true,
+          "isUnique": false
+        },
+        {
+          "name": "cacheIndexesOnWrite",
+          "typeName": "boolean",
+          "cardinality": "SINGLE",
+          "isIndexable": false,
+          "isOptional": true,
+          "isUnique": false
+        },
+        {
+          "name": "evictBlocksOnClose",
+          "typeName": "boolean",
+          "cardinality": "SINGLE",
+          "isIndexable": false,
+          "isOptional": true,
+          "isUnique": false
+        },
+        {
+          "name": "prefetchBlocksOnOpen",
+          "typeName": "boolean",
+          "cardinality": "SINGLE",
+          "isIndexable": false,
+          "isOptional": true,
+          "isUnique": false
+        }
+      ]
+    }
+  ]
+}
\ No newline at end of file

Reply via email to