http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java index b6f55a1..40866fe 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java @@ -23,10 +23,8 @@ import com.google.common.collect.ImmutableList; import com.sun.jersey.api.client.ClientResponse; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasServiceException; -import org.apache.atlas.fs.model.FSDataTypes; import org.apache.atlas.hive.HiveITBase; import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; -import org.apache.atlas.hive.model.HiveDataModelGenerator; import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Struct; @@ -45,7 +43,6 @@ import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.plan.HiveOperation; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.security.UserGroupInformation; -import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; @@ -80,7 +77,7 @@ public class HiveHookIT extends HiveITBase { String dbId = assertDatabaseIsRegistered(dbName); Referenceable definition = atlasClient.getEntity(dbId); - Map params = (Map) definition.get(HiveDataModelGenerator.PARAMETERS); + Map params = (Map) definition.get(HiveMetaStoreBridge.PARAMETERS); Assert.assertNotNull(params); Assert.assertEquals(params.size(), 2); Assert.assertEquals(params.get("p1"), "v1"); @@ -150,8 +147,8 @@ public class HiveHookIT extends HiveITBase { Referenceable colEntity = atlasClient.getEntity(colId); Assert.assertEquals(colEntity.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), String.format("%s.%s.%s@%s", dbName.toLowerCase(), tableName.toLowerCase(), colName.toLowerCase(), CLUSTER_NAME)); - Assert.assertNotNull(colEntity.get(HiveDataModelGenerator.TABLE)); - Assert.assertEquals(((Id) colEntity.get(HiveDataModelGenerator.TABLE))._getId(), tableId); + Assert.assertNotNull(colEntity.get(HiveMetaStoreBridge.TABLE)); + Assert.assertEquals(((Id) colEntity.get(HiveMetaStoreBridge.TABLE))._getId(), tableId); //assert that column.owner = table.owner Referenceable tableRef = atlasClient.getEntity(tableId); @@ -161,8 +158,8 @@ public class HiveHookIT extends HiveITBase { tableName = createTable(); tableId = assertTableIsRegistered(DEFAULT_DB, tableName); tableRef = atlasClient.getEntity(tableId); - Assert.assertEquals(tableRef.get(HiveDataModelGenerator.TABLE_TYPE_ATTR), TableType.MANAGED_TABLE.name()); - Assert.assertEquals(tableRef.get(HiveDataModelGenerator.COMMENT), "table comment"); + Assert.assertEquals(tableRef.get(HiveMetaStoreBridge.TABLE_TYPE_ATTR), TableType.MANAGED_TABLE.name()); + Assert.assertEquals(tableRef.get(HiveMetaStoreBridge.COMMENT), "table comment"); String entityName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName); Assert.assertEquals(tableRef.get(AtlasClient.NAME), tableName.toLowerCase()); Assert.assertEquals(tableRef.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), entityName); @@ -170,13 +167,13 @@ public class HiveHookIT extends HiveITBase { Table t = hiveMetaStoreBridge.hiveClient.getTable(DEFAULT_DB, tableName); long createTime = Long.parseLong(t.getMetadata().getProperty(hive_metastoreConstants.DDL_TIME)) * HiveMetaStoreBridge.MILLIS_CONVERT_FACTOR; - verifyTimestamps(tableRef, HiveDataModelGenerator.CREATE_TIME, createTime); - verifyTimestamps(tableRef, HiveDataModelGenerator.LAST_ACCESS_TIME, createTime); + verifyTimestamps(tableRef, HiveMetaStoreBridge.CREATE_TIME, createTime); + verifyTimestamps(tableRef, HiveMetaStoreBridge.LAST_ACCESS_TIME, createTime); - final Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC); - Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_IS_STORED_AS_SUB_DIRS), false); - Assert.assertNotNull(sdRef.get(HiveDataModelGenerator.TABLE)); - Assert.assertEquals(((Id) sdRef.get(HiveDataModelGenerator.TABLE))._getId(), tableId); + final Referenceable sdRef = (Referenceable) tableRef.get(HiveMetaStoreBridge.STORAGE_DESC); + Assert.assertEquals(sdRef.get(HiveMetaStoreBridge.STORAGE_IS_STORED_AS_SUB_DIRS), false); + Assert.assertNotNull(sdRef.get(HiveMetaStoreBridge.TABLE)); + Assert.assertEquals(((Id) sdRef.get(HiveMetaStoreBridge.TABLE))._getId(), tableId); //Create table where database doesn't exist, will create database instance as well assertDatabaseIsRegistered(DEFAULT_DB); @@ -878,7 +875,7 @@ public class HiveHookIT extends HiveITBase { String tableId = assertTableIsRegistered(DEFAULT_DB, tableName); Referenceable tableEntity = atlasClient.getEntity(tableId); - final String createTime = (String)tableEntity.get(HiveDataModelGenerator.CREATE_TIME); + final String createTime = (String)tableEntity.get(HiveMetaStoreBridge.CREATE_TIME); Assert.assertNotNull(createTime); String columnGuid = assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), NAME)); @@ -916,10 +913,10 @@ public class HiveHookIT extends HiveITBase { assertTableIsRegistered(newDBName, newTableName, new AssertPredicate() { @Override public void assertOnEntity(final Referenceable entity) throws Exception { - Referenceable sd = ((Referenceable) entity.get(HiveDataModelGenerator.STORAGE_DESC)); - String location = (String) sd.get(HiveDataModelGenerator.LOCATION); + Referenceable sd = ((Referenceable) entity.get(HiveMetaStoreBridge.STORAGE_DESC)); + String location = (String) sd.get(HiveMetaStoreBridge.LOCATION); assertTrue(location.contains(newTableName)); - Assert.assertEquals(entity.get(HiveDataModelGenerator.CREATE_TIME), createTime); + Assert.assertEquals(entity.get(HiveMetaStoreBridge.CREATE_TIME), createTime); } }); } @@ -929,7 +926,7 @@ public class HiveHookIT extends HiveITBase { Referenceable tableRef = atlasClient.getEntity(tableId); //with soft delete, the deleted columns are returned as well. So, filter the deleted ones - List<Referenceable> columns = ((List<Referenceable>) tableRef.get(HiveDataModelGenerator.COLUMNS)); + List<Referenceable> columns = ((List<Referenceable>) tableRef.get(HiveMetaStoreBridge.COLUMNS)); List<Referenceable> activeColumns = new ArrayList<>(); for (Referenceable col : columns) { if (col.getId().getState() == Id.EntityState.ACTIVE) { @@ -1046,7 +1043,7 @@ public class HiveHookIT extends HiveITBase { assertColumnIsRegistered(newColQualifiedName, new AssertPredicate() { @Override public void assertOnEntity(Referenceable entity) throws Exception { - assertEquals(entity.get(HiveDataModelGenerator.COMMENT), comment); + assertEquals(entity.get(HiveMetaStoreBridge.COMMENT), comment); } }); @@ -1071,7 +1068,7 @@ public class HiveHookIT extends HiveITBase { assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() { @Override public void assertOnEntity(Referenceable entity) throws Exception { - List<Referenceable> columns = (List<Referenceable>) entity.get(HiveDataModelGenerator.COLUMNS); + List<Referenceable> columns = (List<Referenceable>) entity.get(HiveMetaStoreBridge.COLUMNS); assertEquals(columns.get(0).get(NAME), finalNewColName); assertEquals(columns.get(1).get(NAME), "id"); } @@ -1099,7 +1096,7 @@ public class HiveHookIT extends HiveITBase { assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() { @Override public void assertOnEntity(Referenceable entity) throws Exception { - List<Referenceable> columns = (List<Referenceable>) entity.get(HiveDataModelGenerator.COLUMNS); + List<Referenceable> columns = (List<Referenceable>) entity.get(HiveMetaStoreBridge.COLUMNS); assertEquals(columns.get(1).get(NAME), finalNewColName2); assertEquals(columns.get(0).get(NAME), "id"); } @@ -1289,8 +1286,8 @@ public class HiveHookIT extends HiveITBase { assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() { @Override public void assertOnEntity(Referenceable tableRef) throws Exception { - Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC); - Assert.assertEquals(new Path((String)sdRef.get(HiveDataModelGenerator.LOCATION)).toString(), new Path(testPath).toString()); + Referenceable sdRef = (Referenceable) tableRef.get(HiveMetaStoreBridge.STORAGE_DESC); + Assert.assertEquals(new Path((String)sdRef.get(HiveMetaStoreBridge.LOCATION)).toString(), new Path(testPath).toString()); } }); @@ -1313,18 +1310,18 @@ public class HiveHookIT extends HiveITBase { assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() { @Override public void assertOnEntity(Referenceable tableRef) throws Exception { - Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC); - Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_DESC_INPUT_FMT), + Referenceable sdRef = (Referenceable) tableRef.get(HiveMetaStoreBridge.STORAGE_DESC); + Assert.assertEquals(sdRef.get(HiveMetaStoreBridge.STORAGE_DESC_INPUT_FMT), "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"); - Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_DESC_OUTPUT_FMT), + Assert.assertEquals(sdRef.get(HiveMetaStoreBridge.STORAGE_DESC_OUTPUT_FMT), "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat"); Assert.assertNotNull(sdRef.get("serdeInfo")); Struct serdeInfo = (Struct) sdRef.get("serdeInfo"); Assert.assertEquals(serdeInfo.get("serializationLib"), "org.apache.hadoop.hive.ql.io.orc.OrcSerde"); - Assert.assertNotNull(serdeInfo.get(HiveDataModelGenerator.PARAMETERS)); + Assert.assertNotNull(serdeInfo.get(HiveMetaStoreBridge.PARAMETERS)); Assert.assertEquals( - ((Map<String, String>) serdeInfo.get(HiveDataModelGenerator.PARAMETERS)) + ((Map<String, String>) serdeInfo.get(HiveMetaStoreBridge.PARAMETERS)) .get("serialization.format"), "1"); } @@ -1337,10 +1334,10 @@ public class HiveHookIT extends HiveITBase { * runCommand(query); * tableRef = atlasClient.getEntity(tableId); - * sdRef = (Referenceable)tableRef.get(HiveDataModelGenerator.STORAGE_DESC); - * Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_DESC_INPUT_FMT), "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"); - * Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_DESC_OUTPUT_FMT), "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat"); - * Assert.assertEquals(((Map) sdRef.get(HiveDataModelGenerator.PARAMETERS)).get("orc.compress"), "ZLIB"); + * sdRef = (Referenceable)tableRef.get(HiveMetaStoreBridge.STORAGE_DESC); + * Assert.assertEquals(sdRef.get(HiveMetaStoreBridge.STORAGE_DESC_INPUT_FMT), "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"); + * Assert.assertEquals(sdRef.get(HiveMetaStoreBridge.STORAGE_DESC_OUTPUT_FMT), "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat"); + * Assert.assertEquals(((Map) sdRef.get(HiveMetaStoreBridge.PARAMETERS)).get("orc.compress"), "ZLIB"); */ } @@ -1375,8 +1372,8 @@ public class HiveHookIT extends HiveITBase { private void verifyBucketSortingProperties(Referenceable tableRef, int numBuckets, ImmutableList<String> bucketColNames, ImmutableList<String> sortcolNames) throws Exception { - Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC); - Assert.assertEquals(((scala.math.BigInt) sdRef.get(HiveDataModelGenerator.STORAGE_NUM_BUCKETS)).intValue(), + Referenceable sdRef = (Referenceable) tableRef.get(HiveMetaStoreBridge.STORAGE_DESC); + Assert.assertEquals(((scala.math.BigInt) sdRef.get(HiveMetaStoreBridge.STORAGE_NUM_BUCKETS)).intValue(), numBuckets); Assert.assertEquals(sdRef.get("bucketCols"), bucketColNames); @@ -1661,7 +1658,7 @@ public class HiveHookIT extends HiveITBase { assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() { @Override public void assertOnEntity(Referenceable tableRef) throws Exception { - Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC); + Referenceable sdRef = (Referenceable) tableRef.get(HiveMetaStoreBridge.STORAGE_DESC); Struct serdeInfo = (Struct) sdRef.get("serdeInfo"); Assert.assertEquals(serdeInfo.get("serializationLib"), serdeLib); verifyProperties(serdeInfo, expectedProps, false); @@ -1670,7 +1667,7 @@ public class HiveHookIT extends HiveITBase { } private void verifyProperties(Struct referenceable, Map<String, String> expectedProps, boolean checkIfNotExists) { - Map<String, String> parameters = (Map<String, String>) referenceable.get(HiveDataModelGenerator.PARAMETERS); + Map<String, String> parameters = (Map<String, String>) referenceable.get(HiveMetaStoreBridge.PARAMETERS); if (checkIfNotExists == false) { //Check if properties exist @@ -1744,7 +1741,7 @@ public class HiveHookIT extends HiveITBase { } private String getDSTypeName(Entity entity) { - return Entity.Type.TABLE.equals(entity.getType()) ? HiveDataTypes.HIVE_TABLE.name() : FSDataTypes.HDFS_PATH().toString(); + return Entity.Type.TABLE.equals(entity.getType()) ? HiveDataTypes.HIVE_TABLE.name() : HiveMetaStoreBridge.HDFS_PATH.toString(); } private <T extends Entity> SortedMap<T, Referenceable> getSortedProcessDataSets(Set<T> inputTbls) {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0010-base_model.json ---------------------------------------------------------------------- diff --git a/addons/models/0010-base_model.json b/addons/models/0010-base_model.json new file mode 100644 index 0000000..7f64d85 --- /dev/null +++ b/addons/models/0010-base_model.json @@ -0,0 +1,97 @@ +{ + "enumDefs": [], + "structDefs": [], + "classificationDefs": [], + "entityDefs": [ + { + "name": "Referenceable", + "superTypes": [], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "qualifiedName", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": true + } + ] + }, + { + "name": "Asset", + "superTypes": [], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "name", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "description", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "owner", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": true, + "isUnique": false + } + ] + }, + { + "name": "DataSet", + "superTypes": [ + "Referenceable", + "Asset" + ], + "typeVersion": "1.0", + "attributeDefs": [] + }, + { + "name": "Infrastructure", + "superTypes": [ + "Referenceable", + "Asset" + ], + "typeVersion": "1.0", + "attributeDefs": [] + }, + { + "name": "Process", + "superTypes": [ + "Referenceable", + "Asset" + ], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "inputs", + "typeName": "array<DataSet>", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "outputs", + "typeName": "array<DataSet>", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0020-fs_model.json ---------------------------------------------------------------------- diff --git a/addons/models/0020-fs_model.json b/addons/models/0020-fs_model.json new file mode 100644 index 0000000..ff17261 --- /dev/null +++ b/addons/models/0020-fs_model.json @@ -0,0 +1,191 @@ +{ + "enumDefs": [ + { + "name": "file_action", + "typeVersion": "1.0", + "elementDefs": [ + { + "ordinal": 0, + "value": "NONE" + }, + { + "ordinal": 1, + "value": "EXECUTE" + }, + { + "ordinal": 2, + "value": "WRITE" + }, + { + "ordinal": 3, + "value": "WRITE_EXECUTE" + }, + { + "ordinal": 4, + "value": "READ" + }, + { + "ordinal": 5, + "value": "READ_EXECUTE" + }, + { + "ordinal": 6, + "value": "READ_WRITE" + }, + { + "ordinal": 7, + "value": "ALL" + } + ] + } + ], + "structDefs": [ + { + "name": "fs_permissions", + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "group", + "typeName": "file_action", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "user", + "typeName": "file_action", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "sticky", + "typeName": "boolean", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "others", + "typeName": "file_action", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + } + ] + } + ], + "classificationDefs": [], + "entityDefs": [ + { + "name": "fs_path", + "superTypes": [ + "DataSet" + ], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "path", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "createTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": true, + "isUnique": false + }, + { + "name": "modifiedTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": true, + "isUnique": false + }, + { + "name": "isFile", + "typeName": "boolean", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": true, + "isUnique": false + }, + { + "name": "isSymlink", + "typeName": "boolean", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "fileSize", + "typeName": "long", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "group", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": true, + "isUnique": false + }, + { + "name": "posixPermissions", + "typeName": "fs_permissions", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": true, + "isUnique": false + } + ] + }, + { + "name": "hdfs_path", + "superTypes": [ + "fs_path" + ], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "clusterName", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": true, + "isUnique": false + }, + { + "name": "numberOfReplicas", + "typeName": "int", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": true, + "isUnique": false + }, + { + "name": "extendedAttributes", + "typeName": "map<string,string>", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0030-hive_model.json ---------------------------------------------------------------------- diff --git a/addons/models/0030-hive_model.json b/addons/models/0030-hive_model.json new file mode 100644 index 0000000..2c2e9ed --- /dev/null +++ b/addons/models/0030-hive_model.json @@ -0,0 +1,498 @@ +{ + "enumDefs": [ + { + "name": "hive_principal_type", + "typeVersion": "1.0", + "elementDefs": [ + { + "ordinal": 1, + "value": "USER" + }, + { + "ordinal": 2, + "value": "ROLE" + }, + { + "ordinal": 3, + "value": "GROUP" + } + ] + } + ], + "structDefs": [ + { + "name": "hive_order", + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "order", + "typeName": "int", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "col", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + } + ] + }, + { + "name": "hive_serde", + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "name", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "serializationLib", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "parameters", + "typeName": "map<string,string>", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + } + ], + "classificationDefs": [], + "entityDefs": [ + { + "name": "hive_process", + "superTypes": [ + "Process" + ], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "startTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "endTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "userName", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "operationType", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "queryText", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "queryPlan", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "queryId", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "recentQueries", + "typeName": "array<string>", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "clusterName", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "queryGraph", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + }, + { + "name": "hive_table", + "superTypes": [ + "DataSet" + ], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "db", + "typeName": "hive_db", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "createTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "lastAccessTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "comment", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "retention", + "typeName": "int", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "sd", + "typeName": "hive_storagedesc", + "cardinality": "SINGLE", + "constraintDefs": [ + { + "type": "mappedFromRef", + "params": { + "refAttribute": "table" + } + } + ], + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "partitionKeys", + "typeName": "array<hive_column>", + "cardinality": "SINGLE", + "constraintDefs": [ + { + "type": "foreignKey" + } + ], + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "aliases", + "typeName": "array<string>", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "columns", + "typeName": "array<hive_column>", + "cardinality": "SINGLE", + "constraintDefs": [ + { + "type": "mappedFromRef", + "params": { + "refAttribute": "table" + } + } + ], + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "parameters", + "typeName": "map<string,string>", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "viewOriginalText", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "viewExpandedText", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "tableType", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "temporary", + "typeName": "boolean", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": true, + "isUnique": false + } + ] + }, + { + "name": "hive_storagedesc", + "superTypes": [ + "Referenceable" + ], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "table", + "typeName": "hive_table", + "cardinality": "SINGLE", + "constraintDefs": [ + { + "type": "foreignKey", + "params": { + "onDelete": "cascade" + } + } + ], + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "location", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "inputFormat", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "outputFormat", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "compressed", + "typeName": "boolean", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "numBuckets", + "typeName": "int", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "serdeInfo", + "typeName": "hive_serde", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "bucketCols", + "typeName": "array<string>", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "sortCols", + "typeName": "array<hive_order>", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "parameters", + "typeName": "map<string,string>", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "storedAsSubDirectories", + "typeName": "boolean", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + }, + { + "name": "hive_db", + "superTypes": [ + "Referenceable", + "Asset" + ], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "clusterName", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "location", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "parameters", + "typeName": "map<string,string>", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "ownerType", + "typeName": "hive_principal_type", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + }, + { + "name": "hive_column", + "superTypes": [ + "DataSet" + ], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "type", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "comment", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "table", + "typeName": "hive_table", + "cardinality": "SINGLE", + "constraintDefs": [ + { + "type": "foreignKey", + "params": { + "onDelete": "cascade" + } + } + ], + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0040-sqoop_model.json ---------------------------------------------------------------------- diff --git a/addons/models/0040-sqoop_model.json b/addons/models/0040-sqoop_model.json new file mode 100644 index 0000000..f5c7fd9 --- /dev/null +++ b/addons/models/0040-sqoop_model.json @@ -0,0 +1,97 @@ +{ + "enumDefs": [], + "structDefs": [], + "classificationDefs": [], + "entityDefs": [ + { + "name": "sqoop_process", + "superTypes": [ + "Process" + ], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "operation", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "commandlineOpts", + "typeName": "map<string,string>", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "startTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "endTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "userName", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": true, + "isUnique": false + } + ] + }, + { + "name": "sqoop_dbdatastore", + "superTypes": [ + "DataSet" + ], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "dbStoreType", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "storeUse", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "storeUri", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "source", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": true, + "isUnique": false + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0050-falcon_model.json ---------------------------------------------------------------------- diff --git a/addons/models/0050-falcon_model.json b/addons/models/0050-falcon_model.json new file mode 100644 index 0000000..b7398ef --- /dev/null +++ b/addons/models/0050-falcon_model.json @@ -0,0 +1,147 @@ +{ + "enumDefs": [], + "structDefs": [], + "classificationDefs": [], + "entityDefs": [ + { + "name": "falcon_feed_replication", + "superTypes": [ + "Process" + ], + "typeVersion": "1.0", + "attributeDefs": [] + }, + { + "name": "falcon_cluster", + "superTypes": [ + "Infrastructure" + ], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "colo", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "tags", + "typeName": "map<string,string>", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + }, + { + "name": "falcon_feed", + "superTypes": [ + "DataSet" + ], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "frequency", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "stored-in", + "typeName": "falcon_cluster", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "groups", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "tags", + "typeName": "map<string,string>", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + }, + { + "name": "falcon_process", + "superTypes": [ + "Process" + ], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "frequency", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "runs-on", + "typeName": "falcon_cluster", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "tags", + "typeName": "map<string,string>", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "pipelines", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": true, + "isUnique": false + }, + { + "name": "workflow-properties", + "typeName": "map<string,string>", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + }, + { + "name": "falcon_feed_creation", + "superTypes": [ + "Process" + ], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "stored-in", + "typeName": "falcon_cluster", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0060-hbase_model.json ---------------------------------------------------------------------- diff --git a/addons/models/0060-hbase_model.json b/addons/models/0060-hbase_model.json new file mode 100644 index 0000000..d249d87 --- /dev/null +++ b/addons/models/0060-hbase_model.json @@ -0,0 +1,24 @@ +{ + "enumDefs": [], + "structDefs": [], + "classificationDefs": [], + "entityDefs": [ + { + "name": "hbase_table", + "superTypes": [ + "DataSet" + ], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "uri", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0070-kafka_model.json ---------------------------------------------------------------------- diff --git a/addons/models/0070-kafka_model.json b/addons/models/0070-kafka_model.json new file mode 100644 index 0000000..b7f6e33 --- /dev/null +++ b/addons/models/0070-kafka_model.json @@ -0,0 +1,57 @@ +{ + "enumDefs": [], + "structDefs": [], + "classificationDefs": [], + "entityDefs": [ + { + "name": "kafka_topic", + "superTypes": [ + "DataSet" + ], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "topic", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": true + }, + { + "name": "uri", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + } + ] + }, + { + "name": "jms_topic", + "superTypes": [ + "DataSet" + ], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "topic", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": true + }, + { + "name": "uri", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0080-storm_model.json ---------------------------------------------------------------------- diff --git a/addons/models/0080-storm_model.json b/addons/models/0080-storm_model.json new file mode 100644 index 0000000..59f3228 --- /dev/null +++ b/addons/models/0080-storm_model.json @@ -0,0 +1,150 @@ +{ + "enumDefs": [], + "structDefs": [], + "classificationDefs": [], + "entityDefs": [ + { + "name": "storm_topology", + "superTypes": [ + "Process" + ], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "id", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": true + }, + { + "name": "startTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "endTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "conf", + "typeName": "map<string,string>", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "clusterName", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": true, + "isUnique": false + }, + { + "name": "nodes", + "typeName": "array<storm_node>", + "cardinality": "LIST", + "constraintDefs": [ + { + "type": "foreignKey" + } + ], + "isIndexable": false, + "isOptional": false, + "isUnique": false + } + ] + }, + { + "name": "storm_node", + "superTypes": [], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "name", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "description", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": true, + "isUnique": false + }, + { + "name": "driverClass", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "conf", + "typeName": "map<string,string>", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + }, + { + "name": "storm_spout", + "superTypes": [ + "storm_node" + ], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "outputs", + "typeName": "array<string>", + "cardinality": "LIST", + "isIndexable": false, + "isOptional": false, + "isUnique": false + } + ] + }, + { + "name": "storm_bolt", + "superTypes": [ + "storm_node" + ], + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "inputs", + "typeName": "array<string>", + "cardinality": "LIST", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "outputs", + "typeName": "array<string>", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/patches/001-hive_column_add_position.json ---------------------------------------------------------------------- diff --git a/addons/models/patches/001-hive_column_add_position.json b/addons/models/patches/001-hive_column_add_position.json new file mode 100644 index 0000000..eeab74a --- /dev/null +++ b/addons/models/patches/001-hive_column_add_position.json @@ -0,0 +1,21 @@ +{ + "patches": [ + { + "action": "ADD_ATTRIBUTE", + "typeName": "hive_column", + "applyToVersion": "1.0", + "updateToVersion": "1.1", + "params": null, + "attributeDefs": [ + { + "name": "position", + "typeName": "int", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/sqoop-bridge/pom.xml ---------------------------------------------------------------------- diff --git a/addons/sqoop-bridge/pom.xml b/addons/sqoop-bridge/pom.xml index 9ac411e..e71d64a 100644 --- a/addons/sqoop-bridge/pom.xml +++ b/addons/sqoop-bridge/pom.xml @@ -411,24 +411,6 @@ <version>1.2.1</version> <inherited>false</inherited> <executions> - <execution> - <configuration> - <mainClass>org.apache.atlas.sqoop.model.SqoopDataModelGenerator</mainClass> - <systemProperties> - <systemProperty> - <key>atlas.conf</key> - <value>${project.build.directory}/../../../typesystem/target/test-classes</value> - </systemProperty> - </systemProperties> - <arguments> - <argument>${project.build.directory}/models/sqoop_model.json</argument> - </arguments> - </configuration> - <phase>package</phase> - <goals> - <goal>java</goal> - </goals> - </execution> </executions> </plugin> </plugins> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java ---------------------------------------------------------------------- diff --git a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java index e43fc86..6fb27e5 100644 --- a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java +++ b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java @@ -23,11 +23,9 @@ import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasConstants; import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; -import org.apache.atlas.hive.model.HiveDataModelGenerator; import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.hook.AtlasHook; import org.apache.atlas.notification.hook.HookNotification; -import org.apache.atlas.sqoop.model.SqoopDataModelGenerator; import org.apache.atlas.sqoop.model.SqoopDataTypes; import org.apache.atlas.typesystem.Referenceable; import org.apache.commons.configuration.Configuration; @@ -55,6 +53,20 @@ public class SqoopHook extends SqoopJobDataPublisher { public static final String ATLAS_CLUSTER_NAME = "atlas.cluster.name"; public static final String DEFAULT_CLUSTER_NAME = "primary"; + public static final String USER = "userName"; + public static final String DB_STORE_TYPE = "dbStoreType"; + public static final String DB_STORE_USAGE = "storeUse"; + public static final String SOURCE = "source"; + public static final String DESCRIPTION = "description"; + public static final String STORE_URI = "storeUri"; + public static final String OPERATION = "operation"; + public static final String START_TIME = "startTime"; + public static final String END_TIME = "endTime"; + public static final String CMD_LINE_OPTS = "commandlineOpts"; + // multiple inputs and outputs for process + public static final String INPUTS = "inputs"; + public static final String OUTPUTS = "outputs"; + static { org.apache.hadoop.conf.Configuration.addDefaultResource("sqoop-site.xml"); } @@ -75,7 +87,7 @@ public class SqoopHook extends SqoopJobDataPublisher { tableRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, HiveMetaStoreBridge.getTableQualifiedName(clusterName, dbName, tableName)); tableRef.set(AtlasClient.NAME, tableName.toLowerCase()); - tableRef.set(HiveDataModelGenerator.DB, dbRef); + tableRef.set(HiveMetaStoreBridge.DB, dbRef); return tableRef; } @@ -94,11 +106,11 @@ public class SqoopHook extends SqoopJobDataPublisher { String name = getSqoopDBStoreName(data); storeRef.set(AtlasClient.NAME, name); storeRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name); - storeRef.set(SqoopDataModelGenerator.DB_STORE_TYPE, data.getStoreType()); - storeRef.set(SqoopDataModelGenerator.DB_STORE_USAGE, usage); - storeRef.set(SqoopDataModelGenerator.STORE_URI, data.getUrl()); - storeRef.set(SqoopDataModelGenerator.SOURCE, source); - storeRef.set(SqoopDataModelGenerator.DESCRIPTION, ""); + storeRef.set(SqoopHook.DB_STORE_TYPE, data.getStoreType()); + storeRef.set(SqoopHook.DB_STORE_USAGE, usage); + storeRef.set(SqoopHook.STORE_URI, data.getUrl()); + storeRef.set(SqoopHook.SOURCE, source); + storeRef.set(SqoopHook.DESCRIPTION, ""); storeRef.set(AtlasClient.OWNER, data.getUser()); return storeRef; } @@ -109,24 +121,24 @@ public class SqoopHook extends SqoopJobDataPublisher { final String sqoopProcessName = getSqoopProcessName(data, clusterName); procRef.set(AtlasClient.NAME, sqoopProcessName); procRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, sqoopProcessName); - procRef.set(SqoopDataModelGenerator.OPERATION, data.getOperation()); + procRef.set(SqoopHook.OPERATION, data.getOperation()); if (isImportOperation(data)) { - procRef.set(SqoopDataModelGenerator.INPUTS, dbStoreRef); - procRef.set(SqoopDataModelGenerator.OUTPUTS, hiveTableRef); + procRef.set(SqoopHook.INPUTS, dbStoreRef); + procRef.set(SqoopHook.OUTPUTS, hiveTableRef); } else { - procRef.set(SqoopDataModelGenerator.INPUTS, hiveTableRef); - procRef.set(SqoopDataModelGenerator.OUTPUTS, dbStoreRef); + procRef.set(SqoopHook.INPUTS, hiveTableRef); + procRef.set(SqoopHook.OUTPUTS, dbStoreRef); } - procRef.set(SqoopDataModelGenerator.USER, data.getUser()); - procRef.set(SqoopDataModelGenerator.START_TIME, new Date(data.getStartTime())); - procRef.set(SqoopDataModelGenerator.END_TIME, new Date(data.getEndTime())); + procRef.set(SqoopHook.USER, data.getUser()); + procRef.set(SqoopHook.START_TIME, new Date(data.getStartTime())); + procRef.set(SqoopHook.END_TIME, new Date(data.getEndTime())); Map<String, String> sqoopOptionsMap = new HashMap<>(); Properties options = data.getOptions(); for (Object k : options.keySet()) { sqoopOptionsMap.put((String)k, (String) options.get(k)); } - procRef.set(SqoopDataModelGenerator.CMD_LINE_OPTS, sqoopOptionsMap); + procRef.set(SqoopHook.CMD_LINE_OPTS, sqoopOptionsMap); return procRef; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java ---------------------------------------------------------------------- diff --git a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java deleted file mode 100644 index 3373246..0000000 --- a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java +++ /dev/null @@ -1,187 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.sqoop.model; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; - -import org.apache.atlas.AtlasClient; -import org.apache.atlas.AtlasException; -import org.apache.atlas.addons.ModelDefinitionDump; -import org.apache.atlas.typesystem.TypesDef; -import org.apache.atlas.typesystem.json.TypesSerialization; -import org.apache.atlas.typesystem.types.AttributeDefinition; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.DataTypes; -import org.apache.atlas.typesystem.types.EnumType; -import org.apache.atlas.typesystem.types.EnumTypeDefinition; -import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; -import org.apache.atlas.typesystem.types.Multiplicity; -import org.apache.atlas.typesystem.types.StructTypeDefinition; -import org.apache.atlas.typesystem.types.TraitType; -import org.apache.atlas.typesystem.types.utils.TypesUtil; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; - -/** - * Utility that generates Sqoop data model for both metastore entities and DDL/DML queries. - */ -public class SqoopDataModelGenerator { - - private static final Logger LOG = LoggerFactory.getLogger(SqoopDataModelGenerator.class); - - private final Map<String, HierarchicalTypeDefinition<ClassType>> classTypeDefinitions; - private final Map<String, EnumTypeDefinition> enumTypeDefinitionMap; - private final Map<String, StructTypeDefinition> structTypeDefinitionMap; - private static final DataTypes.MapType STRING_MAP_TYPE = - new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE); - - public static final String USER = "userName"; - public static final String DB_STORE_TYPE = "dbStoreType"; - public static final String DB_STORE_USAGE = "storeUse"; - public static final String SOURCE = "source"; - public static final String DESCRIPTION = "description"; - public static final String STORE_URI = "storeUri"; - public static final String OPERATION = "operation"; - public static final String START_TIME = "startTime"; - public static final String END_TIME = "endTime"; - public static final String CMD_LINE_OPTS = "commandlineOpts"; - // multiple inputs and outputs for process - public static final String INPUTS = "inputs"; - public static final String OUTPUTS = "outputs"; - - public SqoopDataModelGenerator() { - classTypeDefinitions = new HashMap<>(); - enumTypeDefinitionMap = new HashMap<>(); - structTypeDefinitionMap = new HashMap<>(); - } - - public void createDataModel() throws AtlasException { - LOG.info("Generating the Sqoop Data Model...."); - - // enums - - // structs - - // classes - createSqoopDbStoreClass(); - - // DDL/DML Process - createSqoopProcessClass(); - } - - public TypesDef getTypesDef() { - return TypesUtil.getTypesDef(getEnumTypeDefinitions(), getStructTypeDefinitions(), getTraitTypeDefinitions(), - getClassTypeDefinitions()); - } - - public String getDataModelAsJSON() { - return TypesSerialization.toJson(getTypesDef()); - } - - public ImmutableList<EnumTypeDefinition> getEnumTypeDefinitions() { - return ImmutableList.copyOf(enumTypeDefinitionMap.values()); - } - - public ImmutableList<StructTypeDefinition> getStructTypeDefinitions() { - return ImmutableList.copyOf(structTypeDefinitionMap.values()); - } - - public ImmutableList<HierarchicalTypeDefinition<ClassType>> getClassTypeDefinitions() { - return ImmutableList.copyOf(classTypeDefinitions.values()); - } - - public ImmutableList<HierarchicalTypeDefinition<TraitType>> getTraitTypeDefinitions() { - return ImmutableList.of(); - } - - private void createSqoopDbStoreClass() throws AtlasException { - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition(DB_STORE_TYPE, - DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null), - new AttributeDefinition(DB_STORE_USAGE, - DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), - new AttributeDefinition(STORE_URI, - DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), - new AttributeDefinition(SOURCE, - DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null) - }; - - HierarchicalTypeDefinition<ClassType> definition = - new HierarchicalTypeDefinition<>(ClassType.class, SqoopDataTypes.SQOOP_DBDATASTORE.getName(), null, - ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions); - classTypeDefinitions.put(SqoopDataTypes.SQOOP_DBDATASTORE.getName(), definition); - LOG.debug("Created definition for " + SqoopDataTypes.SQOOP_DBDATASTORE.getName()); - } - - - private void createSqoopProcessClass() throws AtlasException { - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition(OPERATION, - DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null), - new AttributeDefinition(CMD_LINE_OPTS, STRING_MAP_TYPE.getName(), Multiplicity.REQUIRED, false, null), - new AttributeDefinition(START_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null), - new AttributeDefinition(END_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null), - new AttributeDefinition(USER, - DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null), - }; - - HierarchicalTypeDefinition<ClassType> definition = - new HierarchicalTypeDefinition<>(ClassType.class, SqoopDataTypes.SQOOP_PROCESS.getName(), null, - ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions); - classTypeDefinitions.put(SqoopDataTypes.SQOOP_PROCESS.getName(), definition); - LOG.debug("Created definition for " + SqoopDataTypes.SQOOP_PROCESS.getName()); - } - - public String getModelAsJson() throws AtlasException { - createDataModel(); - return getDataModelAsJSON(); - } - - public static void main(String[] args) throws Exception { - SqoopDataModelGenerator dataModelGenerator = new SqoopDataModelGenerator(); - String modelAsJson = dataModelGenerator.getModelAsJson(); - - if (args.length == 1) { - ModelDefinitionDump.dumpModelToFile(args[0], modelAsJson); - return; - } - - System.out.println("sqoopDataModelAsJSON = " + modelAsJson); - - TypesDef typesDef = dataModelGenerator.getTypesDef(); - for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) { - System.out.println(String.format("%s(%s) - values %s", enumType.name, EnumType.class.getSimpleName(), - Arrays.toString(enumType.enumValues))); - } - - for (HierarchicalTypeDefinition<ClassType> classType : typesDef.classTypesAsJavaList()) { - System.out.println( - String.format("%s(%s) - super types [%s] - attributes %s", classType.typeName, - ClassType.class.getSimpleName(), StringUtils.join(classType.superTypes, ","), - Arrays.toString(classType.attributeDefinitions))); - } - - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java ---------------------------------------------------------------------- diff --git a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java b/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java index d4fb3af..b1bd115 100644 --- a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java +++ b/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java @@ -18,17 +18,13 @@ package org.apache.atlas.sqoop.hook; -import com.sun.jersey.api.client.ClientResponse; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasClient; -import org.apache.atlas.AtlasServiceException; import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; import org.apache.atlas.hive.model.HiveDataTypes; -import org.apache.atlas.sqoop.model.SqoopDataModelGenerator; import org.apache.atlas.sqoop.model.SqoopDataTypes; import org.apache.atlas.utils.AuthenticationUtil; import org.apache.commons.configuration.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.sqoop.SqoopJobDataPublisher; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONObject; @@ -54,28 +50,6 @@ public class SqoopHookIT { } else { atlasClient = new AtlasClient(configuration.getStringArray(HiveMetaStoreBridge.ATLAS_ENDPOINT)); } - registerDataModels(atlasClient); - } - - private void registerDataModels(AtlasClient client) throws Exception { - // Make sure hive model exists - HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(ApplicationProperties.get(), new HiveConf(), atlasClient); - hiveMetaStoreBridge.registerHiveDataModel(); - SqoopDataModelGenerator dataModelGenerator = new SqoopDataModelGenerator(); - - //Register sqoop data model if its not already registered - try { - client.getType(SqoopDataTypes.SQOOP_PROCESS.getName()); - LOG.info("Sqoop data model is already registered!"); - } catch(AtlasServiceException ase) { - if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) { - //Expected in case types do not exist - LOG.info("Registering Sqoop data model"); - client.createType(dataModelGenerator.getModelAsJson()); - } else { - throw ase; - } - } } @Test http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/storm-bridge/pom.xml ---------------------------------------------------------------------- diff --git a/addons/storm-bridge/pom.xml b/addons/storm-bridge/pom.xml index fcb6e01..6be55b4 100644 --- a/addons/storm-bridge/pom.xml +++ b/addons/storm-bridge/pom.xml @@ -529,24 +529,6 @@ <version>1.2.1</version> <inherited>false</inherited> <executions> - <execution> - <configuration> - <mainClass>org.apache.atlas.storm.model.StormDataModelGenerator</mainClass> - <systemProperties> - <systemProperty> - <key>atlas.conf</key> - <value>${project.build.directory}/../../../typesystem/target/test-classes</value> - </systemProperty> - </systemProperties> - <arguments> - <argument>${project.build.directory}/models/storm_model.json</argument> - </arguments> - </configuration> - <phase>package</phase> - <goals> - <goal>java</goal> - </goals> - </execution> </executions> </plugin> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java ---------------------------------------------------------------------- diff --git a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java b/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java index bb6a476..5193832 100644 --- a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java +++ b/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java @@ -26,9 +26,7 @@ import org.apache.storm.generated.TopologyInfo; import org.apache.storm.utils.Utils; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasConstants; -import org.apache.atlas.fs.model.FSDataTypes; import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; -import org.apache.atlas.hive.model.HiveDataModelGenerator; import org.apache.atlas.hook.AtlasHook; import org.apache.atlas.storm.model.StormDataTypes; import org.apache.atlas.typesystem.Referenceable; @@ -213,7 +211,7 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook { break; case "HdfsBolt": - dataSetReferenceable = new Referenceable(FSDataTypes.HDFS_PATH().toString()); + dataSetReferenceable = new Referenceable(HiveMetaStoreBridge.HDFS_PATH); String hdfsUri = config.get("HdfsBolt.rotationActions") == null ? config.get("HdfsBolt.fileNameFormat.path") : config.get("HdfsBolt.rotationActions"); @@ -241,7 +239,7 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook { final String tableQualifiedName = HiveMetaStoreBridge.getTableQualifiedName(clusterName, databaseName, hiveTableName); dataSetReferenceable.set(AtlasClient.NAME, hiveTableName); - dataSetReferenceable.set(HiveDataModelGenerator.DB, dbReferenceable); + dataSetReferenceable.set(HiveMetaStoreBridge.DB, dbReferenceable); dataSetReferenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableQualifiedName); break; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/storm-bridge/src/main/java/org/apache/atlas/storm/model/StormDataModelGenerator.java ---------------------------------------------------------------------- diff --git a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/model/StormDataModelGenerator.java b/addons/storm-bridge/src/main/java/org/apache/atlas/storm/model/StormDataModelGenerator.java deleted file mode 100644 index 95eead7..0000000 --- a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/model/StormDataModelGenerator.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.storm.model; - -import org.apache.atlas.addons.ModelDefinitionDump; -import org.apache.atlas.typesystem.TypesDef; -import org.apache.atlas.typesystem.json.TypesSerialization; - -import java.io.IOException; - -public class StormDataModelGenerator { - - public static void main(String[] args) throws IOException { - StormDataModel.main(new String[]{}); - TypesDef typesDef = StormDataModel.typesDef(); - String stormTypesAsJSON = TypesSerialization.toJson(typesDef); - - if (args.length == 1) { - ModelDefinitionDump.dumpModelToFile(args[0], stormTypesAsJSON); - return; - } - - System.out.println("stormTypesAsJSON = " + stormTypesAsJSON); - } -}
