http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/intg/src/test/java/org/apache/atlas/TestUtilsV2.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java index 91fdb47..bbccf77 100755 --- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java +++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java @@ -18,8 +18,6 @@ package org.apache.atlas; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; @@ -30,7 +28,6 @@ import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasEnumDef; import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef; -import org.apache.atlas.model.typedef.AtlasRelationshipDef; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality; @@ -92,7 +89,7 @@ public final class TestUtilsV2 { AtlasTypeUtil.createRequiredAttrDef("city", "string")); AtlasEntityDef deptTypeDef = - AtlasTypeUtil.createClassTypeDef(DEPARTMENT_TYPE, "Department"+_description, ImmutableSet.<String>of(), + AtlasTypeUtil.createClassTypeDef(DEPARTMENT_TYPE, "Department"+_description, Collections.<String>emptySet(), AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), new AtlasAttributeDef("employees", String.format("array<%s>", "Employee"), true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, @@ -100,7 +97,7 @@ public final class TestUtilsV2 { add(new AtlasStructDef.AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF)); }})); - AtlasEntityDef personTypeDef = AtlasTypeUtil.createClassTypeDef("Person", "Person"+_description, ImmutableSet.<String>of(), + AtlasEntityDef personTypeDef = AtlasTypeUtil.createClassTypeDef("Person", "Person"+_description, Collections.<String>emptySet(), AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), AtlasTypeUtil.createOptionalAttrDef("address", "Address"), AtlasTypeUtil.createOptionalAttrDef("birthday", "date"), @@ -113,7 +110,7 @@ public final class TestUtilsV2 { AtlasTypeUtil.createOptionalAttrDef("approximationOfPi", "bigdecimal") ); - AtlasEntityDef employeeTypeDef = AtlasTypeUtil.createClassTypeDef("Employee", "Employee"+_description, ImmutableSet.of("Person"), + AtlasEntityDef employeeTypeDef = AtlasTypeUtil.createClassTypeDef("Employee", "Employee"+_description, Collections.singleton("Person"), AtlasTypeUtil.createOptionalAttrDef("orgLevel", "OrgLevel"), new AtlasAttributeDef("department", "Department", false, AtlasAttributeDef.Cardinality.SINGLE, 1, 1, @@ -142,18 +139,18 @@ public final class TestUtilsV2 { put(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, "employees"); }})); - AtlasEntityDef managerTypeDef = AtlasTypeUtil.createClassTypeDef("Manager", "Manager"+_description, ImmutableSet.of("Employee"), + AtlasEntityDef managerTypeDef = AtlasTypeUtil.createClassTypeDef("Manager", "Manager"+_description, Collections.singleton("Employee"), new AtlasAttributeDef("subordinates", String.format("array<%s>", "Employee"), false, AtlasAttributeDef.Cardinality.SET, 1, 10, false, false, Collections.<AtlasConstraintDef>emptyList())); AtlasClassificationDef securityClearanceTypeDef = - AtlasTypeUtil.createTraitTypeDef("SecurityClearance", "SecurityClearance"+_description, ImmutableSet.<String>of(), + AtlasTypeUtil.createTraitTypeDef("SecurityClearance", "SecurityClearance"+_description, Collections.<String>emptySet(), AtlasTypeUtil.createRequiredAttrDef("level", "int")); - AtlasTypesDef ret = new AtlasTypesDef(ImmutableList.of(orgLevelEnum), ImmutableList.of(addressDetails), - ImmutableList.of(securityClearanceTypeDef), - ImmutableList.of(deptTypeDef, personTypeDef, employeeTypeDef, managerTypeDef)); + AtlasTypesDef ret = new AtlasTypesDef(Collections.singletonList(orgLevelEnum), Collections.singletonList(addressDetails), + Collections.singletonList(securityClearanceTypeDef), + Arrays.asList(deptTypeDef, personTypeDef, employeeTypeDef, managerTypeDef)); populateSystemAttributes(ret); @@ -161,7 +158,7 @@ public final class TestUtilsV2 { } public static AtlasTypesDef defineInverseReferenceTestTypes() { - AtlasEntityDef aDef = AtlasTypeUtil.createClassTypeDef("A", ImmutableSet.<String>of(), + AtlasEntityDef aDef = AtlasTypeUtil.createClassTypeDef("A", Collections.<String>emptySet(), AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), new AtlasAttributeDef("b", "B", true, Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList()), // 1-1 new AtlasAttributeDef("oneB", "B", true, Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList()), // 1-* @@ -170,7 +167,7 @@ public final class TestUtilsV2 { Collections.<AtlasConstraintDef>singletonList(new AtlasConstraintDef( AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF, Collections.<String, Object>singletonMap(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, "mappedFromA"))))); // *-* - AtlasEntityDef bDef = AtlasTypeUtil.createClassTypeDef("B", ImmutableSet.<String>of(), + AtlasEntityDef bDef = AtlasTypeUtil.createClassTypeDef("B", Collections.<String>emptySet(), AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), new AtlasAttributeDef("a", "A", true, Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>singletonList(new AtlasConstraintDef( @@ -183,7 +180,7 @@ public final class TestUtilsV2 { AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF, Collections.<String, Object>singletonMap(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, "manyB")))), new AtlasAttributeDef("mappedFromA", "A", true, Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList())); - AtlasTypesDef ret = new AtlasTypesDef(ImmutableList.<AtlasEnumDef>of(), ImmutableList.<AtlasStructDef>of(), ImmutableList.<AtlasClassificationDef>of(), ImmutableList.<AtlasEntityDef>of(aDef, bDef)); + AtlasTypesDef ret = new AtlasTypesDef(Collections.<AtlasEnumDef>emptyList(), Collections.<AtlasStructDef>emptyList(), Collections.<AtlasClassificationDef>emptyList(), Arrays.asList(aDef, bDef)); populateSystemAttributes(ret); @@ -207,7 +204,7 @@ public final class TestUtilsV2 { AtlasEntityDef deptTypeDef = AtlasTypeUtil.createClassTypeDef(DEPARTMENT_TYPE, "Department"+_description, - ImmutableSet.<String>of(), + Collections.<String>emptySet(), AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), AtlasTypeUtil.createOptionalAttrDef("dep-code", "string"), new AtlasAttributeDef("employees", String.format("array<%s>", "Employee"), true, @@ -217,7 +214,7 @@ public final class TestUtilsV2 { }})); AtlasEntityDef personTypeDef = AtlasTypeUtil.createClassTypeDef("Person", "Person"+_description, - ImmutableSet.<String>of(), + Collections.<String>emptySet(), AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), AtlasTypeUtil.createOptionalAttrDef("email", "string"), AtlasTypeUtil.createOptionalAttrDef("address", "Address"), @@ -232,7 +229,7 @@ public final class TestUtilsV2 { ); AtlasEntityDef employeeTypeDef = AtlasTypeUtil.createClassTypeDef("Employee", "Employee"+_description, - ImmutableSet.of("Person"), + Collections.singleton("Person"), AtlasTypeUtil.createOptionalAttrDef("orgLevel", "OrgLevel"), AtlasTypeUtil.createOptionalAttrDef("empCode", "string"), new AtlasAttributeDef("department", "Department", false, @@ -258,19 +255,19 @@ public final class TestUtilsV2 { ); AtlasEntityDef managerTypeDef = AtlasTypeUtil.createClassTypeDef("Manager", "Manager"+_description, - ImmutableSet.of("Employee"), + Collections.singleton("Employee"), new AtlasAttributeDef("subordinates", String.format("array<%s>", "Employee"), false, AtlasAttributeDef.Cardinality.SET, 1, 10, false, false, Collections.<AtlasConstraintDef>emptyList())); AtlasClassificationDef securityClearanceTypeDef = - AtlasTypeUtil.createTraitTypeDef("SecurityClearance", "SecurityClearance"+_description, ImmutableSet.<String>of(), + AtlasTypeUtil.createTraitTypeDef("SecurityClearance", "SecurityClearance"+_description, Collections.<String>emptySet(), AtlasTypeUtil.createRequiredAttrDef("level", "int")); - AtlasTypesDef ret = new AtlasTypesDef(ImmutableList.of(orgLevelEnum), - ImmutableList.of(addressDetails), - ImmutableList.of(securityClearanceTypeDef), - ImmutableList.of(deptTypeDef, personTypeDef, employeeTypeDef, managerTypeDef)); + AtlasTypesDef ret = new AtlasTypesDef(Collections.singletonList(orgLevelEnum), + Collections.singletonList(addressDetails), + Collections.singletonList(securityClearanceTypeDef), + Arrays.asList(deptTypeDef, personTypeDef, employeeTypeDef, managerTypeDef)); populateSystemAttributes(ret); @@ -295,7 +292,7 @@ public final class TestUtilsV2 { AtlasTypeUtil.createRequiredAttrDef("zip", "int")); AtlasEntityDef deptTypeDef = - AtlasTypeUtil.createClassTypeDef(DEPARTMENT_TYPE, "Department"+_description, ImmutableSet.<String>of(), + AtlasTypeUtil.createClassTypeDef(DEPARTMENT_TYPE, "Department"+_description, Collections.<String>emptySet(), AtlasTypeUtil.createRequiredAttrDef("name", "string"), AtlasTypeUtil.createRequiredAttrDef("dep-code", "string"), new AtlasAttributeDef("employees", String.format("array<%s>", "Person"), true, @@ -304,7 +301,7 @@ public final class TestUtilsV2 { add(new AtlasStructDef.AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF)); }})); - AtlasEntityDef personTypeDef = AtlasTypeUtil.createClassTypeDef("Person", "Person"+_description, ImmutableSet.<String>of(), + AtlasEntityDef personTypeDef = AtlasTypeUtil.createClassTypeDef("Person", "Person"+_description, Collections.<String>emptySet(), AtlasTypeUtil.createRequiredAttrDef("name", "string"), AtlasTypeUtil.createRequiredAttrDef("emp-code", "string"), AtlasTypeUtil.createOptionalAttrDef("orgLevel", "OrgLevel"), @@ -338,10 +335,10 @@ public final class TestUtilsV2 { AtlasTypeUtil.createOptionalAttrDef("approximationOfPi", "bigdecimal") ); - AtlasTypesDef ret = new AtlasTypesDef(ImmutableList.of(orgLevelEnum), - ImmutableList.of(addressDetails), - ImmutableList.<AtlasClassificationDef>of(), - ImmutableList.of(deptTypeDef, personTypeDef)); + AtlasTypesDef ret = new AtlasTypesDef(Collections.singletonList(orgLevelEnum), + Collections.singletonList(addressDetails), + Collections.<AtlasClassificationDef>emptyList(), + Arrays.asList(deptTypeDef, personTypeDef)); populateSystemAttributes(ret); @@ -390,7 +387,7 @@ public final class TestUtilsV2 { julius.setAttribute("name", "Julius"); julius.setAttribute("department", hrDeptId); julius.setAttribute("address", juliusAddr); - julius.setAttribute("subordinates", ImmutableList.of()); + julius.setAttribute("subordinates", Collections.emptyList()); /******* Employee - Max (Manager: Jane, Mentor: Julius) *******/ AtlasEntity max = new AtlasEntity(EMPLOYEE_TYPE); @@ -430,8 +427,8 @@ public final class TestUtilsV2 { john.setAttribute("numberOfStarsEstimate", new BigInteger("1000000000000000000000")); john.setAttribute("approximationOfPi", new BigDecimal("3.141592653589793238462643383279502884197169399375105820974944592307816406286")); - jane.setAttribute("subordinates", ImmutableList.of(johnId, maxId)); - hrDept.setAttribute("employees", ImmutableList.of(janeId, juliusId, maxId, johnId)); + jane.setAttribute("subordinates", Arrays.asList(johnId, maxId)); + hrDept.setAttribute("employees", Arrays.asList(janeId, juliusId, maxId, johnId)); entitiesWithExtInfo.addEntity(hrDept); entitiesWithExtInfo.addEntity(jane); @@ -487,7 +484,7 @@ public final class TestUtilsV2 { juliusAddr.setAttribute("street", "Madison Ave"); juliusAddr.setAttribute("city", "Newtonville"); julius.setAttribute("address", juliusAddr); - julius.setAttribute("subordinates", ImmutableList.of()); + julius.setAttribute("subordinates", Collections.emptyList()); AtlasObjectId janeId = AtlasTypeUtil.getAtlasObjectId(jane); AtlasObjectId johnId = AtlasTypeUtil.getAtlasObjectId(john); @@ -516,9 +513,9 @@ public final class TestUtilsV2 { john.setAttribute("manager", janeId); john.setAttribute("mentor", maxId); - hrDept.setAttribute("employees", ImmutableList.of(johnId, janeId, juliusId, maxId)); + hrDept.setAttribute("employees", Arrays.asList(johnId, janeId, juliusId, maxId)); - jane.setAttribute("subordinates", ImmutableList.of(johnId, maxId)); + jane.setAttribute("subordinates", Arrays.asList(johnId, maxId)); deptEmpEntities.put(jane.getGuid(), jane); deptEmpEntities.put(john.getGuid(), john); @@ -549,20 +546,20 @@ public final class TestUtilsV2 { public static AtlasTypesDef simpleType(){ AtlasEntityDef superTypeDefinition = - AtlasTypeUtil.createClassTypeDef("h_type", ImmutableSet.<String>of(), + AtlasTypeUtil.createClassTypeDef("h_type", Collections.<String>emptySet(), AtlasTypeUtil.createOptionalAttrDef("attr", "string")); AtlasStructDef structTypeDefinition = new AtlasStructDef("s_type", "structType", "1.0", Arrays.asList(AtlasTypeUtil.createRequiredAttrDef("name", "string"))); AtlasClassificationDef traitTypeDefinition = - AtlasTypeUtil.createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of()); + AtlasTypeUtil.createTraitTypeDef("t_type", "traitType", Collections.<String>emptySet()); AtlasEnumDef enumTypeDefinition = new AtlasEnumDef("e_type", "enumType", "1.0", Arrays.asList(new AtlasEnumElementDef("ONE", "Element Description", 1))); - AtlasTypesDef ret = AtlasTypeUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition), - ImmutableList.of(traitTypeDefinition), ImmutableList.of(superTypeDefinition)); + AtlasTypesDef ret = AtlasTypeUtil.getTypesDef(Collections.singletonList(enumTypeDefinition), Collections.singletonList(structTypeDefinition), + Collections.singletonList(traitTypeDefinition), Collections.singletonList(superTypeDefinition)); populateSystemAttributes(ret); @@ -571,23 +568,23 @@ public final class TestUtilsV2 { public static AtlasTypesDef simpleTypeUpdated(){ AtlasEntityDef superTypeDefinition = - AtlasTypeUtil.createClassTypeDef("h_type", ImmutableSet.<String>of(), + AtlasTypeUtil.createClassTypeDef("h_type", Collections.<String>emptySet(), AtlasTypeUtil.createOptionalAttrDef("attr", "string")); AtlasEntityDef newSuperTypeDefinition = - AtlasTypeUtil.createClassTypeDef("new_h_type", ImmutableSet.<String>of(), + AtlasTypeUtil.createClassTypeDef("new_h_type", Collections.<String>emptySet(), AtlasTypeUtil.createOptionalAttrDef("attr", "string")); AtlasStructDef structTypeDefinition = new AtlasStructDef("s_type", "structType", "1.0", Arrays.asList(AtlasTypeUtil.createRequiredAttrDef("name", "string"))); AtlasClassificationDef traitTypeDefinition = - AtlasTypeUtil.createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of()); + AtlasTypeUtil.createTraitTypeDef("t_type", "traitType", Collections.<String>emptySet()); AtlasEnumDef enumTypeDefinition = new AtlasEnumDef("e_type", "enumType", Arrays.asList(new AtlasEnumElementDef("ONE", "Element Description", 1))); - AtlasTypesDef ret = AtlasTypeUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition), - ImmutableList.of(traitTypeDefinition), ImmutableList.of(superTypeDefinition, newSuperTypeDefinition)); + AtlasTypesDef ret = AtlasTypeUtil.getTypesDef(Collections.singletonList(enumTypeDefinition), Collections.singletonList(structTypeDefinition), + Collections.singletonList(traitTypeDefinition), Arrays.asList(superTypeDefinition, newSuperTypeDefinition)); populateSystemAttributes(ret); @@ -596,13 +593,13 @@ public final class TestUtilsV2 { public static AtlasTypesDef simpleTypeUpdatedDiff() { AtlasEntityDef newSuperTypeDefinition = - AtlasTypeUtil.createClassTypeDef("new_h_type", ImmutableSet.<String>of(), + AtlasTypeUtil.createClassTypeDef("new_h_type", Collections.<String>emptySet(), AtlasTypeUtil.createOptionalAttrDef("attr", "string")); - AtlasTypesDef ret = AtlasTypeUtil.getTypesDef(ImmutableList.<AtlasEnumDef>of(), - ImmutableList.<AtlasStructDef>of(), - ImmutableList.<AtlasClassificationDef>of(), - ImmutableList.of(newSuperTypeDefinition)); + AtlasTypesDef ret = AtlasTypeUtil.getTypesDef(Collections.<AtlasEnumDef>emptyList(), + Collections.<AtlasStructDef>emptyList(), + Collections.<AtlasClassificationDef>emptyList(), + Collections.singletonList(newSuperTypeDefinition)); populateSystemAttributes(ret); @@ -613,12 +610,12 @@ public final class TestUtilsV2 { public static AtlasTypesDef defineHiveTypes() { String _description = "_description"; AtlasEntityDef superTypeDefinition = - AtlasTypeUtil.createClassTypeDef(SUPER_TYPE_NAME, "SuperType_description", ImmutableSet.<String>of(), + AtlasTypeUtil.createClassTypeDef(SUPER_TYPE_NAME, "SuperType_description", Collections.<String>emptySet(), AtlasTypeUtil.createOptionalAttrDef("namespace", "string"), AtlasTypeUtil.createOptionalAttrDef("cluster", "string"), AtlasTypeUtil.createOptionalAttrDef("colo", "string")); AtlasEntityDef databaseTypeDefinition = - AtlasTypeUtil.createClassTypeDef(DATABASE_TYPE, DATABASE_TYPE + _description,ImmutableSet.of(SUPER_TYPE_NAME), + AtlasTypeUtil.createClassTypeDef(DATABASE_TYPE, DATABASE_TYPE + _description,Collections.singleton(SUPER_TYPE_NAME), AtlasTypeUtil.createUniqueRequiredAttrDef(NAME, "string"), AtlasTypeUtil.createOptionalAttrDef("isReplicated", "boolean"), AtlasTypeUtil.createOptionalAttrDef("created", "string"), @@ -640,7 +637,7 @@ public final class TestUtilsV2 { AtlasEntityDef columnsDefinition = AtlasTypeUtil.createClassTypeDef(COLUMN_TYPE, COLUMN_TYPE + "_description", - ImmutableSet.<String>of(), + Collections.<String>emptySet(), AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), AtlasTypeUtil.createRequiredAttrDef("type", "string"), AtlasTypeUtil.createOptionalAttrDef("description", "string"), @@ -684,7 +681,7 @@ public final class TestUtilsV2 { AtlasEntityDef storageDescClsDef = new AtlasEntityDef(STORAGE_DESC_TYPE, STORAGE_DESC_TYPE + _description, "1.0", - Arrays.asList(attributeDefinitions), ImmutableSet.of(SUPER_TYPE_NAME)); + Arrays.asList(attributeDefinitions), Collections.singleton(SUPER_TYPE_NAME)); AtlasAttributeDef[] partClsAttributes = new AtlasAttributeDef[]{ new AtlasAttributeDef("values", "array<string>", @@ -720,7 +717,7 @@ public final class TestUtilsV2 { AtlasEntityDef partClsDef = new AtlasEntityDef("partition_class_type", "partition_class_type" + _description, "1.0", - Arrays.asList(partClsAttributes), ImmutableSet.of(SUPER_TYPE_NAME)); + Arrays.asList(partClsAttributes), Collections.singleton(SUPER_TYPE_NAME)); AtlasEntityDef processClsType = new AtlasEntityDef(PROCESS_TYPE, PROCESS_TYPE + _description, "1.0", @@ -728,10 +725,10 @@ public final class TestUtilsV2 { AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.<AtlasConstraintDef>emptyList())), - ImmutableSet.<String>of()); + Collections.<String>emptySet()); AtlasEntityDef tableTypeDefinition = - AtlasTypeUtil.createClassTypeDef(TABLE_TYPE, TABLE_TYPE + _description, ImmutableSet.of(SUPER_TYPE_NAME), + AtlasTypeUtil.createClassTypeDef(TABLE_TYPE, TABLE_TYPE + _description, Collections.singleton(SUPER_TYPE_NAME), AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), AtlasTypeUtil.createOptionalAttrDef("description", "string"), AtlasTypeUtil.createRequiredAttrDef("type", "string"), @@ -810,26 +807,25 @@ public final class TestUtilsV2 { )); AtlasClassificationDef piiTypeDefinition = - AtlasTypeUtil.createTraitTypeDef(PII, PII + _description, ImmutableSet.<String>of()); + AtlasTypeUtil.createTraitTypeDef(PII, PII + _description, Collections.<String>emptySet()); AtlasClassificationDef classificationTypeDefinition = - AtlasTypeUtil.createTraitTypeDef(CLASSIFICATION, CLASSIFICATION + _description, ImmutableSet.<String>of(), + AtlasTypeUtil.createTraitTypeDef(CLASSIFICATION, CLASSIFICATION + _description, Collections.<String>emptySet(), AtlasTypeUtil.createRequiredAttrDef("tag", "string")); AtlasClassificationDef fetlClassificationTypeDefinition = - AtlasTypeUtil.createTraitTypeDef("fetl" + CLASSIFICATION, "fetl" + CLASSIFICATION + _description, ImmutableSet.of(CLASSIFICATION), + AtlasTypeUtil.createTraitTypeDef("fetl" + CLASSIFICATION, "fetl" + CLASSIFICATION + _description, Collections.singleton(CLASSIFICATION), AtlasTypeUtil.createRequiredAttrDef("tag", "string")); - AtlasClassificationDef phiTypeDefinition = AtlasTypeUtil.createTraitTypeDef(PHI, PHI + _description, ImmutableSet.<String>of(), + AtlasClassificationDef phiTypeDefinition = AtlasTypeUtil.createTraitTypeDef(PHI, PHI + _description, Collections.<String>emptySet(), AtlasTypeUtil.createRequiredAttrDef("stringAttr", "string"), AtlasTypeUtil.createRequiredAttrDef("booleanAttr", "boolean"), AtlasTypeUtil.createRequiredAttrDef("integerAttr", "int")); - AtlasTypesDef ret = AtlasTypeUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), - ImmutableList.of(structTypeDefinition, partitionDefinition), - ImmutableList.of(classificationTypeDefinition, fetlClassificationTypeDefinition, piiTypeDefinition, phiTypeDefinition), - ImmutableList.of(superTypeDefinition, databaseTypeDefinition, columnsDefinition, tableTypeDefinition, - storageDescClsDef, partClsDef, processClsType)); + AtlasTypesDef ret = AtlasTypeUtil.getTypesDef(Collections.singletonList(enumTypeDefinition), + Arrays.asList(structTypeDefinition, partitionDefinition), + Arrays.asList(classificationTypeDefinition, fetlClassificationTypeDefinition, piiTypeDefinition, phiTypeDefinition), + Arrays.asList(superTypeDefinition, databaseTypeDefinition, columnsDefinition, tableTypeDefinition, storageDescClsDef, partClsDef, processClsType)); populateSystemAttributes(ret); @@ -840,6 +836,10 @@ public final class TestUtilsV2 { return RandomStringUtils.randomAlphanumeric(10); } + public static final String randomString(int count) { + return RandomStringUtils.randomAlphanumeric(count); + } + public static AtlasEntity createDBEntity() { String dbName = RandomStringUtils.randomAlphanumeric(10); return createDBEntity(dbName); @@ -990,11 +990,11 @@ public final class TestUtilsV2 { public static List<AtlasClassificationDef> getClassificationWithValidSuperType() { AtlasClassificationDef securityClearanceTypeDef = - AtlasTypeUtil.createTraitTypeDef("SecurityClearance1", "SecurityClearance_description", ImmutableSet.<String>of(), + AtlasTypeUtil.createTraitTypeDef("SecurityClearance1", "SecurityClearance_description", Collections.<String>emptySet(), AtlasTypeUtil.createRequiredAttrDef("level", "int")); AtlasClassificationDef janitorSecurityClearanceTypeDef = - AtlasTypeUtil.createTraitTypeDef("JanitorClearance", "JanitorClearance_description", ImmutableSet.of("SecurityClearance1"), + AtlasTypeUtil.createTraitTypeDef("JanitorClearance", "JanitorClearance_description", Collections.singleton("SecurityClearance1"), AtlasTypeUtil.createRequiredAttrDef("level", "int")); List<AtlasClassificationDef> ret = Arrays.asList(securityClearanceTypeDef, janitorSecurityClearanceTypeDef); @@ -1006,7 +1006,7 @@ public final class TestUtilsV2 { public static List<AtlasClassificationDef> getClassificationWithName(String name) { AtlasClassificationDef classificationTypeDef = - AtlasTypeUtil.createTraitTypeDef(name, "s_description", ImmutableSet.<String>of(), + AtlasTypeUtil.createTraitTypeDef(name, "s_description", Collections.<String>emptySet(), AtlasTypeUtil.createRequiredAttrDef("level", "int")); @@ -1019,7 +1019,7 @@ public final class TestUtilsV2 { public static AtlasClassificationDef getSingleClassificationWithName(String name) { AtlasClassificationDef classificaitonTypeDef = - AtlasTypeUtil.createTraitTypeDef(name, "s_description", ImmutableSet.<String>of(), + AtlasTypeUtil.createTraitTypeDef(name, "s_description", Collections.<String>emptySet(), AtlasTypeUtil.createRequiredAttrDef("level", "int")); populateSystemAttributes(classificaitonTypeDef); @@ -1032,7 +1032,7 @@ public final class TestUtilsV2 { } public static List<AtlasEntityDef> getEntityWithValidSuperType() { - AtlasEntityDef developerTypeDef = AtlasTypeUtil.createClassTypeDef("Developer", "Developer_description", ImmutableSet.of("Employee"), + AtlasEntityDef developerTypeDef = AtlasTypeUtil.createClassTypeDef("Developer", "Developer_description", Collections.singleton("Employee"), new AtlasAttributeDef("language", String.format("array<%s>", "string"), false, AtlasAttributeDef.Cardinality.SET, 1, 10, false, false, Collections.<AtlasConstraintDef>emptyList())); @@ -1045,7 +1045,7 @@ public final class TestUtilsV2 { } public static List<AtlasEntityDef> getEntityWithName(String name) { - AtlasEntityDef developerTypeDef = AtlasTypeUtil.createClassTypeDef(name, "Developer_description", ImmutableSet.<String>of(), + AtlasEntityDef developerTypeDef = AtlasTypeUtil.createClassTypeDef(name, "Developer_description", Collections.<String>emptySet(), new AtlasAttributeDef("language", String.format("array<%s>", "string"), false, AtlasAttributeDef.Cardinality.SET, 1, 10, false, false, Collections.<AtlasConstraintDef>emptyList())); @@ -1058,7 +1058,7 @@ public final class TestUtilsV2 { } public static AtlasEntityDef getSingleEntityWithName(String name) { - AtlasEntityDef developerTypeDef = AtlasTypeUtil.createClassTypeDef(name, "Developer_description", ImmutableSet.<String>of(), + AtlasEntityDef developerTypeDef = AtlasTypeUtil.createClassTypeDef(name, "Developer_description", Collections.<String>emptySet(), new AtlasAttributeDef("language", String.format("array<%s>", "string"), false, AtlasAttributeDef.Cardinality.SET, 1, 10, false, false, Collections.<AtlasConstraintDef>emptyList()));
http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/intg/src/test/java/org/apache/atlas/type/TestAtlasRelationshipType.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/type/TestAtlasRelationshipType.java b/intg/src/test/java/org/apache/atlas/type/TestAtlasRelationshipType.java index 8f3ac5b..819dfe5 100644 --- a/intg/src/test/java/org/apache/atlas/type/TestAtlasRelationshipType.java +++ b/intg/src/test/java/org/apache/atlas/type/TestAtlasRelationshipType.java @@ -17,7 +17,6 @@ */ package org.apache.atlas.type; -import com.google.common.collect.ImmutableSet; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.typedef.AtlasBaseTypeDef; @@ -33,10 +32,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinali import org.testng.Assert; import org.testng.annotations.Test; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; +import java.util.*; import static org.testng.Assert.fail; @@ -195,22 +191,22 @@ public class TestAtlasRelationshipType { } private void createEmployeeTypes() throws AtlasBaseException { - AtlasEntityDef phoneDef = AtlasTypeUtil.createClassTypeDef(PHONE_TYPE, getDescription(PHONE_TYPE), ImmutableSet.<String>of(), + AtlasEntityDef phoneDef = AtlasTypeUtil.createClassTypeDef(PHONE_TYPE, getDescription(PHONE_TYPE), Collections.<String>emptySet(), AtlasTypeUtil.createRequiredAttrDef("phone_number", "int"), AtlasTypeUtil.createOptionalAttrDef("area_code", "int"), AtlasTypeUtil.createOptionalAttrDef("owner", EMPLOYEE_TYPE)); - AtlasEntityDef employeeDef = AtlasTypeUtil.createClassTypeDef(EMPLOYEE_TYPE, getDescription(EMPLOYEE_TYPE), ImmutableSet.<String>of(), + AtlasEntityDef employeeDef = AtlasTypeUtil.createClassTypeDef(EMPLOYEE_TYPE, getDescription(EMPLOYEE_TYPE), Collections.<String>emptySet(), AtlasTypeUtil.createRequiredAttrDef("name", "string"), AtlasTypeUtil.createOptionalAttrDef("dob", "date"), AtlasTypeUtil.createOptionalAttrDef("age", "int"), AtlasTypeUtil.createRequiredAttrDef("phone_no", PHONE_TYPE)); - AtlasEntityDef departmentDef = AtlasTypeUtil.createClassTypeDef(DEPARTMENT_TYPE, getDescription(DEPARTMENT_TYPE), ImmutableSet.<String>of(), + AtlasEntityDef departmentDef = AtlasTypeUtil.createClassTypeDef(DEPARTMENT_TYPE, getDescription(DEPARTMENT_TYPE), Collections.<String>emptySet(), AtlasTypeUtil.createRequiredAttrDef("name", "string"), AtlasTypeUtil.createOptionalAttrDef("count", "int")); - AtlasEntityDef addressDef = AtlasTypeUtil.createClassTypeDef(ADDRESS_TYPE, getDescription(ADDRESS_TYPE), ImmutableSet.<String>of(), + AtlasEntityDef addressDef = AtlasTypeUtil.createClassTypeDef(ADDRESS_TYPE, getDescription(ADDRESS_TYPE), Collections.<String>emptySet(), AtlasTypeUtil.createOptionalAttrDef("street", "string"), AtlasTypeUtil.createRequiredAttrDef("city", "string"), AtlasTypeUtil.createRequiredAttrDef("state", "string"), http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/intg/src/test/resources/atlas-application.properties ---------------------------------------------------------------------- diff --git a/intg/src/test/resources/atlas-application.properties b/intg/src/test/resources/atlas-application.properties new file mode 100644 index 0000000..b937c33 --- /dev/null +++ b/intg/src/test/resources/atlas-application.properties @@ -0,0 +1,145 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#system property +atlas.data=${sys:user.dir}/target/data + + + +#re-use existing property +atlas.graph.data=${atlas.data}/graph + +#plain property +atlas.service=atlas + +#invalid system property +atlas.db=${atlasdb} + +atlas.TypeSystem.impl=org.apache.atlas.typesystem.types.TypeSystem + + + +######### Atlas Server Configs ######### +atlas.rest.address=http://localhost:31000 + +######### Graph Database Configs ######### + + +# Graph database implementation. Value inserted by maven. +atlas.graphdb.backend=${graphdb.backend.impl} + +# Graph Storage +atlas.graph.storage.backend=${graph.storage.backend} + +# Entity repository implementation +atlas.EntityAuditRepository.impl=${entity.repository.impl} + +# Graph Search Index Backend +atlas.graph.index.search.backend=${graph.index.backend} + +#Berkeley storage directory +atlas.graph.storage.directory=${sys:atlas.data}/berkley + +#hbase +#For standalone mode , specify localhost +#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2 + +atlas.graph.storage.hostname=${graph.storage.hostname} +atlas.graph.storage.hbase.regions-per-server=1 +atlas.graph.storage.lock.wait-time=10000 + +#ElasticSearch +atlas.graph.index.search.directory=${sys:atlas.data}/es +atlas.graph.index.search.elasticsearch.client-only=false +atlas.graph.index.search.elasticsearch.local-mode=true +atlas.graph.index.search.elasticsearch.create.sleep=2000 + +# Solr cloud mode properties +atlas.graph.index.search.solr.mode=cloud +atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address} +atlas.graph.index.search.max-result-set-size=150 + +######### Hive Lineage Configs ######### +## Schema +atlas.lineage.schema.query.hive_table=hive_table where __guid='%s'\, columns +atlas.lineage.schema.query.hive_table_v1=hive_table_v1 where __guid='%s'\, columns + +######### Notification Configs ######### +atlas.notification.embedded=true + +atlas.kafka.zookeeper.connect=localhost:19026 +atlas.kafka.bootstrap.servers=localhost:19027 +atlas.kafka.data=${sys:atlas.data}/kafka +atlas.kafka.zookeeper.session.timeout.ms=4000 +atlas.kafka.zookeeper.sync.time.ms=20 +atlas.kafka.consumer.timeout.ms=4000 +atlas.kafka.auto.commit.interval.ms=100 +atlas.kafka.hook.group.id=atlas +atlas.kafka.entities.group.id=atlas_entities +#atlas.kafka.auto.commit.enable=false + +atlas.kafka.enable.auto.commit=false +atlas.kafka.auto.offset.reset=earliest +atlas.kafka.session.timeout.ms=30000 + + + +######### Entity Audit Configs ######### +atlas.audit.hbase.tablename=ATLAS_ENTITY_AUDIT_EVENTS +atlas.audit.zookeeper.session.timeout.ms=1000 +atlas.audit.hbase.zookeeper.quorum=localhost +atlas.audit.hbase.zookeeper.property.clientPort=19026 + +######### Security Properties ######### + +# SSL config +atlas.enableTLS=false +atlas.server.https.port=31443 + +######### Security Properties ######### + +hbase.security.authentication=simple + +atlas.hook.falcon.synchronous=true + +######### JAAS Configuration ######## + +atlas.jaas.KafkaClient.loginModuleName = com.sun.security.auth.module.Krb5LoginModule +atlas.jaas.KafkaClient.loginModuleControlFlag = required +atlas.jaas.KafkaClient.option.useKeyTab = true +atlas.jaas.KafkaClient.option.storeKey = true +atlas.jaas.KafkaClient.option.serviceName = kafka +atlas.jaas.KafkaClient.option.keyTab = /etc/security/keytabs/atlas.service.keytab +atlas.jaas.KafkaClient.option.principal = atlas/_h...@example.com + +######### High Availability Configuration ######## +atlas.server.ha.enabled=false +#atlas.server.ids=id1 +#atlas.server.address.id1=localhost:21000 + +#########POLICY FILE PATH ######### +# atlas.auth.policy.file=policy-store.txt + +atlas.authentication.method.file=true +atlas.authentication.method.ldap.type=none +# atlas.authentication.method.file.filename=users-credentials.properties +atlas.authentication.method.kerberos=false + +######### Gremlin Search Configuration ######### +# Set to false to disable gremlin search. +atlas.search.gremlin.enable=true \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/intg/src/test/resources/policy-store.txt ---------------------------------------------------------------------- diff --git a/intg/src/test/resources/policy-store.txt b/intg/src/test/resources/policy-store.txt new file mode 100644 index 0000000..048affe --- /dev/null +++ b/intg/src/test/resources/policy-store.txt @@ -0,0 +1,9 @@ +##Policy Format +##r-READ, w-WRITE, u-UPDATE, d-DELETE +##Policy_Name;;User_Name1:Operations_Allowed,User_Name2:Operations_Allowed;;Group_Name1:Operations_Allowed,Group_Name2:Operations_Allowed;;Resource_Type1:Resource_Name,Resource_Type2:Resource_Name +## +adminPolicy;;admin:rwud;;ROLE_ADMIN:rwud;;type:*,entity:*,operation:* +dataScientistPolicy;;;;DATA_SCIENTIST:r;;type:*,entity:* +dataStewardPolicy;;;;DATA_STEWARD:rwu;;type:*,entity:* +hadoopPolicy;;;;hadoop:rwud;;type:*,entity:*,operation:* +rangerTagSyncPolicy;;;;RANGER_TAG_SYNC:r;;type:*,entity:* http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/intg/src/test/resources/sampleInstance.json ---------------------------------------------------------------------- diff --git a/intg/src/test/resources/sampleInstance.json b/intg/src/test/resources/sampleInstance.json new file mode 100755 index 0000000..efcc48d --- /dev/null +++ b/intg/src/test/resources/sampleInstance.json @@ -0,0 +1,72 @@ +{ + "$typeName$": "Department", + "$id$": { + "id": -1420494283853484000, + "$typeName$": "Department", + "version": 0 + }, + "employees": [ + { + "$typeName$": "Person", + "$id$": { + "id": -1420494283853508000, + "$typeName$": "Person", + "version": 0 + }, + "manager": { + "id": -1420494283853511000, + "$typeName$": "Manager", + "version": 0 + }, + "department": { + "id": -1420494283853484000, + "$typeName$": "Department", + "version": 0 + }, + "name": "John" + }, + { + "$typeName$": "Manager", + "$id$": { + "id": -1420494283853511000, + "$typeName$": "Manager", + "version": 0 + }, + "manager": null, + "subordinates": [ + { + "$typeName$": "Person", + "$id$": { + "id": -1420494283853508000, + "$typeName$": "Person", + "version": 0 + }, + "manager": { + "id": -1420494283853511000, + "$typeName$": "Manager", + "version": 0 + }, + "department": { + "id": -1420494283853484000, + "$typeName$": "Department", + "version": 0 + }, + "name": "John" + } + ], + "department": { + "id": -1420494283853484000, + "$typeName$": "Department", + "version": 0 + }, + "name": "Jane", + "$traits$": { + "SecurityClearance": { + "$typeName$": "SecurityClearance", + "level": 1 + } + } + } + ], + "name": "hr" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/intg/src/test/resources/sampleTypes.json ---------------------------------------------------------------------- diff --git a/intg/src/test/resources/sampleTypes.json b/intg/src/test/resources/sampleTypes.json new file mode 100755 index 0000000..d0ac311 --- /dev/null +++ b/intg/src/test/resources/sampleTypes.json @@ -0,0 +1,633 @@ +{ + "enumTypes": [ + { + "name": "HiveObjectType", + "enumValues": [ + { + "value": "GLOBAL", + "ordinal": 1 + }, + { + "value": "DATABASE", + "ordinal": 2 + }, + { + "value": "TABLE", + "ordinal": 3 + }, + { + "value": "PARTITION", + "ordinal": 4 + }, + { + "value": "COLUMN", + "ordinal": 5 + } + ] + }, + { + "name": "LockLevel", + "enumValues": [ + { + "value": "DB", + "ordinal": 1 + }, + { + "value": "TABLE", + "ordinal": 2 + }, + { + "value": "PARTITION", + "ordinal": 3 + } + ] + }, + { + "name": "TxnState", + "enumValues": [ + { + "value": "COMMITTED", + "ordinal": 1 + }, + { + "value": "ABORTED", + "ordinal": 2 + }, + { + "value": "OPEN", + "ordinal": 3 + } + ] + }, + { + "name": "PrincipalType", + "enumValues": [ + { + "value": "USER", + "ordinal": 1 + }, + { + "value": "ROLE", + "ordinal": 2 + }, + { + "value": "GROUP", + "ordinal": 3 + } + ] + } + ], + "structTypes": [ + { + "typeName": "t2", + "attributeDefinitions": [ + { + "name": "a", + "dataTypeName": "int", + "multiplicity": "required", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "s", + "dataTypeName": "t2", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + } + ] + }, + { + "typeName": "t1", + "attributeDefinitions": [ + { + "name": "a", + "dataTypeName": "int", + "multiplicity": "required", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "b", + "dataTypeName": "boolean", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "c", + "dataTypeName": "byte", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "d", + "dataTypeName": "short", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "e", + "dataTypeName": "int", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "f", + "dataTypeName": "int", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "g", + "dataTypeName": "long", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "h", + "dataTypeName": "float", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "i", + "dataTypeName": "double", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "j", + "dataTypeName": "biginteger", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "k", + "dataTypeName": "bigdecimal", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "l", + "dataTypeName": "date", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "m", + "dataTypeName": "array<int>", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "n", + "dataTypeName": "array<bigdecimal>", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "o", + "dataTypeName": "map<string,double>", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + } + ] + }, + { + "typeName": "ts1", + "attributeDefinitions": [ + { + "name": "a", + "dataTypeName": "int", + "multiplicity": "required", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "b", + "dataTypeName": "boolean", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "c", + "dataTypeName": "byte", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "d", + "dataTypeName": "short", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "e", + "dataTypeName": "int", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "f", + "dataTypeName": "int", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "g", + "dataTypeName": "long", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "h", + "dataTypeName": "float", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "i", + "dataTypeName": "double", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "j", + "dataTypeName": "biginteger", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "k", + "dataTypeName": "bigdecimal", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "l", + "dataTypeName": "date", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "m", + "dataTypeName": "array<int>", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "n", + "dataTypeName": "array<bigdecimal>", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "o", + "dataTypeName": "map<string,double>", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + } + ] + } + ], + "traitTypes": [ + { + "superTypes": [ + "B", + "C" + ], + "hierarchicalMetaTypeName": "org.apache.atlas.types.TraitType", + "typeName": "D", + "attributeDefinitions": [ + { + "name": "d", + "dataTypeName": "short", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + } + ] + }, + { + "superTypes": [ + ], + "hierarchicalMetaTypeName": "org.apache.atlas.types.TraitType", + "typeName": "A", + "attributeDefinitions": [ + { + "name": "a", + "dataTypeName": "int", + "multiplicity": "required", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "b", + "dataTypeName": "boolean", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "c", + "dataTypeName": "byte", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "d", + "dataTypeName": "short", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + } + ] + }, + { + "superTypes": [ + "A" + ], + "hierarchicalMetaTypeName": "org.apache.atlas.types.TraitType", + "typeName": "B", + "attributeDefinitions": [ + { + "name": "b", + "dataTypeName": "boolean", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + } + ] + }, + { + "superTypes": [ + "A" + ], + "hierarchicalMetaTypeName": "org.apache.atlas.types.TraitType", + "typeName": "C", + "attributeDefinitions": [ + { + "name": "c", + "dataTypeName": "byte", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + } + ] + }, + { + "superTypes": [ + ], + "hierarchicalMetaTypeName": "org.apache.atlas.types.TraitType", + "typeName": "SecurityClearance", + "attributeDefinitions": [ + { + "name": "level", + "dataTypeName": "int", + "multiplicity": "required", + "isComposite": false, + "reverseAttributeName": null + } + ] + } + ], + "classTypes": [ + { + "superTypes": [ + "Person" + ], + "hierarchicalMetaTypeName": "org.apache.atlas.types.ClassType", + "typeName": "Manager", + "attributeDefinitions": [ + { + "name": "subordinates", + "dataTypeName": "array<Person>", + "multiplicity": "collection", + "isComposite": false, + "reverseAttributeName": "manager" + } + ] + }, + { + "superTypes": [ + ], + "hierarchicalMetaTypeName": "org.apache.atlas.types.ClassType", + "typeName": "Department", + "attributeDefinitions": [ + { + "name": "name", + "dataTypeName": "string", + "multiplicity": "required", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "employees", + "dataTypeName": "array<Person>", + "multiplicity": "collection", + "isComposite": true, + "reverseAttributeName": "department" + } + ] + }, + { + "superTypes": [ + ], + "hierarchicalMetaTypeName": "org.apache.atlas.types.ClassType", + "typeName": "t4", + "attributeDefinitions": [ + { + "name": "a", + "dataTypeName": "int", + "multiplicity": "required", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "b", + "dataTypeName": "boolean", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "c", + "dataTypeName": "byte", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "d", + "dataTypeName": "short", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "enum1", + "dataTypeName": "HiveObjectType", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "e", + "dataTypeName": "int", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "f", + "dataTypeName": "int", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "g", + "dataTypeName": "long", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "enum2", + "dataTypeName": "PrincipalType", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "h", + "dataTypeName": "float", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "i", + "dataTypeName": "double", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "j", + "dataTypeName": "biginteger", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "k", + "dataTypeName": "bigdecimal", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "enum3", + "dataTypeName": "TxnState", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "l", + "dataTypeName": "date", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "m", + "dataTypeName": "array<int>", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "n", + "dataTypeName": "array<bigdecimal>", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "o", + "dataTypeName": "map<string,double>", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "enum4", + "dataTypeName": "LockLevel", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": null + } + ] + }, + { + "superTypes": [ + ], + "hierarchicalMetaTypeName": "org.apache.atlas.types.ClassType", + "typeName": "Person", + "attributeDefinitions": [ + { + "name": "name", + "dataTypeName": "string", + "multiplicity": "required", + "isComposite": false, + "reverseAttributeName": null + }, + { + "name": "department", + "dataTypeName": "Department", + "multiplicity": "required", + "isComposite": false, + "reverseAttributeName": "employees" + }, + { + "name": "manager", + "dataTypeName": "Manager", + "multiplicity": "optional", + "isComposite": false, + "reverseAttributeName": "subordinates" + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/intg/src/test/resources/users-credentials.properties ---------------------------------------------------------------------- diff --git a/intg/src/test/resources/users-credentials.properties b/intg/src/test/resources/users-credentials.properties new file mode 100644 index 0000000..3fc3bb1 --- /dev/null +++ b/intg/src/test/resources/users-credentials.properties @@ -0,0 +1,3 @@ +#username=group::sha256-password +admin=ADMIN::8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918 +rangertagsync=RANGER_TAG_SYNC::e3f67240f5117d1753c940dae9eea772d36ed5fe9bd9c94a300e40413f1afb9d http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/notification/pom.xml ---------------------------------------------------------------------- diff --git a/notification/pom.xml b/notification/pom.xml index 9b36940..079d424 100644 --- a/notification/pom.xml +++ b/notification/pom.xml @@ -42,12 +42,12 @@ <dependency> <groupId>org.apache.atlas</groupId> - <artifactId>atlas-typesystem</artifactId> + <artifactId>atlas-server-api</artifactId> </dependency> <dependency> - <groupId>org.apache.atlas</groupId> - <artifactId>atlas-server-api</artifactId> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> </dependency> <dependency> @@ -57,7 +57,7 @@ <dependency> <groupId>org.apache.kafka</groupId> - <artifactId>kafka_${scala.binary.version}</artifactId> + <artifactId>kafka_${kafka.scala.binary.version}</artifactId> </dependency> <dependency> @@ -77,11 +77,10 @@ <dependency> <groupId>org.apache.atlas</groupId> - <artifactId>atlas-typesystem</artifactId> + <artifactId>atlas-intg</artifactId> <classifier>tests</classifier> <scope>test</scope> </dependency> - </dependencies> <build> @@ -159,7 +158,7 @@ </artifactItem> <artifactItem> <groupId>org.apache.kafka</groupId> - <artifactId>kafka_${scala.binary.version}</artifactId> + <artifactId>kafka_${kafka.scala.binary.version}</artifactId> <version>${kafka.version}</version> </artifactItem> <artifactItem> @@ -168,26 +167,6 @@ <version>${kafka.version}</version> </artifactItem> <artifactItem> - <groupId>org.scala-lang</groupId> - <artifactId>scala-compiler</artifactId> - <version>${scala.version}</version> - </artifactItem> - <artifactItem> - <groupId>org.scala-lang</groupId> - <artifactId>scala-reflect</artifactId> - <version>${scala.version}</version> - </artifactItem> - <artifactItem> - <groupId>org.scala-lang</groupId> - <artifactId>scala-library</artifactId> - <version>${scala.version}</version> - </artifactItem> - <artifactItem> - <groupId>org.scala-lang</groupId> - <artifactId>scalap</artifactId> - <version>${scala.version}</version> - </artifactItem> - <artifactItem> <groupId>com.101tec</groupId> <artifactId>zkclient</artifactId> <version>${zkclient.version}</version> http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/notification/src/main/java/org/apache/atlas/hook/AtlasHook.java ---------------------------------------------------------------------- diff --git a/notification/src/main/java/org/apache/atlas/hook/AtlasHook.java b/notification/src/main/java/org/apache/atlas/hook/AtlasHook.java index 38f3208..bf6a36c 100644 --- a/notification/src/main/java/org/apache/atlas/hook/AtlasHook.java +++ b/notification/src/main/java/org/apache/atlas/hook/AtlasHook.java @@ -21,22 +21,20 @@ package org.apache.atlas.hook; import com.google.common.annotations.VisibleForTesting; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.kafka.NotificationProvider; +import org.apache.atlas.model.notification.HookNotification; +import org.apache.atlas.v1.model.instance.Referenceable; import org.apache.atlas.notification.NotificationException; import org.apache.atlas.notification.NotificationInterface; -import org.apache.atlas.notification.hook.HookNotification; +import org.apache.atlas.v1.model.notification.HookNotificationV1.EntityCreateRequest; import org.apache.atlas.security.InMemoryJAASConfiguration; -import org.apache.atlas.typesystem.Referenceable; -import org.apache.atlas.typesystem.json.InstanceSerialization; import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.security.UserGroupInformation; -import org.codehaus.jettison.json.JSONArray; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.List; @@ -101,18 +99,10 @@ public abstract class AtlasHook { protected abstract String getNumberOfRetriesPropertyKey(); - protected void notifyEntities(String user, Collection<Referenceable> entities) { - JSONArray entitiesArray = new JSONArray(); - - for (Referenceable entity : entities) { - LOG.info("Adding entity for type: {}", entity.getTypeName()); - final String entityJson = InstanceSerialization.toJson(entity, true); - entitiesArray.put(entityJson); - } - - List<HookNotification.HookNotificationMessage> hookNotificationMessages = new ArrayList<>(); - hookNotificationMessages.add(new HookNotification.EntityCreateRequest(user, entitiesArray)); - notifyEntities(hookNotificationMessages); + protected void notifyEntities(String user, List<Referenceable> entities) { + List<HookNotification> hookNotifications = new ArrayList<>(); + hookNotifications.add(new EntityCreateRequest(user, entities)); + notifyEntities(hookNotifications); } /** @@ -124,12 +114,12 @@ public abstract class AtlasHook { * @param messages hook notification messages * @param maxRetries maximum number of retries while sending message to messaging system */ - public static void notifyEntities(List<HookNotification.HookNotificationMessage> messages, int maxRetries) { + public static void notifyEntities(List<HookNotification> messages, int maxRetries) { notifyEntitiesInternal(messages, maxRetries, notificationInterface, logFailedMessages, failedMessagesLogger); } @VisibleForTesting - static void notifyEntitiesInternal(List<HookNotification.HookNotificationMessage> messages, int maxRetries, + static void notifyEntitiesInternal(List<HookNotification> messages, int maxRetries, NotificationInterface notificationInterface, boolean shouldLogFailedMessages, FailedMessagesLogger logger) { if (messages == null || messages.isEmpty()) { @@ -176,7 +166,7 @@ public abstract class AtlasHook { * * @param messages hook notification messages */ - protected void notifyEntities(List<HookNotification.HookNotificationMessage> messages) { + protected void notifyEntities(List<HookNotification> messages) { final int maxRetries = atlasProperties.getInt(getNumberOfRetriesPropertyKey(), 3); notifyEntities(messages, maxRetries); } http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/notification/src/main/java/org/apache/atlas/kafka/AtlasKafkaConsumer.java ---------------------------------------------------------------------- diff --git a/notification/src/main/java/org/apache/atlas/kafka/AtlasKafkaConsumer.java b/notification/src/main/java/org/apache/atlas/kafka/AtlasKafkaConsumer.java index e3bb71c..fd0c4e4 100644 --- a/notification/src/main/java/org/apache/atlas/kafka/AtlasKafkaConsumer.java +++ b/notification/src/main/java/org/apache/atlas/kafka/AtlasKafkaConsumer.java @@ -18,7 +18,8 @@ package org.apache.atlas.kafka; import org.apache.atlas.notification.AbstractNotificationConsumer; -import org.apache.atlas.notification.MessageDeserializer; +import org.apache.atlas.notification.AtlasNotificationMessageDeserializer; +import org.apache.atlas.notification.NotificationInterface; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,13 +42,18 @@ public class AtlasKafkaConsumer<T> extends AbstractNotificationConsumer<T> { private static final Logger LOG = LoggerFactory.getLogger(AtlasKafkaConsumer.class); private final KafkaConsumer kafkaConsumer; - private final boolean autoCommitEnabled; - private long pollTimeoutMilliSeconds = 1000L; + private final boolean autoCommitEnabled; + private long pollTimeoutMilliSeconds = 1000L; - public AtlasKafkaConsumer(MessageDeserializer<T> deserializer, KafkaConsumer kafkaConsumer, boolean autoCommitEnabled, long pollTimeoutMilliSeconds) { + public AtlasKafkaConsumer(NotificationInterface.NotificationType notificationType, KafkaConsumer kafkaConsumer, boolean autoCommitEnabled, long pollTimeoutMilliSeconds) { + this(notificationType.getDeserializer(), kafkaConsumer, autoCommitEnabled, pollTimeoutMilliSeconds); + } + + public AtlasKafkaConsumer(AtlasNotificationMessageDeserializer<T> deserializer, KafkaConsumer kafkaConsumer, boolean autoCommitEnabled, long pollTimeoutMilliSeconds) { super(deserializer); - this.kafkaConsumer = kafkaConsumer; - this.autoCommitEnabled = autoCommitEnabled; + + this.autoCommitEnabled = autoCommitEnabled; + this.kafkaConsumer = kafkaConsumer; this.pollTimeoutMilliSeconds = pollTimeoutMilliSeconds; } http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/notification/src/main/java/org/apache/atlas/kafka/KafkaNotification.java ---------------------------------------------------------------------- diff --git a/notification/src/main/java/org/apache/atlas/kafka/KafkaNotification.java b/notification/src/main/java/org/apache/atlas/kafka/KafkaNotification.java index 6bb8d73..4d6b444 100644 --- a/notification/src/main/java/org/apache/atlas/kafka/KafkaNotification.java +++ b/notification/src/main/java/org/apache/atlas/kafka/KafkaNotification.java @@ -23,6 +23,7 @@ import kafka.server.KafkaServer; import kafka.utils.Time; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasException; +import org.apache.atlas.model.notification.AtlasNotificationMessage; import org.apache.atlas.notification.AbstractNotification; import org.apache.atlas.notification.NotificationConsumer; import org.apache.atlas.notification.NotificationException; @@ -40,6 +41,7 @@ import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.zookeeper.server.NIOServerCnxnFactory; import org.apache.zookeeper.server.ServerCnxnFactory; import org.apache.zookeeper.server.ZooKeeperServer; +import org.codehaus.jackson.type.TypeReference; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.core.annotation.Order; @@ -185,7 +187,8 @@ public class KafkaNotification extends AbstractNotification implements Service { Properties consumerProperties = getConsumerProperties(notificationType); List<NotificationConsumer<T>> consumers = new ArrayList<>(); - AtlasKafkaConsumer kafkaConsumer = new AtlasKafkaConsumer(notificationType.getDeserializer(), getKafkaConsumer(consumerProperties,notificationType, autoCommitEnabled), autoCommitEnabled, pollTimeOutMs ); + AtlasKafkaConsumer kafkaConsumer =new AtlasKafkaConsumer(notificationType, getKafkaConsumer(consumerProperties, notificationType, autoCommitEnabled), autoCommitEnabled, pollTimeOutMs); + consumers.add(kafkaConsumer); return consumers; } http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/notification/src/main/java/org/apache/atlas/notification/AbstractMessageDeserializer.java ---------------------------------------------------------------------- diff --git a/notification/src/main/java/org/apache/atlas/notification/AbstractMessageDeserializer.java b/notification/src/main/java/org/apache/atlas/notification/AbstractMessageDeserializer.java index 37a57d1..a787141 100644 --- a/notification/src/main/java/org/apache/atlas/notification/AbstractMessageDeserializer.java +++ b/notification/src/main/java/org/apache/atlas/notification/AbstractMessageDeserializer.java @@ -18,148 +18,30 @@ package org.apache.atlas.notification; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonDeserializationContext; -import com.google.gson.JsonDeserializer; -import com.google.gson.JsonElement; -import com.google.gson.JsonParseException; -import com.google.gson.reflect.TypeToken; -import org.apache.atlas.typesystem.IReferenceableInstance; -import org.apache.atlas.typesystem.IStruct; -import org.apache.atlas.typesystem.Referenceable; -import org.apache.atlas.typesystem.Struct; -import org.apache.atlas.typesystem.json.InstanceSerialization; -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONException; +import org.apache.atlas.model.notification.AtlasNotificationMessage; +import org.apache.atlas.model.notification.MessageVersion; +import org.codehaus.jackson.type.TypeReference; import org.slf4j.Logger; -import java.lang.reflect.Type; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - /** * Base notification message deserializer. */ public abstract class AbstractMessageDeserializer<T> extends AtlasNotificationMessageDeserializer<T> { - private static final Map<Type, JsonDeserializer> DESERIALIZER_MAP = new HashMap<>(); - - static { - DESERIALIZER_MAP.put(ImmutableList.class, new ImmutableListDeserializer()); - DESERIALIZER_MAP.put(ImmutableMap.class, new ImmutableMapDeserializer()); - DESERIALIZER_MAP.put(JSONArray.class, new JSONArrayDeserializer()); - DESERIALIZER_MAP.put(IStruct.class, new StructDeserializer()); - DESERIALIZER_MAP.put(IReferenceableInstance.class, new ReferenceableDeserializer()); - DESERIALIZER_MAP.put(Referenceable.class, new ReferenceableDeserializer()); - } - - // ----- Constructors ---------------------------------------------------- /** * Create a deserializer. * - * @param notificationMessageType the type of the notification message * @param expectedVersion the expected message version - * @param deserializerMap map of individual deserializers used to define this message deserializer * @param notificationLogger logger for message version mismatch */ - public AbstractMessageDeserializer(Type notificationMessageType, - MessageVersion expectedVersion, - Map<Type, JsonDeserializer> deserializerMap, - Logger notificationLogger) { - super(notificationMessageType, expectedVersion, getDeserializer(deserializerMap), notificationLogger); + public AbstractMessageDeserializer(TypeReference<T> messageType, + TypeReference<AtlasNotificationMessage<T>> notificationMessageType, + MessageVersion expectedVersion, Logger notificationLogger) { + super(messageType, notificationMessageType, expectedVersion, notificationLogger); } // ----- helper methods -------------------------------------------------- - - private static Gson getDeserializer(Map<Type, JsonDeserializer> deserializerMap) { - GsonBuilder builder = new GsonBuilder(); - - for (Map.Entry<Type, JsonDeserializer> entry : DESERIALIZER_MAP.entrySet()) { - builder.registerTypeAdapter(entry.getKey(), entry.getValue()); - } - - for (Map.Entry<Type, JsonDeserializer> entry : deserializerMap.entrySet()) { - builder.registerTypeAdapter(entry.getKey(), entry.getValue()); - } - return builder.create(); - } - - - // ----- deserializer classes -------------------------------------------- - - /** - * Deserializer for ImmutableList. - */ - protected static class ImmutableListDeserializer implements JsonDeserializer<ImmutableList<?>> { - public static final Type LIST_TYPE = new TypeToken<List<?>>() { - }.getType(); - - @Override - public ImmutableList<?> deserialize(JsonElement json, Type type, - JsonDeserializationContext context) { - final List<?> list = context.deserialize(json, LIST_TYPE); - return ImmutableList.copyOf(list); - } - } - - /** - * Deserializer for ImmutableMap. - */ - protected static class ImmutableMapDeserializer implements JsonDeserializer<ImmutableMap<?, ?>> { - - public static final Type MAP_TYPE = new TypeToken<Map<?, ?>>() { - }.getType(); - - @Override - public ImmutableMap<?, ?> deserialize(JsonElement json, Type type, - JsonDeserializationContext context) { - final Map<?, ?> map = context.deserialize(json, MAP_TYPE); - return ImmutableMap.copyOf(map); - } - } - - /** - * Deserializer for JSONArray. - */ - public static final class JSONArrayDeserializer implements JsonDeserializer<JSONArray> { - @Override - public JSONArray deserialize(final JsonElement json, final Type type, - final JsonDeserializationContext context) { - try { - return new JSONArray(json.toString()); - } catch (JSONException e) { - throw new JsonParseException(e.getMessage(), e); - } - } - } - - /** - * Deserializer for Struct. - */ - protected static final class StructDeserializer implements JsonDeserializer<IStruct> { - @Override - public IStruct deserialize(final JsonElement json, final Type type, - final JsonDeserializationContext context) { - return context.deserialize(json, Struct.class); - } - } - - /** - * Deserializer for Referenceable. - */ - protected static final class ReferenceableDeserializer implements JsonDeserializer<IReferenceableInstance> { - @Override - public IReferenceableInstance deserialize(final JsonElement json, final Type type, - final JsonDeserializationContext context) { - - return InstanceSerialization.fromJsonReferenceable(json.toString(), true); - } - } } http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/notification/src/main/java/org/apache/atlas/notification/AbstractNotification.java ---------------------------------------------------------------------- diff --git a/notification/src/main/java/org/apache/atlas/notification/AbstractNotification.java b/notification/src/main/java/org/apache/atlas/notification/AbstractNotification.java index 4f56bd8..8bc7cb4 100644 --- a/notification/src/main/java/org/apache/atlas/notification/AbstractNotification.java +++ b/notification/src/main/java/org/apache/atlas/notification/AbstractNotification.java @@ -18,18 +18,19 @@ package org.apache.atlas.notification; import com.google.common.annotations.VisibleForTesting; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; import com.google.gson.JsonParser; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; import org.apache.atlas.AtlasException; import org.apache.atlas.ha.HAConfiguration; -import org.apache.atlas.notification.AtlasNotificationBaseMessage.CompressionKind; -import org.apache.atlas.typesystem.IReferenceableInstance; -import org.apache.atlas.typesystem.Referenceable; -import org.apache.atlas.typesystem.json.InstanceSerialization; +import org.apache.atlas.model.notification.AtlasNotificationBaseMessage; +import org.apache.atlas.model.notification.AtlasNotificationMessage; +import org.apache.atlas.model.notification.AtlasNotificationStringMessage; +import org.apache.atlas.v1.model.instance.Referenceable; +import org.apache.atlas.model.notification.AtlasNotificationBaseMessage.CompressionKind; +import org.apache.atlas.type.AtlasType; +import org.apache.atlas.model.notification.MessageVersion; import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.StringUtils; import org.codehaus.jettison.json.JSONArray; @@ -45,8 +46,8 @@ import java.util.List; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; -import static org.apache.atlas.notification.AtlasNotificationBaseMessage.MESSAGE_COMPRESSION_ENABLED; -import static org.apache.atlas.notification.AtlasNotificationBaseMessage.MESSAGE_MAX_LENGTH_BYTES; +import static org.apache.atlas.model.notification.AtlasNotificationBaseMessage.MESSAGE_COMPRESSION_ENABLED; +import static org.apache.atlas.model.notification.AtlasNotificationBaseMessage.MESSAGE_MAX_LENGTH_BYTES; /** * Abstract notification interface implementation. @@ -79,15 +80,6 @@ public abstract class AbstractNotification implements NotificationInterface { private final boolean embedded; private final boolean isHAEnabled; - /** - * Used for message serialization. - */ - public static final Gson GSON = new GsonBuilder(). - registerTypeAdapter(IReferenceableInstance.class, new ReferenceableSerializer()). - registerTypeAdapter(Referenceable.class, new ReferenceableSerializer()). - registerTypeAdapter(JSONArray.class, new JSONArraySerializer()). - create(); - // ----- Constructors ---------------------------------------------------- public AbstractNotification(Configuration applicationProperties) throws AtlasException { @@ -160,7 +152,7 @@ public abstract class AbstractNotification implements NotificationInterface { public static String getMessageJson(Object message) { AtlasNotificationMessage<?> notificationMsg = new AtlasNotificationMessage<>(CURRENT_MESSAGE_VERSION, message); - return GSON.toJson(notificationMsg); + return AtlasType.toV1Json(notificationMsg); } private static String getHostAddress() { @@ -190,7 +182,7 @@ public abstract class AbstractNotification implements NotificationInterface { */ public static void createNotificationMessages(Object message, List<String> msgJsonList) { AtlasNotificationMessage<?> notificationMsg = new AtlasNotificationMessage<>(CURRENT_MESSAGE_VERSION, message, getHostAddress(), getCurrentUser()); - String msgJson = GSON.toJson(notificationMsg); + String msgJson = AtlasType.toV1Json(notificationMsg); boolean msgLengthExceedsLimit = (msgJson.length() * MAX_BYTES_PER_CHAR) > MESSAGE_MAX_LENGTH_BYTES; @@ -215,7 +207,7 @@ public abstract class AbstractNotification implements NotificationInterface { if (!msgLengthExceedsLimit) { // no need to split AtlasNotificationStringMessage compressedMsg = new AtlasNotificationStringMessage(encodedBytes, msgId, compressionKind); - msgJson = GSON.toJson(compressedMsg); // msgJson will not have multi-byte characters here, due to use of encodeBase64() above + msgJson = AtlasType.toV1Json(compressedMsg); // msgJson will not have multi-byte characters here, due to use of encodeBase64() above msgBytes = null; // not used after this point } else { // encodedBytes will be split msgJson = null; // not used after this point @@ -241,7 +233,7 @@ public abstract class AbstractNotification implements NotificationInterface { AtlasNotificationStringMessage splitMsg = new AtlasNotificationStringMessage(encodedBytes, offset, length, msgId, compressionKind, i, splitCount); - String splitMsgJson = GSON.toJson(splitMsg); + String splitMsgJson = AtlasType.toV1Json(splitMsg); msgJsonList.add(splitMsgJson); @@ -264,10 +256,10 @@ public abstract class AbstractNotification implements NotificationInterface { /** * Serializer for Referenceable. */ - public static final class ReferenceableSerializer implements JsonSerializer<IReferenceableInstance> { + public static final class ReferenceableSerializer implements JsonSerializer<Referenceable> { @Override - public JsonElement serialize(IReferenceableInstance src, Type typeOfSrc, JsonSerializationContext context) { - String instanceJson = InstanceSerialization.toJson(src, true); + public JsonElement serialize(Referenceable src, Type typeOfSrc, JsonSerializationContext context) { + String instanceJson = AtlasType.toV1Json(src); return new JsonParser().parse(instanceJson).getAsJsonObject(); } } http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/notification/src/main/java/org/apache/atlas/notification/AbstractNotificationConsumer.java ---------------------------------------------------------------------- diff --git a/notification/src/main/java/org/apache/atlas/notification/AbstractNotificationConsumer.java b/notification/src/main/java/org/apache/atlas/notification/AbstractNotificationConsumer.java index 8cf1e8e..c3940ce 100644 --- a/notification/src/main/java/org/apache/atlas/notification/AbstractNotificationConsumer.java +++ b/notification/src/main/java/org/apache/atlas/notification/AbstractNotificationConsumer.java @@ -16,30 +16,19 @@ * limitations under the License. */ package org.apache.atlas.notification; + import org.apache.kafka.common.TopicPartition; + /** * Abstract notification consumer. */ public abstract class AbstractNotificationConsumer<T> implements NotificationConsumer<T> { + protected final AtlasNotificationMessageDeserializer<T> deserializer; - /** - * Deserializer used to deserialize notification messages for this consumer. - */ - protected final MessageDeserializer<T> deserializer; - - - - /** - * Construct an AbstractNotificationConsumer. - * - * @param deserializer the message deserializer used by this consumer - */ - public AbstractNotificationConsumer(MessageDeserializer<T> deserializer) { + protected AbstractNotificationConsumer(AtlasNotificationMessageDeserializer<T> deserializer) { this.deserializer = deserializer; } - - public abstract void commit(TopicPartition partition, long offset); }