This is an automated email from the ASF dual-hosted git repository.
nbonte pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 801d40c ATLAS-4006, ATLAS-4007: Support business metadata in export
and import operations.
801d40c is described below
commit 801d40c3c62161a2bf8ae9fd0b1fea46d04eb7af
Author: Ashutosh Mestry <[email protected]>
AuthorDate: Mon Nov 9 16:19:28 2020 -0800
ATLAS-4006, ATLAS-4007: Support business metadata in export and import
operations.
Signed-off-by: Nikhil P Bonte <[email protected]>
---
.../repository/impexp/AtlasServerService.java | 2 +-
.../atlas/repository/impexp/ExportService.java | 7 +
.../repository/impexp/ExportTypeProcessor.java | 9 ++
.../repository/impexp/ImportTypeDefProcessor.java | 6 +
.../repository/impexp/TypeAttributeDifference.java | 12 ++
.../impexp/TypeAttributeDifferenceTest.java | 24 ++-
.../src/test/resources/json/typesDef-bm.json | 165 +++++++++++++++++++++
7 files changed, 223 insertions(+), 2 deletions(-)
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
index 542106f..5bbda5b 100644
---
a/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
+++
b/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
@@ -105,7 +105,7 @@ public class AtlasServerService {
AtlasObjectId objectId = getObjectId(server);
for (String guid : entityGuids) {
- AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo =
entityStore.getById(guid, false, true);
+ AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo =
entityStore.getById(guid, false, false);
updateAttribute(entityWithExtInfo, attributeName, objectId);
}
}
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
index 0491a85..8af2057 100644
---
a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
+++
b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
@@ -25,6 +25,7 @@ import org.apache.atlas.model.impexp.AtlasExportResult;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.model.typedef.AtlasBusinessMetadataDef;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
@@ -163,6 +164,12 @@ public class ExportService {
typesDef.getRelationshipDefs().add(relationshipDef);
}
+
+ for (String bm : context.businessMetadataTypes) {
+ AtlasBusinessMetadataDef bmDef =
typeRegistry.getBusinessMetadataDefByName(bm);
+
+ typesDef.getBusinessMetadataDefs().add(bmDef);
+ }
}
private AtlasExportResult.OperationStatus[]
processItems(AtlasExportRequest request, ExportContext context) {
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportTypeProcessor.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportTypeProcessor.java
index 664af91..a85db5c 100644
---
a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportTypeProcessor.java
+++
b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportTypeProcessor.java
@@ -121,6 +121,7 @@ class ExportTypeProcessor {
addAttributeTypes(entityType, context);
addRelationshipTypes(entityType, context);
+ addBusinessMetadataType(entityType, context);
if (CollectionUtils.isNotEmpty(entityType.getAllSuperTypes())) {
for (String superType : entityType.getAllSuperTypes()) {
@@ -180,6 +181,14 @@ class ExportTypeProcessor {
}
}
+ private void addBusinessMetadataType(AtlasEntityType entityType,
ExportService.ExportContext context) {
+ for (String bmTypeName : entityType.getBusinessAttributes().keySet()) {
+ AtlasBusinessMetadataType bmType =
typeRegistry.getBusinessMetadataTypeByName(bmTypeName);
+
+ addBusinessMetadataType(bmType, context);
+ }
+ }
+
private void addAttributeTypes(AtlasStructType structType,
ExportService.ExportContext context) {
for (AtlasStructDef.AtlasAttributeDef attributeDef :
structType.getStructDef().getAttributeDefs()) {
addType(attributeDef.getTypeName(), context);
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTypeDefProcessor.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTypeDefProcessor.java
index b658077..32d9c7c 100644
---
a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTypeDefProcessor.java
+++
b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTypeDefProcessor.java
@@ -19,6 +19,7 @@ package org.apache.atlas.repository.impexp;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportResult;
+import org.apache.atlas.model.typedef.AtlasBusinessMetadataDef;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
@@ -72,6 +73,10 @@ public class ImportTypeDefProcessor {
for (AtlasRelationshipDef def : typesDef.getRelationshipDefs()) {
def.setGuid(null);
}
+
+ for (AtlasBusinessMetadataDef def :
typesDef.getBusinessMetadataDefs()) {
+ def.setGuid(null);
+ }
}
private void updateMetricsForTypesDef(AtlasTypesDef typeDefinitionMap,
AtlasImportResult result) {
@@ -80,5 +85,6 @@ public class ImportTypeDefProcessor {
result.incrementMeticsCounter("typedef:entitydef",
typeDefinitionMap.getEntityDefs().size());
result.incrementMeticsCounter("typedef:struct",
typeDefinitionMap.getStructDefs().size());
result.incrementMeticsCounter("typedef:relationship",
typeDefinitionMap.getRelationshipDefs().size());
+ result.incrementMeticsCounter("typedef:businessmetadata",
typeDefinitionMap.getBusinessMetadataDefs().size());
}
}
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java
index 0d78dbe..0d1533d 100644
---
a/repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java
+++
b/repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java
@@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportResult;
+import org.apache.atlas.model.typedef.AtlasBusinessMetadataDef;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
@@ -54,6 +55,7 @@ public class TypeAttributeDifference {
updateClassificationDef(typeDefinitionMap.getClassificationDefs(),
result);
updateEntityDef(typeDefinitionMap.getEntityDefs(), result);
updateRelationshipDefs(typeDefinitionMap.getRelationshipDefs(),
result);
+
updateBusinessMetadataDefs(typeDefinitionMap.getBusinessMetadataDefs(), result);
}
private void updateEntityDef(List<AtlasEntityDef> entityDefs,
AtlasImportResult result) throws AtlasBaseException {
@@ -108,6 +110,16 @@ public class TypeAttributeDifference {
}
}
+ private void updateBusinessMetadataDefs(List<AtlasBusinessMetadataDef>
businessMetadataDefs, AtlasImportResult result) throws AtlasBaseException {
+ for (AtlasBusinessMetadataDef def : businessMetadataDefs) {
+ AtlasBusinessMetadataDef existing =
typeRegistry.getBusinessMetadataDefByName(def.getName());
+ if (existing != null && addAttributes(existing, def)) {
+ typeDefStore.updateStructDefByName(existing.getName(),
existing);
+
result.incrementMeticsCounter("typedef:businessmetadatadef:update");
+ }
+ }
+ }
+
@VisibleForTesting
boolean addElements(AtlasEnumDef existing, AtlasEnumDef incoming) throws
AtlasBaseException {
return addElements(existing, getElementsAbsentInExisting(existing,
incoming));
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java
index f9b7cd1..b1d3cc5 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java
@@ -17,22 +17,33 @@
*/
package org.apache.atlas.repository.impexp;
+import org.apache.atlas.TestUtilsV2;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import
org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.utils.TestLoadModelUtils;
+import org.apache.atlas.utils.TestResourceFileUtils;
import org.mockito.Mock;
-import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+
public class TypeAttributeDifferenceTest {
private TypeAttributeDifference typeAttributeDifference;
@@ -161,4 +172,15 @@ public class TypeAttributeDifferenceTest {
private boolean invokeUpdate(AtlasEnumDef existing, AtlasEnumDef incoming)
throws Exception {
return typeAttributeDifference.addElements(existing, incoming);
}
+
+ @Test
+ public void t1() throws IOException, AtlasBaseException {
+ AtlasTypesDef typesDef = TestResourceFileUtils.readObjectFromJson(".",
"typesDef-bm", AtlasTypesDef.class);
+
+ AtlasImportResult result = new AtlasImportResult();
+ AtlasTypesDef typesToCreate =
AtlasTypeDefStoreInitializer.getTypesToCreate(typesDef, this.typeRegistry);
+ typeDefStore.createTypesDef(typesToCreate);
+ typeAttributeDifference.updateTypes(typesDef, result);
+ assertNotNull(typesDef);
+ }
}
diff --git a/repository/src/test/resources/json/typesDef-bm.json
b/repository/src/test/resources/json/typesDef-bm.json
new file mode 100644
index 0000000..3bde422
--- /dev/null
+++ b/repository/src/test/resources/json/typesDef-bm.json
@@ -0,0 +1,165 @@
+{
+ "enumDefs": [],
+ "structDefs": [],
+ "classificationDefs": [],
+ "entityDefs": [
+ {
+ "category": "ENTITY",
+ "name": "rdbms_table",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "name",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": -1,
+ "valuesMaxCount": -1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "name": "rdbms_storage",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "name",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": -1,
+ "valuesMaxCount": -1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "name": "rdbms_db",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "name",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": -1,
+ "valuesMaxCount": -1,
+ "isUnique": true,
+ "isIndexable": true
+ },
+ {
+ "name": "sd",
+ "typeName": "rdbms_storage",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": -1,
+ "valuesMaxCount": -1,
+ "isUnique": false,
+ "isIndexable": false,
+ "constraints": [
+ {
+ "type": "ownedRef"
+ }
+ ],
+ "options": {
+ "isSoftReference": "true"
+ }
+ },
+ {
+ "name": "tables",
+ "typeName": "array<rdbms_table>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": -1,
+ "valuesMaxCount": -1,
+ "isUnique": false,
+ "isIndexable": false,
+ "constraints": [
+ {
+ "type": "ownedRef"
+ }
+ ],
+ "options": {
+ "isSoftReference": "true"
+ }
+ },
+ {
+ "name": "regions",
+ "typeName": "map<string,rdbms_table>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": -1,
+ "valuesMaxCount": -1,
+ "isUnique": false,
+ "isIndexable": false,
+ "constraints": [
+ {
+ "type": "ownedRef"
+ }
+ ],
+ "options": {
+ "isSoftReference": "true"
+ }
+ }
+ ],
+ "superTypes": [],
+ "businessAttributeDefs": {
+ "B1": [{
+ "name": "A1",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true,
+ "includeInNotification": false,
+ "searchWeight": 5,
+ "options": {
+ "applicableEntityTypes": "[\"hive_db\"]",
+ "maxStrLength": "50"
+ }
+ }]
+ }
+ }
+ ],
+ "businessMetadataDefs": [
+ {
+ "category": "BUSINESS_METADATA",
+ "guid": "a0d52995-9ef7-41e7-bfc0-652e0cf3587c",
+ "createdBy": "admin",
+ "updatedBy": "admin",
+ "createTime": 1603487492875,
+ "updateTime": 1603487515791,
+ "version": 2,
+ "name": "B1",
+ "description": "Test B1",
+ "typeVersion": "1.1",
+ "attributeDefs": [
+ {
+ "name": "A1",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true,
+ "includeInNotification": false,
+ "searchWeight": 5,
+ "options": {
+ "applicableEntityTypes": "[\"rdbms_table\"]",
+ "maxStrLength": "50"
+ }
+ }
+ ]
+ }
+ ]
+}
