[atlas] branch master updated: ATLAS-3073: Investigate and fix IT failures in AdminExportImportTestIT
This is an automated email from the ASF dual-hosted git repository.
amestry pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 9e2d326 ATLAS-3073: Investigate and fix IT failures in
AdminExportImportTestIT
9e2d326 is described below
commit 9e2d326688a093c7d361063f24fc7bd5affd4e01
Author: Ashutosh Mestry
AuthorDate: Wed Mar 13 18:20:56 2019 -0700
ATLAS-3073: Investigate and fix IT failures in AdminExportImportTestIT
---
addons/models/1000-Hadoop/1030-hive_model.json | 2 +-
.../models/1000-Hadoop/patches/008-remove-hive-legacy-attributes.json | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/addons/models/1000-Hadoop/1030-hive_model.json
b/addons/models/1000-Hadoop/1030-hive_model.json
index 7207a41..e964433 100644
--- a/addons/models/1000-Hadoop/1030-hive_model.json
+++ b/addons/models/1000-Hadoop/1030-hive_model.json
@@ -464,7 +464,7 @@
"name": "hive_table_db",
"serviceType": "hive",
"typeVersion": "1.2",
-"relationshipCategory": "COMPOSITION",
+"relationshipCategory": "AGGREGATION",
"relationshipLabel": "__hive_table.db",
"endDef1": {
"type": "hive_table",
diff --git
a/addons/models/1000-Hadoop/patches/008-remove-hive-legacy-attributes.json
b/addons/models/1000-Hadoop/patches/008-remove-hive-legacy-attributes.json
index 2c863f0..4544ee3 100644
--- a/addons/models/1000-Hadoop/patches/008-remove-hive-legacy-attributes.json
+++ b/addons/models/1000-Hadoop/patches/008-remove-hive-legacy-attributes.json
@@ -8,7 +8,8 @@
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"params": {
-"relationshipLabel": "__hive_table.db"
+"relationshipLabel": "__hive_table.db",
+"relationshipCategory": "AGGREGATION"
}
},
{
[atlas] branch branch-1.0 updated: ATLAS-3069: avoid repeated warnings of missing relationship-def
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-1.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-1.0 by this push:
new 6c9dcc9 ATLAS-3069: avoid repeated warnings of missing
relationship-def
6c9dcc9 is described below
commit 6c9dcc9d563102175048528643d4249cfa92380a
Author: Sridhar Swamy
AuthorDate: Fri Mar 8 15:35:49 2019 -0800
ATLAS-3069: avoid repeated warnings of missing relationship-def
Signed-off-by: Madhan Neethiraj
(cherry picked from commit ca22400ebf425088097b20b33873c15a71d5f908)
---
.../java/org/apache/atlas/type/AtlasEntityType.java | 3 +--
.../org/apache/atlas/type/AtlasTypeRegistry.java| 21 +
2 files changed, 18 insertions(+), 6 deletions(-)
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
index b5360c1..87a327a 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
@@ -182,8 +182,7 @@ public class AtlasEntityType extends AtlasStructType {
// validate if RelationshipDefs is defined for all entityDefs
if (attributeEntityType != null &&
!hasRelationshipAttribute(attributeName)) {
-LOG.warn("No RelationshipDef defined between {} and {} on
attribute: {}.{}", getTypeName(),
- attributeEntityType.getTypeName(), getTypeName(),
attributeName);
+typeRegistry.reportMissingRelationshipDef(getTypeName(),
attributeEntityType.getTypeName(), attributeName);
}
}
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
index 21f9429..db63ed1 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
@@ -56,16 +56,20 @@ public class AtlasTypeRegistry {
protected RegistryData registryData;
private final TypeRegistryUpdateSynchronizer updateSynchronizer;
+private final SetmissingRelationshipDefs;
+
public AtlasTypeRegistry() {
-registryData = new RegistryData();
-updateSynchronizer = new TypeRegistryUpdateSynchronizer(this);
+registryData= new RegistryData();
+updateSynchronizer = new TypeRegistryUpdateSynchronizer(this);
+missingRelationshipDefs = new HashSet<>();
}
// used only by AtlasTransientTypeRegistry
protected AtlasTypeRegistry(AtlasTypeRegistry other) {
-registryData = new RegistryData();
-updateSynchronizer = other.updateSynchronizer;
+registryData= new RegistryData();
+updateSynchronizer = other.updateSynchronizer;
+missingRelationshipDefs = other.missingRelationshipDefs;
}
public Collection getAllTypeNames() { return
registryData.allTypes.getAllTypeNames(); }
@@ -224,6 +228,15 @@ public class AtlasTypeRegistry {
updateSynchronizer.releaseTypeRegistryForUpdate(transientTypeRegistry,
commitUpdates);
}
+public void reportMissingRelationshipDef(String entityType1, String
entityType2, String attributeName) {
+String key = entityType1 + "->" + entityType2 + ":" + attributeName;
+
+if (!missingRelationshipDefs.contains(key)) {
+LOG.warn("No RelationshipDef defined between {} and {} on
attribute: {}.{}", entityType1, entityType2, entityType1, attributeName);
+
+missingRelationshipDefs.add(key);
+}
+}
static class RegistryData {
final TypeCache
allTypes;
[atlas] branch master updated: ATLAS-3069: avoid repeated warnings of missing relationship-def
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new ca22400 ATLAS-3069: avoid repeated warnings of missing
relationship-def
ca22400 is described below
commit ca22400ebf425088097b20b33873c15a71d5f908
Author: Sridhar Swamy
AuthorDate: Fri Mar 8 15:35:49 2019 -0800
ATLAS-3069: avoid repeated warnings of missing relationship-def
Signed-off-by: Madhan Neethiraj
---
.../java/org/apache/atlas/type/AtlasEntityType.java | 3 +--
.../org/apache/atlas/type/AtlasTypeRegistry.java| 21 +
2 files changed, 18 insertions(+), 6 deletions(-)
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
index b5360c1..87a327a 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
@@ -182,8 +182,7 @@ public class AtlasEntityType extends AtlasStructType {
// validate if RelationshipDefs is defined for all entityDefs
if (attributeEntityType != null &&
!hasRelationshipAttribute(attributeName)) {
-LOG.warn("No RelationshipDef defined between {} and {} on
attribute: {}.{}", getTypeName(),
- attributeEntityType.getTypeName(), getTypeName(),
attributeName);
+typeRegistry.reportMissingRelationshipDef(getTypeName(),
attributeEntityType.getTypeName(), attributeName);
}
}
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
index 21f9429..db63ed1 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
@@ -56,16 +56,20 @@ public class AtlasTypeRegistry {
protected RegistryData registryData;
private final TypeRegistryUpdateSynchronizer updateSynchronizer;
+private final SetmissingRelationshipDefs;
+
public AtlasTypeRegistry() {
-registryData = new RegistryData();
-updateSynchronizer = new TypeRegistryUpdateSynchronizer(this);
+registryData= new RegistryData();
+updateSynchronizer = new TypeRegistryUpdateSynchronizer(this);
+missingRelationshipDefs = new HashSet<>();
}
// used only by AtlasTransientTypeRegistry
protected AtlasTypeRegistry(AtlasTypeRegistry other) {
-registryData = new RegistryData();
-updateSynchronizer = other.updateSynchronizer;
+registryData= new RegistryData();
+updateSynchronizer = other.updateSynchronizer;
+missingRelationshipDefs = other.missingRelationshipDefs;
}
public Collection getAllTypeNames() { return
registryData.allTypes.getAllTypeNames(); }
@@ -224,6 +228,15 @@ public class AtlasTypeRegistry {
updateSynchronizer.releaseTypeRegistryForUpdate(transientTypeRegistry,
commitUpdates);
}
+public void reportMissingRelationshipDef(String entityType1, String
entityType2, String attributeName) {
+String key = entityType1 + "->" + entityType2 + ":" + attributeName;
+
+if (!missingRelationshipDefs.contains(key)) {
+LOG.warn("No RelationshipDef defined between {} and {} on
attribute: {}.{}", entityType1, entityType2, entityType1, attributeName);
+
+missingRelationshipDefs.add(key);
+}
+}
static class RegistryData {
final TypeCache
allTypes;
[atlas] branch master updated: ATLAS-3057:- Atlas Index Repair tool for JanusGraph
This is an automated email from the ASF dual-hosted git repository. nixon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/atlas.git The following commit(s) were added to refs/heads/master by this push: new 3ed1f5a ATLAS-3057:- Atlas Index Repair tool for JanusGraph 3ed1f5a is described below commit 3ed1f5a082ed665ce54f7dffc683b0f7b191e44d Author: nikhilbonte AuthorDate: Tue Feb 26 16:27:43 2019 +0530 ATLAS-3057:- Atlas Index Repair tool for JanusGraph Signed-off-by: nixonrodrigues --- docs/src/site/markdown/AtlasRepairIndex.md | 59 + docs/src/site/twiki/index.twiki| 1 + tools/atlas-index-repair/pom.xml | 61 + .../java/org/apache/atlas/tools/RepairIndex.java | 250 + .../src/main/resources/atlas-log4j.xml | 42 .../src/main/resources/repair_index.py | 153 + 6 files changed, 566 insertions(+) diff --git a/docs/src/site/markdown/AtlasRepairIndex.md b/docs/src/site/markdown/AtlasRepairIndex.md new file mode 100644 index 000..3a9d40f --- /dev/null +++ b/docs/src/site/markdown/AtlasRepairIndex.md @@ -0,0 +1,59 @@ + + + ##Introduction + + The document describes the use of the Atlas Index Repair Utility for JanusGraph, with HBase as back-end data store and Solr as index store. + Need for this Tool + In rare, cases it is possible that during entity creation, the entity is stored in the data store, but the corresponding indexes are not created in Solr. Since Atlas relies heavily on Solr in the operation of its Basic Search, this will result in entity not being returned by a search. Note that Advanced Search is not affected by this. + Location + The tool is part of the normal Atlas installation, it is located under the tools/atlas-index-repair directory. + Steps to Execute Tool + #Complete Restore + If the user needs to restore all the indexes, this can be accomplished by executing the tool with no command-line parameters: + + >atlas-index-repair/repair_index.py + + This will result in vertex_index, edge_index and fulltext_index to be re-built completely. It is recommended that existing contents of these indexes be deleted before executing this restore. + ##Caveats + Note that the full index repair is a time consuming process. Depending on the size of data the process may take days to complete. During the restore process the Basic Search functionality will not be available. Be sure to allocate sufficient time for this activity. + #Selective Restore + To perform selective restore for an Atlas entity, specify the GUID of that entity: + >atlas-index-repair/repair_index.py [-g \] + + Example: + > atlas-index-repair/repair_index.py -g 13d77457-2a45-4e92-ad53-a172c7cb70a5 + + Note that Atlas will use REST APIs to fetch the entity, which will need correct authentication mechanism to be specified based on the installation. + + For an Atlas installation with username and password use: + >atlas-index-repair/repair_index.py [-g \] [-u \] [-p \] + * guid: [optional] specify guid for which indexes are to be updated + * user: [optional] specify username for atlas instance + * password: [optional] specify password for atlas instance + + Example: + >atlas-index-repair/repair_index.py -u admin -p admin123 -g 13d77457-2a45-4e92-ad53-a172c7cb70a5 + + For Atlas installation that uses kerberos as authentication mode, + use: kinit -kt /etc/security/keytabs/atlas.service.keytab atlas/fqdn@DOMAIN + + Example: + >kinit -kt /etc/security/keytabs/atlas.service.keytab atlas/f...@example.com + > + >atlas-index-repair/repair_index.py -g 13d77457-2a45-4e92-ad53-a172c7cb70a5 diff --git a/docs/src/site/twiki/index.twiki b/docs/src/site/twiki/index.twiki index 7d8c982..05a33fa 100755 --- a/docs/src/site/twiki/index.twiki +++ b/docs/src/site/twiki/index.twiki @@ -72,6 +72,7 @@ capabilities around these data assets for data scientists, analysts and the data * [[Bridge-Kafka][Kafka Bridge]] * [[HighAvailability][Fault Tolerance And High Availability Options]] * [[Migration-0.8-to-1.0][Migration from Apache Atlas 0.8]] + * [[AtlasRepairIndex][Index repair tool]] ---++ API Documentation diff --git a/tools/atlas-index-repair/pom.xml b/tools/atlas-index-repair/pom.xml new file mode 100644 index 000..940e97a --- /dev/null +++ b/tools/atlas-index-repair/pom.xml @@ -0,0 +1,61 @@ + + + +http://maven.apache.org/POM/4.0.0; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd;> +4.0.0 + +apache-atlas +org.apache.atlas +2.0.0-SNAPSHOT +../../ + +atlas-index-repair-tool +Apache Atlas index repair Module +Apache Atlas index repair tool +jar + + + + +org.apache.atlas
[atlas] branch master updated: ATLAS-3070: Create patch framework to persist typedef patches applied to atlas
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 4925ec3 ATLAS-3070: Create patch framework to persist typedef patches
applied to atlas
4925ec3 is described below
commit 4925ec35933b8be6a75e758654629f55bb12ee60
Author: Sarath Subramanian
AuthorDate: Wed Mar 13 01:43:59 2019 -0700
ATLAS-3070: Create patch framework to persist typedef patches applied to
atlas
---
.../001-base_model_replication_attributes.json | 2 +
.../patches/002-base_model_add_service_type.json | 66 +-
.../patches/001-hive_column_add_position.json | 2 +
.../patches/002-hive_column_table_add_options.json | 4 +
...hive_column_update_table_remove_constraint.json | 2 +
...04-hbase_table_column_family_add_attribute.json | 4 +
...ble_column_family_add_additional_attribute.json | 4 +
.../patches/006-kafka_topic_add_attribute.json | 2 +
.../patches/007-hadoop_model_add_service_type.json | 110 +-
.../patches/008-remove-hive-legacy-attributes.json | 10 +
.../001-rdbms_column_table_add_options.json| 4 +
.../patches/002-rdbms_model_add_service_type.json | 32 ++-
.../003-remove-rdbms-legacy-attributes.json| 18 ++
.../patches/001-cloud_model_add_service_type.json | 22 ++
.../org/apache/atlas/repository/Constants.java | 9 +
.../org/apache/atlas/model/patches/AtlasPatch.java | 225 +
.../java/org/apache/atlas/type/AtlasTypeUtil.java | 2 +
.../repository/graph/GraphBackedSearchIndexer.java | 6 +
.../bootstrap/AtlasTypeDefStoreInitializer.java| 224
.../store/graph/v2/AtlasGraphUtilsV2.java | 98 -
.../store/graph/v2/AtlasTypeDefGraphStoreV2.java | 2 +-
.../apache/atlas/web/resources/AdminResource.java | 20 +-
22 files changed, 819 insertions(+), 49 deletions(-)
diff --git
a/addons/models/-Area0/patches/001-base_model_replication_attributes.json
b/addons/models/-Area0/patches/001-base_model_replication_attributes.json
index 92c561f..1bedea7 100644
---
a/addons/models/-Area0/patches/001-base_model_replication_attributes.json
+++
b/addons/models/-Area0/patches/001-base_model_replication_attributes.json
@@ -1,6 +1,8 @@
{
"patches": [
{
+ "id": "TYPEDEF_PATCH__001",
+ "description": "Add 'replicatedFrom' and 'replicatedTo' attributes to
Referenceable",
"action": "ADD_ATTRIBUTE",
"typeName": "Referenceable",
"applyToVersion": "1.0",
diff --git
a/addons/models/-Area0/patches/002-base_model_add_service_type.json
b/addons/models/-Area0/patches/002-base_model_add_service_type.json
index 6d4e80d..13e55fa 100644
--- a/addons/models/-Area0/patches/002-base_model_add_service_type.json
+++ b/addons/models/-Area0/patches/002-base_model_add_service_type.json
@@ -1,13 +1,17 @@
{
"patches": [
{
+ "id": "TYPEDEF_PATCH__002",
+ "description": "Set serviceType 'atlas_core' to Referenceable",
"action": "SET_SERVICE_TYPE",
"typeName":"Referenceable",
- "applyToVersion": "1.0",
- "updateToVersion": "1.1",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
"serviceType": "atlas_core"
},
{
+ "id": "TYPEDEF_PATCH__003",
+ "description": "Set serviceType 'atlas_core' to __internal",
"action": "SET_SERVICE_TYPE",
"typeName":"__internal",
"applyToVersion": "1.0",
@@ -15,6 +19,8 @@
"serviceType": "atlas_core"
},
{
+ "id": "TYPEDEF_PATCH__004",
+ "description": "Set serviceType 'atlas_core' to Asset",
"action": "SET_SERVICE_TYPE",
"typeName":"Asset",
"applyToVersion": "1.1",
@@ -22,6 +28,8 @@
"serviceType": "atlas_core"
},
{
+ "id": "TYPEDEF_PATCH__005",
+ "description": "Set serviceType 'atlas_core' to DataSet",
"action": "SET_SERVICE_TYPE",
"typeName":"DataSet",
"applyToVersion": "1.1",
@@ -29,6 +37,8 @@
"serviceType": "atlas_core"
},
{
+ "id": "TYPEDEF_PATCH__006",
+ "description": "Set serviceType 'atlas_core' to Infrastructure",
"action": "SET_SERVICE_TYPE",
"typeName":"Infrastructure",
"applyToVersion": "1.1",
@@ -36,6 +46,8 @@
"serviceType": "atlas_core"
},
{
+ "id": "TYPEDEF_PATCH__007",
+ "description": "Set serviceType 'atlas_core' to Process",
"action": "SET_SERVICE_TYPE",
"typeName":"Process",
"applyToVersion": "1.1",
@@ -43,6 +55,8 @@
"serviceType":
