[atlas] branch master updated: ATLAS-3112: Allow Indexing of array attributes (LIST or SET) in indexing store
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new d234de2 ATLAS-3112: Allow Indexing of array attributes (LIST or SET)
in indexing store
d234de2 is described below
commit d234de2d7ef4a935db313a852ce80b8aab126c3b
Author: Sarath Subramanian
AuthorDate: Wed Apr 3 10:11:38 2019 -0700
ATLAS-3112: Allow Indexing of array attributes (LIST or SET) in indexing
store
---
.../repository/graphdb/AtlasGraphManagement.java | 3 +-
.../graphdb/janus/AtlasJanusGraphManagement.java | 8 --
.../graphdb/janus/AbstractGraphDatabaseTest.java | 2 +-
.../repository/graph/GraphBackedSearchIndexer.java | 24
.../apache/atlas/repository/graph/GraphHelper.java | 4 +--
.../store/graph/v2/AtlasGraphUtilsV2.java | 27 ++
.../store/graph/v2/EntityGraphMapper.java | 32 --
.../store/graph/v2/EntityStateChecker.java | 4 +--
8 files changed, 70 insertions(+), 34 deletions(-)
diff --git
a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
index b1e2c5d..cfa1cb4 100644
---
a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
+++
b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
@@ -155,6 +155,7 @@ public interface AtlasGraphManagement {
*
* @param vertexIndex
* @param propertyKey
+ * @param propertyClass
*/
-void addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey);
+void addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey, Class
propertyClass);
}
diff --git
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
index 4e18432..7b034ba 100644
---
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
+++
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
@@ -195,11 +195,15 @@ public class AtlasJanusGraphManagement implements
AtlasGraphManagement {
}
@Override
-public void addMixedIndex(String indexName, AtlasPropertyKey propertyKey) {
+public void addMixedIndex(String indexName, AtlasPropertyKey propertyKey,
Class propertyClass) {
PropertyKey janusKey=
AtlasJanusObjectFactory.createPropertyKey(propertyKey);
JanusGraphIndex vertexIndex = management.getGraphIndex(indexName);
-management.addIndexKey(vertexIndex, janusKey);
+if (propertyClass == String.class) {
+management.addIndexKey(vertexIndex, janusKey,
Mapping.STRING.asParameter());
+} else {
+management.addIndexKey(vertexIndex, janusKey);
+}
}
@Override
diff --git
a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AbstractGraphDatabaseTest.java
b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AbstractGraphDatabaseTest.java
index f72b412..a139d08 100644
---
a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AbstractGraphDatabaseTest.java
+++
b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AbstractGraphDatabaseTest.java
@@ -99,7 +99,7 @@ public abstract class AbstractGraphDatabaseTest {
AtlasPropertyKey key = management.makePropertyKey(propertyName,
propertyClass, cardinality);
try {
if (propertyClass != Integer.class) {
-management.addMixedIndex(BACKING_INDEX_NAME, key);
+management.addMixedIndex(BACKING_INDEX_NAME, key,
propertyClass);
}
} catch(Throwable t) {
//ok
diff --git
a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index c57f8e3..9f75d2a 100755
---
a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++
b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -271,12 +271,13 @@ public class GraphBackedSearchIndexer implements
SearchIndexer, ActiveStateChang
createVertexIndex(management, VERTEX_ID_IN_IMPORT_KEY,
UniqueKind.NONE, Long.class, SINGLE, true, false);
createVertexIndex(management, ENTITY_TYPE_PROPERTY_KEY,
UniqueKind.NONE, String.class, SINGLE, true, false);
-createVertexIndex(management, SUPER_TYPES_PROPERTY_KEY,
UniqueKind.NONE, String.class, SET, true, false);
[atlas] branch master updated: ATLAS-3017: Add Atlas server statistics rest endpoint
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 984445e ATLAS-3017: Add Atlas server statistics rest endpoint
984445e is described below
commit 984445e9111c743f3f255fc4653fc02555a8f095
Author: nikhilbonte
AuthorDate: Wed Apr 3 05:31:16 2019 -0700
ATLAS-3017: Add Atlas server statistics rest endpoint
Signed-off-by: Sarath Subramanian
---
.../js/templates/common/Statistics_tmpl.html | 49 +++-
dashboardv2/public/js/views/common/Statistics.js | 7 +-
.../org/apache/atlas/model/AtlasStatistics.java| 78 +++
.../org/apache/atlas/services/MetricsService.java | 18 +-
.../java/org/apache/atlas/util/StatisticsUtil.java | 254 +
.../notification/NotificationHookConsumer.java | 12 +-
.../web/service/ActiveInstanceElectorService.java | 9 +-
.../NotificationHookConsumerKafkaTest.java | 10 +-
.../notification/NotificationHookConsumerTest.java | 26 ++-
.../service/ActiveInstanceElectorServiceTest.java | 32 +--
10 files changed, 446 insertions(+), 49 deletions(-)
diff --git a/dashboardv2/public/js/templates/common/Statistics_tmpl.html
b/dashboardv2/public/js/templates/common/Statistics_tmpl.html
index 13e64a7..71643c7 100644
--- a/dashboardv2/public/js/templates/common/Statistics_tmpl.html
+++ b/dashboardv2/public/js/templates/common/Statistics_tmpl.html
@@ -18,8 +18,8 @@
- Active Entities (0)
-
+Active Entities (0)
+
@@ -28,10 +28,10 @@
-
-Entity
-Count
-
+
+Entity
+Count
+
@@ -42,8 +42,8 @@
- Deleted Entities (0)
-
+Deleted Entities (0)
+
@@ -52,10 +52,35 @@
-
-Entity
-Count
-
+
+Entity
+Count
+
+
+
+
+
+
+
+
+
+
+
+
+Server Statistics
+
+
+
+
+
+
+
+
+
+
+Parameter
+Value
+
diff --git a/dashboardv2/public/js/views/common/Statistics.js
b/dashboardv2/public/js/views/common/Statistics.js
index fc1596a..009babe 100644
--- a/dashboardv2/public/js/views/common/Statistics.js
+++ b/dashboardv2/public/js/views/common/Statistics.js
@@ -38,7 +38,8 @@ define(['require',
entityActive: "[data-id='entityActive'] tbody",
entityDelete: "[data-id='entityDelete'] tbody",
entityActiveHeader: "[data-id='entityActive'] .count",
-entityDeletedHeader: "[data-id='entityDelete'] .count"
+entityDeletedHeader: "[data-id='entityDelete'] .count",
+stats: "[data-id='stats'] tbody"
},
/** ui events hash */
events: function() {},
@@ -71,7 +72,8 @@ define(['require',
var data = _.first(data.toJSON()),
no_records = 'No records found!',
activeEntityTable =
_.isEmpty(data.entity.entityActive) ? no_records : that.getTable({ valueObject:
data.entity.entityActive }),
-deleteEntityTable =
_.isEmpty(data.entity.entityDeleted) ? no_records : that.getTable({
valueObject: data.entity.entityDeleted });
+deleteEntityTable =
_.isEmpty(data.entity.entityDeleted) ? no_records : that.getTable({
valueObject: data.entity.entityDeleted}),
+stats = _.isEmpty(data.general.stats) ? no_records
: that.getTable({ valueObject: data.general.stats, formatIntVal: false});
var totalActive = 0,
totalDeleted = 0;
if (data.entity && data.general.entityCount) {
@@ -84,6 +86,7 @@ define(['require',
}
that.ui.entityActive.html(activeEntityTable);
[atlas] branch master updated: ATLAS-3113: Use index query to search for active entities and better logging in java patch framework
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 66f57da ATLAS-3113: Use index query to search for active entities and
better logging in java patch framework
66f57da is described below
commit 66f57da82afd1d348eca4e0c9deccc3838ebe114
Author: Sarath Subramanian
AuthorDate: Wed Apr 3 05:06:18 2019 -0700
ATLAS-3113: Use index query to search for active entities and better
logging in java patch framework
---
.../patches/UniqueAttributePatchHandler.java| 21 +
.../store/graph/v2/AtlasGraphUtilsV2.java | 18 +-
2 files changed, 26 insertions(+), 13 deletions(-)
diff --git
a/repository/src/main/java/org/apache/atlas/repository/patches/UniqueAttributePatchHandler.java
b/repository/src/main/java/org/apache/atlas/repository/patches/UniqueAttributePatchHandler.java
index 0e707a5..f2238f1 100644
---
a/repository/src/main/java/org/apache/atlas/repository/patches/UniqueAttributePatchHandler.java
+++
b/repository/src/main/java/org/apache/atlas/repository/patches/UniqueAttributePatchHandler.java
@@ -22,6 +22,7 @@ import org.apache.atlas.repository.IndexException;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer.UniqueKind;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
+import org.apache.atlas.repository.graphdb.AtlasIndexQuery.Result;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
@@ -56,7 +57,7 @@ public class UniqueAttributePatchHandler extends
AtlasJavaPatchHandler {
for (AtlasEntityType entityType : allEntityTypes) {
String typeName =
entityType.getTypeName();
Map uniqAttributes=
entityType.getUniqAttributes();
-int entitiesProcessed = 0;
+int patchAppliedCount = 0;
LOG.info("Applying java patch: {} for type: {}", getPatchId(),
typeName);
@@ -67,12 +68,16 @@ public class UniqueAttributePatchHandler extends
AtlasJavaPatchHandler {
// register unique attribute property keys in graph
registerUniqueAttrPropertyKeys(attributes);
-Iterator iterator =
findActiveEntityVerticesByType(typeName);
+Iterator> iterator =
findActiveEntityVerticesByType(typeName);
-while (iterator.hasNext()) {
-AtlasVertex entityVertex = iterator.next();
+int entityCount = 0;
+
+while (iterator != null && iterator.hasNext()) {
+AtlasVertex entityVertex = iterator.next().getVertex();
boolean patchApplied = false;
+entityCount++;
+
for (AtlasAttribute attribute : attributes) {
String uniquePropertyKey =
attribute.getVertexUniquePropertyName();
Collection propertyKeys =
entityVertex.getPropertyKeys();
@@ -104,11 +109,11 @@ public class UniqueAttributePatchHandler extends
AtlasJavaPatchHandler {
}
if (patchApplied) {
-entitiesProcessed++;
+patchAppliedCount++;
}
-if (entitiesProcessed % 1000 == 0) {
-LOG.info("Java patch: {} : processed {} {}
entities.", getPatchId(), entitiesProcessed, typeName);
+if (entityCount % 1000 == 0) {
+LOG.info("Java patch: {} : applied {}; processed
{} {} entities.", getPatchId(), patchAppliedCount, entityCount, typeName);
}
}
} catch (IndexException e) {
@@ -120,7 +125,7 @@ public class UniqueAttributePatchHandler extends
AtlasJavaPatchHandler {
}
}
-LOG.info("Applied java patch ({}) for type: {}; Total processed:
{}", getPatchId(), typeName, entitiesProcessed);
+LOG.info("Applied java patch ({}) for type: {}; Total processed:
{}", getPatchId(), typeName, patchAppliedCount);
}
if (patchFailed) {
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
index 2882f09..80141b4 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
+++
