Repository: atlas Updated Branches: refs/heads/branch-0.8 570f7a8c9 -> fba929a64
ATLAS-2827: Indexed string attr compaction when length exceeds 33482223 Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/fba929a6 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/fba929a6 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/fba929a6 Branch: refs/heads/branch-0.8 Commit: fba929a64d5fa10c7938107cb455d28e49e06d0f Parents: 570f7a8 Author: apoorvnaik <[email protected]> Authored: Fri Aug 17 08:48:40 2018 -0700 Committer: apoorvnaik <[email protected]> Committed: Fri Aug 17 08:49:31 2018 -0700 ---------------------------------------------------------------------- .../store/graph/v1/EntityGraphMapper.java | 27 +++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/fba929a6/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java index ac2d4c9..47960f3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java @@ -39,6 +39,7 @@ import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.type.AtlasArrayType; +import org.apache.atlas.type.AtlasBuiltInTypes; import org.apache.atlas.type.AtlasClassificationType; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasMapType; @@ -47,6 +48,7 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute; import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeUtil; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; @@ -67,7 +69,9 @@ import static org.apache.atlas.repository.graph.GraphHelper.string; @Component public class EntityGraphMapper { private static final Logger LOG = LoggerFactory.getLogger(EntityGraphMapper.class); - private static final String SOFT_REF_FORMAT = "%s:%s"; + + private static final String SOFT_REF_FORMAT = "%s:%s"; + private static final int INDEXED_STR_MAX_ALLOWED_LEN = 33482223; private final GraphHelper graphHelper = GraphHelper.getInstance(); private final AtlasGraph graph; @@ -426,9 +430,26 @@ public class EntityGraphMapper { private Object mapPrimitiveValue(AttributeMutationContext ctx) { - AtlasGraphUtilsV1.setProperty(ctx.getReferringVertex(), ctx.getVertexProperty(), ctx.getValue()); + boolean isIndexableStrAttr = ctx.getAttributeDef().getIsIndexable() && ctx.getAttrType() instanceof AtlasBuiltInTypes.AtlasStringType; + + Object ret = ctx.getValue(); + + // Titan bug, when an indexed string attribute has a value longer than a certain length then the reverse indexed key generated by JanusGraph + // exceeds the HBase row length's hard limit (Short.MAX). This trimming and hashing procedure is to circumvent that limitation + if (ret != null && isIndexableStrAttr) { + String value = (String) ctx.getValue(); + + if (value.length() > INDEXED_STR_MAX_ALLOWED_LEN) { + LOG.warn("Indexed-String-Attribute: {} exceeds {} characters, trimming and appending checksum", + ctx.getAttribute().getQualifiedName(), INDEXED_STR_MAX_ALLOWED_LEN); + String sha256Hex = DigestUtils.shaHex(value); // Storing SHA checksum in case verification is needed after retrieval + ret = value.substring(0, (INDEXED_STR_MAX_ALLOWED_LEN - 1) - sha256Hex.length()) + ":" + sha256Hex; + } + } - return ctx.getValue(); + AtlasGraphUtilsV1.setProperty(ctx.getReferringVertex(), ctx.getVertexProperty(), ret); + + return ret; } private AtlasEdge mapStructValue(AttributeMutationContext ctx, EntityMutationContext context) throws AtlasBaseException {
